feat: freeze all python code, restructure build

Add all python code run on the device to the manifest.py to freeze
it. This has two benefits:
- It precompiles the bytecode, so it is smaller and faster
- All code (C and python) is now in the firmware image, leaving the
  littlefs filesystem on the flash free for user settings.

This requires changing the way the hardware variants are handled. There
is now only one _filesystem_ image, but instead there are different
_firmware_ images for each variant.

Signed-off-by: Matthias Blankertz <matthias@blankertz.org>

software/boards/RPI_PICO_W/manifest-Rev1.py

@@ -0,0 +1,3 @@
+include("manifest.py")
+
+module("hwconfig.py", "../../src/hwconfig_Rev1")

software/boards/RPI_PICO_W/manifest-breadboard.py

@@ -0,0 +1,3 @@
+include("manifest.py")
+
+module("hwconfig.py", "../../src/hwconfig_breadboard")

software/boards/RPI_PICO_W/manifest.py

@@ -15,3 +15,10 @@ module("microdot.py", "../../lib/microdot/src/microdot/")
 # TonberryPico modules
 module("audiocore.py", "../../modules/audiocore")
 module("rp2_neopixel.py", "../../modules")
+
+module("main.py", "../../src")
+module("app.py", "../../src")
+module("mp3player.py", "../../src")
+module("webserver.py", "../../src")
+package("utils", base_path="../../src")
+package("nfc", base_path="../../src")

software/build.sh

@@ -6,9 +6,6 @@ set -eu
 
 ( cd lib/micropython
   make -C mpy-cross -j "$(nproc)"
-  make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W clean
-  make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W \
-       USER_C_MODULES="$TOPDIR"/modules/micropython.cmake -j "$(nproc)"
 
   # build tonberry specific unix port of micropython
   make -C ports/unix VARIANT_DIR="$TOPDIR"/boards/tonberry_unix clean
@@ -19,31 +16,40 @@ set -eu
   make -j "$(nproc)"
 )
 
-PICOTOOL=picotool
-if ! command -v $PICOTOOL >/dev/null 2>&1; then
-    echo "system picotool not found, checking SDK build dir"
-    PICOTOOL=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/_deps/picotool-build/picotool
-    if ! command -v $PICOTOOL >/dev/null 2>&1; then
-       echo "No picotool found, exiting"
-       exit 1
-    fi
-fi
 BUILDDIR=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/
+OUTDIR=$(pwd)/build
+mkdir -p "$OUTDIR"
 FS_STAGE_DIR=$(mktemp -d)
+mkdir  "$FS_STAGE_DIR"/fs
 trap 'rm -rf $FS_STAGE_DIR' EXIT
-for hwconfig in src/hwconfig_*.py; do
+tools/mklittlefs/mklittlefs -p 256 -s 868352 -c "$FS_STAGE_DIR"/fs "$FS_STAGE_DIR"/filesystem.bin
+
+for hwconfig in boards/RPI_PICO_W/manifest-*.py; do
     hwconfig_base=$(basename "$hwconfig")
-    hwname=${hwconfig_base##hwconfig_}
+    hwname=${hwconfig_base##manifest-}
     hwname=${hwname%%.py}
-    find src/ -iname '*.py' \! -iname 'hwconfig_*.py' | cpio -pdm "$FS_STAGE_DIR"
+    hwconfig_abs=$(realpath "$hwconfig")
-    cp "$hwconfig" "$FS_STAGE_DIR"/src/hwconfig.py
+    ( cd lib/micropython
-    tools/mklittlefs/mklittlefs -p 256 -s 868352 -c "$FS_STAGE_DIR"/src $BUILDDIR/filesystem.bin
+      make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W clean
+      make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W \
+           USER_C_MODULES="$TOPDIR"/modules/micropython.cmake \
+           FROZEN_MANIFEST="$hwconfig_abs" -j "$(nproc)"
+    )
+    PICOTOOL=picotool
+    if ! command -v $PICOTOOL >/dev/null 2>&1; then
+        echo "system picotool not found, checking SDK build dir"
+        PICOTOOL=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/_deps/picotool-build/picotool
+        if ! command -v $PICOTOOL >/dev/null 2>&1; then
+            echo "No picotool found, exiting"
+            exit 1
+        fi
+    fi
     truncate -s 2M $BUILDDIR/firmware-filesystem.bin
     dd if=$BUILDDIR/firmware.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k
-    dd if=$BUILDDIR/filesystem.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k seek=1200
+    dd if="$FS_STAGE_DIR"/filesystem.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k seek=1200
-    $PICOTOOL uf2 convert $BUILDDIR/firmware-filesystem.bin $BUILDDIR/firmware-filesystem-"$hwname".uf2
+    cp $BUILDDIR/firmware.uf2 "$OUTDIR"/firmware-"$hwname".uf2
-    rm -r "${FS_STAGE_DIR:?}"/*
+    $PICOTOOL uf2 convert $BUILDDIR/firmware-filesystem.bin "$OUTDIR"/firmware-filesystem-"$hwname".uf2
 done
 
-echo "Output in $BUILDDIR/firmware.uf2"
+echo "Output in" "${OUTDIR}"/firmware-*.uf2
-echo "Images with filesystem in" ${BUILDDIR}firmware-filesystem-*.uf2
+echo "Images with filesystem in" "${OUTDIR}"/firmware-filesystem-*.uf2

software/flash.sh

@@ -2,6 +2,8 @@
 
 set -eu
 
+TOPDIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
 check_command()
 {
     name=$1
@@ -16,7 +18,7 @@ check_command lsusb
 check_command picotool
 
 DEVICEPATH=/dev/disk/by-label/RPI-RP2
-IMAGEPATH=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/
+IMAGEPATH=${TOPDIR}/build
 REVISION=Rev1
 
 flash_via_mountpoint()
@@ -83,7 +85,7 @@ if [ $# -gt 0 ]; then
     usage
 fi
 
-IMAGEFILE="$IMAGEPATH"/firmware-filesystem-$REVISION.uf2
+IMAGEFILE="$IMAGEPATH"/firmware-$REVISION.uf2
 
 if [ "$FLASH_VIA_MOUNTPOINT" -eq 0 ]; then
     flash_via_picotool