feat: freeze all python code, restructure build

Add all python code run on the device to the manifest.py to freeze
it. This has two benefits:
- It precompiles the bytecode, so it is smaller and faster
- All code (C and python) is now in the firmware image, leaving the
  littlefs filesystem on the flash free for user settings.

This requires changing the way the hardware variants are handled. There
is now only one _filesystem_ image, but instead there are different
_firmware_ images for each variant.

Signed-off-by: Matthias Blankertz <matthias@blankertz.org>

software/boards/RPI_PICO_W/manifest-Rev1.py

@@ -0,0 +1,3 @@
+include("manifest.py")
+
+module("hwconfig.py", "../../src/hwconfig_Rev1")

software/boards/RPI_PICO_W/manifest-breadboard.py

@@ -0,0 +1,3 @@
+include("manifest.py")
+
+module("hwconfig.py", "../../src/hwconfig_breadboard")

software/boards/RPI_PICO_W/manifest.py

@@ -15,3 +15,10 @@ module("microdot.py", "../../lib/microdot/src/microdot/")
 # TonberryPico modules
 module("audiocore.py", "../../modules/audiocore")
 module("rp2_neopixel.py", "../../modules")
+
+module("main.py", "../../src")
+module("app.py", "../../src")
+module("mp3player.py", "../../src")
+module("webserver.py", "../../src")
+package("utils", base_path="../../src")
+package("nfc", base_path="../../src")

software/build.sh

@@ -6,9 +6,6 @@ set -eu
 
 ( cd lib/micropython
   make -C mpy-cross -j "$(nproc)"
-  make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W clean
-  make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W \
-       USER_C_MODULES="$TOPDIR"/modules/micropython.cmake -j "$(nproc)"
 
   # build tonberry specific unix port of micropython
   make -C ports/unix VARIANT_DIR="$TOPDIR"/boards/tonberry_unix clean
@@ -19,6 +16,25 @@ set -eu
   make -j "$(nproc)"
 )
 
+BUILDDIR=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/
+OUTDIR=$(pwd)/build
+mkdir -p "$OUTDIR"
+FS_STAGE_DIR=$(mktemp -d)
+mkdir  "$FS_STAGE_DIR"/fs
+trap 'rm -rf $FS_STAGE_DIR' EXIT
+tools/mklittlefs/mklittlefs -p 256 -s 868352 -c "$FS_STAGE_DIR"/fs "$FS_STAGE_DIR"/filesystem.bin
+
+for hwconfig in boards/RPI_PICO_W/manifest-*.py; do
+    hwconfig_base=$(basename "$hwconfig")
+    hwname=${hwconfig_base##manifest-}
+    hwname=${hwname%%.py}
+    hwconfig_abs=$(realpath "$hwconfig")
+    ( cd lib/micropython
+      make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W clean
+      make -C ports/rp2 BOARD=TONBERRY_RPI_PICO_W BOARD_DIR="$TOPDIR"/boards/RPI_PICO_W \
+           USER_C_MODULES="$TOPDIR"/modules/micropython.cmake \
+           FROZEN_MANIFEST="$hwconfig_abs" -j "$(nproc)"
+    )
     PICOTOOL=picotool
     if ! command -v $PICOTOOL >/dev/null 2>&1; then
         echo "system picotool not found, checking SDK build dir"
@@ -28,22 +44,12 @@ if ! command -v $PICOTOOL >/dev/null 2>&1; then
             exit 1
         fi
     fi
-BUILDDIR=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/
-FS_STAGE_DIR=$(mktemp -d)
-trap 'rm -rf $FS_STAGE_DIR' EXIT
-for hwconfig in src/hwconfig_*.py; do
-    hwconfig_base=$(basename "$hwconfig")
-    hwname=${hwconfig_base##hwconfig_}
-    hwname=${hwname%%.py}
-    find src/ -iname '*.py' \! -iname 'hwconfig_*.py' | cpio -pdm "$FS_STAGE_DIR"
-    cp "$hwconfig" "$FS_STAGE_DIR"/src/hwconfig.py
-    tools/mklittlefs/mklittlefs -p 256 -s 868352 -c "$FS_STAGE_DIR"/src $BUILDDIR/filesystem.bin
     truncate -s 2M $BUILDDIR/firmware-filesystem.bin
     dd if=$BUILDDIR/firmware.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k
-    dd if=$BUILDDIR/filesystem.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k seek=1200
-    $PICOTOOL uf2 convert $BUILDDIR/firmware-filesystem.bin $BUILDDIR/firmware-filesystem-"$hwname".uf2
-    rm -r "${FS_STAGE_DIR:?}"/*
+    dd if="$FS_STAGE_DIR"/filesystem.bin of=$BUILDDIR/firmware-filesystem.bin bs=1k seek=1200
+    cp $BUILDDIR/firmware.uf2 "$OUTDIR"/firmware-"$hwname".uf2
+    $PICOTOOL uf2 convert $BUILDDIR/firmware-filesystem.bin "$OUTDIR"/firmware-filesystem-"$hwname".uf2
 done
 
-echo "Output in $BUILDDIR/firmware.uf2"
-echo "Images with filesystem in" ${BUILDDIR}firmware-filesystem-*.uf2
+echo "Output in" "${OUTDIR}"/firmware-*.uf2
+echo "Images with filesystem in" "${OUTDIR}"/firmware-filesystem-*.uf2

software/flash.sh

@@ -2,6 +2,8 @@
 
 set -eu
 
+TOPDIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
 check_command()
 {
     name=$1
@@ -16,7 +18,7 @@ check_command lsusb
 check_command picotool
 
 DEVICEPATH=/dev/disk/by-label/RPI-RP2
-IMAGEPATH=lib/micropython/ports/rp2/build-TONBERRY_RPI_PICO_W/
+IMAGEPATH=${TOPDIR}/build
 REVISION=Rev1
 
 flash_via_mountpoint()
@@ -83,7 +85,7 @@ if [ $# -gt 0 ]; then
     usage
 fi
 
-IMAGEFILE="$IMAGEPATH"/firmware-filesystem-$REVISION.uf2
+IMAGEFILE="$IMAGEPATH"/firmware-$REVISION.uf2
 
 if [ "$FLASH_VIA_MOUNTPOINT" -eq 0 ]; then
     flash_via_picotool