From 23deac138b2dbe711b79c28af8ee629e8290ffbd Mon Sep 17 00:00:00 2001 From: Austin Lane Date: Tue, 7 Oct 2025 12:01:13 -0400 Subject: [PATCH] Meshtastic build manifest --- .clusterfuzzlite/build.sh | 2 +- .github/actions/build-variant/action.yml | 2 +- .github/workflows/build_firmware.yml | 18 +- .github/workflows/build_one_arch.yml | 4 +- .github/workflows/build_one_target.yml | 2 +- .github/workflows/main_matrix.yml | 25 +-- .github/workflows/merge_queue.yml | 21 +-- .github/workflows/pr_tests.yml | 2 +- .github/workflows/test_native.yml | 14 +- .github/workflows/tests.yml | 2 +- arch/esp32/esp32.ini | 6 + arch/nrf52/nrf52.ini | 4 + arch/stm32/stm32.ini | 2 +- bin/build-esp32.sh | 29 ++-- bin/build-native.sh | 12 +- bin/build-nrf52.sh | 49 +++--- bin/build-rp2xx0.sh | 19 ++- bin/build-stm32wl.sh | 19 ++- bin/device-install.bat | 154 +++++------------- bin/device-install.sh | 150 +++++------------ bin/device-update.bat | 10 +- bin/device-update.sh | 4 +- bin/native-gdbserver.sh | 2 +- bin/native-run.sh | 2 +- bin/platformio-custom.py | 147 ++++++++--------- bin/platformio-pre.py | 16 ++ bin/test-simulator.sh | 2 +- debian/rules | 1 - extra_scripts/disable_adafruit_usb.py | 3 +- extra_scripts/esp32_extra.py | 80 +++++++++ extra_scripts/esp32_pre.py | 73 +++++++++ extra_scripts/nrf52_extra.py | 50 ++++++ .../{extra_stm32.py => stm32_extra.py} | 2 + meshtasticd.spec.rpkg | 2 +- platformio.ini | 4 +- variants/native/portduino/platformio.ini | 2 +- .../nrf52840/wio-sdk-wm1110/platformio.ini | 2 +- 37 files changed, 514 insertions(+), 424 deletions(-) create mode 100644 bin/platformio-pre.py create mode 100755 extra_scripts/esp32_extra.py create mode 100755 extra_scripts/esp32_pre.py create mode 100755 extra_scripts/nrf52_extra.py rename extra_scripts/{extra_stm32.py => stm32_extra.py} (95%) diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 10a2db0bd..86ab775f9 100644 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -51,7 +51,7 @@ for f in .clusterfuzzlite/*_fuzzer.cpp; do fuzzer=$(basename "$f" .cpp) cp -f "$f" src/fuzzer.cpp pio run -vvv --environment "$PIO_ENV" - program="$PLATFORMIO_WORKSPACE_DIR/build/$PIO_ENV/program" + program="$PLATFORMIO_WORKSPACE_DIR/build/$PIO_ENV/meshtasticd" cp "$program" "$OUT/$fuzzer" # Copy shared libraries used by the fuzzer. diff --git a/.github/actions/build-variant/action.yml b/.github/actions/build-variant/action.yml index f611908ee..71016f8ad 100644 --- a/.github/actions/build-variant/action.yml +++ b/.github/actions/build-variant/action.yml @@ -102,7 +102,7 @@ runs: - name: Store binaries as an artifact uses: actions/upload-artifact@v4 with: - name: firmware-${{ inputs.arch }}-${{ inputs.board }}-${{ steps.version.outputs.long }}.zip + name: firmware-${{ inputs.arch }}-${{ inputs.board }}-${{ steps.version.outputs.long }} overwrite: true path: | ${{ inputs.artifact-paths }} diff --git a/.github/workflows/build_firmware.yml b/.github/workflows/build_firmware.yml index b62729332..40693bfee 100644 --- a/.github/workflows/build_firmware.yml +++ b/.github/workflows/build_firmware.yml @@ -55,15 +55,29 @@ jobs: ota_firmware_source: ${{ steps.ota_dir.outputs.src || '' }} ota_firmware_target: ${{ steps.ota_dir.outputs.tgt || '' }} + - name: Echo manifest from release/firmware-*.mt.json to job summary + if: ${{ always() }} + env: + PIO_ENV: ${{ inputs.pio_env }} + run: | + echo "## Manifest: \`$PIO_ENV\`" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + cat release/firmware-*.mt.json >> $GITHUB_STEP_SUMMARY + echo '' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + - name: Store binaries as an artifact uses: actions/upload-artifact@v4 id: upload with: - name: firmware-${{ inputs.platform }}-${{ inputs.pio_env }}-${{ inputs.version }}.zip + name: firmware-${{ inputs.platform }}-${{ inputs.pio_env }}-${{ inputs.version }} overwrite: true path: | + release/*.mt.json release/*.bin release/*.elf release/*.uf2 release/*.hex - release/*-ota.zip + release/*.zip + release/device-*.sh + release/device-*.bat diff --git a/.github/workflows/build_one_arch.yml b/.github/workflows/build_one_arch.yml index 9c57f8b7d..a3f416711 100644 --- a/.github/workflows/build_one_arch.yml +++ b/.github/workflows/build_one_arch.yml @@ -134,7 +134,7 @@ jobs: ./firmware-*.bin ./firmware-*.uf2 ./firmware-*.hex - ./firmware-*-ota.zip + ./firmware-*.zip ./device-*.sh ./device-*.bat ./littlefs-*.bin @@ -163,7 +163,7 @@ jobs: - name: Repackage in single elfs zip uses: actions/upload-artifact@v4 with: - name: debug-elfs-${{inputs.arch}}-${{ needs.version.outputs.long }}.zip + name: debug-elfs-${{inputs.arch}}-${{ needs.version.outputs.long }} overwrite: true path: ./*.elf retention-days: 30 diff --git a/.github/workflows/build_one_target.yml b/.github/workflows/build_one_target.yml index 15b3fdba9..3891b4924 100644 --- a/.github/workflows/build_one_target.yml +++ b/.github/workflows/build_one_target.yml @@ -140,7 +140,7 @@ jobs: ./firmware-*.bin ./firmware-*.uf2 ./firmware-*.hex - ./firmware-*-ota.zip + ./firmware-*.zip ./device-*.sh ./device-*.bat ./littlefs-*.bin diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 02a4c23b8..ff7b00218 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -177,19 +177,17 @@ jobs: - name: Display structure of downloaded files run: ls -R - - name: Move files up - run: mv -b -t ./ ./bin/device-*.sh ./bin/device-*.bat - - name: Repackage in single firmware zip uses: actions/upload-artifact@v4 with: name: firmware-${{matrix.arch}}-${{ needs.version.outputs.long }} overwrite: true path: | + ./firmware-*.mt.json ./firmware-*.bin ./firmware-*.uf2 ./firmware-*.hex - ./firmware-*-ota.zip + ./firmware-*.zip ./device-*.sh ./device-*.bat ./littlefs-*.bin @@ -218,7 +216,7 @@ jobs: - name: Repackage in single elfs zip uses: actions/upload-artifact@v4 with: - name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip + name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }} overwrite: true path: ./*.elf retention-days: 30 @@ -244,11 +242,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - name: Create release uses: softprops/action-gh-release@v2 id: create_release @@ -313,11 +306,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - uses: actions/download-artifact@v5 with: pattern: firmware-${{matrix.arch}}-${{ needs.version.outputs.long }} @@ -337,7 +325,7 @@ jobs: - uses: actions/download-artifact@v5 with: - name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip + name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }} merge-multiple: true path: ./elfs @@ -368,11 +356,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - uses: actions/download-artifact@v5 with: pattern: firmware-{${{ env.targets }}}-${{ needs.version.outputs.long }} diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index e8c3d3450..07c9e2fa8 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -168,7 +168,7 @@ jobs: ./firmware-*.bin ./firmware-*.uf2 ./firmware-*.hex - ./firmware-*-ota.zip + ./firmware-*.zip ./device-*.sh ./device-*.bat ./littlefs-*.bin @@ -197,7 +197,7 @@ jobs: - name: Repackage in single elfs zip uses: actions/upload-artifact@v4 with: - name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip + name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }} overwrite: true path: ./*.elf retention-days: 30 @@ -223,11 +223,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - name: Create release uses: softprops/action-gh-release@v2 id: create_release @@ -292,11 +287,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - uses: actions/download-artifact@v5 with: pattern: firmware-${{matrix.arch}}-${{ needs.version.outputs.long }} @@ -316,7 +306,7 @@ jobs: - uses: actions/download-artifact@v5 with: - name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip + name: debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }} merge-multiple: true path: ./elfs @@ -347,11 +337,6 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.x - - uses: actions/download-artifact@v5 with: pattern: firmware-{${{ env.targets }}}-${{ needs.version.outputs.long }} diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 4e285852d..874fec881 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -52,7 +52,7 @@ jobs: if: needs.native-tests.result != 'skipped' uses: actions/download-artifact@v5 with: - name: platformio-test-report-${{ steps.version.outputs.long }}.zip + name: platformio-test-report-${{ steps.version.outputs.long }} merge-multiple: true - name: Parse test results and create detailed summary diff --git a/.github/workflows/test_native.yml b/.github/workflows/test_native.yml index 9cf1c9e53..d963e9785 100644 --- a/.github/workflows/test_native.yml +++ b/.github/workflows/test_native.yml @@ -40,7 +40,7 @@ jobs: - name: Integration test run: | - .pio/build/coverage/program -s & + .pio/build/coverage/meshtasticd -s & PID=$! timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done" echo "Simulator started, launching python test..." @@ -62,7 +62,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() # run this step even if previous step failed with: - name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.long }}.zip + name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.long }} overwrite: true path: ./coverage_*.info @@ -96,7 +96,7 @@ jobs: if: always() # run this step even if previous step failed uses: actions/upload-artifact@v4 with: - name: platformio-test-report-${{ steps.version.outputs.long }}.zip + name: platformio-test-report-${{ steps.version.outputs.long }} overwrite: true path: ./testreport.xml @@ -111,7 +111,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() # run this step even if previous step failed with: - name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.long }}.zip + name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.long }} overwrite: true path: ./coverage_*.info @@ -139,7 +139,7 @@ jobs: - name: Download test artifacts uses: actions/download-artifact@v5 with: - name: platformio-test-report-${{ steps.version.outputs.long }}.zip + name: platformio-test-report-${{ steps.version.outputs.long }} merge-multiple: true - name: Test Report @@ -152,7 +152,7 @@ jobs: - name: Download coverage artifacts uses: actions/download-artifact@v5 with: - pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.long }}.zip + pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.long }} path: code-coverage-report merge-multiple: true @@ -165,5 +165,5 @@ jobs: - name: Save Code Coverage Report uses: actions/upload-artifact@v4 with: - name: code-coverage-report-${{ steps.version.outputs.long }}.zip + name: code-coverage-report-${{ steps.version.outputs.long }} path: code-coverage-report diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 942659348..92300ad97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - # - uses: actions/setup-python@v5 + # - uses: actions/setup-python@v6 # with: # python-version: '3.10' diff --git a/arch/esp32/esp32.ini b/arch/esp32/esp32.ini index 810c9780e..9f3a97c53 100644 --- a/arch/esp32/esp32.ini +++ b/arch/esp32/esp32.ini @@ -2,10 +2,16 @@ [esp32_base] extends = arduino_base custom_esp32_kind = esp32 +custom_mtjson_part = platform = # renovate: datasource=custom.pio depName=platformio/espressif32 packageName=platformio/platform/espressif32 platformio/espressif32@6.11.0 +extra_scripts = + ${env.extra_scripts} + pre:extra_scripts/esp32_pre.py + extra_scripts/esp32_extra.py + build_src_filter = ${arduino_base.build_src_filter} - - - - - diff --git a/arch/nrf52/nrf52.ini b/arch/nrf52/nrf52.ini index 36effe017..71552fdd5 100644 --- a/arch/nrf52/nrf52.ini +++ b/arch/nrf52/nrf52.ini @@ -11,6 +11,10 @@ platform_packages = ; Don't renovate toolchain-gccarmnoneeabi platformio/toolchain-gccarmnoneeabi@~1.90301.0 +extra_scripts = + ${env.extra_scripts} + extra_scripts/nrf52_extra.py + build_type = debug build_flags = -include arch/nrf52/cpp_overrides/lfs_util.h diff --git a/arch/stm32/stm32.ini b/arch/stm32/stm32.ini index 8b7d256b3..8d3c4e056 100644 --- a/arch/stm32/stm32.ini +++ b/arch/stm32/stm32.ini @@ -8,7 +8,7 @@ platform_packages = platformio/framework-arduinoststm32@https://github.com/stm32duino/Arduino_Core_STM32/archive/2.10.1.zip extra_scripts = ${env.extra_scripts} - post:extra_scripts/extra_stm32.py + extra_scripts/stm32_extra.py build_type = release diff --git a/bin/build-esp32.sh b/bin/build-esp32.sh index 92836db23..8c684aa7e 100755 --- a/bin/build-esp32.sh +++ b/bin/build-esp32.sh @@ -5,7 +5,8 @@ set -e VERSION=`bin/buildinfo.py long` SHORT_VERSION=`bin/buildinfo.py short` -OUTDIR=release/ +BUILDDIR=.pio/build/$1 +OUTDIR=release rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true @@ -14,7 +15,7 @@ rm -r $OUTDIR/* || true platformio pkg install -e $1 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS" -rm -f .pio/build/$1/firmware.* +rm -f $BUILDDIR/firmware* # The shell vars the build tool expects to find export APP_VERSION=$VERSION @@ -22,16 +23,14 @@ export APP_VERSION=$VERSION basename=firmware-$1-$VERSION pio run --environment $1 # -v -SRCELF=.pio/build/$1/firmware.elf -cp $SRCELF $OUTDIR/$basename.elf + +cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf echo "Copying ESP32 bin file" -SRCBIN=.pio/build/$1/firmware.factory.bin -cp $SRCBIN $OUTDIR/$basename.bin +cp $BUILDDIR/$basename.factory.bin $OUTDIR/$basename.factory.bin echo "Copying ESP32 update bin file" -SRCBIN=.pio/build/$1/firmware.bin -cp $SRCBIN $OUTDIR/$basename-update.bin +cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin echo "Building Filesystem for ESP32 targets" # If you want to build the webui, uncomment the following lines @@ -40,7 +39,13 @@ echo "Building Filesystem for ESP32 targets" # # Remove webserver files from the filesystem and rebuild # ls -l data/static # Diagnostic list of files # rm -rf data/static -pio run --environment $1 -t buildfs -cp .pio/build/$1/littlefs.bin $OUTDIR/littlefs-$1-$VERSION.bin -cp bin/device-install.* $OUTDIR -cp bin/device-update.* $OUTDIR \ No newline at end of file +pio run --environment $1 -t buildfs --disable-auto-clean +cp $BUILDDIR/littlefs-$1-$VERSION.bin $OUTDIR/littlefs-$1-$VERSION.bin +cp bin/device-install.* $OUTDIR/ +cp bin/device-update.* $OUTDIR/ + +# Generate the manifest file +echo "Generating Meshtastic manifest" +TIMEFORMAT="Generated manifest in %E seconds" +time pio run --environment $1 -t mtjson --silent --disable-auto-clean +cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json diff --git a/bin/build-native.sh b/bin/build-native.sh index fff86e87e..f35e46a87 100755 --- a/bin/build-native.sh +++ b/bin/build-native.sh @@ -17,15 +17,19 @@ VERSION=$(bin/buildinfo.py long) SHORT_VERSION=$(bin/buildinfo.py short) PIO_ENV=${1:-native} -OUTDIR=release/ +BUILDDIR=.pio/build/$PIO_ENV +OUTDIR=release -rm -f $OUTDIR/firmware* +rm -f $OUTDIR/meshtasticd* mkdir -p $OUTDIR/ rm -r $OUTDIR/* || true +basename=meshtasticd-$1-$VERSION + # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale pio pkg install --environment "$PIO_ENV" || platformioFailed pio run --environment "$PIO_ENV" || platformioFailed -cp ".pio/build/$PIO_ENV/program" "$OUTDIR/meshtasticd_linux_$(uname -m)" -cp bin/native-install.* $OUTDIR + +cp "$BUILDDIR/meshtasticd" "$OUTDIR/meshtasticd_linux_$(uname -m)" +cp bin/native-install.* $OUTDIR/ diff --git a/bin/build-nrf52.sh b/bin/build-nrf52.sh index deca209d2..c605fb1e0 100755 --- a/bin/build-nrf52.sh +++ b/bin/build-nrf52.sh @@ -5,7 +5,8 @@ set -e VERSION=$(bin/buildinfo.py long) SHORT_VERSION=$(bin/buildinfo.py short) -OUTDIR=release/ +BUILDDIR=.pio/build/$1 +OUTDIR=release rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true @@ -14,7 +15,7 @@ rm -r $OUTDIR/* || true platformio pkg install -e $1 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS" -rm -f .pio/build/$1/firmware.* +rm -f $BUILDDIR/firmware* # The shell vars the build tool expects to find export APP_VERSION=$VERSION @@ -22,32 +23,32 @@ export APP_VERSION=$VERSION basename=firmware-$1-$VERSION pio run --environment $1 # -v -SRCELF=.pio/build/$1/firmware.elf -cp $SRCELF $OUTDIR/$basename.elf -echo "Generating NRF52 dfu file" -DFUPKG=.pio/build/$1/firmware.zip -cp $DFUPKG $OUTDIR/$basename-ota.zip +cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf -echo "Generating NRF52 uf2 file" -SRCHEX=.pio/build/$1/firmware.hex +echo "Copying NRF52 dfu (OTA) file" +cp $BUILDDIR/$basename.zip $OUTDIR/$basename.zip -# if WM1110 target, merge hex with softdevice 7.3.0 +echo "Copying NRF52 UF2 file" +cp $BUILDDIR/$basename.uf2 $OUTDIR/$basename.uf2 +cp bin/*.uf2 $OUTDIR/ + +SRCHEX=$BUILDDIR/$basename.hex + +# if WM1110 target, copy the merged.hex if (echo $1 | grep -q "wio-sdk-wm1110"); then - echo "Merging with softdevice" - bin/mergehex -m bin/s140_nrf52_7.3.0_softdevice.hex $SRCHEX -o .pio/build/$1/$basename.hex - SRCHEX=.pio/build/$1/$basename.hex - bin/uf2conv.py $SRCHEX -c -o $OUTDIR/$basename.uf2 -f 0xADA52840 - cp $SRCHEX $OUTDIR - cp bin/*.uf2 $OUTDIR -else - bin/uf2conv.py $SRCHEX -c -o $OUTDIR/$basename.uf2 -f 0xADA52840 - cp bin/device-install.* $OUTDIR - cp bin/device-update.* $OUTDIR - cp bin/*.uf2 $OUTDIR + echo "Copying .merged.hex file" + SRCHEX=$BUILDDIR/$basename.merged.hex + cp $SRCHEX $OUTDIR/ fi if (echo $1 | grep -q "rak4631"); then - echo "Copying hex file" - cp .pio/build/$1/firmware.hex $OUTDIR/$basename.hex -fi \ No newline at end of file + echo "Copying .hex file" + cp $SRCHEX $OUTDIR/ +fi + +# Generate the manifest file +echo "Generating Meshtastic manifest" +TIMEFORMAT="Generated manifest in %E seconds" +time pio run --environment $1 -t mtjson --silent --disable-auto-clean +cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json diff --git a/bin/build-rp2xx0.sh b/bin/build-rp2xx0.sh index cb4865914..ae26fdfbf 100755 --- a/bin/build-rp2xx0.sh +++ b/bin/build-rp2xx0.sh @@ -5,7 +5,8 @@ set -e VERSION=`bin/buildinfo.py long` SHORT_VERSION=`bin/buildinfo.py short` -OUTDIR=release/ +BUILDDIR=.pio/build/$1 +OUTDIR=release rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true @@ -14,7 +15,7 @@ rm -r $OUTDIR/* || true platformio pkg install -e $1 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS" -rm -f .pio/build/$1/firmware.* +rm -f $BUILDDIR/firmware* # The shell vars the build tool expects to find export APP_VERSION=$VERSION @@ -22,12 +23,14 @@ export APP_VERSION=$VERSION basename=firmware-$1-$VERSION pio run --environment $1 # -v -SRCELF=.pio/build/$1/firmware.elf -cp $SRCELF $OUTDIR/$basename.elf + +cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf echo "Copying uf2 file" -SRCBIN=.pio/build/$1/firmware.uf2 -cp $SRCBIN $OUTDIR/$basename.uf2 +cp $BUILDDIR/$basename.uf2 $OUTDIR/$basename.uf2 -cp bin/device-install.* $OUTDIR -cp bin/device-update.* $OUTDIR +# Generate the manifest file +echo "Generating Meshtastic manifest" +TIMEFORMAT="Generated manifest in %E seconds" +time pio run --environment $1 -t mtjson --silent --disable-auto-clean +cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json diff --git a/bin/build-stm32wl.sh b/bin/build-stm32wl.sh index f62df4842..b85da04a6 100755 --- a/bin/build-stm32wl.sh +++ b/bin/build-stm32wl.sh @@ -5,7 +5,8 @@ set -e VERSION=$(bin/buildinfo.py long) SHORT_VERSION=$(bin/buildinfo.py short) -OUTDIR=release/ +BUILDDIR=.pio/build/$1 +OUTDIR=release rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true @@ -14,7 +15,7 @@ rm -r $OUTDIR/* || true platformio pkg install -e $1 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS" -rm -f .pio/build/$1/firmware.* +rm -f $BUILDDIR/firmware* # The shell vars the build tool expects to find export APP_VERSION=$VERSION @@ -22,8 +23,14 @@ export APP_VERSION=$VERSION basename=firmware-$1-$VERSION pio run --environment $1 # -v -SRCELF=.pio/build/$1/firmware.elf -cp $SRCELF $OUTDIR/$basename.elf -SRCBIN=.pio/build/$1/firmware.bin -cp $SRCBIN $OUTDIR/$basename.bin +cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf + +echo "Copying STM32 bin file" +cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin + +# Generate the manifest file +echo "Generating Meshtastic manifest" +TIMEFORMAT="Generated manifest in %E seconds" +time pio run --environment $1 -t mtjson --silent --disable-auto-clean +cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json diff --git a/bin/device-install.bat b/bin/device-install.bat index 9c206d718..c200a3201 100755 --- a/bin/device-install.bat +++ b/bin/device-install.bat @@ -5,22 +5,14 @@ TITLE Meshtastic device-install SET "SCRIPT_NAME=%~nx0" SET "DEBUG=0" SET "PYTHON=" -SET "TFT_BUILD=0" -SET "BIGDB8=0" -SET "MUIDB8=0" -SET "BIGDB16=0" SET "ESPTOOL_BAUD=115200" SET "ESPTOOL_CMD=" SET "LOGCOUNTER=0" SET "BPS_RESET=0" - -@REM FIXME: Determine mcu from PlatformIO variant, this is unmaintainable. -SET "S3=s3 v3 t-deck wireless-paper wireless-tracker station-g2 unphone t-eth-elite tlora-pager mesh-tab dreamcatcher ESP32-S3-Pico seeed-sensecap-indicator heltec_capsule_sensor_v3 vision-master icarus tracksenger elecrow-adv" -SET "C3=esp32c3" -@REM FIXME: Determine flash size from PlatformIO variant, this is unmaintainable. -SET "BIGDB_8MB=crowpanel-esp32s3 heltec_capsule_sensor_v3 heltec-v3 heltec-vision-master-e213 heltec-vision-master-e290 heltec-vision-master-t190 heltec-wireless-paper heltec-wireless-tracker heltec-wsl-v3 icarus seeed-xiao-s3 tbeam-s3-core tracksenger" -SET "MUIDB_8MB=picomputer-s3 unphone seeed-sensecap-indicator" -SET "BIGDB_16MB=t-deck mesh-tab t-energy-s3 dreamcatcher ESP32-S3-Pico m5stack-cores3 station-g2 t-eth-elite tlora-pager t-watch-s3 elecrow-adv" +@REM Default offsets. +@REM https://github.com/meshtastic/web-flasher/blob/main/stores/firmwareStore.ts#L202 +SET "OTA_OFFSET=0x260000" +SET "SPIFFS_OFFSET=0x300000" GOTO getopts :help @@ -29,7 +21,7 @@ ECHO. ECHO Usage: %SCRIPT_NAME% -f filename [-p PORT] [-P python] [--1200bps-reset] ECHO. ECHO Options: -ECHO -f filename The firmware .bin file to flash. Custom to your device type and region. (required) +ECHO -f filename The firmware .factory.bin file to flash. Custom to your device type and region. (required) ECHO The file must be located in this current directory. ECHO -p PORT Set the environment variable for ESPTOOL_PORT. ECHO If not set, ESPTOOL iterates all ports (Dangerous). @@ -40,12 +32,12 @@ ECHO --1200bps-reset Attempt to place the device in correct mode. (1200bps ECHO Some hardware requires this twice. ECHO. ECHO Example: %SCRIPT_NAME% -p COM17 --1200bps-reset -ECHO Example: %SCRIPT_NAME% -f firmware-t-deck-tft-2.6.0.0b106d4.bin -p COM11 -ECHO Example: %SCRIPT_NAME% -f firmware-unphone-2.6.0.0b106d4.bin -p COM11 +ECHO Example: %SCRIPT_NAME% -f firmware-t-deck-tft-2.6.0.0b106d4.factory.bin -p COM11 +ECHO Example: %SCRIPT_NAME% -f firmware-unphone-2.6.0.0b106d4.factory.bin -p COM11 GOTO eof :version -ECHO %SCRIPT_NAME% [Version 2.6.2] +ECHO %SCRIPT_NAME% [Version 2.7.0] ECHO Meshtastic GOTO eof @@ -78,8 +70,8 @@ IF "__!FILENAME!__"=="____" ( CALL :LOG_MESSAGE ERROR "Filename containing spaces are not supported." GOTO help ) - IF "__!FILENAME:firmware-=!__"=="__!FILENAME!__" ( - CALL :LOG_MESSAGE ERROR "Filename must be a firmware-* file." + IF NOT "__!FILENAME:.factory.bin=!__"=="__!FILENAME!__" ( + CALL :LOG_MESSAGE ERROR "Filename must be a firmware-*.factory.bin file." GOTO help ) @REM Remove ".\" or "./" file prefix if present. @@ -93,12 +85,26 @@ IF NOT EXIST !FILENAME! ( GOTO eof ) -IF NOT "!FILENAME:update=!"=="!FILENAME!" ( - CALL :LOG_MESSAGE DEBUG "We are working with a *update* file. !FILENAME!" - CALL :LOG_MESSAGE INFO "Use script device-update.bat to flash update !FILENAME!." - GOTO eof +CALL :LOG_MESSAGE DEBUG "Checking for metadata..." +@REM Derive metadata filename from firmware filename. +SET "METAFILE=!FILENAME:.factory.bin=!.mt.json" +IF EXIST !METAFILE! ( + @REM Print parsed json with powershell + CALL :LOG_MESSAGE INFO "Firmware metadata: !METAFILE!" + powershell -NoProfile -Command "(Get-Content '!METAFILE!' | ConvertFrom-Json | Out-String).Trim()" + + @REM Save metadata values to variables for later use. + FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^ + "(Get-Content '!METAFILE!' | ConvertFrom-Json).mcu"`) DO SET "MCU=%%A" + FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^ + "(Get-Content '!METAFILE!' | ConvertFrom-Json).part | Where-Object { $_.subtype -eq 'ota_1' } | Select-Object -ExpandProperty offset"` + ) DO SET "OTA_OFFSET=%%A" + FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^ + "(Get-Content '!METAFILE!' | ConvertFrom-Json).part | Where-Object { $_.subtype -eq 'spiffs' } | Select-Object -ExpandProperty offset"` + ) DO SET "SPIFFS_OFFSET=%%A" ) ELSE ( - CALL :LOG_MESSAGE DEBUG "We are NOT working with a *update* file. !FILENAME!" + CALL :LOG_MESSAGE ERROR "No metadata file found: !METAFILE!" + GOTO eof ) :skip-filename @@ -108,7 +114,7 @@ IF NOT "__%PYTHON%__"=="____" ( SET "ESPTOOL_CMD=!PYTHON! -m esptool" CALL :LOG_MESSAGE DEBUG "Python interpreter supplied." ) ELSE ( - CALL :LOG_MESSAGE DEBUG "Python interpreter NOT supplied. Looking for esptool... + CALL :LOG_MESSAGE DEBUG "Python interpreter NOT supplied. Looking for esptool..." WHERE esptool >nul 2>&1 IF %ERRORLEVEL% EQU 0 ( @REM WHERE exits with code 0 if esptool is found. @@ -146,100 +152,26 @@ IF %BPS_RESET% EQU 1 ( GOTO eof ) -@REM Check if FILENAME contains "-tft-" and set target partitionScheme accordingly. -@REM https://github.com/meshtastic/web-flasher/blob/main/types/resources.ts#L3 -IF NOT "!FILENAME:-tft-=!"=="!FILENAME!" ( - CALL :LOG_MESSAGE DEBUG "We are working with a *-tft-* file. !FILENAME!" - SET "TFT_BUILD=1" +@REM Extract PROGNAME from %FILENAME% for later use. +SET "PROGNAME=!FILENAME:.factory.bin=!" +CALL :LOG_MESSAGE DEBUG "Computed PROGNAME: !PROGNAME!" + +IF "__!MCU!__" == "__esp32s3__" ( + @REM We are working with ESP32-S3 + SET "OTA_FILENAME=bleota-s3.bin" +) ELSE IF "__!MCU!__" == "__esp32c3__" ( + @REM We are working with ESP32-C3 + SET "OTA_FILENAME=bleota-c3.bin" ) ELSE ( - CALL :LOG_MESSAGE DEBUG "We are NOT working with a *-tft-* file. !FILENAME!" + @REM Everything else + SET "OTA_FILENAME=bleota.bin" ) - -FOR %%a IN (%BIGDB_8MB%) DO ( - IF NOT "!FILENAME:%%a=!"=="!FILENAME!" ( - @REM We are working with any of %BIGDB_8MB%. - SET "BIGDB8=1" - GOTO end_loop_bigdb_8mb - ) -) -:end_loop_bigdb_8mb - -FOR %%a IN (%MUIDB_8MB%) DO ( - IF NOT "!FILENAME:%%a=!"=="!FILENAME!" ( - @REM We are working with any of %MUIDB_8MB%. - SET "MUIDB8=1" - GOTO end_loop_muidb_8mb - ) -) -:end_loop_muidb_8mb - -FOR %%a IN (%BIGDB_16MB%) DO ( - IF NOT "!FILENAME:%%a=!"=="!FILENAME!" ( - @REM We are working with any of %BIGDB_16MB%. - SET "BIGDB16=1" - GOTO end_loop_bigdb_16mb - ) -) -:end_loop_bigdb_16mb - -IF %BIGDB8% EQU 1 CALL :LOG_MESSAGE INFO "BigDB 8mb partition selected." -IF %MUIDB8% EQU 1 CALL :LOG_MESSAGE INFO "MUIDB 8mb partition selected." -IF %BIGDB16% EQU 1 CALL :LOG_MESSAGE INFO "BigDB 16mb partition selected." - -@REM Extract BASENAME from %FILENAME% for later use. -SET "BASENAME=!FILENAME:firmware-=!" -CALL :LOG_MESSAGE DEBUG "Computed firmware basename: !BASENAME!" - -@REM Account for S3 and C3 board's different OTA partition. -FOR %%a IN (%S3%) DO ( - IF NOT "!FILENAME:%%a=!"=="!FILENAME!" ( - @REM We are working with any of %S3%. - SET "OTA_FILENAME=bleota-s3.bin" - GOTO :end_loop_s3 - ) -) - -FOR %%a IN (%C3%) DO ( - IF NOT "!FILENAME:%%a=!"=="!FILENAME!" ( - @REM We are working with any of %C3%. - SET "OTA_FILENAME=bleota-c3.bin" - GOTO :end_loop_c3 - ) -) - -@REM Everything else -SET "OTA_FILENAME=bleota.bin" -:end_loop_s3 -:end_loop_c3 CALL :LOG_MESSAGE DEBUG "Set OTA_FILENAME to: !OTA_FILENAME!" @REM Set SPIFFS filename with "littlefs-" prefix. -SET "SPIFFS_FILENAME=littlefs-%BASENAME%" +SET "SPIFFS_FILENAME=littlefs-!PROGNAME:firmware-=!.bin" CALL :LOG_MESSAGE DEBUG "Set SPIFFS_FILENAME to: !SPIFFS_FILENAME!" -@REM Default offsets. -@REM https://github.com/meshtastic/web-flasher/blob/main/stores/firmwareStore.ts#L202 -SET "OTA_OFFSET=0x260000" -SET "SPIFFS_OFFSET=0x300000" - -@REM Offsets for BigDB 8mb. -IF %BIGDB8% EQU 1 ( - SET "OTA_OFFSET=0x340000" - SET "SPIFFS_OFFSET=0x670000" -) - -@REM Offsets for MUIDB 8mb. -IF %MUIDB8% EQU 1 ( - SET "OTA_OFFSET=0x5D0000" - SET "SPIFFS_OFFSET=0x670000" -) - -@REM Offsets for BigDB 16mb. -IF %BIGDB16% EQU 1 ( - SET "OTA_OFFSET=0x650000" - SET "SPIFFS_OFFSET=0xc90000" -) - CALL :LOG_MESSAGE DEBUG "Set OTA_OFFSET to: !OTA_OFFSET!" CALL :LOG_MESSAGE DEBUG "Set SPIFFS_OFFSET to: !SPIFFS_OFFSET!" diff --git a/bin/device-install.sh b/bin/device-install.sh index ede75bbba..1778a952d 100755 --- a/bin/device-install.sh +++ b/bin/device-install.sh @@ -2,67 +2,15 @@ PYTHON=${PYTHON:-$(which python3 python | head -n 1)} BPS_RESET=false -TFT_BUILD=false MCU="" # Constants RESET_BAUD=1200 FIRMWARE_OFFSET=0x00 - -# Variant groups -BIGDB_8MB=( - "crowpanel-esp32s3" - "heltec_capsule_sensor_v3" - "heltec-v3" - "heltec-vision-master-e213" - "heltec-vision-master-e290" - "heltec-vision-master-t190" - "heltec-wireless-paper" - "heltec-wireless-tracker" - "heltec-wsl-v3" - "icarus" - "seeed-xiao-s3" - "tbeam-s3-core" - "tracksenger" -) -MUIDB_8MB=( - "picomputer-s3" - "unphone" - "seeed-sensecap-indicator" -) -BIGDB_16MB=( - "t-deck" - "mesh-tab" - "t-energy-s3" - "dreamcatcher" - "ESP32-S3-Pico" - "m5stack-cores3" - "station-g2" - "t-eth-elite" - "tlora-pager" - "t-watch-s3" - "elecrow-adv" -) -S3_VARIANTS=( - "s3" - "-v3" - "t-deck" - "wireless-paper" - "wireless-tracker" - "station-g2" - "unphone" - "t-eth-elite" - "tlora-pager" - "mesh-tab" - "dreamcatcher" - "ESP32-S3-Pico" - "seeed-sensecap-indicator" - "heltec_capsule_sensor_v3" - "vision-master" - "icarus" - "tracksenger" - "elecrow-adv" -) +# Default littlefs* offset. +OFFSET=0x300000 +# Default OTA Offset +OTA_OFFSET=0x260000 # Determine the correct esptool command to use if "$PYTHON" -m esptool version >/dev/null 2>&1; then @@ -76,6 +24,14 @@ else exit 1 fi +# Check for jq +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq not found" >&2 + echo "Install jq with your package manager." >&2 + echo "e.g. 'apt install jq', 'dnf install jq', 'brew install jq', etc." >&2 + exit 1 +fi + set -e # Usage info @@ -87,7 +43,7 @@ Flash image file to device, but first erasing and writing system information. -h Display this help and exit. -p ESPTOOL_PORT Set the environment variable for ESPTOOL_PORT. If not set, ESPTOOL iterates all ports (Dangerous). -P PYTHON Specify alternate python interpreter to use to invoke esptool. (Default: "$PYTHON") - -f FILENAME The firmware .bin file to flash. Custom to your device type and region. + -f FILENAME The firmware *.factory.bin file to flash. Custom to your device type and region. --1200bps-reset Attempt to place the device in correct mode. Some hardware requires this twice. (1200bps Reset) EOF @@ -136,69 +92,43 @@ fi shift } -if [[ "$FILENAME" != firmware-* ]]; then - echo "Filename must be a firmware-* file." +if [[ $(basename "$FILENAME") != firmware-*.factory.bin ]]; then + echo "Filename must be a firmware-*.factory.bin file." exit 1 fi -# Check if FILENAME contains "-tft-" and set target partitionScheme accordingly. -if [[ "${FILENAME//-tft-/}" != "$FILENAME" ]]; then - TFT_BUILD=true -fi +# Extract PROGNAME from %FILENAME% for later use. +PROGNAME="${FILENAME/.factory.bin/}" +# Derive metadata filename from %PROGNAME%. +METAFILE="${PROGNAME}.mt.json" -# Extract BASENAME from %FILENAME% for later use. -BASENAME="${FILENAME/firmware-/}" - -if [ -f "${FILENAME}" ] && [ -n "${FILENAME##*"update"*}" ]; then - # Default littlefs* offset. - OFFSET=0x300000 - - # Default OTA Offset - OTA_OFFSET=0x260000 - - # littlefs* offset for BigDB 8mb and OTA OFFSET. - for variant in "${BIGDB_8MB[@]}"; do - if [ -z "${FILENAME##*"$variant"*}" ]; then - OFFSET=0x670000 - OTA_OFFSET=0x340000 - fi - done - - for variant in "${MUIDB_8MB[@]}"; do - if [ -z "${FILENAME##*"$variant"*}" ]; then - OFFSET=0x670000 - OTA_OFFSET=0x5D0000 - fi - done - - # littlefs* offset for BigDB 16mb and OTA OFFSET. - for variant in "${BIGDB_16MB[@]}"; do - if [ -z "${FILENAME##*"$variant"*}" ]; then - OFFSET=0xc90000 - OTA_OFFSET=0x650000 - fi - done - - # Account for S3 board's different OTA partition - # FIXME: Use PlatformIO info to determine MCU type, this is unmaintainable - for variant in "${S3_VARIANTS[@]}"; do - if [ -z "${FILENAME##*"$variant"*}" ]; then - MCU="esp32s3" - fi - done - - if [ "$MCU" != "esp32s3" ]; then - if [ -n "${FILENAME##*"esp32c3"*}" ]; then - OTAFILE=bleota.bin - else - OTAFILE=bleota-c3.bin +if [[ -f "$FILENAME" && "$FILENAME" == *.factory.bin ]]; then + # Display metadata if it exists + if [[ -f "$METAFILE" ]]; then + echo "Firmware metadata: ${METAFILE}" + jq . "$METAFILE" + # Extract relevant fields from metadata + if [[ $(jq -r '.part' "$METAFILE") != "null" ]]; then + OTA_OFFSET=$(jq -r '.part[] | select(.subtype == "ota_1") | .offset' "$METAFILE") + SPIFFS_OFFSET=$(jq -r '.part[] | select(.subtype == "spiffs") | .offset' "$METAFILE") fi + MCU=$(jq -r '.mcu' "$METAFILE") else + echo "ERROR: No metadata file found at ${METAFILE}" + exit 1 + fi + + # Determine OTA filename based on MCU type + if [ "$MCU" == "esp32s3" ]; then OTAFILE=bleota-s3.bin + elif [ "$MCU" == "esp32c3" ]; then + OTAFILE=bleota-c3.bin + else + OTAFILE=bleota.bin fi # Set SPIFFS filename with "littlefs-" prefix. - SPIFFSFILE=littlefs-${BASENAME} + SPIFFSFILE="littlefs-${PROGNAME/firmware-/}.bin" if [[ ! -f "$FILENAME" ]]; then echo "Error: file ${FILENAME} wasn't found. Terminating." diff --git a/bin/device-update.bat b/bin/device-update.bat index a263da992..a9f7a9e1e 100755 --- a/bin/device-update.bat +++ b/bin/device-update.bat @@ -30,11 +30,11 @@ ECHO --change-mode Attempt to place the device in correct mode. (1200bps ECHO Some hardware requires this twice. ECHO. ECHO Example: %SCRIPT_NAME% -p COM17 --change-mode -ECHO Example: %SCRIPT_NAME% -f firmware-t-deck-tft-2.6.0.0b106d4-update.bin -p COM11 +ECHO Example: %SCRIPT_NAME% -f firmware-t-deck-tft-2.6.0.0b106d4.bin -p COM11 GOTO eof :version -ECHO %SCRIPT_NAME% [Version 2.6.2] +ECHO %SCRIPT_NAME% [Version 2.7.0] ECHO Meshtastic GOTO eof @@ -78,12 +78,12 @@ IF NOT EXIST !FILENAME! ( GOTO eof ) -IF "!FILENAME:update=!"=="!FILENAME!" ( - CALL :LOG_MESSAGE DEBUG "We are NOT working with a *update* file. !FILENAME!" +IF NOT "__!FILENAME:.factory.bin=!__"=="__!FILENAME!__" ( + CALL :LOG_MESSAGE DEBUG "We are working with a *.factory.bin* file. !FILENAME!" CALL :LOG_MESSAGE INFO "Use script device-install.bat to flash !FILENAME!." GOTO eof ) ELSE ( - CALL :LOG_MESSAGE DEBUG "We are working with a *update* file. !FILENAME!" + CALL :LOG_MESSAGE DEBUG "We are not working with a *.factory.bin* file. !FILENAME!" ) :skip-filename diff --git a/bin/device-update.sh b/bin/device-update.sh index f64280a5b..1c3d6be70 100755 --- a/bin/device-update.sh +++ b/bin/device-update.sh @@ -29,7 +29,7 @@ Flash image file to device, leave existing system intact." -h Display this help and exit -p ESPTOOL_PORT Set the environment variable for ESPTOOL_PORT. If not set, ESPTOOL iterates all ports (Dangerous). -P PYTHON Specify alternate python interpreter to use to invoke esptool. (Default: "$PYTHON") - -f FILENAME The *update.bin file to flash. Custom to your device type. + -f FILENAME The *.bin file to flash. Custom to your device type. --change-mode Attempt to place the device in correct mode. Some hardware requires this twice. (1200bps Reset) EOF @@ -78,7 +78,7 @@ fi shift } -if [ -f "${FILENAME}" ] && [ -z "${FILENAME##*"update"*}" ]; then +if [[ -f "$FILENAME" && "$FILENAME" != *.factory.bin ]]; then echo "Trying to flash update ${FILENAME}" $ESPTOOL_CMD --baud $FLASH_BAUD write-flash $UPDATE_OFFSET "${FILENAME}" else diff --git a/bin/native-gdbserver.sh b/bin/native-gdbserver.sh index f779d6670..a45a2dc26 100755 --- a/bin/native-gdbserver.sh +++ b/bin/native-gdbserver.sh @@ -2,4 +2,4 @@ set -e pio run --environment native -gdbserver --once localhost:2345 .pio/build/native/program "$@" +gdbserver --once localhost:2345 .pio/build/native/meshtasticd "$@" diff --git a/bin/native-run.sh b/bin/native-run.sh index 6566fc591..a8309c2d3 100755 --- a/bin/native-run.sh +++ b/bin/native-run.sh @@ -2,4 +2,4 @@ set -e pio run --environment native -.pio/build/native/program "$@" +.pio/build/native/meshtasticd "$@" diff --git a/bin/platformio-custom.py b/bin/platformio-custom.py index 4a1887d9d..151cf0a97 100644 --- a/bin/platformio-custom.py +++ b/bin/platformio-custom.py @@ -2,98 +2,77 @@ # trunk-ignore-all(ruff/F821) # trunk-ignore-all(flake8/F821): For SConstruct imports import sys -from os.path import join +from os.path import join, basename, isfile import subprocess import json import re -import time from datetime import datetime from readprops import readProps Import("env") platform = env.PioPlatform() +progname = env.get("PROGNAME") +lfsbin = f"{progname.replace('firmware-', 'littlefs-')}.bin" - -def esp32_create_combined_bin(source, target, env): - # this sub is borrowed from ESPEasy build toolchain. It's licensed under GPL V3 - # https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py - print("Generating combined binary for serial flashing") - - app_offset = 0x10000 - - new_file_name = env.subst("$BUILD_DIR/${PROGNAME}.factory.bin") - sections = env.subst(env.get("FLASH_EXTRA_IMAGES")) - firmware_name = env.subst("$BUILD_DIR/${PROGNAME}.bin") - chip = env.get("BOARD_MCU") - flash_size = env.BoardConfig().get("upload.flash_size") - flash_freq = env.BoardConfig().get("build.f_flash", "40m") - flash_freq = flash_freq.replace("000000L", "m") - flash_mode = env.BoardConfig().get("build.flash_mode", "dio") - memory_type = env.BoardConfig().get("build.arduino.memory_type", "qio_qspi") - if flash_mode == "qio" or flash_mode == "qout": - flash_mode = "dio" - if memory_type == "opi_opi" or memory_type == "opi_qspi": - flash_mode = "dout" - cmd = [ - "--chip", - chip, - "merge_bin", - "-o", - new_file_name, - "--flash_mode", - flash_mode, - "--flash_freq", - flash_freq, - "--flash_size", - flash_size, +def manifest_gather(source, target, env): + out = [] + check_paths = [ + progname, + f"{progname}.elf", + f"{progname}.bin", + f"{progname}.factory.bin", + f"{progname}.hex", + f"{progname}.merged.hex", + f"{progname}.uf2", + f"{progname}.factory.uf2", + f"{progname}.zip", + lfsbin ] + for p in check_paths: + f = env.File(env.subst(f"$BUILD_DIR/{p}")) + if f.exists(): + d = { + "name": p, + "md5": f.get_content_hash(), # Returns MD5 hash + "bytes": f.get_size() # Returns file size in bytes + } + out.append(d) + print(d) + manifest_write(out, env) - print(" Offset | File") - for section in sections: - sect_adr, sect_file = section.split(" ", 1) - print(f" - {sect_adr} | {sect_file}") - cmd += [sect_adr, sect_file] +def manifest_write(files, env): + manifest = { + "version": verObj["long"], + "build_epoch": build_epoch, + "board": env.get("PIOENV"), + "mcu": env.get("BOARD_MCU"), + "repo": repo_owner, + "files": files, + "part": None, + "has_mui": False, + "has_inkhud": False, + } + # Get partition table (generated in esp32_pre.py) if it exists + if env.get("custom_mtjson_part"): + # custom_mtjson_part is a JSON string, convert it back to a dict + pj = json.loads(env.get("custom_mtjson_part")) + manifest["part"] = pj + # Enable has_mui for TFT builds + if ("HAS_TFT", 1) in env.get("CPPDEFINES", []): + manifest["has_mui"] = True + if "MESHTASTIC_INCLUDE_INKHUD" in env.get("CPPDEFINES", []): + manifest["has_inkhud"] = True - print(f" - {hex(app_offset)} | {firmware_name}") - cmd += [hex(app_offset), firmware_name] - - print("Using esptool.py arguments: %s" % " ".join(cmd)) - - esptool.main(cmd) - - -if platform.name == "espressif32": - sys.path.append(join(platform.get_package_dir("tool-esptoolpy"))) - import esptool - - env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) - - esp32_kind = env.GetProjectOption("custom_esp32_kind") - if esp32_kind == "esp32": - # Free up some IRAM by removing auxiliary SPI flash chip drivers. - # Wrapped stub symbols are defined in src/platform/esp32/iram-quirk.c. - env.Append( - LINKFLAGS=[ - "-Wl,--wrap=esp_flash_chip_gd", - "-Wl,--wrap=esp_flash_chip_issi", - "-Wl,--wrap=esp_flash_chip_winbond", - ] - ) - else: - # For newer ESP32 targets, using newlib nano works better. - env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"]) - -if platform.name == "nordicnrf52": - env.AddPostAction("$BUILD_DIR/${PROGNAME}.hex", - env.VerboseAction(f"\"{sys.executable}\" ./bin/uf2conv.py \"$BUILD_DIR/firmware.hex\" -c -f 0xADA52840 -o \"$BUILD_DIR/firmware.uf2\"", - "Generating UF2 file")) + # Write the manifest to the build directory + with open(env.subst("$BUILD_DIR/${PROGNAME}.mt.json"), "w") as f: + json.dump(manifest, f, indent=2) Import("projenv") prefsLoc = projenv["PROJECT_DIR"] + "/version.properties" verObj = readProps(prefsLoc) -print("Using meshtastic platformio-custom.py, firmware version " + verObj["long"] + " on " + env.get("PIOENV")) +print(f"Using meshtastic platformio-custom.py, firmware version {verObj['long']} on {env.get('PIOENV')}") # get repository owner if git is installed try: @@ -139,10 +118,10 @@ flags = [ "-DBUILD_EPOCH=" + str(build_epoch), ] + pref_flags -print ("Using flags:") +print("Using flags:") for flag in flags: print(flag) - + projenv.Append( CCFLAGS=flags, ) @@ -181,3 +160,19 @@ def load_boot_logo(source, target, env): # Load the boot logo on TFT builds if ("HAS_TFT", 1) in env.get("CPPDEFINES", []): env.AddPreAction('$BUILD_DIR/littlefs.bin', load_boot_logo) + +# Rename (mv) littlefs.bin to include the PROGNAME +# This ensures the littlefs.bin is named consistently with the firmware +env.AddPostAction('$BUILD_DIR/littlefs.bin', env.VerboseAction( + f'mv $BUILD_DIR/littlefs.bin $BUILD_DIR/{lfsbin}', + f'Renaming littlefs.bin to {lfsbin}' +)) + +env.AddCustomTarget( + name="mtjson", + dependencies=None, + actions=[manifest_gather], + title="Meshtastic Manifest", + description="Generating Meshtastic manifest JSON + Checksums", + always_build=True, +) diff --git a/bin/platformio-pre.py b/bin/platformio-pre.py new file mode 100644 index 000000000..4e51a6544 --- /dev/null +++ b/bin/platformio-pre.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +Import("env") +platform = env.PioPlatform() + +if platform.name == "native": + env.Replace(PROGNAME="meshtasticd") +else: + from readprops import readProps + prefsLoc = env["PROJECT_DIR"] + "/version.properties" + verObj = readProps(prefsLoc) + env.Replace(PROGNAME=f"firmware-{env.get('PIOENV')}-{verObj['long']}") + +# Print the new program name for verification +print(f"PROGNAME: {env.get('PROGNAME')}") diff --git a/bin/test-simulator.sh b/bin/test-simulator.sh index 3c5f8f811..92ed21a7a 100755 --- a/bin/test-simulator.sh +++ b/bin/test-simulator.sh @@ -3,7 +3,7 @@ set -e echo "Starting simulator" -.pio/build/native/program & +.pio/build/native/meshtasticd -s & sleep 20 # 5 seconds was not enough echo "Simulator started, launching python test..." diff --git a/debian/rules b/debian/rules index 0b5d1ac57..ebb572153 100755 --- a/debian/rules +++ b/debian/rules @@ -28,5 +28,4 @@ override_dh_auto_build: # Build with platformio $(PIO_ENV) platformio run -e native-tft # Move the binary and default config to the correct name - mv .pio/build/native-tft/program .pio/build/native-tft/meshtasticd cp bin/config-dist.yaml bin/config.yaml diff --git a/extra_scripts/disable_adafruit_usb.py b/extra_scripts/disable_adafruit_usb.py index 596242184..3b901e2db 100644 --- a/extra_scripts/disable_adafruit_usb.py +++ b/extra_scripts/disable_adafruit_usb.py @@ -1,10 +1,9 @@ +#!/usr/bin/env python3 # trunk-ignore-all(flake8/F821) # trunk-ignore-all(ruff/F821) Import("env") -# NOTE: This is not currently used, but can serve as an example on how to write extra_scripts - # print("Current CLI targets", COMMAND_LINE_TARGETS) # print("Current Build targets", BUILD_TARGETS) # print("CPP defs", env.get("CPPDEFINES")) diff --git a/extra_scripts/esp32_extra.py b/extra_scripts/esp32_extra.py new file mode 100755 index 000000000..8841ad1dc --- /dev/null +++ b/extra_scripts/esp32_extra.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +# trunk-ignore-all(ruff/E402): Hacky esptool import +# trunk-ignore-all(flake8/E402): Hacky esptool import +import sys +from os.path import join + +Import("env") +platform = env.PioPlatform() + +sys.path.append(join(platform.get_package_dir("tool-esptoolpy"))) +import esptool + + +def esp32_create_combined_bin(source, target, env): + # this sub is borrowed from ESPEasy build toolchain. It's licensed under GPL V3 + # https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py + print("Generating combined binary for serial flashing") + + app_offset = 0x10000 + + new_file_name = env.subst("$BUILD_DIR/${PROGNAME}.factory.bin") + sections = env.subst(env.get("FLASH_EXTRA_IMAGES")) + firmware_name = env.subst("$BUILD_DIR/${PROGNAME}.bin") + chip = env.get("BOARD_MCU") + board = env.BoardConfig() + flash_size = board.get("upload.flash_size") + flash_freq = board.get("build.f_flash", "40m") + flash_freq = flash_freq.replace("000000L", "m") + flash_mode = board.get("build.flash_mode", "dio") + memory_type = board.get("build.arduino.memory_type", "qio_qspi") + if flash_mode == "qio" or flash_mode == "qout": + flash_mode = "dio" + if memory_type == "opi_opi" or memory_type == "opi_qspi": + flash_mode = "dout" + cmd = [ + "--chip", + chip, + "merge_bin", + "-o", + new_file_name, + "--flash_mode", + flash_mode, + "--flash_freq", + flash_freq, + "--flash_size", + flash_size, + ] + + print(" Offset | File") + for section in sections: + sect_adr, sect_file = section.split(" ", 1) + print(f" - {sect_adr} | {sect_file}") + cmd += [sect_adr, sect_file] + + print(f" - {hex(app_offset)} | {firmware_name}") + cmd += [hex(app_offset), firmware_name] + + print("Using esptool.py arguments: %s" % " ".join(cmd)) + + esptool.main(cmd) + + +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) + +esp32_kind = env.GetProjectOption("custom_esp32_kind") +if esp32_kind == "esp32": + # Free up some IRAM by removing auxiliary SPI flash chip drivers. + # Wrapped stub symbols are defined in src/platform/esp32/iram-quirk.c. + env.Append( + LINKFLAGS=[ + "-Wl,--wrap=esp_flash_chip_gd", + "-Wl,--wrap=esp_flash_chip_issi", + "-Wl,--wrap=esp_flash_chip_winbond", + ] + ) +else: + # For newer ESP32 targets, using newlib nano works better. + env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"]) diff --git a/extra_scripts/esp32_pre.py b/extra_scripts/esp32_pre.py new file mode 100755 index 000000000..8e21770e9 --- /dev/null +++ b/extra_scripts/esp32_pre.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +import json +import sys +from os.path import isfile + +Import("env") + + +# From https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py +def _parse_size(value): + if isinstance(value, int): + return value + elif value.isdigit(): + return int(value) + elif value.startswith("0x"): + return int(value, 16) + elif value[-1].upper() in ("K", "M"): + base = 1024 if value[-1].upper() == "K" else 1024 * 1024 + return int(value[:-1]) * base + return value + + +def _parse_partitions(env): + partitions_csv = env.subst("$PARTITIONS_TABLE_CSV") + if not isfile(partitions_csv): + sys.stderr.write( + "Could not find the file %s with partitions " "table.\n" % partitions_csv + ) + env.Exit(1) + return + + result = [] + # The first offset is 0x9000 because partition table is flashed to 0x8000 and + # occupies an entire flash sector, which size is 0x1000 + next_offset = 0x9000 + with open(partitions_csv) as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + tokens = [t.strip() for t in line.split(",")] + if len(tokens) < 5: + continue + + bound = 0x10000 if tokens[1] in ("0", "app") else 4 + calculated_offset = (next_offset + bound - 1) & ~(bound - 1) + partition = { + "name": tokens[0], + "type": tokens[1], + "subtype": tokens[2], + "offset": tokens[3] or calculated_offset, + "size": tokens[4], + "flags": tokens[5] if len(tokens) > 5 else None, + } + result.append(partition) + next_offset = _parse_size(partition["offset"]) + _parse_size( + partition["size"] + ) + + return result + + +def mtjson_esp32_part(target, source, env): + part = _parse_partitions(env) + pj = json.dumps(part) + # print(f"JSON_PARTITIONS: {pj}") + # Dump json string to 'custom_mtjson_part' variable to use later when writing the manifest + env.Replace(custom_mtjson_part=pj) + + +env.AddPreAction("mtjson", mtjson_esp32_part) diff --git a/extra_scripts/nrf52_extra.py b/extra_scripts/nrf52_extra.py new file mode 100755 index 000000000..8e95e42bf --- /dev/null +++ b/extra_scripts/nrf52_extra.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports + +import sys +from os.path import basename + +Import("env") + + +# Custom HEX from ELF +# Convert hex to uf2 for nrf52 +def nrf52_hex_to_uf2(source, target, env): + hex_path = target[0].get_abspath() + # When using merged hex, drop 'merged' from uf2 filename + uf2_path = hex_path.replace(".merged.", ".") + uf2_path = uf2_path.replace(".hex", ".uf2") + env.Execute( + env.VerboseAction( + f'"{sys.executable}" ./bin/uf2conv.py "{hex_path}" -c -f 0xADA52840 -o "{uf2_path}"', + f"Generating UF2 file from {basename(hex_path)}", + ) + ) + + +def nrf52_mergehex(source, target, env): + hex_path = target[0].get_abspath() + merged_hex_path = hex_path.replace(".hex", ".merged.hex") + merge_with = None + if "wio-sdk-wm1110" == str(env.get("PIOENV")): + merge_with = env.subst("$PROJECT_DIR/bin/s140_nrf52_7.3.0_softdevice.hex") + else: + print("merge_with not defined for this target") + + if merge_with is not None: + env.Execute( + env.VerboseAction( + f'"$PROJECT_DIR/bin/mergehex" -m "{hex_path}" "{merge_with}" -o "{merged_hex_path}"', + "Merging HEX with SoftDevice", + ) + ) + print(f'Merged file saved at "{basename(merged_hex_path)}"') + nrf52_hex_to_uf2([hex_path, merge_with], [env.File(merged_hex_path)], env) + + +# if WM1110 target, merge hex with softdevice 7.3.0 +if "wio-sdk-wm1110" == env.get("PIOENV"): + env.AddPostAction("$BUILD_DIR/${PROGNAME}.hex", nrf52_mergehex) +else: + env.AddPostAction("$BUILD_DIR/${PROGNAME}.hex", nrf52_hex_to_uf2) diff --git a/extra_scripts/extra_stm32.py b/extra_scripts/stm32_extra.py similarity index 95% rename from extra_scripts/extra_stm32.py rename to extra_scripts/stm32_extra.py index f3bd8c514..afceb7d81 100755 --- a/extra_scripts/extra_stm32.py +++ b/extra_scripts/stm32_extra.py @@ -1,7 +1,9 @@ +#!/usr/bin/env python3 # trunk-ignore-all(ruff/F821) # trunk-ignore-all(flake8/F821): For SConstruct imports Import("env") + # Custom HEX from ELF env.AddPostAction( "$BUILD_DIR/${PROGNAME}.elf", diff --git a/meshtasticd.spec.rpkg b/meshtasticd.spec.rpkg index eb4ab5ae7..d901d0a61 100644 --- a/meshtasticd.spec.rpkg +++ b/meshtasticd.spec.rpkg @@ -69,7 +69,7 @@ platformio run -e native-tft %install # Install meshtasticd binary mkdir -p %{buildroot}%{_bindir} -install -m 0755 .pio/build/native-tft/program %{buildroot}%{_bindir}/meshtasticd +install -m 0755 .pio/build/native-tft/meshtasticd %{buildroot}%{_bindir}/meshtasticd # Install portduino VFS dir install -p -d -m 0770 %{buildroot}%{_localstatedir}/lib/meshtasticd diff --git a/platformio.ini b/platformio.ini index 5b7f5ddcf..2a10676fb 100644 --- a/platformio.ini +++ b/platformio.ini @@ -14,7 +14,9 @@ description = Meshtastic [env] test_build_src = true -extra_scripts = bin/platformio-custom.py +extra_scripts = + pre:bin/platformio-pre.py + bin/platformio-custom.py ; note: we add src to our include search path so that lmic_project_config can override ; note: TINYGPS_OPTION_NO_CUSTOM_FIELDS is VERY important. We don't use custom fields and somewhere in that pile ; of code is a heap corruption bug! diff --git a/variants/native/portduino/platformio.ini b/variants/native/portduino/platformio.ini index 4e6a592de..fc58eb92c 100644 --- a/variants/native/portduino/platformio.ini +++ b/variants/native/portduino/platformio.ini @@ -109,5 +109,5 @@ extends = env:native build_flags = -lgcov --coverage -fprofile-abs-path -fsanitize=address ${env:native.build_flags} ; https://docs.platformio.org/en/latest/projectconf/sections/env/options/test/test_testing_command.html test_testing_command = - ${platformio.build_dir}/${this.__env__}/program + ${platformio.build_dir}/${this.__env__}/meshtasticd -s diff --git a/variants/nrf52840/wio-sdk-wm1110/platformio.ini b/variants/nrf52840/wio-sdk-wm1110/platformio.ini index 028129783..2deeeedcf 100644 --- a/variants/nrf52840/wio-sdk-wm1110/platformio.ini +++ b/variants/nrf52840/wio-sdk-wm1110/platformio.ini @@ -4,7 +4,7 @@ extends = nrf52840_base board = wio-sdk-wm1110 extra_scripts = - ${env.extra_scripts} + ${nrf52840_base.extra_scripts} extra_scripts/disable_adafruit_usb.py # Remove adafruit USB serial from the build (it is incompatible with using the ch340 serial chip on this board)