From 71591fb06a5d4018bd6ed3ab4c9673c2a03287ac Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 21 Jan 2025 19:53:32 -0500 Subject: [PATCH 1/9] Build docker images with other linux (#5837) --- .github/workflows/build_docker.yml | 51 -------- .github/workflows/daily_packaging.yml | 6 + .github/workflows/docker_build.yml | 70 +++++++++++ .github/workflows/docker_manifest.yml | 167 +++++++++++++++++++++++++ .github/workflows/main_matrix.yml | 35 +++++- .github/workflows/release_channels.yml | 7 ++ 6 files changed, 281 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/build_docker.yml create mode 100644 .github/workflows/docker_build.yml create mode 100644 .github/workflows/docker_manifest.yml diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml deleted file mode 100644 index 18787f16a..000000000 --- a/.github/workflows/build_docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Docker - -on: workflow_call - -permissions: - contents: write - packages: write - -jobs: - build-native: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Get release version string - run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT - id: version - - - name: Docker login - if: ${{ github.event_name != 'pull_request_target' && github.event_name != 'pull_request' }} - uses: docker/login-action@v3 - with: - username: meshtastic - password: ${{ secrets.DOCKER_FIRMWARE_TOKEN }} - - - name: Docker setup - if: ${{ github.event_name != 'pull_request_target' && github.event_name != 'pull_request' }} - uses: docker/setup-buildx-action@v3 - - - name: Docker build and push tagged versions - if: ${{ github.event_name == 'workflow_dispatch' }} - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - push: true - tags: meshtastic/meshtasticd:${{ steps.version.outputs.long }} - - - name: Docker build and push - if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request_target' && github.event_name != 'pull_request' }} - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - push: true - tags: meshtastic/meshtasticd:latest diff --git a/.github/workflows/daily_packaging.yml b/.github/workflows/daily_packaging.yml index 14daae74d..cb8f866c6 100644 --- a/.github/workflows/daily_packaging.yml +++ b/.github/workflows/daily_packaging.yml @@ -20,6 +20,12 @@ permissions: packages: write jobs: + docker-multiarch: + uses: ./.github/workflows/docker_manifest.yml + with: + release_channel: daily + secrets: inherit + package-ppa: strategy: fail-fast: false diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml new file mode 100644 index 000000000..83c67bb32 --- /dev/null +++ b/.github/workflows/docker_build.yml @@ -0,0 +1,70 @@ +name: Build Docker + +# Build Docker image, push untagged (digest-only) + +on: + workflow_call: + inputs: + distro: + description: Distro to target + required: true + type: string + # choices: [debian, alpine] + platform: + description: Platform to target + required: true + type: string + runs-on: + description: Runner to use + required: true + type: string + push: + description: Push images to registry + required: false + type: boolean + default: false + outputs: + digest: + description: Digest of built image + value: ${{ jobs.docker-build.outputs.digest }} + +permissions: + contents: write + packages: write + +jobs: + docker-build: + outputs: + digest: ${{ steps.docker_variant.outputs.digest }} + runs-on: ${{ inputs.runs-on }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Docker login + if: ${{ inputs.push }} + uses: docker/login-action@v3 + with: + username: meshtastic + password: ${{ secrets.DOCKER_FIRMWARE_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Docker setup + uses: docker/setup-buildx-action@v3 + + - name: Docker build and push + uses: docker/build-push-action@v6 + id: docker_variant + with: + context: . + file: | + ${{ contains(inputs.distro, 'debian') && './Dockerfile' || contains(inputs.distro, 'alpine') && './alpine.Dockerfile' }} + push: ${{ inputs.push }} + tags: "" # Intentionally empty, push with digest only + platforms: ${{ inputs.platform }} diff --git a/.github/workflows/docker_manifest.yml b/.github/workflows/docker_manifest.yml new file mode 100644 index 000000000..30dcfb067 --- /dev/null +++ b/.github/workflows/docker_manifest.yml @@ -0,0 +1,167 @@ +name: Build Docker Multi-Arch Manifest + +on: + workflow_call: + inputs: + release_channel: + description: Release channel to target + required: true + type: string + +permissions: + contents: write + packages: write + +jobs: + docker-debian-amd64: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/amd64 + runs-on: ubuntu-24.04 + push: true + + docker-debian-arm64: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/arm64 + runs-on: ubuntu-24.04-arm + push: true + + docker-debian-armv7: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/arm/v7 + runs-on: ubuntu-24.04-arm + push: true + + docker-alpine-amd64: + uses: ./.github/workflows/docker_build.yml + with: + distro: alpine + platform: linux/amd64 + runs-on: ubuntu-24.04 + push: true + + docker-alpine-arm64: + uses: ./.github/workflows/docker_build.yml + with: + distro: alpine + platform: linux/arm64 + runs-on: ubuntu-24.04-arm + push: true + + docker-alpine-armv7: + uses: ./.github/workflows/docker_build.yml + with: + distro: alpine + platform: linux/arm/v7 + runs-on: ubuntu-24.04-arm + push: true + + docker-manifest: + needs: + # Debian + - docker-debian-amd64 + - docker-debian-arm64 + - docker-debian-armv7 + # Alpine + - docker-alpine-amd64 + - docker-alpine-arm64 + - docker-alpine-armv7 + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Get release version string + run: | + echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + echo "short=$(./bin/buildinfo.py short)" >> $GITHUB_OUTPUT + id: version + + - name: Enumerate tags + shell: python + run: | + import os + + short = "${{ steps.version.outputs.short }}" + long = "${{ steps.version.outputs.long }}" + release_channel = "${{ inputs.release_channel }}" + tags = { + "beta": { + "debian": [ + f"{short}", f"{long}", f"{short}-beta", f"{long}-beta", "beta", "latest", + f"{short}-debian", f"{long}-debian", f"{short}-beta-debian", f"{long}-beta-debian", "beta-debian" + ], + "alpine": [ + f"{short}-alpine", f"{long}-alpine", f"{short}-beta-alpine", f"{long}-beta-alpine", "beta-alpine" + ] + }, + "alpha": { + "debian": [ + f"{short}-alpha", f"{long}-alpha", "alpha", + f"{short}-alpha-debian", f"{long}-alpha-debian", "alpha-debian" + ], + "alpine": [ + f"{short}-alpha-alpine", f"{long}-alpha-alpine", "alpha-alpine" + ] + }, + "daily": { + "debian": ["daily", "daily-debian"], + "alpine": ["daily-alpine"] + } + } + + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + fh.write(f"debian={','.join(tags[release_channel]['debian'])}\n") + fh.write(f"alpine={','.join(tags[release_channel]['alpine'])}\n") + id: tags + + - name: Docker login + uses: docker/login-action@v3 + with: + username: meshtastic + password: ${{ secrets.DOCKER_FIRMWARE_TOKEN }} + + - name: Docker meta (Debian) + id: meta_debian + uses: docker/metadata-action@v5 + with: + images: meshtastic/meshtasticd + tags: ${{ steps.tags.outputs.debian }} + + - name: Create Docker manifest (Debian) + id: manifest_debian + uses: int128/docker-manifest-create-action@v2 + with: + tags: ${{ steps.meta_debian.outputs.tags }} + push: true + sources: | + meshtastic/meshtasticd@${{ needs.docker-debian-amd64.outputs.digest }} + meshtastic/meshtasticd@${{ needs.docker-debian-arm64.outputs.digest }} + meshtastic/meshtasticd@${{ needs.docker-debian-armv7.outputs.digest }} + + - name: Docker meta (Alpine) + id: meta_alpine + uses: docker/metadata-action@v5 + with: + images: meshtastic/meshtasticd + tags: ${{ steps.tags.outputs.alpine }} + + - name: Create Docker manifest (Alpine) + id: manifest_alpine + uses: int128/docker-manifest-create-action@v2 + with: + tags: ${{ steps.meta_alpine.outputs.tags }} + push: true + sources: | + meshtastic/meshtasticd@${{ needs.docker-alpine-amd64.outputs.digest }} + meshtastic/meshtasticd@${{ needs.docker-alpine-arm64.outputs.digest }} + meshtastic/meshtasticd@${{ needs.docker-alpine-armv7.outputs.digest }} diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 0a0ea9954..a9678f4fc 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -147,10 +147,37 @@ jobs: test-native: uses: ./.github/workflows/test_native.yml - build-docker: - if: ${{ github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/build_docker.yml - secrets: inherit + docker-debian-amd64: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/amd64 + runs-on: ubuntu-24.04 + push: false + + docker-alpine-amd64: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/amd64 + runs-on: ubuntu-24.04 + push: false + + docker-debian-arm64: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/arm64 + runs-on: ubuntu-24.04-arm + push: false + + docker-debian-armv7: + uses: ./.github/workflows/docker_build.yml + with: + distro: debian + platform: linux/arm/v7 + runs-on: ubuntu-24.04-arm + push: false after-checks: runs-on: ubuntu-latest diff --git a/.github/workflows/release_channels.yml b/.github/workflows/release_channels.yml index afb7319ed..b59a0316c 100644 --- a/.github/workflows/release_channels.yml +++ b/.github/workflows/release_channels.yml @@ -9,6 +9,13 @@ permissions: packages: write jobs: + build-docker: + uses: ./.github/workflows/docker_manifest.yml + with: + release_channel: |- + ${{ contains(github.event.release.name, 'Beta') && 'beta' || contains(github.event.release.name, 'Alpha') && 'alpha' }} + secrets: inherit + package-ppa: strategy: fail-fast: false From 0fdbf70452158d292d71bdd8b02499d8bd4b2c2a Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 21 Jan 2025 22:26:10 -0500 Subject: [PATCH 2/9] Small fix: Correctly pass secrets in Docker builds (#5905) --- .github/workflows/docker_build.yml | 3 +++ .github/workflows/docker_manifest.yml | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 83c67bb32..43072b777 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -4,6 +4,9 @@ name: Build Docker on: workflow_call: + secrets: + DOCKER_FIRMWARE_TOKEN: + required: false # Only required for push inputs: distro: description: Distro to target diff --git a/.github/workflows/docker_manifest.yml b/.github/workflows/docker_manifest.yml index 30dcfb067..9183dfd6c 100644 --- a/.github/workflows/docker_manifest.yml +++ b/.github/workflows/docker_manifest.yml @@ -2,6 +2,9 @@ name: Build Docker Multi-Arch Manifest on: workflow_call: + secrets: + DOCKER_FIRMWARE_TOKEN: + required: true inputs: release_channel: description: Release channel to target @@ -20,6 +23,7 @@ jobs: platform: linux/amd64 runs-on: ubuntu-24.04 push: true + secrets: inherit docker-debian-arm64: uses: ./.github/workflows/docker_build.yml @@ -28,6 +32,7 @@ jobs: platform: linux/arm64 runs-on: ubuntu-24.04-arm push: true + secrets: inherit docker-debian-armv7: uses: ./.github/workflows/docker_build.yml @@ -36,6 +41,7 @@ jobs: platform: linux/arm/v7 runs-on: ubuntu-24.04-arm push: true + secrets: inherit docker-alpine-amd64: uses: ./.github/workflows/docker_build.yml @@ -44,6 +50,7 @@ jobs: platform: linux/amd64 runs-on: ubuntu-24.04 push: true + secrets: inherit docker-alpine-arm64: uses: ./.github/workflows/docker_build.yml @@ -52,6 +59,7 @@ jobs: platform: linux/arm64 runs-on: ubuntu-24.04-arm push: true + secrets: inherit docker-alpine-armv7: uses: ./.github/workflows/docker_build.yml @@ -60,6 +68,7 @@ jobs: platform: linux/arm/v7 runs-on: ubuntu-24.04-arm push: true + secrets: inherit docker-manifest: needs: From fdc87d492c0bc164a50f02da2f1ab2806013704e Mon Sep 17 00:00:00 2001 From: Eric Severance Date: Wed, 22 Jan 2025 00:45:34 -0800 Subject: [PATCH 3/9] Add quotes around ${platformio.build_dir} (#5906) Fixes #5898 (hopefully) --- platformio.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio.ini b/platformio.ini index ea4de4db1..1c51e53b4 100644 --- a/platformio.ini +++ b/platformio.ini @@ -20,7 +20,7 @@ extra_scripts = bin/platformio-custom.py build_flags = -Wno-missing-field-initializers -Wno-format - -Isrc -Isrc/mesh -Isrc/mesh/generated -Isrc/gps -Isrc/buzz -Wl,-Map,${platformio.build_dir}/output.map + -Isrc -Isrc/mesh -Isrc/mesh/generated -Isrc/gps -Isrc/buzz -Wl,-Map,"${platformio.build_dir}"/output.map -DUSE_THREAD_NAMES -DTINYGPS_OPTION_NO_CUSTOM_FIELDS -DPB_ENABLE_MALLOC=1 From 7fb22cf678d0e40d7e44e3e9ecd8cb10f734e8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 22 Jan 2025 14:11:58 +0100 Subject: [PATCH 4/9] ignore platformio core files when building in place --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 803aee139..b63f431d1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ web.tar *.code-workspace .idea +.platformio +.local +.cache .DS_Store Thumbs.db From 01892cbd1eaa824628315f3c79078763f3e8cc91 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 22 Jan 2025 09:55:57 -0500 Subject: [PATCH 5/9] Docker: tag intermediate containers (#5910) --- .github/workflows/docker_build.yml | 21 ++++++++++++++++++++- .github/workflows/docker_manifest.yml | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 43072b777..eec0785c0 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -48,6 +48,11 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} + - name: Get release version string + run: | + echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + - name: Docker login if: ${{ inputs.push }} uses: docker/login-action@v3 @@ -61,6 +66,20 @@ jobs: - name: Docker setup uses: docker/setup-buildx-action@v3 + - name: Sanitize platform string + id: sanitize_platform + # Replace slashes with underscores + run: echo "cleaned_platform=${{ inputs.platform }}" | sed 's/\//_/g' >> $GITHUB_OUTPUT + + - name: Docker tag + id: meta + uses: docker/metadata-action@v5 + with: + images: meshtastic/meshtasticd + tags: | + GHA-${{ steps.version.outputs.long }}-${{ inputs.distro }}-${{ steps.sanitize_platform.outputs.cleaned_platform }} + flavor: latest=false + - name: Docker build and push uses: docker/build-push-action@v6 id: docker_variant @@ -69,5 +88,5 @@ jobs: file: | ${{ contains(inputs.distro, 'debian') && './Dockerfile' || contains(inputs.distro, 'alpine') && './alpine.Dockerfile' }} push: ${{ inputs.push }} - tags: "" # Intentionally empty, push with digest only + tags: ${{ steps.meta.outputs.tags }} # Tag is only meant to be consumed by the "manifest" job platforms: ${{ inputs.platform }} diff --git a/.github/workflows/docker_manifest.yml b/.github/workflows/docker_manifest.yml index 9183dfd6c..28dbf8c21 100644 --- a/.github/workflows/docker_manifest.yml +++ b/.github/workflows/docker_manifest.yml @@ -145,6 +145,7 @@ jobs: with: images: meshtastic/meshtasticd tags: ${{ steps.tags.outputs.debian }} + flavor: latest=false - name: Create Docker manifest (Debian) id: manifest_debian From 8e8b22edb0ae0aa56f92bbf49967b667a9acc747 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 22 Jan 2025 12:09:29 -0500 Subject: [PATCH 6/9] Debian: Switch OBS repo to `network:Meshtastic` (#5912) --- .github/workflows/daily_packaging.yml | 2 +- .github/workflows/release_channels.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily_packaging.yml b/.github/workflows/daily_packaging.yml index cb8f866c6..11fe2043a 100644 --- a/.github/workflows/daily_packaging.yml +++ b/.github/workflows/daily_packaging.yml @@ -40,7 +40,7 @@ jobs: package-obs: uses: ./.github/workflows/package_obs.yml with: - obs_project: home:meshtastic:daily + obs_project: network:Meshtastic:daily series: unstable secrets: inherit diff --git a/.github/workflows/release_channels.yml b/.github/workflows/release_channels.yml index b59a0316c..a3a105d6d 100644 --- a/.github/workflows/release_channels.yml +++ b/.github/workflows/release_channels.yml @@ -32,7 +32,7 @@ jobs: uses: ./.github/workflows/package_obs.yml with: obs_project: |- - home:meshtastic:${{ contains(github.event.release.name, 'Beta') && 'beta' || contains(github.event.release.name, 'Alpha') && 'alpha' }} + network:Meshtastic:${{ contains(github.event.release.name, 'Beta') && 'beta' || contains(github.event.release.name, 'Alpha') && 'alpha' }} series: |- ${{ contains(github.event.release.name, 'Beta') && 'beta' || contains(github.event.release.name, 'Alpha') && 'alpha' }} secrets: inherit From 3b40fe9805041a69381442bce1b2273bdd91b99d Mon Sep 17 00:00:00 2001 From: Austin Date: Thu, 23 Jan 2025 17:03:03 -0500 Subject: [PATCH 7/9] Docker: Switch tags to newline-seperated (#5919) --- .github/workflows/docker_manifest.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker_manifest.yml b/.github/workflows/docker_manifest.yml index 28dbf8c21..d1d1a5634 100644 --- a/.github/workflows/docker_manifest.yml +++ b/.github/workflows/docker_manifest.yml @@ -128,9 +128,14 @@ jobs: } } - with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: - fh.write(f"debian={','.join(tags[release_channel]['debian'])}\n") - fh.write(f"alpine={','.join(tags[release_channel]['alpine'])}\n") + with open(os.environ["GITHUB_OUTPUT"], "a") as fh: + fh.write("debian< Date: Thu, 23 Jan 2025 19:12:20 -0600 Subject: [PATCH 8/9] NRF52 - Remove file totally before opening write (#5916) * Remove prefs first * Remove file first * Remove truncate * No longer needed * Missed a param * That wasn't supposed to be there * Remove vestigal lfs assert * Durr --- src/SafeFile.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/SafeFile.cpp b/src/SafeFile.cpp index 94232e81d..c942aa0ee 100644 --- a/src/SafeFile.cpp +++ b/src/SafeFile.cpp @@ -8,9 +8,8 @@ static File openFile(const char *filename, bool fullAtomic) concurrency::LockGuard g(spiLock); LOG_DEBUG("Opening %s, fullAtomic=%d", filename, fullAtomic); #ifdef ARCH_NRF52 - File file = FSCom.open(filename, FILE_O_WRITE); - file.seek(0); - return file; + FSCom.remove(filename); + return FSCom.open(filename, FILE_O_WRITE); #endif if (!fullAtomic) FSCom.remove(filename); // Nuke the old file to make space (ignore if it !exists) @@ -59,9 +58,6 @@ bool SafeFile::close() return false; spiLock->lock(); -#ifdef ARCH_NRF52 - f.truncate(); -#endif f.close(); spiLock->unlock(); From d1f7739bbea229049ac92d32022888fbd8b8e382 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Thu, 23 Jan 2025 19:56:59 -0600 Subject: [PATCH 9/9] Peg NRF52 arduino to meshtastic fork with LFE bluetooth fix (#5924) --- arch/nrf52/nrf52.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/nrf52/nrf52.ini b/arch/nrf52/nrf52.ini index 57b276978..b68977c78 100644 --- a/arch/nrf52/nrf52.ini +++ b/arch/nrf52/nrf52.ini @@ -4,7 +4,7 @@ platform = platformio/nordicnrf52@^10.7.0 extends = arduino_base platform_packages = ; our custom Git version until they merge our PR - framework-arduinoadafruitnrf52 @ https://github.com/geeksville/Adafruit_nRF52_Arduino.git + framework-arduinoadafruitnrf52 @ https://github.com/meshtastic/Adafruit_nRF52_Arduino.git#e13f5820002a4fb2a5e6754b42ace185277e5adf toolchain-gccarmnoneeabi@~1.90301.0 build_type = debug