diff --git a/.github/actions/setup-base/action.yml b/.github/actions/setup-base/action.yml new file mode 100644 index 000000000..1791f80ae --- /dev/null +++ b/.github/actions/setup-base/action.yml @@ -0,0 +1,41 @@ +name: 'Setup Build Base Composite Action' +description: 'Base build actions for Meshtastic Platform IO steps' + +runs: + using: "composite" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: "recursive" + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Install cppcheck + shell: bash + run: | + sudo apt-get install -y cppcheck + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Cache python libs + uses: actions/cache@v3 + id: cache-pip # needed in if test + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + + - name: Upgrade python tools + shell: bash + run: | + python -m pip install --upgrade pip + pip install -U platformio adafruit-nrfutil + pip install -U meshtastic --pre + + - name: Upgrade platformio + shell: bash + run: | + pio upgrade \ No newline at end of file diff --git a/.github/workflows/build_esp32.yml b/.github/workflows/build_esp32.yml new file mode 100644 index 000000000..6d5dd9863 --- /dev/null +++ b/.github/workflows/build_esp32.yml @@ -0,0 +1,54 @@ +name: Build ESP32 + +on: + workflow_call: + inputs: + board: + required: true + type: string + +jobs: + build-esp32: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build base + id: base + uses: ./.github/actions/setup-base + + - name: Pull web ui + uses: dsaltares/fetch-gh-release-asset@master + with: + repo: "meshtastic/web" + file: "build.tar" + target: "build.tar" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Unpack web ui + run: | + tar -xf build.tar -C data/static + rm build.tar + + - name: Build ESP32 + run: bin/build-esp32.sh ${{ inputs.board }} + + - name: Pull OTA Firmware + uses: dsaltares/fetch-gh-release-asset@master + with: + repo: "meshtastic/firmware-ota" + file: "firmware.bin" + target: "release/bleota.bin" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get release version string + shell: bash + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Store binaries as an artifact + uses: actions/upload-artifact@v3 + with: + name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip + path: | + release/*.bin + release/*.elf diff --git a/.github/workflows/build_nrf52.yml b/.github/workflows/build_nrf52.yml new file mode 100644 index 000000000..cdf43c22e --- /dev/null +++ b/.github/workflows/build_nrf52.yml @@ -0,0 +1,33 @@ +name: Build NRF52 + +on: + workflow_call: + inputs: + board: + required: true + type: string + +jobs: + build-nrf52: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build base + id: base + uses: ./.github/actions/setup-base + + - name: Build NRF52 + run: bin/build-nrf52.sh ${{ inputs.board }} + + - name: Get release version string + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Store binaries as an artifact + uses: actions/upload-artifact@v3 + with: + name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip + path: | + release/*.uf2 + release/*.elf + release/*.zip \ No newline at end of file diff --git a/.github/workflows/build_rpi2040.yml b/.github/workflows/build_rpi2040.yml new file mode 100644 index 000000000..fb7e3db5b --- /dev/null +++ b/.github/workflows/build_rpi2040.yml @@ -0,0 +1,32 @@ +name: Build RPI2040 + +on: + workflow_call: + inputs: + board: + required: true + type: string + +jobs: + build-rpi2040: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build base + id: base + uses: ./.github/actions/setup-base + + - name: Build Raspberry Pi 2040 + run: ./bin/build-rpi2040.sh ${{ inputs.board }} + + - name: Get release version string + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Store binaries as an artifact + uses: actions/upload-artifact@v3 + with: + name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip + path: | + release/*.uf2 + release/*.elf \ No newline at end of file diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index d9d6634ff..2440b2ed3 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -35,37 +35,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Install cppcheck - run: | - sudo apt-get install -y cppcheck - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Cache python libs - uses: actions/cache@v3 - id: cache-pip # needed in if test - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - - - name: Upgrade python tools and install platformio - run: | - python -m pip install --upgrade pip - pip install -U platformio - - - name: Upgrade platformio - run: | - pio upgrade + - uses: actions/checkout@v3 + - name: Build base + id: base + uses: ./.github/actions/setup-base - name: Check ${{ matrix.board }} run: bin/check-all.sh ${{ matrix.board }} @@ -94,73 +67,9 @@ jobs: - board: m5stack-core - board: m5stack-coreink - board: tbeam-s3-core - - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Cache python libs - uses: actions/cache@v3 - id: cache-pip # needed in if test - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - - - name: Upgrade python tools - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil - - - name: Upgrade platformio - run: | - pio upgrade - - - name: Pull web ui - uses: dsaltares/fetch-gh-release-asset@master - with: - repo: "meshtastic/web" - file: "build.tar" - target: "build.tar" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Unpack web ui - run: | - tar -xf build.tar -C data/static - rm build.tar - - - name: Build ESP32 - run: bin/build-esp32.sh ${{ matrix.board }} - - - name: Pull OTA Firmware - uses: dsaltares/fetch-gh-release-asset@master - with: - repo: "meshtastic/firmware-ota" - file: "firmware.bin" - target: "release/bleota.bin" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Get release version string - run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT - id: version - - - name: Store binaries as an artifact - uses: actions/upload-artifact@v3 - with: - name: firmware-${{ matrix.board }}-${{ steps.version.outputs.version }}.zip - path: | - release/*.bin - release/*.elf - retention-days: 30 + uses: ./.github/workflows/build_esp32.yml + with: + board: ${{ matrix.board }} build-nrf52: strategy: @@ -173,53 +82,9 @@ jobs: - board: t-echo - board: pca10059_diy_eink - board: feather_diy - - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Cache python libs - uses: actions/cache@v3 - id: cache-pip # needed in if test - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - - - name: Upgrade python tools - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil - - - name: Upgrade platformio - run: | - pio upgrade - - - name: Build NRF52 - run: bin/build-nrf52.sh ${{ matrix.board }} - - - name: Get release version string - run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT - id: version - - - name: Store binaries as an artifact - uses: actions/upload-artifact@v3 - with: - name: firmware-${{ matrix.board }}-${{ steps.version.outputs.version }}.zip - path: | - release/*.uf2 - release/*.elf - release/*.zip - retention-days: 30 + uses: ./.github/workflows/build_nrf52.yml + with: + board: ${{ matrix.board }} build-rpi2040: strategy: @@ -228,84 +93,17 @@ jobs: matrix: include: - board: pico - - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Cache python libs - uses: actions/cache@v3 - id: cache-pip # needed in if test - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - - - name: Upgrade python tools - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil - - - name: Upgrade platformio - run: | - pio upgrade - - - name: Build Raspberry Pi 2040 - run: ./bin/build-rpi2040.sh ${{ matrix.board }} - - - name: Get release version string - run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT - id: version - - - name: Store binaries as an artifact - uses: actions/upload-artifact@v3 - with: - name: firmware-${{ matrix.board }}-${{ steps.version.outputs.version }}.zip - path: | - release/*.uf2 - release/*.elf - retention-days: 30 + uses: ./.github/workflows/build_rpi2040.yml + with: + board: ${{ matrix.board }} build-native: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Cache python libs - uses: actions/cache@v3 - id: cache-pip # needed in if test - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - - - name: Upgrade python tools - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil - pip install -U meshtastic --pre - - - name: Upgrade platformio - run: | - pio upgrade + - uses: actions/checkout@v3 + - name: Build base + id: base + uses: ./.github/actions/setup-base # We now run integration test before other build steps (to quickly see runtime failures) - name: Build for native @@ -332,8 +130,27 @@ jobs: release/meshtasticd_linux_amd64 release/device-*.sh release/device-*.bat - retention-days: 30 + + - name: Docker login + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: docker/login-action@v2 + with: + username: meshtastic + password: ${{ secrets.DOCKER_TOKEN }} + - name: Docker setup + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: docker/setup-buildx-action@v2 + + - name: Docker build and push + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: docker/build-push-action@v3 + with: + context: . + file: ./Dockerfile + push: true + tags: meshtastic/device-simulator:latest + after-checks: runs-on: ubuntu-latest needs: [check] diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 000000000..6ae867e8b --- /dev/null +++ b/.semgrepignore @@ -0,0 +1 @@ +.github/workflows/main_matrix.yml diff --git a/Dockerfile b/Dockerfile index 0ce4e3326..8e3cd2154 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,41 @@ FROM debian:bullseye-slim AS builder -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install wget python3 g++ zip python3-venv git vim -RUN wget https://raw.githubusercontent.com/platformio/platformio-core-installer/master/get-platformio.py -O get-platformio.py; chmod +x get-platformio.py -RUN python3 get-platformio.py -RUN git clone https://github.com/meshtastic/firmware --recurse-submodules -RUN cd firmware -RUN chmod +x ./firmware/bin/build-native.sh -RUN . ~/.platformio/penv/bin/activate; cd firmware; sh ./bin/build-native.sh + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install build deps +USER root +RUN apt-get update && \ + apt-get -y install wget python3 g++ zip python3-venv git vim ca-certificates + +# create a non-priveleged user & group +RUN groupadd -g 1000 mesh && useradd -ml -u 1000 -g 1000 mesh + +USER mesh +RUN wget https://raw.githubusercontent.com/platformio/platformio-core-installer/master/get-platformio.py -qO /tmp/get-platformio.py && \ + chmod +x /tmp/get-platformio.py && \ + python3 /tmp/get-platformio.py && \ + git clone https://github.com/meshtastic/firmware --recurse-submodules /tmp/firmware && \ + cd /tmp/firmware && \ + chmod +x /tmp/firmware/bin/build-native.sh && \ + source ~/.platformio/penv/bin/activate && \ + ./bin/build-native.sh FROM frolvlad/alpine-glibc -WORKDIR /root/ -COPY --from=builder /firmware/release/meshtasticd_linux_amd64 ./ -RUN apk --update add --no-cache g++ -CMD sh -cx "./meshtasticd_linux_amd64 --hwid '$RANDOM'" \ No newline at end of file + +RUN apk --update add --no-cache g++ shadow && \ + groupadd -g 1000 mesh && useradd -ml -u 1000 -g 1000 mesh + +COPY --from=builder /tmp/firmware/release/meshtasticd_linux_amd64 /home/mesh/ + +USER mesh +WORKDIR /home/mesh +CMD sh -cx "./meshtasticd_linux_amd64 --hwid '$RANDOM'" + +HEALTHCHECK NONE diff --git a/README.md b/README.md index 6432803e4..5bae2f345 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository contains the device firmware for the Meshtastic project. -**[Building Instructions](https://meshtastic.org/docs/developers/Firmware/build)** +**[Building Instructions](https://meshtastic.org/docs/development/firmware/build)** **[Flashing Instructions](https://meshtastic.org/docs/getting-started/flashing-firmware/)** ## Stats diff --git a/bin/build-esp32.sh b/bin/build-esp32.sh index 12961864f..5a2044a35 100755 --- a/bin/build-esp32.sh +++ b/bin/build-esp32.sh @@ -10,9 +10,6 @@ OUTDIR=release/ rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true -# Make sure our submodules are current -git submodule update - # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale platformio pkg update diff --git a/bin/build-native.sh b/bin/build-native.sh index b620a01d8..8bc262860 100755 --- a/bin/build-native.sh +++ b/bin/build-native.sh @@ -12,9 +12,6 @@ rm -f $OUTDIR/firmware* mkdir -p $OUTDIR/ rm -r $OUTDIR/* || true -# Make sure our submodules are current -git submodule update - # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale platformio pkg update diff --git a/bin/build-nrf52.sh b/bin/build-nrf52.sh index 6c723e28f..a9980f486 100755 --- a/bin/build-nrf52.sh +++ b/bin/build-nrf52.sh @@ -10,9 +10,6 @@ OUTDIR=release/ rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true -# Make sure our submodules are current -git submodule update - # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale platformio pkg update diff --git a/bin/build-rpi2040.sh b/bin/build-rpi2040.sh index 670f570f1..fe0725085 100755 --- a/bin/build-rpi2040.sh +++ b/bin/build-rpi2040.sh @@ -10,9 +10,6 @@ OUTDIR=release/ rm -f $OUTDIR/firmware* rm -r $OUTDIR/* || true -# Make sure our submodules are current -git submodule update - # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale platformio pkg update