Github action runs on job matrix now for parallel operation (both build and check) (#1202)

* Build matrix for parallel jobs

.github/workflows/main.yml

@@ -1,20 +1,8 @@
-name: Continuous Integration
+name: Continuous Integration (Legacy serial build)
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
-  push:
-    branches: [ master ]
-    paths-ignore:
-      - '**.md'
-      - '**.yml'
-      - 'version.properties'
-      
-  # Note: This is different from "pull_request". Need to specify ref when doing checkouts.
-  pull_request_target:
-    branches: [ master ]
-    paths-ignore:
-      - '**.md'
-      - '**.yml'
-
+  workflow_dispatch:
+  
 jobs:
 
   ci-check:

.github/workflows/main_matrix.yml

@@ -0,0 +1,332 @@
+name: Continuous Integration
+on:
+  # # Triggers the workflow on push but only for the master branch
+  push:
+    branches: [ master ]
+    paths-ignore:
+      - '**.md'
+      - '**.yml'
+      - 'version.properties'
+      
+  # Note: This is different from "pull_request". Need to specify ref when doing checkouts.
+  pull_request_target:
+    branches: [ master ]
+    paths-ignore:
+      - '**.md'
+      - '**.yml'
+
+  workflow_dispatch:
+
+jobs:
+
+  check:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - board: rak11200
+          - board: tlora-v2
+          - board: tlora-v1
+          - board: tlora_v1_3
+          - board: tlora-v2-1-1.6
+          - board: tbeam
+          - board: heltec-v1
+          - board: heltec-v2.0
+          - board: heltec-v2.1
+          - board: tbeam0.7
+          - board: meshtastic-diy-v1
+          - board: rak4631_5005
+          - board: rak4631_19003
+          - board: t-echo
+  
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{github.event.pull_request.head.repo.full_name}}
+
+    - name: Install cppcheck
+      run: |
+        sudo apt-get install -y cppcheck
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.x
+
+    - name: Cache python libs
+      uses: actions/cache@v1
+      id: cache-pip # needed in if test
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+
+    - name: Upgrade python tools and install platformio
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U platformio
+
+    - name: Upgrade platformio
+      run: |
+        pio upgrade
+
+    - name: Check ${{ matrix.board }}
+      run: bin/check-all.sh ${{ matrix.board }}
+
+  build-esp32:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - board: rak11200
+          - board: tlora-v2
+          - board: tlora-v1
+          - board: tlora_v1_3
+          - board: tlora-v2-1-1.6
+          - board: tbeam
+          - board: heltec-v1
+          - board: heltec-v2.0
+          - board: heltec-v2.1
+          - board: tbeam0.7
+          - board: meshtastic-diy-v1
+    
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{github.event.pull_request.head.repo.full_name}}
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.x
+
+    - name: Cache python libs
+      uses: actions/cache@v1
+      id: cache-pip # needed in if test
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+
+    - name: Upgrade python tools
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U platformio meshtastic adafruit-nrfutil
+
+    - name: Upgrade platformio
+      run: |
+        pio upgrade
+
+    - name: Pull web ui
+      uses: dsaltares/fetch-gh-release-asset@master
+      with:
+        repo: "meshtastic/meshtastic-web"
+        file: "build.tar"
+        target: "build.tar"
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Unpack web ui
+      run: |
+        tar -xf build.tar -C data/static
+        rm build.tar
+
+    - name: Build ESP32
+      run: bin/build-esp32.sh ${{ matrix.board }}
+
+    - name: Get release version string
+      run: echo "::set-output name=version::$(./bin/buildinfo.py long)"
+      id: version
+
+    - name: Store binaries as an artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: firmware-${{ matrix.board }}-${{ steps.version.outputs.version }}.zip
+        path: |
+          release/*.bin
+          release/*.elf
+        retention-days: 90
+
+  build-nrf52:
+    strategy:
+      fail-fast: false
+      matrix:
+        include: 
+          - board: rak4631_5005
+          - board: rak4631_19003
+          - board: t-echo
+    
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{github.event.pull_request.head.repo.full_name}}
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.x
+
+    - name: Cache python libs
+      uses: actions/cache@v1
+      id: cache-pip # needed in if test
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+
+    - name: Upgrade python tools
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U platformio meshtastic adafruit-nrfutil
+
+    - name: Upgrade platformio
+      run: |
+        pio upgrade
+
+    - name: Build NRF52
+      run: bin/build-nrf52.sh ${{ matrix.board }}
+
+    - name: Get release version string
+      run: echo "::set-output name=version::$(./bin/buildinfo.py long)"
+      id: version
+
+    - name: Store binaries as an artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: firmware-${{ matrix.board }}-${{ steps.version.outputs.version }}.zip
+        path: |
+          release/*.uf2
+          release/*.elf
+        retention-days: 90
+
+  build-native:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{github.event.pull_request.head.repo.full_name}}
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.x
+
+    - name: Cache python libs
+      uses: actions/cache@v1
+      id: cache-pip # needed in if test
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+
+    - name: Upgrade python tools
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U platformio meshtastic adafruit-nrfutil
+
+    - name: Upgrade platformio
+      run: |
+        pio upgrade
+
+    # We now run integration test before other build steps (to quickly see runtime failures)
+    - name: Build for native
+      run: platformio run -e native
+    - name: Integration test
+      run: |
+        .pio/build/native/program &
+        sleep 20 # 5 seconds was not enough
+        echo "Simulator started, launching python test..."
+        python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
+
+    - name: Build Native
+      run: bin/build-native.sh
+
+    - name: Get release version string
+      run: echo "::set-output name=version::$(./bin/buildinfo.py long)"
+      id: version
+
+    - name: Store binaries as an artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: firmware-native-${{ steps.version.outputs.version }}.zip
+        path: | 
+          release/meshtasticd_linux_amd64
+          release/device-*.sh
+          release/device-*.bat
+        retention-days: 90
+
+  gather-artifacts:
+    runs-on: ubuntu-latest
+    needs: [build-esp32, build-nrf52, build-native]
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+
+      - uses: actions/download-artifact@v2
+        with:
+          path: ./
+
+      - name: Get release version string
+        run: echo "::set-output name=version::$(./bin/buildinfo.py long)"
+        id: version
+
+      - name: Move files up
+        run: mv -b -t ./ ./*tbeam-*/spiffs*.bin ./*tbeam-*/system-info.bin ./**/firmware*.bin ./**/*.uf2 ./**/*.elf ./**/meshtasticd_linux_amd64 ./*native*/*device-*.sh ./*native*/*device-*.bat
+
+      - name: Repackage in single firmware zip
+        uses: actions/upload-artifact@v2
+        with:
+          name: firmware-${{ steps.version.outputs.version }}
+          path: | 
+            ./*.bin
+            ./*.uf2
+            ./meshtasticd_linux_amd64
+            ./device-*.sh
+            ./device-*.bat
+          retention-days: 90
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: firmware-${{ steps.version.outputs.version }}
+          path: ./output
+
+      # For diagnostics
+      - name: Show artifacts 
+        run: ls -lR
+
+      - name: Zip firmware
+        run: zip -j -r ./firmware-${{ steps.version.outputs.version }}.zip ./output
+
+      - name: Repackage in single elfs zip
+        uses: actions/upload-artifact@v2
+        with:
+          name: debug-elfs-${{ steps.version.outputs.version }}.zip
+          path: ./*.elf
+          retention-days: 90
+
+      - name: Create request artifacts
+        if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request' }}
+        uses: gavv/pull-request-artifacts@v1.0.0
+        with:
+          commit: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts-branch: artifacts
+          artifacts: ./firmware-${{ steps.version.outputs.version }}.zip

bin/build-all.sh

@@ -108,4 +108,4 @@ echo Generating $ARCHIVEDIR/elfs-$VERSION.zip
 rm -f $ARCHIVEDIR/elfs-$VERSION.zip
 zip --junk-paths $ARCHIVEDIR/elfs-$VERSION.zip $OUTDIR/elfs/universal/firmware-*-$VERSION.* 
 
-echo BUILT ALL
+echo BUILT ALL

bin/build-esp32.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION=`bin/buildinfo.py long`
+SHORT_VERSION=`bin/buildinfo.py short`
+
+OUTDIR=release/
+
+rm -f $OUTDIR/firmware*
+rm -r $OUTDIR/* || true
+
+# Make sure our submodules are current
+git submodule update 
+
+# Important to pull latest version of libs into all device flavors, otherwise some devices might be stale
+platformio lib update 
+
+echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
+rm -f .pio/build/$1/firmware.*
+
+# The shell vars the build tool expects to find
+export APP_VERSION=$VERSION
+
+# Are we building a universal/regionless rom?
+export HW_VERSION="1.0"
+basename=firmware-$1-$VERSION
+
+pio run --environment $1 # -v
+SRCELF=.pio/build/$1/firmware.elf
+cp $SRCELF $OUTDIR/$basename.elf
+
+echo "Copying ESP32 bin file"
+SRCBIN=.pio/build/$1/firmware.bin
+cp $SRCBIN $OUTDIR/$basename.bin
+
+echo "Building SPIFFS for ESP32 targets"
+pio run --environment tbeam -t buildfs
+cp .pio/build/tbeam/spiffs.bin $OUTDIR/spiffs-$VERSION.bin
+cp images/system-info.bin $OUTDIR/system-info.bin
+
+cp bin/device-install.* $OUTDIR
+cp bin/device-update.* $OUTDIR

bin/build-native.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION=`bin/buildinfo.py long`
+SHORT_VERSION=`bin/buildinfo.py short`
+
+OUTDIR=release/
+
+rm -f $OUTDIR/firmware*
+
+mkdir -p $OUTDIR/
+rm -r $OUTDIR/* || true
+
+# Make sure our submodules are current
+git submodule update 
+
+# Important to pull latest version of libs into all device flavors, otherwise some devices might be stale
+platformio lib update 
+
+pio run --environment native
+cp .pio/build/native/program $OUTDIR/meshtasticd_linux_amd64
+
+cp bin/device-install.* $OUTDIR
+cp bin/device-update.* $OUTDIR
+

bin/build-nrf52.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION=`bin/buildinfo.py long`
+SHORT_VERSION=`bin/buildinfo.py short`
+
+OUTDIR=release/
+
+rm -f $OUTDIR/firmware*
+rm -r $OUTDIR/* || true
+
+# Make sure our submodules are current
+git submodule update 
+
+# Important to pull latest version of libs into all device flavors, otherwise some devices might be stale
+platformio lib update 
+
+echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
+rm -f .pio/build/$1/firmware.*
+
+# The shell vars the build tool expects to find
+export APP_VERSION=$VERSION
+
+export HW_VERSION="1.0"
+basename=firmware-$1-$VERSION
+
+pio run --environment $1 # -v
+SRCELF=.pio/build/$1/firmware.elf
+cp $SRCELF $OUTDIR/$basename.elf
+
+echo "Generating NRF52 uf2 file"
+SRCHEX=.pio/build/$1/firmware.hex
+bin/uf2conv.py $SRCHEX -c -o $OUTDIR/$basename.uf2 -f 0xADA52840
+
+cp bin/device-install.* $OUTDIR
+cp bin/device-update.* $OUTDIR