Run tests as part of the main CI (#5712)

* Create an shared action to install native dependecies

* Create a workflow for running native tests

* Artifact names contain version

* Add test-native to main_matrix.yml

* No permission are required for test_native.yml

* Add permissions for dorny/test-reporter

* No permissions when running tests

* s/Generate Reports/Generate Test Reports/

.github/actions/setup-native/action.yml

@@ -0,0 +1,14 @@
+name: Setup native build
+description: Install libraries needed for building the Native/Portduino build
+
+runs:
+  using: composite
+  steps:
+    - name: Setup base
+      id: base
+      uses: ./.github/actions/setup-base
+
+    - name: Install libs needed for native build
+      shell: bash
+      run: |
+        sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev

.github/workflows/build_native.yml

@@ -10,12 +10,6 @@ jobs:
   build-native:
     runs-on: ubuntu-latest
     steps:
-      - name: Install libs needed for native build
-        shell: bash
-        run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -23,17 +17,9 @@ jobs:
           ref: ${{github.event.pull_request.head.ref}}
           repository: ${{github.event.pull_request.head.repo.full_name}}
 
-      - name: Upgrade python tools
+      - name: Setup native build
-        shell: bash
+        id: base
-        run: |
+        uses: ./.github/actions/setup-native
-          python -m pip install --upgrade pip
-          pip install -U platformio adafruit-nrfutil
-          pip install -U meshtastic --pre
-
-      - name: Upgrade platformio
-        shell: bash
-        run: |
-          pio upgrade
 
       - name: Build Native
         run: bin/build-native.sh

.github/workflows/main_matrix.yml

@@ -137,6 +137,9 @@ jobs:
   package-native:
     uses: ./.github/workflows/package_amd64.yml
 
+  test-native:
+    uses: ./.github/workflows/test_native.yml
+
   build-docker:
     if: ${{ github.event_name == 'workflow_dispatch' }}
     uses: ./.github/workflows/build_docker.yml

.github/workflows/test_native.yml

@@ -0,0 +1,153 @@
+name: Run Tests on Native platform
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+permissions: {}
+
+env:
+  LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"
+
+jobs:
+  simulator-tests:
+    name: Native Simulator Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup native build
+        id: base
+        uses: ./.github/actions/setup-native
+
+      - name: Install simulator dependencies
+        run: pip install -U dotmap
+
+      # We now run integration test before other build steps (to quickly see runtime failures)
+      - name: Build for native/coverage
+        run: platformio run -e coverage
+
+      - name: Capture initial coverage information
+        shell: bash
+        run: |
+          sudo apt-get install -y lcov
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
+
+      - name: Integration test
+        run: |
+          .pio/build/coverage/program &
+          PID=$!
+          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
+          echo "Simulator started, launching python test..."
+          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
+          wait
+
+      - name: Capture coverage information
+        if: always() # run this step even if previous step failed
+        run: lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
+
+      - name: Get release version string
+        if: always() # run this step even if previous step failed
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: Save coverage information
+        uses: actions/upload-artifact@v4
+        if: always() # run this step even if previous step failed
+        with:
+          name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./coverage_*.info
+
+  platformio-tests:
+    name: Native PlatformIO Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup native build
+        id: base
+        uses: ./.github/actions/setup-native
+
+      - name: Get release version string
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: PlatformIO Tests
+        run: platformio test -e coverage --junit-output-path testreport.xml
+
+      - name: Save test results
+        if: always() # run this step even if previous step failed
+        uses: actions/upload-artifact@v4
+        with:
+          name: platformio-test-report-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./testreport.xml
+
+      - name: Capture coverage information
+        if: always() # run this step even if previous step failed
+        run: |
+          sudo apt-get install -y lcov
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
+
+      - name: Save coverage information
+        uses: actions/upload-artifact@v4
+        if: always() # run this step even if previous step failed
+        with:
+          name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./coverage_*.info
+
+  generate-reports:
+    name: Generate Test Reports
+    runs-on: ubuntu-latest
+    permissions: # Needed for dorny/test-reporter.
+      contents: read
+      actions: read
+      checks: write
+    needs:
+      - simulator-tests
+      - platformio-tests
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get release version string
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: Download test artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: platformio-test-report-${{ steps.version.outputs.version }}.zip
+          merge-multiple: true
+
+      - name: Test Report
+        uses: dorny/test-reporter@v1.9.1
+        with:
+          name: PlatformIO Tests
+          path: testreport.xml
+          reporter: java-junit
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.version }}.zip
+          path: code-coverage-report
+          merge-multiple: true
+
+      - name: Generate Code Coverage Report
+        run: |
+          sudo apt-get install -y lcov
+          lcov --quiet --add-tracefile code-coverage-report/coverage_base.info --add-tracefile code-coverage-report/coverage_integration.info --add-tracefile code-coverage-report/coverage_tests.info --output-file code-coverage-report/coverage_src.info
+          genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report
+
+      - name: Save Code Coverage Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report-${{ steps.version.outputs.version }}.zip
+          path: code-coverage-report

.github/workflows/tests.yml

@@ -6,79 +6,8 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  test-simulator:
+  native-tests:
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/test_native.yml
-    env:
-      LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"
-    steps:
-      - name: Install libs needed for native build
-        shell: bash
-        run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev
-          sudo apt-get install -y lcov
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Upgrade python tools
-        shell: bash
-        run: |
-          python -m pip install --upgrade pip
-          pip install -U platformio adafruit-nrfutil dotmap
-          pip install -U meshtastic --pre
-
-      - name: Upgrade platformio
-        shell: bash
-        run: |
-          pio upgrade
-
-      - name: Build Native
-        run: bin/build-native.sh
-
-      # We now run integration test before other build steps (to quickly see runtime failures)
-      - name: Build for native/coverage
-        run: |
-          platformio run -e coverage
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
-
-      - name: Integration test
-        run: |
-          .pio/build/coverage/program &
-          PID=$!
-          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
-          echo "Simulator started, launching python test..."
-          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
-          wait
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
-
-      - name: PlatformIO Tests
-        run: |
-          platformio test -e coverage --junit-output-path testreport.xml
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
-
-      - name: Test Report
-        uses: dorny/test-reporter@v1.9.1
-        if: success() || failure() # run this step even if previous step failed
-        with:
-          name: PlatformIO Tests
-          path: testreport.xml
-          reporter: java-junit
-
-      - name: Generate Code Coverage Report
-        run: |
-          lcov --quiet --add-tracefile coverage_base.info --add-tracefile coverage_integration.info --add-tracefile coverage_tests.info --output-file coverage_src.info
-          mkdir code-coverage-report
-          genhtml --quiet --legend --prefix "${PWD}" coverage_src.info --output-directory code-coverage-report
-          mv coverage_*.info code-coverage-report
-
-      - name: Save Code Coverage Report
-        uses: actions/upload-artifact@v4
-        with:
-          name: code-coverage-report
-          path: code-coverage-report
 
   hardware-tests:
     runs-on: test-runner