Merge branch 'master' into tft-gui-work

.github/actions/setup-native/action.yml

@@ -0,0 +1,14 @@
+name: Setup native build
+description: Install libraries needed for building the Native/Portduino build
+
+runs:
+  using: composite
+  steps:
+    - name: Setup base
+      id: base
+      uses: ./.github/actions/setup-base
+
+    - name: Install libs needed for native build
+      shell: bash
+      run: |
+        sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev

.github/workflows/build_native.yml

@@ -10,12 +10,6 @@ jobs:
   build-native:
     runs-on: ubuntu-latest
     steps:
-      - name: Install libs needed for native build
-        shell: bash
-        run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -23,17 +17,9 @@ jobs:
           ref: ${{github.event.pull_request.head.ref}}
           repository: ${{github.event.pull_request.head.repo.full_name}}
 
-      - name: Upgrade python tools
+      - name: Setup native build
-        shell: bash
+        id: base
-        run: |
+        uses: ./.github/actions/setup-native
-          python -m pip install --upgrade pip
-          pip install -U platformio adafruit-nrfutil
-          pip install -U meshtastic --pre
-
-      - name: Upgrade platformio
-        shell: bash
-        run: |
-          pio upgrade
 
       - name: Build Native
         run: bin/build-native.sh

.github/workflows/main_matrix.yml

@@ -137,6 +137,9 @@ jobs:
   package-native:
     uses: ./.github/workflows/package_amd64.yml
 
+  test-native:
+    uses: ./.github/workflows/test_native.yml
+
   build-docker:
     if: ${{ github.event_name == 'workflow_dispatch' }}
     uses: ./.github/workflows/build_docker.yml

.github/workflows/test_native.yml

@@ -0,0 +1,169 @@
+name: Run Tests on Native platform
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+permissions: {}
+
+env:
+  LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"
+
+jobs:
+  simulator-tests:
+    name: Native Simulator Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+          submodules: recursive
+
+      - name: Setup native build
+        id: base
+        uses: ./.github/actions/setup-native
+
+      - name: Install simulator dependencies
+        run: pip install -U dotmap
+
+      # We now run integration test before other build steps (to quickly see runtime failures)
+      - name: Build for native/coverage
+        run: platformio run -e coverage
+
+      - name: Capture initial coverage information
+        shell: bash
+        run: |
+          sudo apt-get install -y lcov
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
+          sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.
+
+      - name: Integration test
+        run: |
+          .pio/build/coverage/program &
+          PID=$!
+          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
+          echo "Simulator started, launching python test..."
+          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
+          wait
+
+      - name: Capture coverage information
+        if: always() # run this step even if previous step failed
+        run: |
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
+          sed -i -e "s#${PWD}#.#" coverage_integration.info # Make paths relative.
+
+      - name: Get release version string
+        if: always() # run this step even if previous step failed
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: Save coverage information
+        uses: actions/upload-artifact@v4
+        if: always() # run this step even if previous step failed
+        with:
+          name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./coverage_*.info
+
+  platformio-tests:
+    name: Native PlatformIO Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+          submodules: recursive
+
+      - name: Setup native build
+        id: base
+        uses: ./.github/actions/setup-native
+
+      - name: Get release version string
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      # Disable (comment-out) BUILD_EPOCH. It causes a full rebuild between tests and resets the
+      # coverage information each time.
+      - name: Disable BUILD_EPOCH
+        run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini
+
+      - name: PlatformIO Tests
+        run: platformio test -e coverage --junit-output-path testreport.xml
+
+      - name: Save test results
+        if: always() # run this step even if previous step failed
+        uses: actions/upload-artifact@v4
+        with:
+          name: platformio-test-report-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./testreport.xml
+
+      - name: Capture coverage information
+        if: always() # run this step even if previous step failed
+        run: |
+          sudo apt-get install -y lcov
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
+          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.
+
+      - name: Save coverage information
+        uses: actions/upload-artifact@v4
+        if: always() # run this step even if previous step failed
+        with:
+          name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.version }}.zip
+          overwrite: true
+          path: ./coverage_*.info
+
+  generate-reports:
+    name: Generate Test Reports
+    runs-on: ubuntu-latest
+    permissions: # Needed for dorny/test-reporter.
+      contents: read
+      actions: read
+      checks: write
+    needs:
+      - simulator-tests
+      - platformio-tests
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+
+      - name: Get release version string
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: Download test artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: platformio-test-report-${{ steps.version.outputs.version }}.zip
+          merge-multiple: true
+
+      - name: Test Report
+        uses: dorny/test-reporter@v1.9.1
+        with:
+          name: PlatformIO Tests
+          path: testreport.xml
+          reporter: java-junit
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.version }}.zip
+          path: code-coverage-report
+          merge-multiple: true
+
+      - name: Generate Code Coverage Report
+        run: |
+          sudo apt-get install -y lcov
+          lcov --quiet --add-tracefile code-coverage-report/coverage_base.info --add-tracefile code-coverage-report/coverage_integration.info --add-tracefile code-coverage-report/coverage_tests.info --output-file code-coverage-report/coverage_src.info
+          genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report
+
+      - name: Save Code Coverage Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report-${{ steps.version.outputs.version }}.zip
+          path: code-coverage-report

.github/workflows/tests.yml

@@ -6,79 +6,8 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  test-simulator:
+  native-tests:
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/test_native.yml
-    env:
-      LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"
-    steps:
-      - name: Install libs needed for native build
-        shell: bash
-        run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev
-          sudo apt-get install -y lcov
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Upgrade python tools
-        shell: bash
-        run: |
-          python -m pip install --upgrade pip
-          pip install -U platformio adafruit-nrfutil dotmap
-          pip install -U meshtastic --pre
-
-      - name: Upgrade platformio
-        shell: bash
-        run: |
-          pio upgrade
-
-      - name: Build Native
-        run: bin/build-native.sh
-
-      # We now run integration test before other build steps (to quickly see runtime failures)
-      - name: Build for native/coverage
-        run: |
-          platformio run -e coverage
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
-
-      - name: Integration test
-        run: |
-          .pio/build/coverage/program &
-          PID=$!
-          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
-          echo "Simulator started, launching python test..."
-          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
-          wait
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
-
-      - name: PlatformIO Tests
-        run: |
-          platformio test -e coverage --junit-output-path testreport.xml
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
-
-      - name: Test Report
-        uses: dorny/test-reporter@v1.9.1
-        if: success() || failure() # run this step even if previous step failed
-        with:
-          name: PlatformIO Tests
-          path: testreport.xml
-          reporter: java-junit
-
-      - name: Generate Code Coverage Report
-        run: |
-          lcov --quiet --add-tracefile coverage_base.info --add-tracefile coverage_integration.info --add-tracefile coverage_tests.info --output-file coverage_src.info
-          mkdir code-coverage-report
-          genhtml --quiet --legend --prefix "${PWD}" coverage_src.info --output-directory code-coverage-report
-          mv coverage_*.info code-coverage-report
-
-      - name: Save Code Coverage Report
-        uses: actions/upload-artifact@v4
-        with:
-          name: code-coverage-report
-          path: code-coverage-report
 
   hardware-tests:
     runs-on: test-runner

src/configuration.h

@@ -178,13 +178,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #define TCA9535_ADDR 0x20
 #define TCA9555_ADDR 0x26
 
-// -----------------------------------------------------------------------------
-// GPS
-// -----------------------------------------------------------------------------
-#ifndef GPS_THREAD_INTERVAL
-#define GPS_THREAD_INTERVAL 200
-#endif
-
 // -----------------------------------------------------------------------------
 // Touchscreen
 // -----------------------------------------------------------------------------
@@ -206,6 +199,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #define VEXT_ON_VALUE LOW
 #endif
 
+// -----------------------------------------------------------------------------
+// GPS
+// -----------------------------------------------------------------------------
+
 #ifndef GPS_BAUDRATE
 #define GPS_BAUDRATE 9600
 #define GPS_BAUDRATE_FIXED 0
@@ -213,6 +210,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #define GPS_BAUDRATE_FIXED 1
 #endif
 
+#ifndef GPS_THREAD_INTERVAL
+#define GPS_THREAD_INTERVAL 200
+#endif
+
 /* Step #2: follow with defines common to the architecture;
    also enable HAS_ option not specifically disabled by variant.h */
 #include "architecture.h"

src/mesh/MeshModule.cpp

@@ -4,6 +4,7 @@
 #include "NodeDB.h"
 #include "configuration.h"
 #include "modules/RoutingModule.h"
+#include <algorithm>
 #include <assert.h>
 
 std::vector<MeshModule *> *MeshModule::modules;
@@ -29,7 +30,9 @@ void MeshModule::setup() {}
 
 MeshModule::~MeshModule()
 {
-    assert(0); // FIXME - remove from list of modules once someone needs this feature
+    auto it = std::find(modules->begin(), modules->end(), this);
+    assert(it != modules->end());
+    modules->erase(it);
 }
 
 meshtastic_MeshPacket *MeshModule::allocAckNak(meshtastic_Routing_Error err, NodeNum to, PacketId idFrom, ChannelIndex chIndex,

src/mqtt/MQTT.cpp

@@ -2,6 +2,7 @@
 #include "MeshService.h"
 #include "NodeDB.h"
 #include "PowerFSM.h"
+#include "ServiceEnvelope.h"
 #include "configuration.h"
 #include "main.h"
 #include "mesh/Channels.h"
@@ -25,7 +26,6 @@
 #endif
 #include <Throttle.h>
 #include <assert.h>
-#include <pb_decode.h>
 #include <utility>
 
 #include <IPAddress.h>
@@ -47,23 +47,6 @@ static uint8_t bytes[meshtastic_MqttClientProxyMessage_size + 30]; // 12 for cha
 
 static bool isMqttServerAddressPrivate = false;
 
-// meshtastic_ServiceEnvelope that automatically releases dynamically allocated memory when it goes out of scope.
-struct DecodedServiceEnvelope : public meshtastic_ServiceEnvelope {
-    DecodedServiceEnvelope() = delete;
-    DecodedServiceEnvelope(const uint8_t *payload, size_t length)
-        : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_default),
-          validDecode(pb_decode_from_bytes(payload, length, &meshtastic_ServiceEnvelope_msg, this))
-    {
-    }
-    ~DecodedServiceEnvelope()
-    {
-        if (validDecode)
-            pb_release(&meshtastic_ServiceEnvelope_msg, this);
-    }
-    // Clients must check that this is true before using.
-    const bool validDecode;
-};
-
 inline void onReceiveProto(char *topic, byte *payload, size_t length)
 {
     const DecodedServiceEnvelope e(payload, length);
@@ -299,7 +282,9 @@ void mqttInit()
 }
 
 #if HAS_NETWORKING
-MQTT::MQTT() : concurrency::OSThread("mqtt"), pubSub(mqttClient), mqttQueue(MAX_MQTT_QUEUE)
+MQTT::MQTT() : MQTT(std::unique_ptr<MQTTClient>(new MQTTClient())) {}
+MQTT::MQTT(std::unique_ptr<MQTTClient> _mqttClient)
+    : concurrency::OSThread("mqtt"), mqttQueue(MAX_MQTT_QUEUE), mqttClient(std::move(_mqttClient)), pubSub(*mqttClient)
 #else
 MQTT::MQTT() : concurrency::OSThread("mqtt"), mqttQueue(MAX_MQTT_QUEUE)
 #endif
@@ -437,13 +422,13 @@ void MQTT::reconnect()
             }
         } else {
             LOG_INFO("Use non-TLS-encrypted session");
-            pubSub.setClient(mqttClient);
+            pubSub.setClient(*mqttClient);
         }
 #else
-        pubSub.setClient(mqttClient);
+        pubSub.setClient(*mqttClient);
 #endif
 #elif HAS_NETWORKING
-        pubSub.setClient(mqttClient);
+        pubSub.setClient(*mqttClient);
 #endif
 
         std::pair<String, uint16_t> hostAndPort = parseHostAndPort(serverAddr, serverPort);
@@ -461,7 +446,7 @@ void MQTT::reconnect()
             enabled = true; // Start running background process again
             runASAP = true;
             reconnectCount = 0;
-            isMqttServerAddressPrivate = isPrivateIpAddress(mqttClient.remoteIP());
+            isMqttServerAddressPrivate = isPrivateIpAddress(mqttClient->remoteIP());
 
             publishNodeInfo();
             sendSubscriptions();

src/mqtt/MQTT.h

@@ -22,6 +22,7 @@
 
 #if HAS_NETWORKING
 #include <PubSubClient.h>
+#include <memory>
 #endif
 
 #define MAX_MQTT_QUEUE 16
@@ -32,24 +33,7 @@
  */
 class MQTT : private concurrency::OSThread
 {
-    // supposedly the current version is busted:
-    // http://www.iotsharing.com/2017/08/how-to-use-esp32-mqtts-with-mqtts-mosquitto-broker-tls-ssl.html
-#if HAS_WIFI
-    WiFiClient mqttClient;
-#if !defined(ARCH_PORTDUINO)
-#if (defined(ESP_ARDUINO_VERSION_MAJOR) && ESP_ARDUINO_VERSION_MAJOR < 3) || defined(RPI_PICO)
-    WiFiClientSecure wifiSecureClient;
-#endif
-#endif
-#endif
-#if HAS_ETHERNET
-    EthernetClient mqttClient;
-#endif
-
   public:
-#if HAS_NETWORKING
-    PubSubClient pubSub;
-#endif
     MQTT();
 
     /**
@@ -93,7 +77,29 @@ class MQTT : private concurrency::OSThread
 
     virtual int32_t runOnce() override;
 
+#ifndef PIO_UNIT_TESTING
   private:
+#endif
+    // supposedly the current version is busted:
+    // http://www.iotsharing.com/2017/08/how-to-use-esp32-mqtts-with-mqtts-mosquitto-broker-tls-ssl.html
+#if HAS_WIFI
+    using MQTTClient = WiFiClient;
+#if !defined(ARCH_PORTDUINO)
+#if (defined(ESP_ARDUINO_VERSION_MAJOR) && ESP_ARDUINO_VERSION_MAJOR < 3) || defined(RPI_PICO)
+    WiFiClientSecure wifiSecureClient;
+#endif
+#endif
+#endif
+#if HAS_ETHERNET
+    using MQTTClient = EthernetClient;
+#endif
+
+#if HAS_NETWORKING
+    std::unique_ptr<MQTTClient> mqttClient;
+    PubSubClient pubSub;
+    explicit MQTT(std::unique_ptr<MQTTClient> mqttClient);
+#endif
+
     std::string cryptTopic = "/2/e/";   // msh/2/e/CHANNELID/NODEID
     std::string jsonTopic = "/2/json/"; // msh/2/json/CHANNELID/NODEID
     std::string mapTopic = "/2/map/";   // For protobuf-encoded MapReport messages

src/mqtt/ServiceEnvelope.cpp

@@ -0,0 +1,23 @@
+#include "ServiceEnvelope.h"
+#include "mesh-pb-constants.h"
+#include <pb_decode.h>
+
+DecodedServiceEnvelope::DecodedServiceEnvelope(const uint8_t *payload, size_t length)
+    : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_default),
+      validDecode(pb_decode_from_bytes(payload, length, &meshtastic_ServiceEnvelope_msg, this))
+{
+}
+
+DecodedServiceEnvelope::DecodedServiceEnvelope(DecodedServiceEnvelope &&other)
+    : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_zero), validDecode(other.validDecode)
+{
+    std::swap(packet, other.packet);
+    std::swap(channel_id, other.channel_id);
+    std::swap(gateway_id, other.gateway_id);
+}
+
+DecodedServiceEnvelope::~DecodedServiceEnvelope()
+{
+    if (validDecode)
+        pb_release(&meshtastic_ServiceEnvelope_msg, this);
+}

src/mqtt/ServiceEnvelope.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "mesh/generated/meshtastic/mqtt.pb.h"
+
+// meshtastic_ServiceEnvelope that automatically releases dynamically allocated memory when it goes out of scope.
+struct DecodedServiceEnvelope : public meshtastic_ServiceEnvelope {
+    DecodedServiceEnvelope(const uint8_t *payload, size_t length);
+    DecodedServiceEnvelope(DecodedServiceEnvelope &) = delete;
+    DecodedServiceEnvelope(DecodedServiceEnvelope &&);
+    ~DecodedServiceEnvelope();
+    // Clients must check that this is true before using.
+    const bool validDecode;
+};

src/platform/portduino/PortduinoGlue.cpp

@@ -18,7 +18,6 @@
 #include <fstream>
 #include <iostream>
 #include <map>
-#include <sstream>
 #include <sys/ioctl.h>
 #include <unistd.h>
 
@@ -232,9 +231,9 @@ void portduinoSetup()
             dmac[3] = hash[3];
             dmac[4] = hash[4];
             dmac[5] = hash[5];
-            std::stringstream mactmp;
+            char macBuf[13] = {0};
-            mactmp << std::hex << +dmac[0] << +dmac[1] << +dmac[2] << +dmac[3] << +dmac[4] << +dmac[5];
+            sprintf(macBuf, "%02X%02X%02X%02X%02X%02X", dmac[0], dmac[1], dmac[2], dmac[3], dmac[4], dmac[5]);
-            settingsStrings[mac_address] = mactmp.str();
+            settingsStrings[mac_address] = macBuf;
         }
     }
 
@@ -244,7 +243,7 @@ void portduinoSetup()
         std::cout << "Please set a MAC Address in config.yaml using either MACAddress or MACAddressSource." << std::endl;
         exit(EXIT_FAILURE);
     }
-    std::cout << "MAC Address: " << std::hex << +dmac[0] << +dmac[1] << +dmac[2] << +dmac[3] << +dmac[4] << +dmac[5] << std::endl;
+    printf("MAC ADDRESS: %02X:%02X:%02X:%02X:%02X:%02X\n", dmac[0], dmac[1], dmac[2], dmac[3], dmac[4], dmac[5]);
     // Rather important to set this, if not running simulated.
     randomSeed(time(NULL));