diff --git a/.github/actions/setup-native/action.yml b/.github/actions/setup-native/action.yml new file mode 100644 index 000000000..36c95d943 --- /dev/null +++ b/.github/actions/setup-native/action.yml @@ -0,0 +1,14 @@ +name: Setup native build +description: Install libraries needed for building the Native/Portduino build + +runs: + using: composite + steps: + - name: Setup base + id: base + uses: ./.github/actions/setup-base + + - name: Install libs needed for native build + shell: bash + run: | + sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev diff --git a/.github/workflows/build_native.yml b/.github/workflows/build_native.yml index 74bc074aa..11ba09ad7 100644 --- a/.github/workflows/build_native.yml +++ b/.github/workflows/build_native.yml @@ -10,12 +10,6 @@ jobs: build-native: runs-on: ubuntu-latest steps: - - name: Install libs needed for native build - shell: bash - run: | - sudo apt-get update --fix-missing - sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev - - name: Checkout code uses: actions/checkout@v4 with: @@ -23,17 +17,9 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Upgrade python tools - shell: bash - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil - pip install -U meshtastic --pre - - - name: Upgrade platformio - shell: bash - run: | - pio upgrade + - name: Setup native build + id: base + uses: ./.github/actions/setup-native - name: Build Native run: bin/build-native.sh diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 0109bef1a..a437411b5 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -137,6 +137,9 @@ jobs: package-native: uses: ./.github/workflows/package_amd64.yml + test-native: + uses: ./.github/workflows/test_native.yml + build-docker: if: ${{ github.event_name == 'workflow_dispatch' }} uses: ./.github/workflows/build_docker.yml diff --git a/.github/workflows/test_native.yml b/.github/workflows/test_native.yml new file mode 100644 index 000000000..8c1527279 --- /dev/null +++ b/.github/workflows/test_native.yml @@ -0,0 +1,164 @@ +name: Run Tests on Native platform + +on: + workflow_call: + workflow_dispatch: + +permissions: {} + +env: + LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}" + +jobs: + simulator-tests: + name: Native Simulator Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + submodules: recursive + + - name: Setup native build + id: base + uses: ./.github/actions/setup-native + + - name: Install simulator dependencies + run: pip install -U dotmap + + # We now run integration test before other build steps (to quickly see runtime failures) + - name: Build for native/coverage + run: platformio run -e coverage + + - name: Capture initial coverage information + shell: bash + run: | + sudo apt-get install -y lcov + lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info + sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative. + + - name: Integration test + run: | + .pio/build/coverage/program & + PID=$! + timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done" + echo "Simulator started, launching python test..." + python3 -c 'from meshtastic.test import testSimulator; testSimulator()' + wait + + - name: Capture coverage information + if: always() # run this step even if previous step failed + run: | + lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info + sed -i -e "s#${PWD}#.#" coverage_integration.info # Make paths relative. + + - name: Get release version string + if: always() # run this step even if previous step failed + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Save coverage information + uses: actions/upload-artifact@v4 + if: always() # run this step even if previous step failed + with: + name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.version }}.zip + overwrite: true + path: ./coverage_*.info + + platformio-tests: + name: Native PlatformIO Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + submodules: recursive + + - name: Setup native build + id: base + uses: ./.github/actions/setup-native + + - name: Get release version string + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: PlatformIO Tests + run: platformio test -e coverage --junit-output-path testreport.xml + + - name: Save test results + if: always() # run this step even if previous step failed + uses: actions/upload-artifact@v4 + with: + name: platformio-test-report-${{ steps.version.outputs.version }}.zip + overwrite: true + path: ./testreport.xml + + - name: Capture coverage information + if: always() # run this step even if previous step failed + run: | + sudo apt-get install -y lcov + lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info + sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative. + + - name: Save coverage information + uses: actions/upload-artifact@v4 + if: always() # run this step even if previous step failed + with: + name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.version }}.zip + overwrite: true + path: ./coverage_*.info + + generate-reports: + name: Generate Test Reports + runs-on: ubuntu-latest + permissions: # Needed for dorny/test-reporter. + contents: read + actions: read + checks: write + needs: + - simulator-tests + - platformio-tests + if: always() + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Get release version string + run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Download test artifacts + uses: actions/download-artifact@v4 + with: + name: platformio-test-report-${{ steps.version.outputs.version }}.zip + merge-multiple: true + + - name: Test Report + uses: dorny/test-reporter@v1.9.1 + with: + name: PlatformIO Tests + path: testreport.xml + reporter: java-junit + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.version }}.zip + path: code-coverage-report + merge-multiple: true + + - name: Generate Code Coverage Report + run: | + sudo apt-get install -y lcov + lcov --quiet --add-tracefile code-coverage-report/coverage_base.info --add-tracefile code-coverage-report/coverage_integration.info --add-tracefile code-coverage-report/coverage_tests.info --output-file code-coverage-report/coverage_src.info + genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report + + - name: Save Code Coverage Report + uses: actions/upload-artifact@v4 + with: + name: code-coverage-report-${{ steps.version.outputs.version }}.zip + path: code-coverage-report diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae9f82543..c9489db1a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,79 +6,8 @@ on: workflow_dispatch: {} jobs: - test-simulator: - runs-on: ubuntu-latest - env: - LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}" - steps: - - name: Install libs needed for native build - shell: bash - run: | - sudo apt-get update --fix-missing - sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev - sudo apt-get install -y lcov - - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Upgrade python tools - shell: bash - run: | - python -m pip install --upgrade pip - pip install -U platformio adafruit-nrfutil dotmap - pip install -U meshtastic --pre - - - name: Upgrade platformio - shell: bash - run: | - pio upgrade - - - name: Build Native - run: bin/build-native.sh - - # We now run integration test before other build steps (to quickly see runtime failures) - - name: Build for native/coverage - run: | - platformio run -e coverage - lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info - - - name: Integration test - run: | - .pio/build/coverage/program & - PID=$! - timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done" - echo "Simulator started, launching python test..." - python3 -c 'from meshtastic.test import testSimulator; testSimulator()' - wait - lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info - - - name: PlatformIO Tests - run: | - platformio test -e coverage --junit-output-path testreport.xml - lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info - - - name: Test Report - uses: dorny/test-reporter@v1.9.1 - if: success() || failure() # run this step even if previous step failed - with: - name: PlatformIO Tests - path: testreport.xml - reporter: java-junit - - - name: Generate Code Coverage Report - run: | - lcov --quiet --add-tracefile coverage_base.info --add-tracefile coverage_integration.info --add-tracefile coverage_tests.info --output-file coverage_src.info - mkdir code-coverage-report - genhtml --quiet --legend --prefix "${PWD}" coverage_src.info --output-directory code-coverage-report - mv coverage_*.info code-coverage-report - - - name: Save Code Coverage Report - uses: actions/upload-artifact@v4 - with: - name: code-coverage-report - path: code-coverage-report + native-tests: + uses: ./.github/workflows/test_native.yml hardware-tests: runs-on: test-runner diff --git a/src/configuration.h b/src/configuration.h index 994f1e72e..2c77b55e3 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -178,13 +178,6 @@ along with this program. If not, see . #define TCA9535_ADDR 0x20 #define TCA9555_ADDR 0x26 -// ----------------------------------------------------------------------------- -// GPS -// ----------------------------------------------------------------------------- -#ifndef GPS_THREAD_INTERVAL -#define GPS_THREAD_INTERVAL 200 -#endif - // ----------------------------------------------------------------------------- // Touchscreen // ----------------------------------------------------------------------------- @@ -206,6 +199,10 @@ along with this program. If not, see . #define VEXT_ON_VALUE LOW #endif +// ----------------------------------------------------------------------------- +// GPS +// ----------------------------------------------------------------------------- + #ifndef GPS_BAUDRATE #define GPS_BAUDRATE 9600 #define GPS_BAUDRATE_FIXED 0 @@ -213,6 +210,10 @@ along with this program. If not, see . #define GPS_BAUDRATE_FIXED 1 #endif +#ifndef GPS_THREAD_INTERVAL +#define GPS_THREAD_INTERVAL 200 +#endif + /* Step #2: follow with defines common to the architecture; also enable HAS_ option not specifically disabled by variant.h */ #include "architecture.h" diff --git a/src/mesh/MeshModule.cpp b/src/mesh/MeshModule.cpp index 9de54ade5..2f2863fa5 100644 --- a/src/mesh/MeshModule.cpp +++ b/src/mesh/MeshModule.cpp @@ -4,6 +4,7 @@ #include "NodeDB.h" #include "configuration.h" #include "modules/RoutingModule.h" +#include #include std::vector *MeshModule::modules; @@ -29,7 +30,9 @@ void MeshModule::setup() {} MeshModule::~MeshModule() { - assert(0); // FIXME - remove from list of modules once someone needs this feature + auto it = std::find(modules->begin(), modules->end(), this); + assert(it != modules->end()); + modules->erase(it); } meshtastic_MeshPacket *MeshModule::allocAckNak(meshtastic_Routing_Error err, NodeNum to, PacketId idFrom, ChannelIndex chIndex, diff --git a/src/mqtt/MQTT.cpp b/src/mqtt/MQTT.cpp index 4260ae201..46fb607b5 100644 --- a/src/mqtt/MQTT.cpp +++ b/src/mqtt/MQTT.cpp @@ -2,6 +2,7 @@ #include "MeshService.h" #include "NodeDB.h" #include "PowerFSM.h" +#include "ServiceEnvelope.h" #include "configuration.h" #include "main.h" #include "mesh/Channels.h" @@ -25,7 +26,6 @@ #endif #include #include -#include #include #include @@ -47,23 +47,6 @@ static uint8_t bytes[meshtastic_MqttClientProxyMessage_size + 30]; // 12 for cha static bool isMqttServerAddressPrivate = false; -// meshtastic_ServiceEnvelope that automatically releases dynamically allocated memory when it goes out of scope. -struct DecodedServiceEnvelope : public meshtastic_ServiceEnvelope { - DecodedServiceEnvelope() = delete; - DecodedServiceEnvelope(const uint8_t *payload, size_t length) - : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_default), - validDecode(pb_decode_from_bytes(payload, length, &meshtastic_ServiceEnvelope_msg, this)) - { - } - ~DecodedServiceEnvelope() - { - if (validDecode) - pb_release(&meshtastic_ServiceEnvelope_msg, this); - } - // Clients must check that this is true before using. - const bool validDecode; -}; - inline void onReceiveProto(char *topic, byte *payload, size_t length) { const DecodedServiceEnvelope e(payload, length); @@ -299,7 +282,9 @@ void mqttInit() } #if HAS_NETWORKING -MQTT::MQTT() : concurrency::OSThread("mqtt"), pubSub(mqttClient), mqttQueue(MAX_MQTT_QUEUE) +MQTT::MQTT() : MQTT(std::unique_ptr(new MQTTClient())) {} +MQTT::MQTT(std::unique_ptr _mqttClient) + : concurrency::OSThread("mqtt"), mqttQueue(MAX_MQTT_QUEUE), mqttClient(std::move(_mqttClient)), pubSub(*mqttClient) #else MQTT::MQTT() : concurrency::OSThread("mqtt"), mqttQueue(MAX_MQTT_QUEUE) #endif @@ -437,13 +422,13 @@ void MQTT::reconnect() } } else { LOG_INFO("Use non-TLS-encrypted session"); - pubSub.setClient(mqttClient); + pubSub.setClient(*mqttClient); } #else - pubSub.setClient(mqttClient); + pubSub.setClient(*mqttClient); #endif #elif HAS_NETWORKING - pubSub.setClient(mqttClient); + pubSub.setClient(*mqttClient); #endif std::pair hostAndPort = parseHostAndPort(serverAddr, serverPort); @@ -461,7 +446,7 @@ void MQTT::reconnect() enabled = true; // Start running background process again runASAP = true; reconnectCount = 0; - isMqttServerAddressPrivate = isPrivateIpAddress(mqttClient.remoteIP()); + isMqttServerAddressPrivate = isPrivateIpAddress(mqttClient->remoteIP()); publishNodeInfo(); sendSubscriptions(); diff --git a/src/mqtt/MQTT.h b/src/mqtt/MQTT.h index cb1fffcc9..cf52ad877 100644 --- a/src/mqtt/MQTT.h +++ b/src/mqtt/MQTT.h @@ -22,6 +22,7 @@ #if HAS_NETWORKING #include +#include #endif #define MAX_MQTT_QUEUE 16 @@ -32,24 +33,7 @@ */ class MQTT : private concurrency::OSThread { - // supposedly the current version is busted: - // http://www.iotsharing.com/2017/08/how-to-use-esp32-mqtts-with-mqtts-mosquitto-broker-tls-ssl.html -#if HAS_WIFI - WiFiClient mqttClient; -#if !defined(ARCH_PORTDUINO) -#if (defined(ESP_ARDUINO_VERSION_MAJOR) && ESP_ARDUINO_VERSION_MAJOR < 3) || defined(RPI_PICO) - WiFiClientSecure wifiSecureClient; -#endif -#endif -#endif -#if HAS_ETHERNET - EthernetClient mqttClient; -#endif - public: -#if HAS_NETWORKING - PubSubClient pubSub; -#endif MQTT(); /** @@ -93,7 +77,29 @@ class MQTT : private concurrency::OSThread virtual int32_t runOnce() override; +#ifndef PIO_UNIT_TESTING private: +#endif + // supposedly the current version is busted: + // http://www.iotsharing.com/2017/08/how-to-use-esp32-mqtts-with-mqtts-mosquitto-broker-tls-ssl.html +#if HAS_WIFI + using MQTTClient = WiFiClient; +#if !defined(ARCH_PORTDUINO) +#if (defined(ESP_ARDUINO_VERSION_MAJOR) && ESP_ARDUINO_VERSION_MAJOR < 3) || defined(RPI_PICO) + WiFiClientSecure wifiSecureClient; +#endif +#endif +#endif +#if HAS_ETHERNET + using MQTTClient = EthernetClient; +#endif + +#if HAS_NETWORKING + std::unique_ptr mqttClient; + PubSubClient pubSub; + explicit MQTT(std::unique_ptr mqttClient); +#endif + std::string cryptTopic = "/2/e/"; // msh/2/e/CHANNELID/NODEID std::string jsonTopic = "/2/json/"; // msh/2/json/CHANNELID/NODEID std::string mapTopic = "/2/map/"; // For protobuf-encoded MapReport messages diff --git a/src/mqtt/ServiceEnvelope.cpp b/src/mqtt/ServiceEnvelope.cpp new file mode 100644 index 000000000..ee55f22f6 --- /dev/null +++ b/src/mqtt/ServiceEnvelope.cpp @@ -0,0 +1,23 @@ +#include "ServiceEnvelope.h" +#include "mesh-pb-constants.h" +#include + +DecodedServiceEnvelope::DecodedServiceEnvelope(const uint8_t *payload, size_t length) + : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_default), + validDecode(pb_decode_from_bytes(payload, length, &meshtastic_ServiceEnvelope_msg, this)) +{ +} + +DecodedServiceEnvelope::DecodedServiceEnvelope(DecodedServiceEnvelope &&other) + : meshtastic_ServiceEnvelope(meshtastic_ServiceEnvelope_init_zero), validDecode(other.validDecode) +{ + std::swap(packet, other.packet); + std::swap(channel_id, other.channel_id); + std::swap(gateway_id, other.gateway_id); +} + +DecodedServiceEnvelope::~DecodedServiceEnvelope() +{ + if (validDecode) + pb_release(&meshtastic_ServiceEnvelope_msg, this); +} \ No newline at end of file diff --git a/src/mqtt/ServiceEnvelope.h b/src/mqtt/ServiceEnvelope.h new file mode 100644 index 000000000..6ab0695db --- /dev/null +++ b/src/mqtt/ServiceEnvelope.h @@ -0,0 +1,13 @@ +#pragma once + +#include "mesh/generated/meshtastic/mqtt.pb.h" + +// meshtastic_ServiceEnvelope that automatically releases dynamically allocated memory when it goes out of scope. +struct DecodedServiceEnvelope : public meshtastic_ServiceEnvelope { + DecodedServiceEnvelope(const uint8_t *payload, size_t length); + DecodedServiceEnvelope(DecodedServiceEnvelope &) = delete; + DecodedServiceEnvelope(DecodedServiceEnvelope &&); + ~DecodedServiceEnvelope(); + // Clients must check that this is true before using. + const bool validDecode; +}; \ No newline at end of file