From b1125513f3bbfc857f905826c349d8428a0e82cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 16 Oct 2024 21:11:24 +0200 Subject: [PATCH] Use SD card as store&forward memory --- src/graphics/Screen.cpp | 10 +- src/memGet.cpp | 4 + src/mesh/MeshService.cpp | 4 +- src/mesh/MeshService.h | 4 +- src/mesh/NodeDB.cpp | 8 +- src/modules/Modules.cpp | 9 +- src/modules/StoreForwardModule.cpp | 194 ++++++++++++++++++++++------- src/modules/StoreForwardModule.h | 11 ++ 8 files changed, 180 insertions(+), 64 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index efa3ec78f..f1c2bd873 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -49,6 +49,9 @@ along with this program. If not, see . #include "modules/ExternalNotificationModule.h" #include "modules/TextMessageModule.h" #include "modules/WaypointModule.h" +#if !MESHTASTIC_EXCLUDE_STOREFORWARD +#include "modules/StoreForwardModule.h" +#endif #include "sleep.h" #include "target_specific.h" @@ -58,11 +61,9 @@ along with this program. If not, see . #ifdef ARCH_ESP32 #include "esp_task_wdt.h" -#include "modules/StoreForwardModule.h" #endif #if ARCH_PORTDUINO -#include "modules/StoreForwardModule.h" #include "platform/portduino/PortduinoGlue.h" #endif @@ -2397,9 +2398,9 @@ void DebugInfo::drawFrame(OLEDDisplay *display, OLEDDisplayUiState *state, int16 display->setColor(WHITE); // Draw the channel name display->drawString(x, y + FONT_HEIGHT_SMALL, channelStr); +#if !MESHTASTIC_EXCLUDE_STOREFORWARD // Draw our hardware ID to assist with bluetooth pairing. Either prefix with Info or S&F Logo if (moduleConfig.store_forward.enabled) { -#ifdef ARCH_ESP32 if (!Throttle::isWithinTimespanMs(storeForwardModule->lastHeartbeat, (storeForwardModule->heartbeatInterval * 1200))) { // no heartbeat, overlap a bit #if (defined(USE_EINK) || defined(ILI9341_DRIVER) || defined(ILI9342_DRIVER) || defined(ST7701_CS) || defined(ST7735_CS) || \ @@ -2426,6 +2427,9 @@ void DebugInfo::drawFrame(OLEDDisplay *display, OLEDDisplayUiState *state, int16 imgSF); #endif } +#else + // No store and forward, show a exclamation mark + if (false) { #endif } else { // TODO: Raspberry Pi supports more than just the one screen size diff --git a/src/memGet.cpp b/src/memGet.cpp index ef1102f1e..33fb94e1c 100644 --- a/src/memGet.cpp +++ b/src/memGet.cpp @@ -57,6 +57,8 @@ uint32_t MemGet::getFreePsram() { #ifdef ARCH_ESP32 return ESP.getFreePsram(); +#elif defined(HAS_SDCARD) + return SD.totalBytes() - SD.usedBytes(); #elif defined(ARCH_PORTDUINO) return 4194252; #else @@ -73,6 +75,8 @@ uint32_t MemGet::getPsramSize() { #ifdef ARCH_ESP32 return ESP.getPsramSize(); +#elif defined(HAS_SDCARD) + return SD.totalBytes(); #elif defined(ARCH_PORTDUINO) return 4194252; #else diff --git a/src/mesh/MeshService.cpp b/src/mesh/MeshService.cpp index d224c05dc..c561b3817 100644 --- a/src/mesh/MeshService.cpp +++ b/src/mesh/MeshService.cpp @@ -284,7 +284,6 @@ void MeshService::sendToPhone(meshtastic_MeshPacket *p) { perhapsDecode(p); -#ifdef ARCH_ESP32 #if !MESHTASTIC_EXCLUDE_STOREFORWARD if (moduleConfig.store_forward.enabled && storeForwardModule->isServer() && p->decoded.portnum == meshtastic_PortNum_TEXT_MESSAGE_APP) { @@ -292,7 +291,6 @@ void MeshService::sendToPhone(meshtastic_MeshPacket *p) fromNum++; // Notify observers for packet from radio return; } -#endif #endif if (toPhoneQueue.numFree() == 0) { @@ -418,4 +416,4 @@ uint32_t MeshService::GetTimeSinceMeshPacket(const meshtastic_MeshPacket *mp) delta = 0; return delta; -} +} \ No newline at end of file diff --git a/src/mesh/MeshService.h b/src/mesh/MeshService.h index 1ccca4e6d..52ba60663 100644 --- a/src/mesh/MeshService.h +++ b/src/mesh/MeshService.h @@ -13,11 +13,9 @@ #if defined(ARCH_PORTDUINO) && !HAS_RADIO #include "../platform/portduino/SimRadio.h" #endif -#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) #if !MESHTASTIC_EXCLUDE_STOREFORWARD #include "modules/StoreForwardModule.h" #endif -#endif extern Allocator &queueStatusPool; extern Allocator &mqttClientProxyMessagePool; @@ -165,4 +163,4 @@ class MeshService friend class RoutingModule; }; -extern MeshService *service; +extern MeshService *service; \ No newline at end of file diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 558c5b825..aa7141351 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -21,6 +21,9 @@ #include "mesh-pb-constants.h" #include "meshUtils.h" #include "modules/NeighborInfoModule.h" +// #if !MESHTASTIC_EXCLUDE_STOREFORWARD +// #include "modules/StoreForwardModule.h" +// #endif #include #include #include @@ -32,13 +35,12 @@ #if HAS_WIFI #include "mesh/wifi/WiFiAPClient.h" #endif -#include "modules/StoreForwardModule.h" + #include #include #endif #ifdef ARCH_PORTDUINO -#include "modules/StoreForwardModule.h" #include "platform/portduino/PortduinoGlue.h" #endif @@ -1208,4 +1210,4 @@ void recordCriticalError(meshtastic_CriticalErrorCode code, uint32_t address, co LOG_ERROR("A critical failure occurred, portduino is exiting..."); exit(2); #endif -} +} \ No newline at end of file diff --git a/src/modules/Modules.cpp b/src/modules/Modules.cpp index ad3f0ace4..55b1bdc48 100644 --- a/src/modules/Modules.cpp +++ b/src/modules/Modules.cpp @@ -47,9 +47,6 @@ #endif #if ARCH_PORTDUINO #include "input/LinuxInputImpl.h" -#if !MESHTASTIC_EXCLUDE_STOREFORWARD -#include "modules/StoreForwardModule.h" -#endif #endif #if HAS_TELEMETRY #include "modules/Telemetry/DeviceTelemetry.h" @@ -63,6 +60,9 @@ #if HAS_TELEMETRY && !MESHTASTIC_EXCLUDE_POWER_TELEMETRY #include "modules/Telemetry/PowerTelemetry.h" #endif +#if !MESHTASTIC_EXCLUDE_STOREFORWARD +#include "modules/StoreForwardModule.h" +#endif #ifdef ARCH_ESP32 #if defined(USE_SX1280) && !MESHTASTIC_EXCLUDE_AUDIO #include "modules/esp32/AudioModule.h" @@ -70,9 +70,6 @@ #if !MESHTASTIC_EXCLUDE_PAXCOUNTER #include "modules/esp32/PaxcounterModule.h" #endif -#if !MESHTASTIC_EXCLUDE_STOREFORWARD -#include "modules/StoreForwardModule.h" -#endif #endif #if defined(ARCH_ESP32) || defined(ARCH_NRF52) || defined(ARCH_RP2040) #if !MESHTASTIC_EXCLUDE_EXTERNALNOTIFICATION diff --git a/src/modules/StoreForwardModule.cpp b/src/modules/StoreForwardModule.cpp index 039523207..8f76bd047 100644 --- a/src/modules/StoreForwardModule.cpp +++ b/src/modules/StoreForwardModule.cpp @@ -32,7 +32,7 @@ StoreForwardModule *storeForwardModule; int32_t StoreForwardModule::runOnce() { -#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) +#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) || defined(HAS_SDCARD) if (moduleConfig.store_forward.enabled && is_server) { // Send out the message queue. if (this->busy) { @@ -89,6 +89,27 @@ void StoreForwardModule::populatePSRAM() LOG_DEBUG("After PSRAM init: heap %d/%d PSRAM %d/%d", memGet.getFreeHeap(), memGet.getHeapSize(), memGet.getFreePsram(), memGet.getPsramSize()); LOG_DEBUG("numberOfPackets for packetHistory - %u", numberOfPackets); + this->storageType = StorageType::PSRAM; +} + +/** + * if we have an SDCARD, format it for store&forward use + */ +void StoreForwardModule::populateSDCard() +{ +#if defined(HAS_SDCARD) + if (SD.cardType() != CARD_NONE) { + if (!SD.exists("/storeforward")) { + LOG_INFO("Creating StoreForward directory"); + SD.mkdir("/storeforward"); + } + this->storageType = StorageType::SDCARD; + uint32_t numberOfPackets = (this->records ? this->records : (((SD.totalBytes() / 3) * 2) / sizeof(PacketHistoryStruct))); + // only allocate space for one temp copy + this->packetHistory = (PacketHistoryStruct *)malloc(sizeof(PacketHistoryStruct)); + LOG_DEBUG("numberOfPackets for packetHistory - %u", numberOfPackets); + } +#endif } /** @@ -135,11 +156,26 @@ uint32_t StoreForwardModule::getNumAvailablePackets(NodeNum dest, uint32_t last_ lastRequest.emplace(dest, 0); } for (uint32_t i = lastRequest[dest]; i < this->packetHistoryTotalCount; i++) { - if (this->packetHistory[i].time && (this->packetHistory[i].time > last_time)) { - // Client is only interested in packets not from itself and only in broadcast packets or packets towards it. - if (this->packetHistory[i].from != dest && - (this->packetHistory[i].to == NODENUM_BROADCAST || this->packetHistory[i].to == dest)) { - count++; + if (this->storageType == StorageType::PSRAM) { + if (this->packetHistory[i].time && (this->packetHistory[i].time > last_time)) { + // Client is only interested in packets not from itself and only in broadcast packets or packets towards it. + if (this->packetHistory[i].from != dest && + (this->packetHistory[i].to == NODENUM_BROADCAST || this->packetHistory[i].to == dest)) { + count++; + } + } + } else { + auto handler = SD.open("/storeforward/" + String(i), FILE_READ); + if (handler) { + handler.read((uint8_t *)&this->packetHistory[0], sizeof(PacketHistoryStruct)); + handler.close(); + if (this->packetHistory[0].time && (this->packetHistory[0].time > last_time)) { + // Client is only interested in packets not from itself and only in broadcast packets or packets towards it. + if (this->packetHistory[0].from != dest && + (this->packetHistory[0].to == NODENUM_BROADCAST || this->packetHistory[0].to == dest)) { + count++; + } + } } } } @@ -187,19 +223,33 @@ void StoreForwardModule::historyAdd(const meshtastic_MeshPacket &mp) const auto &p = mp.decoded; if (this->packetHistoryTotalCount == this->records) { - LOG_WARN("S&F - PSRAM Full. Starting overwrite."); + LOG_WARN("S&F - Storage Full. Starting overwrite."); this->packetHistoryTotalCount = 0; for (auto &i : lastRequest) { i.second = 0; // Clear the last request index for each client device } } - this->packetHistory[this->packetHistoryTotalCount].time = getTime(); - this->packetHistory[this->packetHistoryTotalCount].to = mp.to; - this->packetHistory[this->packetHistoryTotalCount].channel = mp.channel; - this->packetHistory[this->packetHistoryTotalCount].from = getFrom(&mp); - this->packetHistory[this->packetHistoryTotalCount].payload_size = p.payload.size; - memcpy(this->packetHistory[this->packetHistoryTotalCount].payload, p.payload.bytes, meshtastic_Constants_DATA_PAYLOAD_LEN); + if (this->storageType == StorageType::PSRAM) { + this->packetHistory[this->packetHistoryTotalCount].time = getTime(); + this->packetHistory[this->packetHistoryTotalCount].to = mp.to; + this->packetHistory[this->packetHistoryTotalCount].channel = mp.channel; + this->packetHistory[this->packetHistoryTotalCount].from = getFrom(&mp); + this->packetHistory[this->packetHistoryTotalCount].payload_size = p.payload.size; + memcpy(this->packetHistory[this->packetHistoryTotalCount].payload, p.payload.bytes, + meshtastic_Constants_DATA_PAYLOAD_LEN); + } else { + // Save to SDCARD + this->packetHistory[0].time = getTime(); + this->packetHistory[0].to = mp.to; + this->packetHistory[0].channel = mp.channel; + this->packetHistory[0].from = getFrom(&mp); + this->packetHistory[0].payload_size = p.payload.size; + memcpy(this->packetHistory[0].payload, p.payload.bytes, meshtastic_Constants_DATA_PAYLOAD_LEN); + auto handler = SD.open("/storeforward/" + String(this->packetHistoryTotalCount), FILE_WRITE); + handler.write((uint8_t *)&this->packetHistory, sizeof(PacketHistoryStruct)); + handler.close(); + } this->packetHistoryTotalCount++; } @@ -233,46 +283,93 @@ bool StoreForwardModule::sendPayload(NodeNum dest, uint32_t last_time) meshtastic_MeshPacket *StoreForwardModule::preparePayload(NodeNum dest, uint32_t last_time, bool local) { for (uint32_t i = lastRequest[dest]; i < this->packetHistoryTotalCount; i++) { - if (this->packetHistory[i].time && (this->packetHistory[i].time > last_time)) { - /* Copy the messages that were received by the server in the last msAgo - to the packetHistoryTXQueue structure. - Client not interested in packets from itself and only in broadcast packets or packets towards it. */ - if (this->packetHistory[i].from != dest && - (this->packetHistory[i].to == NODENUM_BROADCAST || this->packetHistory[i].to == dest)) { + if (this->storageType == StorageType::PSRAM) { - meshtastic_MeshPacket *p = allocDataPacket(); + if (this->packetHistory[i].time && (this->packetHistory[i].time > last_time)) { + /* Copy the messages that were received by the server in the last msAgo + to the packetHistoryTXQueue structure. + Client not interested in packets from itself and only in broadcast packets or packets towards it. */ + if (this->packetHistory[i].from != dest && + (this->packetHistory[i].to == NODENUM_BROADCAST || this->packetHistory[i].to == dest)) { - p->to = local ? this->packetHistory[i].to : dest; // PhoneAPI can handle original `to` - p->from = this->packetHistory[i].from; - p->channel = this->packetHistory[i].channel; - p->rx_time = this->packetHistory[i].time; + meshtastic_MeshPacket *p = allocDataPacket(); - // Let's assume that if the server received the S&F request that the client is in range. - // TODO: Make this configurable. - p->want_ack = false; + p->to = local ? this->packetHistory[i].to : dest; // PhoneAPI can handle original `to` + p->from = this->packetHistory[i].from; + p->channel = this->packetHistory[i].channel; + p->rx_time = this->packetHistory[i].time; - if (local) { // PhoneAPI gets normal TEXT_MESSAGE_APP - p->decoded.portnum = meshtastic_PortNum_TEXT_MESSAGE_APP; - memcpy(p->decoded.payload.bytes, this->packetHistory[i].payload, this->packetHistory[i].payload_size); - p->decoded.payload.size = this->packetHistory[i].payload_size; - } else { - meshtastic_StoreAndForward sf = meshtastic_StoreAndForward_init_zero; - sf.which_variant = meshtastic_StoreAndForward_text_tag; - sf.variant.text.size = this->packetHistory[i].payload_size; - memcpy(sf.variant.text.bytes, this->packetHistory[i].payload, this->packetHistory[i].payload_size); - if (this->packetHistory[i].to == NODENUM_BROADCAST) { - sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_BROADCAST; + // Let's assume that if the server received the S&F request that the client is in range. + // TODO: Make this configurable. + p->want_ack = false; + + if (local) { // PhoneAPI gets normal TEXT_MESSAGE_APP + p->decoded.portnum = meshtastic_PortNum_TEXT_MESSAGE_APP; + memcpy(p->decoded.payload.bytes, this->packetHistory[i].payload, this->packetHistory[i].payload_size); + p->decoded.payload.size = this->packetHistory[i].payload_size; } else { - sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_DIRECT; + meshtastic_StoreAndForward sf = meshtastic_StoreAndForward_init_zero; + sf.which_variant = meshtastic_StoreAndForward_text_tag; + sf.variant.text.size = this->packetHistory[i].payload_size; + memcpy(sf.variant.text.bytes, this->packetHistory[i].payload, this->packetHistory[i].payload_size); + if (this->packetHistory[i].to == NODENUM_BROADCAST) { + sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_BROADCAST; + } else { + sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_DIRECT; + } + + p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), + &meshtastic_StoreAndForward_msg, &sf); } - p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), - &meshtastic_StoreAndForward_msg, &sf); + lastRequest[dest] = i + 1; // Update the last request index for the client device + + return p; } + } + } else { + auto handler = SD.open("/storeforward/" + String(i), FILE_READ); + if (handler) { + handler.read((uint8_t *)&this->packetHistory[0], sizeof(PacketHistoryStruct)); + handler.close(); + if (this->packetHistory[0].time && (this->packetHistory[0].time > last_time)) { + if (this->packetHistory[0].from != dest && + (this->packetHistory[0].to == NODENUM_BROADCAST || this->packetHistory[0].to == dest)) { - lastRequest[dest] = i + 1; // Update the last request index for the client device + meshtastic_MeshPacket *p = allocDataPacket(); - return p; + p->to = local ? this->packetHistory[0].to : dest; // PhoneAPI can handle original `to` + p->from = this->packetHistory[0].from; + p->channel = this->packetHistory[0].channel; + p->rx_time = this->packetHistory[0].time; + + // Let's assume that if the server received the S&F request that the client is in range. + p->want_ack = false; + + if (local) { // PhoneAPI gets normal TEXT_MESSAGE_APP + p->decoded.portnum = meshtastic_PortNum_TEXT_MESSAGE_APP; + memcpy(p->decoded.payload.bytes, this->packetHistory[0].payload, this->packetHistory[0].payload_size); + p->decoded.payload.size = this->packetHistory[0].payload_size; + } else { + meshtastic_StoreAndForward sf = meshtastic_StoreAndForward_init_zero; + sf.which_variant = meshtastic_StoreAndForward_text_tag; + sf.variant.text.size = this->packetHistory[0].payload_size; + memcpy(sf.variant.text.bytes, this->packetHistory[0].payload, this->packetHistory[0].payload_size); + if (this->packetHistory[0].to == NODENUM_BROADCAST) { + sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_BROADCAST; + } else { + sf.rr = meshtastic_StoreAndForward_RequestResponse_ROUTER_TEXT_DIRECT; + } + + p->decoded.payload.size = pb_encode_to_bytes( + p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &meshtastic_StoreAndForward_msg, &sf); + } + + lastRequest[dest] = i + 1; // Update the last request index for the client device + + return p; + } + } } } } @@ -377,7 +474,7 @@ void StoreForwardModule::statsSend(uint32_t to) */ ProcessMessage StoreForwardModule::handleReceived(const meshtastic_MeshPacket &mp) { -#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) +#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) || defined(HAS_SDCARD) if (moduleConfig.store_forward.enabled) { if ((mp.decoded.portnum == meshtastic_PortNum_TEXT_MESSAGE_APP) && is_server) { @@ -556,7 +653,7 @@ StoreForwardModule::StoreForwardModule() ProtobufModule("StoreForward", meshtastic_PortNum_STORE_FORWARD_APP, &meshtastic_StoreAndForward_msg) { -#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) +#if defined(ARCH_ESP32) || defined(ARCH_PORTDUINO) || defined(HAS_SDCARD) isPromiscuous = true; // Brown chicken brown cow @@ -601,12 +698,17 @@ StoreForwardModule::StoreForwardModule() this->populatePSRAM(); is_server = true; } else { - LOG_INFO("."); LOG_INFO("S&F: not enough PSRAM free, disabling."); } } else { LOG_INFO("S&F: device doesn't have PSRAM, disabling."); } +#ifdef HAS_SDCARD + // If we have an SDCARD, format it for store&forward use + this->populateSDCard(); + LOG_INFO("S&F: SDCARD initialized"); + is_server = true; +#endif // Client } else { diff --git a/src/modules/StoreForwardModule.h b/src/modules/StoreForwardModule.h index e3273470b..3a5032737 100644 --- a/src/modules/StoreForwardModule.h +++ b/src/modules/StoreForwardModule.h @@ -9,6 +9,10 @@ #include #include +#ifdef HAS_SDCARD +#include +#endif + struct PacketHistoryStruct { uint32_t time; uint32_t to; @@ -18,6 +22,9 @@ struct PacketHistoryStruct { pb_size_t payload_size; }; +// enum for the storage type +enum StorageType { PSRAM, SDCARD }; + class StoreForwardModule : private concurrency::OSThread, public ProtobufModule { bool busy = 0; @@ -80,6 +87,10 @@ class StoreForwardModule : private concurrency::OSThread, public ProtobufModule< private: void populatePSRAM(); + void populateSDCard(); + + // Storage Type + StorageType storageType = PSRAM; // S&F Defaults uint32_t historyReturnMax = 25; // Return maximum of 25 records by default.