Merge pull request #2061 from meshtastic/protobuf-04

Protobuf 04

arch/rp2040/rp2040.ini

@@ -1,6 +1,8 @@
 ; Common settings for rp2040 Processor based targets
 [rp2040_base]
-platform = https://github.com/maxgerhardt/platform-raspberrypi.git#5ce1a228e7cae453f366deb8962252b9b7356bbc
+platform = https://github.com/maxgerhardt/platform-raspberrypi.git#20c7dbfcfe6677c5305fa28ecf5e3870321cb157
+platform_packages = 
+  earlephilhower/toolchain-rp2040-earlephilhower@^5.100300.221223
 extends = arduino_base
 board_build.core = earlephilhower
 board_build.filesystem_size = 0.5m

src/mesh/MeshModule.cpp

@@ -44,7 +44,7 @@ MeshPacket *MeshModule::allocAckNak(Routing_Error err, NodeNum to, PacketId idFr
     // auto p = allocDataProtobuf(c);
     MeshPacket *p = router->allocForSending();
     p->decoded.portnum = PortNum_ROUTING_APP;
-    p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), Routing_fields, &c);
+    p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &Routing_msg, &c);
 
     p->priority = MeshPacket_Priority_ACK;
 

src/mesh/NodeDB.cpp

@@ -401,7 +401,7 @@ bool loadProto(const char *filename, size_t protoSize, size_t objSize, const pb_
 void NodeDB::loadFromDisk()
 {
     // static DeviceState scratch; We no longer read into a tempbuf because this structure is 15KB of valuable RAM
-    if (!loadProto(prefFileName, DeviceState_size, sizeof(devicestate), DeviceState_fields, &devicestate)) {
+    if (!loadProto(prefFileName, DeviceState_size, sizeof(DeviceState), &DeviceState_msg, &devicestate)) {
         installDefaultDeviceState(); // Our in RAM copy might now be corrupt
     } else {
         if (devicestate.version < DEVICESTATE_MIN_VER) {
@@ -412,7 +412,7 @@ void NodeDB::loadFromDisk()
         }
     }
 
-    if (!loadProto(configFileName, LocalConfig_size, sizeof(LocalConfig), LocalConfig_fields, &config)) {
+    if (!loadProto(configFileName, LocalConfig_size, sizeof(LocalConfig), &LocalConfig_msg, &config)) {
         installDefaultConfig(); // Our in RAM copy might now be corrupt
     } else {
         if (config.version < DEVICESTATE_MIN_VER) {
@@ -423,7 +423,7 @@ void NodeDB::loadFromDisk()
         }
     }
 
-    if (!loadProto(moduleConfigFileName, LocalModuleConfig_size, sizeof(LocalModuleConfig), LocalModuleConfig_fields, &moduleConfig)) {
+    if (!loadProto(moduleConfigFileName, LocalModuleConfig_size, sizeof(LocalModuleConfig), &LocalModuleConfig_msg, &moduleConfig)) {
         installDefaultModuleConfig(); // Our in RAM copy might now be corrupt
     } else {
         if (moduleConfig.version < DEVICESTATE_MIN_VER) {
@@ -434,7 +434,7 @@ void NodeDB::loadFromDisk()
         }
     }
 
-    if (!loadProto(channelFileName, ChannelFile_size, sizeof(ChannelFile), ChannelFile_fields, &channelFile)) {
+    if (!loadProto(channelFileName, ChannelFile_size, sizeof(ChannelFile), &ChannelFile_msg, &channelFile)) {
         installDefaultChannels(); // Our in RAM copy might now be corrupt
     } else {
         if (channelFile.version < DEVICESTATE_MIN_VER) {
@@ -445,12 +445,12 @@ void NodeDB::loadFromDisk()
         }
     }
 
-    if (loadProto(oemConfigFile, OEMStore_size, sizeof(OEMStore), OEMStore_fields, &oemStore))
+    if (loadProto(oemConfigFile, OEMStore_size, sizeof(OEMStore), &OEMStore_msg, &oemStore))
         DEBUG_MSG("Loaded OEMStore\n");
 }
 
 /** Save a protobuf from a file, return true for success */
-bool saveProto(const char *filename, size_t protoSize, size_t objSize, const pb_msgdesc_t *fields, const void *dest_struct)
+bool saveProto(const char *filename, size_t protoSize, const pb_msgdesc_t *fields, const void *dest_struct)
 {
     bool okay = false;
 #ifdef FSCom
@@ -498,7 +498,7 @@ void NodeDB::saveChannelsToDisk()
 #ifdef FSCom
         FSCom.mkdir("/prefs");
 #endif
-        saveProto(channelFileName, ChannelFile_size, sizeof(channelFile), ChannelFile_fields, &channelFile);
+        saveProto(channelFileName, ChannelFile_size, &ChannelFile_msg, &channelFile);
     }
 }
 
@@ -508,7 +508,7 @@ void NodeDB::saveDeviceStateToDisk()
 #ifdef FSCom
         FSCom.mkdir("/prefs");
 #endif
-        saveProto(prefFileName, DeviceState_size, sizeof(devicestate), DeviceState_fields, &devicestate);
+        saveProto(prefFileName, DeviceState_size, &DeviceState_msg, &devicestate);
     }
 }
 
@@ -530,7 +530,7 @@ void NodeDB::saveToDisk(int saveWhat)
             config.has_power = true;
             config.has_network = true;
             config.has_bluetooth = true;
-            saveProto(configFileName, LocalConfig_size, sizeof(config), LocalConfig_fields, &config);
+            saveProto(configFileName, LocalConfig_size, &LocalConfig_msg, &config);
         }
 
         if (saveWhat & SEGMENT_MODULECONFIG) {
@@ -541,7 +541,7 @@ void NodeDB::saveToDisk(int saveWhat)
             moduleConfig.has_serial = true;
             moduleConfig.has_store_forward = true;
             moduleConfig.has_telemetry = true;
-            saveProto(moduleConfigFileName, LocalModuleConfig_size, sizeof(moduleConfig), LocalModuleConfig_fields, &moduleConfig);
+            saveProto(moduleConfigFileName, LocalModuleConfig_size, &LocalModuleConfig_msg, &moduleConfig);
         }
 
         if (saveWhat & SEGMENT_CHANNELS) {

src/mesh/PhoneAPI.cpp

@@ -77,7 +77,7 @@ bool PhoneAPI::handleToRadio(const uint8_t *buf, size_t bufLength)
     // return (lastContactMsec != 0) &&
 
     memset(&toRadioScratch, 0, sizeof(toRadioScratch));
-    if (pb_decode_from_bytes(buf, bufLength, ToRadio_fields, &toRadioScratch)) {
+    if (pb_decode_from_bytes(buf, bufLength, &ToRadio_msg, &toRadioScratch)) {
         switch (toRadioScratch.which_payload_variant) {
         case ToRadio_packet_tag:
             return handleToRadioPacket(toRadioScratch.packet);
@@ -291,7 +291,7 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf)
     // Do we have a message from the mesh?
     if (fromRadioScratch.which_payload_variant != 0) {
         // Encapsulate as a FromRadio packet
-        size_t numbytes = pb_encode_to_bytes(buf, FromRadio_size, FromRadio_fields, &fromRadioScratch);
+        size_t numbytes = pb_encode_to_bytes(buf, FromRadio_size, &FromRadio_msg, &fromRadioScratch);
 
         DEBUG_MSG("encoding toPhone packet to phone variant=%d, %d bytes\n", fromRadioScratch.which_payload_variant, numbytes);
         return numbytes;

src/mesh/RadioInterface.cpp

@@ -175,7 +175,7 @@ uint32_t RadioInterface::getRetransmissionMsec(const MeshPacket *p)
 {
     assert(slotTimeMsec); // Better be non zero
     static uint8_t bytes[MAX_RHPACKETLEN];
-    size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded);
+    size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), &Data_msg, &p->decoded);
     uint32_t packetAirtime = getPacketTime(numbytes + sizeof(PacketHeader));
     // Make sure enough time has elapsed for this packet to be sent and an ACK is received.
     // DEBUG_MSG("Waiting for flooding message with airtime %d and slotTime is %d\n", packetAirtime, slotTimeMsec);

src/mesh/Router.cpp

@@ -306,7 +306,7 @@ bool perhapsDecode(MeshPacket *p)
 
             // Take those raw bytes and convert them back into a well structured protobuf we can understand
             memset(&p->decoded, 0, sizeof(p->decoded));
-            if (!pb_decode_from_bytes(bytes, rawSize, Data_fields, &p->decoded)) {
+            if (!pb_decode_from_bytes(bytes, rawSize, &Data_msg, &p->decoded)) {
                 DEBUG_MSG("Invalid protobufs in received mesh packet (bad psk?)!\n");
             } else if (p->decoded.portnum == PortNum_UNKNOWN_APP) {
                 DEBUG_MSG("Invalid portnum (bad psk?)!\n");
@@ -360,7 +360,7 @@ Routing_Error perhapsEncode(MeshPacket *p)
     if (p->which_payload_variant == MeshPacket_decoded_tag) {
         static uint8_t bytes[MAX_RHPACKETLEN]; // we have to use a scratch buffer because a union
 
-        size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded);
+        size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), &Data_msg, &p->decoded);
 
         // Only allow encryption on the text message app.
         //  TODO: Allow modules to opt into compression.

src/mesh/StreamAPI.cpp

@@ -115,7 +115,7 @@ void StreamAPI::emitRebooted()
     fromRadioScratch.rebooted = true;
 
     // DEBUG_MSG("Emitting reboot packet for serial shell\n");
-    emitTxBuffer(pb_encode_to_bytes(txBuf + HEADER_LEN, FromRadio_size, FromRadio_fields, &fromRadioScratch));
+    emitTxBuffer(pb_encode_to_bytes(txBuf + HEADER_LEN, FromRadio_size, &FromRadio_msg, &fromRadioScratch));
 }
 
 /// Hookable to find out when connection changes

src/modules/AdminModule.cpp

@@ -515,7 +515,7 @@ void AdminModule::saveChanges(int saveWhat, bool shouldReboot)
     }
 }
 
-AdminModule::AdminModule() : ProtobufModule("Admin", PortNum_ADMIN_APP, AdminMessage_fields)
+AdminModule::AdminModule() : ProtobufModule("Admin", PortNum_ADMIN_APP, &AdminMessage_msg)
 {
     // restrict to the admin channel for rx
     boundChannel = Channels::adminChannel;

src/modules/CannedMessageModule.cpp

@@ -45,8 +45,7 @@ CannedMessageModule *cannedMessageModule;
 
 // TODO: move it into NodeDB.h!
 extern bool loadProto(const char *filename, size_t protoSize, size_t objSize, const pb_msgdesc_t *fields, void *dest_struct);
-extern bool saveProto(const char *filename, size_t protoSize, size_t objSize, const pb_msgdesc_t *fields,
+extern bool saveProto(const char *filename, size_t protoSize, const pb_msgdesc_t *fields, const void *dest_struct);
-                      const void *dest_struct);
 
 CannedMessageModule::CannedMessageModule()
     : SinglePortModule("canned", PortNum_TEXT_MESSAGE_APP), concurrency::OSThread("CannedMessageModule")
@@ -480,7 +479,7 @@ void CannedMessageModule::drawFrame(OLEDDisplay *display, OLEDDisplayUiState *st
 void CannedMessageModule::loadProtoForModule()
 {
     if (!loadProto(cannedMessagesConfigFile, CannedMessageModuleConfig_size, sizeof(cannedMessagesConfigFile),
-                   CannedMessageModuleConfig_fields, &cannedMessageModuleConfig)) {
+                   &CannedMessageModuleConfig_msg, &cannedMessageModuleConfig)) {
         installDefaultCannedMessageModuleConfig();
     }
 }
@@ -499,8 +498,8 @@ bool CannedMessageModule::saveProtoForModule()
     FS.mkdir("/prefs");
 #endif
 
-    okay &= saveProto(cannedMessagesConfigFile, CannedMessageModuleConfig_size, sizeof(cannedMessageModuleConfig),
+    okay &= saveProto(cannedMessagesConfigFile, CannedMessageModuleConfig_size,
-                      CannedMessageModuleConfig_fields, &cannedMessageModuleConfig);
+        &CannedMessageModuleConfig_msg, &cannedMessageModuleConfig);
 
     return okay;
 }

src/modules/NodeInfoModule.cpp

@@ -51,7 +51,7 @@ MeshPacket *NodeInfoModule::allocReply()
 }
 
 NodeInfoModule::NodeInfoModule()
-    : ProtobufModule("nodeinfo", PortNum_NODEINFO_APP, User_fields), concurrency::OSThread("NodeInfoModule")
+    : ProtobufModule("nodeinfo", PortNum_NODEINFO_APP, &User_msg), concurrency::OSThread("NodeInfoModule")
 {
     isPromiscuous = true; // We always want to update our nodedb, even if we are sniffing on others
     setIntervalFromNow(30 * 1000); // Send our initial owner announcement 30 seconds after we start (to give network time to setup)

src/modules/PositionModule.cpp

@@ -10,7 +10,7 @@
 PositionModule *positionModule;
 
 PositionModule::PositionModule()
-    : ProtobufModule("position", PortNum_POSITION_APP, Position_fields), concurrency::OSThread("PositionModule")
+    : ProtobufModule("position", PortNum_POSITION_APP, &Position_msg), concurrency::OSThread("PositionModule")
 {
     isPromiscuous = true;          // We always want to update our nodedb, even if we are sniffing on others
     setIntervalFromNow(60 * 1000); // Send our initial position 60 seconds after we start (to give GPS time to setup)

src/modules/RemoteHardwareModule.cpp

@@ -47,7 +47,7 @@ static uint64_t digitalReads(uint64_t mask)
 }
 
 RemoteHardwareModule::RemoteHardwareModule()
-    : ProtobufModule("remotehardware", PortNum_REMOTE_HARDWARE_APP, HardwareMessage_fields), concurrency::OSThread(
+    : ProtobufModule("remotehardware", PortNum_REMOTE_HARDWARE_APP, &HardwareMessage_msg), concurrency::OSThread(
                                                                                                  "remotehardware")
 {
 }

src/modules/RoutingModule.cpp

@@ -41,7 +41,7 @@ void RoutingModule::sendAckNak(Routing_Error err, NodeNum to, PacketId idFrom, C
     router->sendLocal(p); // we sometimes send directly to the local node
 }
 
-RoutingModule::RoutingModule() : ProtobufModule("routing", PortNum_ROUTING_APP, Routing_fields)
+RoutingModule::RoutingModule() : ProtobufModule("routing", PortNum_ROUTING_APP, &Routing_msg)
 {
     isPromiscuous = true;
     encryptedOk = true;

src/modules/SerialModule.cpp

@@ -294,7 +294,7 @@ ProcessMessage SerialModuleRadio::handleReceived(const MeshPacket &mp)
                 Position *decoded = NULL;
                 if (mp.which_payload_variant == MeshPacket_decoded_tag && mp.decoded.portnum == ourPortNum) {
                     memset(&scratch, 0, sizeof(scratch));
-                    if (pb_decode_from_bytes(p.payload.bytes, p.payload.size, Position_fields, &scratch)) {
+                    if (pb_decode_from_bytes(p.payload.bytes, p.payload.size, &Position_msg, &scratch)) {
                         decoded = &scratch;
                     }
                     // send position packet as WPL to the serial port

src/modules/Telemetry/EnvironmentTelemetry.cpp

@@ -155,7 +155,7 @@ void EnvironmentTelemetryModule::drawFrame(OLEDDisplay *display, OLEDDisplayUiSt
     const char *lastSender = getSenderShortName(*lastMeasurementPacket);
 
     auto &p = lastMeasurementPacket->decoded;
-    if (!pb_decode_from_bytes(p.payload.bytes, p.payload.size, Telemetry_fields, &lastMeasurement)) {
+    if (!pb_decode_from_bytes(p.payload.bytes, p.payload.size, &Telemetry_msg, &lastMeasurement)) {
         display->setFont(FONT_SMALL);
         display->drawString(x, y += fontHeight(FONT_MEDIUM), "Measurement Error");
         DEBUG_MSG("Unable to decode last packet");

src/modules/TraceRouteModule.cpp

@@ -24,14 +24,14 @@ void TraceRouteModule::updateRoute(MeshPacket* p)
         RouteDiscovery scratch;
         RouteDiscovery *updated = NULL;
         memset(&scratch, 0, sizeof(scratch));
-        pb_decode_from_bytes(incoming.payload.bytes, incoming.payload.size, RouteDiscovery_fields, &scratch);
+        pb_decode_from_bytes(incoming.payload.bytes, incoming.payload.size, &RouteDiscovery_msg, &scratch);
         updated = &scratch;
 
         appendMyID(updated);
         printRoute(updated, p->from, NODENUM_BROADCAST);
       
         // Set updated route to the payload of the to be flooded packet
-        p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), RouteDiscovery_fields, updated);
+        p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &RouteDiscovery_msg, updated);
     }
 }
 
@@ -69,7 +69,7 @@ MeshPacket* TraceRouteModule::allocReply()
     RouteDiscovery scratch;
     RouteDiscovery *updated = NULL;
     memset(&scratch, 0, sizeof(scratch));
-    pb_decode_from_bytes(p.payload.bytes, p.payload.size, RouteDiscovery_fields, &scratch);
+    pb_decode_from_bytes(p.payload.bytes, p.payload.size, &RouteDiscovery_msg, &scratch);
     updated = &scratch;
 
     printRoute(updated, req.from, req.to);
@@ -81,6 +81,6 @@ MeshPacket* TraceRouteModule::allocReply()
 }
 
 
-TraceRouteModule::TraceRouteModule() : ProtobufModule("traceroute", PortNum_TRACEROUTE_APP, RouteDiscovery_fields) {
+TraceRouteModule::TraceRouteModule() : ProtobufModule("traceroute", PortNum_TRACEROUTE_APP, &RouteDiscovery_msg) {
     ourPortNum = PortNum_TRACEROUTE_APP; 
 }

src/mqtt/MQTT.cpp

@@ -70,17 +70,17 @@ void MQTT::onPublish(char *topic, byte *payload, unsigned int length)
                 //invent the "sendposition" type for a valid envelope
                 if (json["payload"]->IsObject() && json["type"]->IsString() && (json["sender"]->AsString().compare(owner.id) != 0)) {
                     JSONObject posit;
-                    posit=json["payload"]->AsObject(); //get nested JSON Position
+                    posit = json["payload"]->AsObject(); //get nested JSON Position
-                    Position pos =Position_init_default;
+                    Position pos = Position_init_default;
-                    pos.latitude_i=posit["latitude_i"]->AsNumber();
+                    pos.latitude_i = posit["latitude_i"]->AsNumber();
-                    pos.longitude_i=posit["longitude_i"]->AsNumber();
+                    pos.longitude_i = posit["longitude_i"]->AsNumber();
-                    pos.altitude=posit["altitude"]->AsNumber();
+                    pos.altitude = posit["altitude"]->AsNumber();
-                    pos.time=posit["time"]->AsNumber();
+                    pos.time = posit["time"]->AsNumber();
 
                     // construct protobuf data packet using POSITION, send it to the mesh
                     MeshPacket *p = router->allocForSending();
                     p->decoded.portnum = PortNum_POSITION_APP;
-                    p->decoded.payload.size=pb_encode_to_bytes(p->decoded.payload.bytes,sizeof(p->decoded.payload.bytes),Position_fields, &pos); //make the Data protobuf from position
+                    p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &Position_msg, &pos); //make the Data protobuf from position
                     service.sendToMesh(p, RX_SRC_LOCAL);
 
                 } else {
@@ -95,7 +95,7 @@ void MQTT::onPublish(char *topic, byte *payload, unsigned int length)
         }
         delete json_value;
     } else {
-        if (!pb_decode_from_bytes(payload, length, ServiceEnvelope_fields, &e)) {
+        if (!pb_decode_from_bytes(payload, length, &ServiceEnvelope_msg, &e)) {
             DEBUG_MSG("Invalid MQTT service envelope, topic %s, len %u!\n", topic, length);
             return;
         }else {
@@ -251,7 +251,7 @@ int32_t MQTT::runOnce()
                     // FIXME - this size calculation is super sloppy, but it will go away once we dynamically alloc meshpackets
                     ServiceEnvelope *env = mqttQueue.dequeuePtr(0);
                     static uint8_t bytes[MeshPacket_size + 64];
-                    size_t numBytes = pb_encode_to_bytes(bytes, sizeof(bytes), ServiceEnvelope_fields, env);
+                    size_t numBytes = pb_encode_to_bytes(bytes, sizeof(bytes), &ServiceEnvelope_msg, env);
 
                     String topic = cryptTopic + env->channel_id + "/" + owner.id;
                     DEBUG_MSG("publish %s, %u bytes from queue\n", topic.c_str(), numBytes);
@@ -305,7 +305,7 @@ void MQTT::onSend(const MeshPacket &mp, ChannelIndex chIndex)
 
             // FIXME - this size calculation is super sloppy, but it will go away once we dynamically alloc meshpackets
             static uint8_t bytes[MeshPacket_size + 64];
-            size_t numBytes = pb_encode_to_bytes(bytes, sizeof(bytes), ServiceEnvelope_fields, env);
+            size_t numBytes = pb_encode_to_bytes(bytes, sizeof(bytes), &ServiceEnvelope_msg, env);
 
             String topic = cryptTopic + channelId + "/" + owner.id;
             DEBUG_MSG("publish %s, %u bytes\n", topic.c_str(), numBytes);

src/platform/portduino/SimRadio.cpp

@@ -201,7 +201,7 @@ void SimRadio::startSend(MeshPacket * txp)
     } else {
         DEBUG_MSG("Payload size is larger than compressed message allows! Sending empty payload.\n");
     }
-    p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), Compressed_fields, &c);
+    p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &Compressed_msg, &c);
     p->decoded.portnum = PortNum_SIMULATOR_APP;
     service.sendToPhone(p); // Sending back to simulator
 }