From 4d82a0146b3913141ef8f8f796b980802f62162b Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 2 Jan 2022 19:50:43 -0800 Subject: [PATCH 1/5] Initial checkin of Online OTA SPIFFS update --- platformio.ini | 1 + src/mesh/RadioLibInterface.h | 1 + src/mesh/http/ContentHandler.cpp | 137 ++++++++++++++++++++++++++++++- src/mesh/http/ContentHandler.h | 5 +- 4 files changed, 139 insertions(+), 5 deletions(-) diff --git a/platformio.ini b/platformio.ini index 9c301ea33..155460b1c 100644 --- a/platformio.ini +++ b/platformio.ini @@ -115,6 +115,7 @@ lib_deps = paulstoffregen/OneWire@^2.3.5 robtillaart/DS18B20@^0.1.11 h2zero/NimBLE-Arduino@1.3.1 + tobozo/ESP32-targz@^1.1.4 # Hmm - this doesn't work yet # board_build.ldscript = linker/esp32.extram.bss.ld lib_ignore = diff --git a/src/mesh/RadioLibInterface.h b/src/mesh/RadioLibInterface.h index 54d9e6744..de3505524 100644 --- a/src/mesh/RadioLibInterface.h +++ b/src/mesh/RadioLibInterface.h @@ -8,6 +8,7 @@ #define RADIOLIB_SOFTWARE_SERIAL_UNSUPPORTED #endif +#define RADIOLIB_EXCLUDE_HTTP #include // ESP32 has special rules about ISR code diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 9a655839f..38fab4848 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -41,6 +41,13 @@ using namespace httpsserver; #include "mesh/http/ContentHandler.h" +#include +#include +HTTPClient http; + +#define DEST_FS_USES_SPIFFS +#include + // We need to specify some content-type mapping, so the resources get delivered with the // right content type and are displayed correctly in the browser char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"}, @@ -50,9 +57,58 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".css", "text/css"}, {".ico", "image/vnd.microsoft.icon"}, {".svg", "image/svg+xml"}, {"", ""}}; +const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; +const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) + // Our API to handle messages to and from the radio. HttpAPI webAPI; +WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const char *cert = NULL) +{ + if (cert == NULL) { + // New versions don't have setInsecure + // client->setInsecure(); + } else { + client->setCACert(cert); + } + const char *UserAgent = "ESP32-HTTP-GzUpdater-Client"; + http.setReuse(true); // handle 301 redirects gracefully + http.setUserAgent(UserAgent); + http.setConnectTimeout(10000); // 10s timeout = 10000 + if (!http.begin(*client, url)) { + log_e("Can't open url %s", url); + return nullptr; + } + const char *headerKeys[] = {"location", "redirect", "Content-Type", "Content-Length", "Content-Disposition"}; + const size_t numberOfHeaders = 5; + http.collectHeaders(headerKeys, numberOfHeaders); + int httpCode = http.GET(); + // file found at server + if (httpCode == HTTP_CODE_FOUND || httpCode == HTTP_CODE_MOVED_PERMANENTLY) { + String newlocation = ""; + String headerLocation = http.header("location"); + String headerRedirect = http.header("redirect"); + if (headerLocation != "") { + newlocation = headerLocation; + Serial.printf("302 (location): %s => %s\n", url, headerLocation.c_str()); + } else if (headerRedirect != "") { + Serial.printf("301 (redirect): %s => %s\n", url, headerLocation.c_str()); + newlocation = headerRedirect; + } + http.end(); + if (newlocation != "") { + log_w("Found 302/301 location header: %s", newlocation.c_str()); + return getTarHTTPClientPtr(client, newlocation.c_str(), cert); + } else { + log_e("Empty redirect !!"); + return nullptr; + } + } + if (httpCode != 200) + return nullptr; + return http.getStreamPtr(); +} + void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) { @@ -66,6 +122,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) ResourceNode *nodeHotspotApple = new ResourceNode("/hotspot-detect.html", "GET", &handleHotspot); ResourceNode *nodeHotspotAndroid = new ResourceNode("/generate_204", "GET", &handleHotspot); + ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/update", "GET", &handleUpdateSPIFFS); + ResourceNode *nodeRestart = new ResourceNode("/restart", "POST", &handleRestart); ResourceNode *nodeFormUpload = new ResourceNode("/upload", "POST", &handleFormUpload); @@ -90,7 +148,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) secureServer->registerNode(nodeJsonSpiffsBrowseStatic); secureServer->registerNode(nodeJsonDelete); secureServer->registerNode(nodeJsonReport); - secureServer->registerNode(nodeRoot); + secureServer->registerNode(nodeUpdateSPIFFS); + secureServer->registerNode(nodeRoot); // This has to be last // Insecure nodes insecureServer->registerNode(nodeAPIv1ToRadioOptions); @@ -105,7 +164,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) insecureServer->registerNode(nodeJsonSpiffsBrowseStatic); insecureServer->registerNode(nodeJsonDelete); insecureServer->registerNode(nodeJsonReport); - insecureServer->registerNode(nodeRoot); + insecureServer->registerNode(nodeUpdateSPIFFS); + insecureServer->registerNode(nodeRoot); // This has to be last } void handleAPIv1FromRadio(HTTPRequest *req, HTTPResponse *res) @@ -620,6 +680,79 @@ void handleHotspot(HTTPRequest *req, HTTPResponse *res) res->println("\n"); } +void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) +{ + res->setHeader("Content-Type", "text/html"); + res->setHeader("Access-Control-Allow-Origin", "*"); + res->setHeader("Access-Control-Allow-Methods", "GET"); + + res->println("Updating. Don't leave this page!"); + + DEBUG_MSG("hi!\n"); + + File root = SPIFFS.open("/"); + File file = root.openNextFile(); + + DEBUG_MSG("Deleting files from /static\n"); + + while (file) { + String filePath = String(file.name()); + if (filePath.indexOf("/static") == 0) { + DEBUG_MSG("%s\n", file.name()); + SPIFFS.remove(file.name()); + } + file = root.openNextFile(); + } + + // return; + + WiFiClientSecure *client = new WiFiClientSecure; + Stream *streamptr = getTarHTTPClientPtr(client, tarURL, certificate); + + if (streamptr != nullptr) { + + TarUnpacker *TARUnpacker = new TarUnpacker(); + TARUnpacker->haltOnError(true); // stop on fail (manual restart/reset required) + TARUnpacker->setTarVerify(true); // true = enables health checks but slows down the overall process + TARUnpacker->setupFSCallbacks(targzTotalBytesFn, targzFreeBytesFn); // prevent the partition from exploding, recommended + TARUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback); // gz log verbosity + TARUnpacker->setTarProgressCallback( + BaseUnpacker::defaultProgressCallback); // prints the untarring progress for each individual file + TARUnpacker->setTarStatusProgressCallback( + BaseUnpacker::defaultTarStatusProgressCallback); // print the filenames as they're expanded + TARUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback); // tar log verbosity + + String contentLengthStr = http.header("Content-Length"); + contentLengthStr.trim(); + int64_t streamSize = -1; + if (contentLengthStr != "") { + streamSize = atoi(contentLengthStr.c_str()); + Serial.printf("Stream size %d\n", streamSize); + res->printf("Stream size %d\n", streamSize); + } + + if (!TARUnpacker->tarStreamExpander(streamptr, streamSize, SPIFFS, "/static")) { + Serial.printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError()); + } else { + // print leftover bytes if any (probably zero-fill from the server) + while (http.connected()) { + size_t streamSize = streamptr->available(); + if (streamSize) { + Serial.printf("%02x ", streamptr->read()); + } else + break; + } + + res->println("Done"); + } + + } else { + Serial.println("Failed to establish http connection"); + } + + res->println("Done"); +} + void handleRestart(HTTPRequest *req, HTTPResponse *res) { res->setHeader("Content-Type", "text/html"); diff --git a/src/mesh/http/ContentHandler.h b/src/mesh/http/ContentHandler.h index cc07cb833..f456d0a1f 100644 --- a/src/mesh/http/ContentHandler.h +++ b/src/mesh/http/ContentHandler.h @@ -1,5 +1,6 @@ #pragma once + void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer); // Declare some handler functions for the various URLs on the server @@ -14,9 +15,7 @@ void handleSpiffsBrowseStatic(HTTPRequest *req, HTTPResponse *res); void handleSpiffsDeleteStatic(HTTPRequest *req, HTTPResponse *res); void handleBlinkLED(HTTPRequest *req, HTTPResponse *res); void handleReport(HTTPRequest *req, HTTPResponse *res); - -void middlewareSpeedUp240(HTTPRequest *req, HTTPResponse *res, std::function next); -void middlewareSpeedUp160(HTTPRequest *req, HTTPResponse *res, std::function next); +void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res); // Interface to the PhoneAPI to access the protobufs with messages class HttpAPI : public PhoneAPI From 893472e36a671047006e5f32dbdf0a62e35ed428 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 2 Jan 2022 20:05:13 -0800 Subject: [PATCH 2/5] Update text and tar URL --- src/mesh/http/ContentHandler.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 38fab4848..737335b83 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -57,7 +57,7 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".css", "text/css"}, {".ico", "image/vnd.microsoft.icon"}, {".svg", "image/svg+xml"}, {"", ""}}; -const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; +const char *tarURL = "https://github.com/meshtastic/meshtastic-web/releases/download/latest/build.tar"; const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) // Our API to handle messages to and from the radio. @@ -368,7 +368,7 @@ void handleStatic(HTTPRequest *req, HTTPResponse *res) DEBUG_MSG("File not available - %s\n", filenameGzip.c_str()); res->println("Web server is running.

The content you are looking for can't be found. Please see: FAQ.

stats"); + "href=/json/report>stats

Experemntal Web Content OTA Update"); } else { res->setHeader("Content-Encoding", "gzip"); } @@ -742,8 +742,6 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) } else break; } - - res->println("Done"); } } else { From b2011a1889e585f62332b4b308475e3f158e6ad9 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 2 Jan 2022 20:37:52 -0800 Subject: [PATCH 3/5] Switching url to casler.org. github has too many redirections. --- src/mesh/http/ContentHandler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 737335b83..3f622b65c 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -57,7 +57,8 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".css", "text/css"}, {".ico", "image/vnd.microsoft.icon"}, {".svg", "image/svg+xml"}, {"", ""}}; -const char *tarURL = "https://github.com/meshtastic/meshtastic-web/releases/download/latest/build.tar"; +//const char *tarURL = "https://github.com/meshtastic/meshtastic-web/releases/download/latest/build.tar"; +const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) // Our API to handle messages to and from the radio. From 6d0368b13d421331a434714d7f88a9fc6a996d5b Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 2 Jan 2022 22:10:55 -0800 Subject: [PATCH 4/5] Update URL to proper meshtastic web download location & disable halting --- src/mesh/http/ContentHandler.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 3f622b65c..3e35ee5fd 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -57,8 +57,8 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".css", "text/css"}, {".ico", "image/vnd.microsoft.icon"}, {".svg", "image/svg+xml"}, {"", ""}}; -//const char *tarURL = "https://github.com/meshtastic/meshtastic-web/releases/download/latest/build.tar"; -const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; +//const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; +const char *tarURL = "https://api-production-871d.up.railway.app/mirror/webui"; const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) // Our API to handle messages to and from the radio. @@ -687,9 +687,7 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) res->setHeader("Access-Control-Allow-Origin", "*"); res->setHeader("Access-Control-Allow-Methods", "GET"); - res->println("Updating. Don't leave this page!"); - - DEBUG_MSG("hi!\n"); + res->println("Downloading Meshtastic Web Content..."); File root = SPIFFS.open("/"); File file = root.openNextFile(); @@ -713,8 +711,8 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) if (streamptr != nullptr) { TarUnpacker *TARUnpacker = new TarUnpacker(); - TARUnpacker->haltOnError(true); // stop on fail (manual restart/reset required) - TARUnpacker->setTarVerify(true); // true = enables health checks but slows down the overall process + TARUnpacker->haltOnError(false); // stop on fail (manual restart/reset required) + TARUnpacker->setTarVerify(false); // true = enables health checks but slows down the overall process TARUnpacker->setupFSCallbacks(targzTotalBytesFn, targzFreeBytesFn); // prevent the partition from exploding, recommended TARUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback); // gz log verbosity TARUnpacker->setTarProgressCallback( From 063d7a7d8117271b084f18f925143d723381cf78 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 2 Jan 2022 22:17:26 -0800 Subject: [PATCH 5/5] Cleanup API endpoint debug output --- src/mesh/http/ContentHandler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 3e35ee5fd..c051e0cda 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -172,7 +172,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) void handleAPIv1FromRadio(HTTPRequest *req, HTTPResponse *res) { - DEBUG_MSG("+++++++++++++++ webAPI handleAPIv1FromRadio\n"); + DEBUG_MSG("webAPI handleAPIv1FromRadio\n"); /* For documentation, see: @@ -217,12 +217,12 @@ void handleAPIv1FromRadio(HTTPRequest *req, HTTPResponse *res) res->write(txBuf, len); } - DEBUG_MSG("--------------- webAPI handleAPIv1FromRadio, len %d\n", len); + DEBUG_MSG("webAPI handleAPIv1FromRadio, len %d\n", len); } void handleAPIv1ToRadio(HTTPRequest *req, HTTPResponse *res) { - DEBUG_MSG("+++++++++++++++ webAPI handleAPIv1ToRadio\n"); + DEBUG_MSG("webAPI handleAPIv1ToRadio\n"); /* For documentation, see: @@ -249,7 +249,7 @@ void handleAPIv1ToRadio(HTTPRequest *req, HTTPResponse *res) webAPI.handleToRadio(buffer, s); res->write(buffer, s); - DEBUG_MSG("--------------- webAPI handleAPIv1ToRadio\n"); + DEBUG_MSG("webAPI handleAPIv1ToRadio\n"); } void handleSpiffsBrowseStatic(HTTPRequest *req, HTTPResponse *res)