From d5fc9054021de3d1e5d77078dfcbded32a1ebc37 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 5 Jan 2022 19:29:45 -0800 Subject: [PATCH 1/4] Don't delete contents of /static unless tar is downloaded --- src/mesh/http/ContentHandler.cpp | 42 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index c051e0cda..efa47a767 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -57,7 +57,7 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".css", "text/css"}, {".ico", "image/vnd.microsoft.icon"}, {".svg", "image/svg+xml"}, {"", ""}}; -//const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; +// const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; const char *tarURL = "https://api-production-871d.up.railway.app/mirror/webui"; const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) @@ -689,26 +689,29 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) res->println("Downloading Meshtastic Web Content..."); - File root = SPIFFS.open("/"); - File file = root.openNextFile(); - - DEBUG_MSG("Deleting files from /static\n"); - - while (file) { - String filePath = String(file.name()); - if (filePath.indexOf("/static") == 0) { - DEBUG_MSG("%s\n", file.name()); - SPIFFS.remove(file.name()); - } - file = root.openNextFile(); - } - - // return; - WiFiClientSecure *client = new WiFiClientSecure; Stream *streamptr = getTarHTTPClientPtr(client, tarURL, certificate); + delay(5); // Let other network operations run + if (streamptr != nullptr) { + DEBUG_MSG("Connection to content server ... success!\n"); + + File root = SPIFFS.open("/"); + File file = root.openNextFile(); + + DEBUG_MSG("Deleting files from /static\n"); + + while (file) { + String filePath = String(file.name()); + if (filePath.indexOf("/static") == 0) { + DEBUG_MSG("%s\n", file.name()); + SPIFFS.remove(file.name()); + } + file = root.openNextFile(); + } + + delay(5); // Let other network operations run TarUnpacker *TARUnpacker = new TarUnpacker(); TARUnpacker->haltOnError(false); // stop on fail (manual restart/reset required) @@ -731,7 +734,10 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) } if (!TARUnpacker->tarStreamExpander(streamptr, streamSize, SPIFFS, "/static")) { + res->printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError()); Serial.printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError()); + + return; } else { // print leftover bytes if any (probably zero-fill from the server) while (http.connected()) { @@ -744,7 +750,9 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) } } else { + res->printf("Failed to establish http connection\n"); Serial.println("Failed to establish http connection"); + return; } res->println("Done"); From dbdbe75e9f6f8ce2e8f754a4daeaa4cec79b498d Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 5 Jan 2022 19:44:21 -0800 Subject: [PATCH 2/4] Rename httpClient --- src/mesh/http/ContentHandler.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index efa47a767..189dfef0e 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -43,7 +43,7 @@ using namespace httpsserver; #include #include -HTTPClient http; +HTTPClient httpClient; #define DEST_FS_USES_SPIFFS #include @@ -73,22 +73,22 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const client->setCACert(cert); } const char *UserAgent = "ESP32-HTTP-GzUpdater-Client"; - http.setReuse(true); // handle 301 redirects gracefully - http.setUserAgent(UserAgent); - http.setConnectTimeout(10000); // 10s timeout = 10000 - if (!http.begin(*client, url)) { + httpClient.setReuse(true); // handle 301 redirects gracefully + httpClient.setUserAgent(UserAgent); + httpClient.setConnectTimeout(10000); // 10s timeout = 10000 + if (!httpClient.begin(*client, url)) { log_e("Can't open url %s", url); return nullptr; } const char *headerKeys[] = {"location", "redirect", "Content-Type", "Content-Length", "Content-Disposition"}; const size_t numberOfHeaders = 5; - http.collectHeaders(headerKeys, numberOfHeaders); - int httpCode = http.GET(); + httpClient.collectHeaders(headerKeys, numberOfHeaders); + int httpCode = httpClient.GET(); // file found at server if (httpCode == HTTP_CODE_FOUND || httpCode == HTTP_CODE_MOVED_PERMANENTLY) { String newlocation = ""; - String headerLocation = http.header("location"); - String headerRedirect = http.header("redirect"); + String headerLocation = httpClient.header("location"); + String headerRedirect = httpClient.header("redirect"); if (headerLocation != "") { newlocation = headerLocation; Serial.printf("302 (location): %s => %s\n", url, headerLocation.c_str()); @@ -96,7 +96,7 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const Serial.printf("301 (redirect): %s => %s\n", url, headerLocation.c_str()); newlocation = headerRedirect; } - http.end(); + httpClient.end(); if (newlocation != "") { log_w("Found 302/301 location header: %s", newlocation.c_str()); return getTarHTTPClientPtr(client, newlocation.c_str(), cert); @@ -107,7 +107,7 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const } if (httpCode != 200) return nullptr; - return http.getStreamPtr(); + return httpClient.getStreamPtr(); } void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) @@ -700,12 +700,12 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) File root = SPIFFS.open("/"); File file = root.openNextFile(); - DEBUG_MSG("Deleting files from /static\n"); + DEBUG_MSG("Deleting files from /static : \n"); while (file) { String filePath = String(file.name()); if (filePath.indexOf("/static") == 0) { - DEBUG_MSG("%s\n", file.name()); + DEBUG_MSG(" %s\n", file.name()); SPIFFS.remove(file.name()); } file = root.openNextFile(); @@ -724,7 +724,7 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) BaseUnpacker::defaultTarStatusProgressCallback); // print the filenames as they're expanded TARUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback); // tar log verbosity - String contentLengthStr = http.header("Content-Length"); + String contentLengthStr = httpClient.header("Content-Length"); contentLengthStr.trim(); int64_t streamSize = -1; if (contentLengthStr != "") { @@ -739,14 +739,16 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) return; } else { + /* // print leftover bytes if any (probably zero-fill from the server) - while (http.connected()) { + while (httpClient.connected()) { size_t streamSize = streamptr->available(); if (streamSize) { Serial.printf("%02x ", streamptr->read()); } else break; } + */ } } else { From 3a178228939aa40fa2afb2225c80cf3b565bfd82 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 5 Jan 2022 22:12:32 -0800 Subject: [PATCH 3/4] Fixed TLS "memory allocation failure" --- src/mesh/http/ContentHandler.cpp | 51 +++++++++++++++++++++++++++----- src/mesh/http/ContentHandler.h | 2 ++ src/mesh/http/WebServer.cpp | 4 +++ src/mesh/http/WebServer.h | 1 + 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 189dfef0e..043d3275d 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -4,6 +4,7 @@ #include "airtime.h" #include "main.h" #include "mesh/http/ContentHelper.h" +#include "mesh/http/WebServer.h" #include "mesh/http/WiFiAPClient.h" #include "power.h" #include "sleep.h" @@ -123,7 +124,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) ResourceNode *nodeHotspotApple = new ResourceNode("/hotspot-detect.html", "GET", &handleHotspot); ResourceNode *nodeHotspotAndroid = new ResourceNode("/generate_204", "GET", &handleHotspot); - ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/update", "GET", &handleUpdateSPIFFS); + ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/spiffs/update", "POST", &handleUpdateSPIFFS); + ResourceNode *nodeDeleteSPIFFS = new ResourceNode("/spiffs/delete", "GET", &handleDeleteSPIFFSContent); ResourceNode *nodeRestart = new ResourceNode("/restart", "POST", &handleRestart); ResourceNode *nodeFormUpload = new ResourceNode("/upload", "POST", &handleFormUpload); @@ -150,6 +152,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) secureServer->registerNode(nodeJsonDelete); secureServer->registerNode(nodeJsonReport); secureServer->registerNode(nodeUpdateSPIFFS); + secureServer->registerNode(nodeDeleteSPIFFS); secureServer->registerNode(nodeRoot); // This has to be last // Insecure nodes @@ -166,6 +169,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) insecureServer->registerNode(nodeJsonDelete); insecureServer->registerNode(nodeJsonReport); insecureServer->registerNode(nodeUpdateSPIFFS); + insecureServer->registerNode(nodeDeleteSPIFFS); insecureServer->registerNode(nodeRoot); // This has to be last } @@ -368,8 +372,10 @@ void handleStatic(HTTPRequest *req, HTTPResponse *res) if (!file.available()) { DEBUG_MSG("File not available - %s\n", filenameGzip.c_str()); res->println("Web server is running.

The content you are looking for can't be found. Please see: FAQ.

stats

Experemntal Web Content OTA Update"); + "href=https://meshtastic.org/docs/getting-started/faq#wifi--web-browser>FAQ.

Experimental " + "Web Content OTA Update -- Click " + "this just once and wait. Be patient!
"); } else { res->setHeader("Content-Encoding", "gzip"); } @@ -685,7 +691,7 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) { res->setHeader("Content-Type", "text/html"); res->setHeader("Access-Control-Allow-Origin", "*"); - res->setHeader("Access-Control-Allow-Methods", "GET"); + // res->setHeader("Access-Control-Allow-Methods", "POST"); res->println("Downloading Meshtastic Web Content..."); @@ -730,7 +736,7 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) if (contentLengthStr != "") { streamSize = atoi(contentLengthStr.c_str()); Serial.printf("Stream size %d\n", streamSize); - res->printf("Stream size %d\n", streamSize); + res->printf("Stream size %d

\n", streamSize); } if (!TARUnpacker->tarStreamExpander(streamptr, streamSize, SPIFFS, "/static")) { @@ -757,7 +763,38 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res) return; } - res->println("Done"); + res->println("Done! Restarting the device. Click this in 10 seconds"); + + /* + * This is a work around for a bug where we run out of memory. + * TODO: Fixme! + */ + // ESP.restart(); + webServerThread->requestRestart = (millis() / 1000) + 5; +} + +void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res) +{ + res->setHeader("Content-Type", "text/html"); + res->setHeader("Access-Control-Allow-Origin", "*"); + res->setHeader("Access-Control-Allow-Methods", "GET"); + + res->println("Deleting SPIFFS Content in /static/*"); + + File root = SPIFFS.open("/"); + File file = root.openNextFile(); + + DEBUG_MSG("Deleting files from /static : \n"); + + while (file) { + String filePath = String(file.name()); + if (filePath.indexOf("/static") == 0) { + DEBUG_MSG(" %s\n", file.name()); + SPIFFS.remove(file.name()); + } + file = root.openNextFile(); + } + } void handleRestart(HTTPRequest *req, HTTPResponse *res) @@ -769,7 +806,7 @@ void handleRestart(HTTPRequest *req, HTTPResponse *res) DEBUG_MSG("***** Restarted on HTTP(s) Request *****\n"); res->println("Restarting"); - ESP.restart(); + webServerThread->requestRestart = (millis() / 1000) + 5; } void handleBlinkLED(HTTPRequest *req, HTTPResponse *res) diff --git a/src/mesh/http/ContentHandler.h b/src/mesh/http/ContentHandler.h index f456d0a1f..2a89b3df6 100644 --- a/src/mesh/http/ContentHandler.h +++ b/src/mesh/http/ContentHandler.h @@ -16,6 +16,8 @@ void handleSpiffsDeleteStatic(HTTPRequest *req, HTTPResponse *res); void handleBlinkLED(HTTPRequest *req, HTTPResponse *res); void handleReport(HTTPRequest *req, HTTPResponse *res); void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res); +void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res); + // Interface to the PhoneAPI to access the protobufs with messages class HttpAPI : public PhoneAPI diff --git a/src/mesh/http/WebServer.cpp b/src/mesh/http/WebServer.cpp index cbf25271d..d9d2ca6b4 100644 --- a/src/mesh/http/WebServer.cpp +++ b/src/mesh/http/WebServer.cpp @@ -175,6 +175,10 @@ int32_t WebServerThread::runOnce() // DEBUG_MSG("WebServerThread::runOnce()\n"); handleWebResponse(); + if (requestRestart && (millis() / 1000) > requestRestart) { + ESP.restart(); + } + // Loop every 5ms. return (5); } diff --git a/src/mesh/http/WebServer.h b/src/mesh/http/WebServer.h index 63a61367d..b036a303d 100644 --- a/src/mesh/http/WebServer.h +++ b/src/mesh/http/WebServer.h @@ -13,6 +13,7 @@ class WebServerThread : private concurrency::OSThread public: WebServerThread(); + uint32_t requestRestart = 0; protected: virtual int32_t runOnce(); From 76d0ad2907f4b9dfd41fe825096dea4c5092a5b7 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 5 Jan 2022 22:27:49 -0800 Subject: [PATCH 4/4] Add page with links to end points on /spiffs --- src/mesh/http/ContentHandler.cpp | 23 ++++++++++++++++++----- src/mesh/http/ContentHandler.h | 1 + 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 043d3275d..4799494ef 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -124,6 +124,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) ResourceNode *nodeHotspotApple = new ResourceNode("/hotspot-detect.html", "GET", &handleHotspot); ResourceNode *nodeHotspotAndroid = new ResourceNode("/generate_204", "GET", &handleHotspot); + ResourceNode *nodeSPIFFS = new ResourceNode("/spiffs", "GET", &handleSPIFFS); ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/spiffs/update", "POST", &handleUpdateSPIFFS); ResourceNode *nodeDeleteSPIFFS = new ResourceNode("/spiffs/delete", "GET", &handleDeleteSPIFFSContent); @@ -153,6 +154,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) secureServer->registerNode(nodeJsonReport); secureServer->registerNode(nodeUpdateSPIFFS); secureServer->registerNode(nodeDeleteSPIFFS); + secureServer->registerNode(nodeSPIFFS); secureServer->registerNode(nodeRoot); // This has to be last // Insecure nodes @@ -170,6 +172,7 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) insecureServer->registerNode(nodeJsonReport); insecureServer->registerNode(nodeUpdateSPIFFS); insecureServer->registerNode(nodeDeleteSPIFFS); + insecureServer->registerNode(nodeSPIFFS); insecureServer->registerNode(nodeRoot); // This has to be last } @@ -371,11 +374,12 @@ void handleStatic(HTTPRequest *req, HTTPResponse *res) res->setHeader("Content-Type", "text/html"); if (!file.available()) { DEBUG_MSG("File not available - %s\n", filenameGzip.c_str()); - res->println("Web server is running.

The content you are looking for can't be found. Please see: FAQ.

Experimental " - "Web Content OTA Update -- Click " - "this just once and wait. Be patient!
"); + res->println( + "Web server is running.

The content you are looking for can't be found. Please see: FAQ.

Experimental " + "Web Content OTA Update -- Click " + "this just once and wait. Be patient!
"); } else { res->setHeader("Content-Encoding", "gzip"); } @@ -794,7 +798,16 @@ void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res) } file = root.openNextFile(); } +} +void handleSPIFFS(HTTPRequest *req, HTTPResponse *res) +{ + res->setHeader("Content-Type", "text/html"); + res->setHeader("Access-Control-Allow-Origin", "*"); + res->setHeader("Access-Control-Allow-Methods", "GET"); + + res->println("Delete Web Content

Be patient!"); } void handleRestart(HTTPRequest *req, HTTPResponse *res) diff --git a/src/mesh/http/ContentHandler.h b/src/mesh/http/ContentHandler.h index 2a89b3df6..4dfb86944 100644 --- a/src/mesh/http/ContentHandler.h +++ b/src/mesh/http/ContentHandler.h @@ -17,6 +17,7 @@ void handleBlinkLED(HTTPRequest *req, HTTPResponse *res); void handleReport(HTTPRequest *req, HTTPResponse *res); void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res); void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res); +void handleSPIFFS(HTTPRequest *req, HTTPResponse *res); // Interface to the PhoneAPI to access the protobufs with messages