Merge pull request #1064 from mc-hamster/SPIFFS_UPDATE

Fixed TLS "memory allocation failure" & Don't delete contents of /static unless tar is downloaded

src/mesh/http/ContentHandler.cpp

@@ -4,6 +4,7 @@
 #include "airtime.h"
 #include "main.h"
 #include "mesh/http/ContentHelper.h"
+#include "mesh/http/WebServer.h"
 #include "mesh/http/WiFiAPClient.h"
 #include "power.h"
 #include "sleep.h"
@@ -43,7 +44,7 @@ using namespace httpsserver;
 
 #include <HTTPClient.h>
 #include <WiFiClientSecure.h>
-HTTPClient http;
+HTTPClient httpClient;
 
 #define DEST_FS_USES_SPIFFS
 #include <ESP32-targz.h>
@@ -57,7 +58,7 @@ char contentTypes[][2][32] = {{".txt", "text/plain"},     {".html", "text/html"}
                               {".css", "text/css"},       {".ico", "image/vnd.microsoft.icon"},
                               {".svg", "image/svg+xml"},  {"", ""}};
 
-//const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar";
+// const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar";
 const char *tarURL = "https://api-production-871d.up.railway.app/mirror/webui";
 const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security)
 
@@ -73,22 +74,22 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const
         client->setCACert(cert);
     }
     const char *UserAgent = "ESP32-HTTP-GzUpdater-Client";
-    http.setReuse(true); // handle 301 redirects gracefully
+    httpClient.setReuse(true); // handle 301 redirects gracefully
-    http.setUserAgent(UserAgent);
+    httpClient.setUserAgent(UserAgent);
-    http.setConnectTimeout(10000); // 10s timeout = 10000
+    httpClient.setConnectTimeout(10000); // 10s timeout = 10000
-    if (!http.begin(*client, url)) {
+    if (!httpClient.begin(*client, url)) {
         log_e("Can't open url %s", url);
         return nullptr;
     }
     const char *headerKeys[] = {"location", "redirect", "Content-Type", "Content-Length", "Content-Disposition"};
     const size_t numberOfHeaders = 5;
-    http.collectHeaders(headerKeys, numberOfHeaders);
+    httpClient.collectHeaders(headerKeys, numberOfHeaders);
-    int httpCode = http.GET();
+    int httpCode = httpClient.GET();
     // file found at server
     if (httpCode == HTTP_CODE_FOUND || httpCode == HTTP_CODE_MOVED_PERMANENTLY) {
         String newlocation = "";
-        String headerLocation = http.header("location");
+        String headerLocation = httpClient.header("location");
-        String headerRedirect = http.header("redirect");
+        String headerRedirect = httpClient.header("redirect");
         if (headerLocation != "") {
             newlocation = headerLocation;
             Serial.printf("302 (location): %s => %s\n", url, headerLocation.c_str());
@@ -96,7 +97,7 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const
             Serial.printf("301 (redirect): %s => %s\n", url, headerLocation.c_str());
             newlocation = headerRedirect;
         }
-        http.end();
+        httpClient.end();
         if (newlocation != "") {
             log_w("Found 302/301 location header: %s", newlocation.c_str());
             return getTarHTTPClientPtr(client, newlocation.c_str(), cert);
@@ -107,7 +108,7 @@ WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const
     }
     if (httpCode != 200)
         return nullptr;
-    return http.getStreamPtr();
+    return httpClient.getStreamPtr();
 }
 
 void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer)
@@ -123,7 +124,9 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer)
     ResourceNode *nodeHotspotApple = new ResourceNode("/hotspot-detect.html", "GET", &handleHotspot);
     ResourceNode *nodeHotspotAndroid = new ResourceNode("/generate_204", "GET", &handleHotspot);
 
-    ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/update", "GET", &handleUpdateSPIFFS);
+    ResourceNode *nodeSPIFFS = new ResourceNode("/spiffs", "GET", &handleSPIFFS);
+    ResourceNode *nodeUpdateSPIFFS = new ResourceNode("/spiffs/update", "POST", &handleUpdateSPIFFS);
+    ResourceNode *nodeDeleteSPIFFS = new ResourceNode("/spiffs/delete", "GET", &handleDeleteSPIFFSContent);
 
     ResourceNode *nodeRestart = new ResourceNode("/restart", "POST", &handleRestart);
     ResourceNode *nodeFormUpload = new ResourceNode("/upload", "POST", &handleFormUpload);
@@ -150,6 +153,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer)
     secureServer->registerNode(nodeJsonDelete);
     secureServer->registerNode(nodeJsonReport);
     secureServer->registerNode(nodeUpdateSPIFFS);
+    secureServer->registerNode(nodeDeleteSPIFFS);
+    secureServer->registerNode(nodeSPIFFS);
     secureServer->registerNode(nodeRoot); // This has to be last
 
     // Insecure nodes
@@ -166,6 +171,8 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer)
     insecureServer->registerNode(nodeJsonDelete);
     insecureServer->registerNode(nodeJsonReport);
     insecureServer->registerNode(nodeUpdateSPIFFS);
+    insecureServer->registerNode(nodeDeleteSPIFFS);
+    insecureServer->registerNode(nodeSPIFFS);
     insecureServer->registerNode(nodeRoot); // This has to be last
 }
 
@@ -367,9 +374,12 @@ void handleStatic(HTTPRequest *req, HTTPResponse *res)
             res->setHeader("Content-Type", "text/html");
             if (!file.available()) {
                 DEBUG_MSG("File not available - %s\n", filenameGzip.c_str());
-                res->println("Web server is running.<br><br>The content you are looking for can't be found. Please see: <a "
+                res->println(
-                             "href=https://meshtastic.org/docs/getting-started/faq#wifi--web-browser>FAQ</a>.<br><br><a "
+                    "Web server is running.<br><br>The content you are looking for can't be found. Please see: <a "
-                             "href=/json/report>stats</a><br><br><a href=/update>Experemntal Web Content OTA Update</a>");
+                    "href=https://meshtastic.org/docs/getting-started/faq#wifi--web-browser>FAQ</a>.<br><br>Experimental "
+                    "Web Content OTA Update</a> -- Click "
+                    "this just once and wait. Be patient!<form action=/spiffs/update "
+                    "method=post><input type=submit value=UPDATE></form>");
             } else {
                 res->setHeader("Content-Encoding", "gzip");
             }
@@ -685,30 +695,33 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res)
 {
     res->setHeader("Content-Type", "text/html");
     res->setHeader("Access-Control-Allow-Origin", "*");
-    res->setHeader("Access-Control-Allow-Methods", "GET");
+    // res->setHeader("Access-Control-Allow-Methods", "POST");
 
     res->println("Downloading Meshtastic Web Content...");
 
-    File root = SPIFFS.open("/");
-    File file = root.openNextFile();
-
-    DEBUG_MSG("Deleting files from /static\n");
-
-    while (file) {
-        String filePath = String(file.name());
-        if (filePath.indexOf("/static") == 0) {
-            DEBUG_MSG("%s\n", file.name());
-            SPIFFS.remove(file.name());
-        }
-        file = root.openNextFile();
-    }
-
-    // return;
-
     WiFiClientSecure *client = new WiFiClientSecure;
     Stream *streamptr = getTarHTTPClientPtr(client, tarURL, certificate);
 
+    delay(5); // Let other network operations run
+
     if (streamptr != nullptr) {
+        DEBUG_MSG("Connection to content server ... success!\n");
+
+        File root = SPIFFS.open("/");
+        File file = root.openNextFile();
+
+        DEBUG_MSG("Deleting files from /static : \n");
+
+        while (file) {
+            String filePath = String(file.name());
+            if (filePath.indexOf("/static") == 0) {
+                DEBUG_MSG("    %s\n", file.name());
+                SPIFFS.remove(file.name());
+            }
+            file = root.openNextFile();
+        }
+
+        delay(5); // Let other network operations run
 
         TarUnpacker *TARUnpacker = new TarUnpacker();
         TARUnpacker->haltOnError(false);  // stop on fail (manual restart/reset required)
@@ -721,33 +734,80 @@ void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res)
             BaseUnpacker::defaultTarStatusProgressCallback);                        // print the filenames as they're expanded
         TARUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback); // tar log verbosity
 
-        String contentLengthStr = http.header("Content-Length");
+        String contentLengthStr = httpClient.header("Content-Length");
         contentLengthStr.trim();
         int64_t streamSize = -1;
         if (contentLengthStr != "") {
             streamSize = atoi(contentLengthStr.c_str());
             Serial.printf("Stream size %d\n", streamSize);
-            res->printf("Stream size %d\n", streamSize);
+            res->printf("Stream size %d<br><br>\n", streamSize);
         }
 
         if (!TARUnpacker->tarStreamExpander(streamptr, streamSize, SPIFFS, "/static")) {
+            res->printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError());
             Serial.printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError());
+
+            return;
         } else {
+            /*
             // print leftover bytes if any (probably zero-fill from the server)
-            while (http.connected()) {
+            while (httpClient.connected()) {
                 size_t streamSize = streamptr->available();
                 if (streamSize) {
                     Serial.printf("%02x ", streamptr->read());
                 } else
                     break;
             }
+            */
         }
 
     } else {
+        res->printf("Failed to establish http connection\n");
         Serial.println("Failed to establish http connection");
+        return;
     }
 
-    res->println("<a href=/>Done</a>");
+    res->println("Done! Restarting the device. <a href=/>Click this in 10 seconds</a>");
+
+    /*
+     * This is a work around for a bug where we run out of memory.
+     *   TODO: Fixme!
+     */
+    // ESP.restart();
+    webServerThread->requestRestart = (millis() / 1000) + 5;
+}
+
+void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res)
+{
+    res->setHeader("Content-Type", "text/html");
+    res->setHeader("Access-Control-Allow-Origin", "*");
+    res->setHeader("Access-Control-Allow-Methods", "GET");
+
+    res->println("Deleting SPIFFS Content in /static/*");
+
+    File root = SPIFFS.open("/");
+    File file = root.openNextFile();
+
+    DEBUG_MSG("Deleting files from /static : \n");
+
+    while (file) {
+        String filePath = String(file.name());
+        if (filePath.indexOf("/static") == 0) {
+            DEBUG_MSG("    %s\n", file.name());
+            SPIFFS.remove(file.name());
+        }
+        file = root.openNextFile();
+    }
+}
+
+void handleSPIFFS(HTTPRequest *req, HTTPResponse *res)
+{
+    res->setHeader("Content-Type", "text/html");
+    res->setHeader("Access-Control-Allow-Origin", "*");
+    res->setHeader("Access-Control-Allow-Methods", "GET");
+
+    res->println("<a href=/spiffs/delete>Delete Web Content</a><p><form action=/spiffs/update "
+                 "method=post><input type=submit value=UPDATE_WEB_CONTENT></form>Be patient!");
 }
 
 void handleRestart(HTTPRequest *req, HTTPResponse *res)
@@ -759,7 +819,7 @@ void handleRestart(HTTPRequest *req, HTTPResponse *res)
     DEBUG_MSG("***** Restarted on HTTP(s) Request *****\n");
     res->println("Restarting");
 
-    ESP.restart();
+    webServerThread->requestRestart = (millis() / 1000) + 5;
 }
 
 void handleBlinkLED(HTTPRequest *req, HTTPResponse *res)

src/mesh/http/ContentHandler.h

@@ -16,6 +16,9 @@ void handleSpiffsDeleteStatic(HTTPRequest *req, HTTPResponse *res);
 void handleBlinkLED(HTTPRequest *req, HTTPResponse *res);
 void handleReport(HTTPRequest *req, HTTPResponse *res);
 void handleUpdateSPIFFS(HTTPRequest *req, HTTPResponse *res);
+void handleDeleteSPIFFSContent(HTTPRequest *req, HTTPResponse *res);
+void handleSPIFFS(HTTPRequest *req, HTTPResponse *res);
+
 
 // Interface to the PhoneAPI to access the protobufs with messages
 class HttpAPI : public PhoneAPI

src/mesh/http/WebServer.cpp

@@ -175,6 +175,10 @@ int32_t WebServerThread::runOnce()
     // DEBUG_MSG("WebServerThread::runOnce()\n");
     handleWebResponse();
 
+    if (requestRestart && (millis() / 1000) > requestRestart) {
+        ESP.restart();
+    }
+
     // Loop every 5ms.
     return (5);
 }

src/mesh/http/WebServer.h

@@ -13,6 +13,7 @@ class WebServerThread : private concurrency::OSThread
 
   public:
     WebServerThread();
+    uint32_t requestRestart = 0;
 
   protected:
     virtual int32_t runOnce();