From c16f7693fdb74fd6026470cd0a8864497070ef8d Mon Sep 17 00:00:00 2001 From: rjwats Date: Sun, 19 Jul 2020 19:32:08 +0100 Subject: [PATCH] Migrate to LittleFS under ESP8266 Make ESP8266 use LittleFS instead of deprecated SPIFFS Make framework use the correct filesystem automatically and handle the call the FS.begin() Change default MQTT keepalive to 60 seconds Fix lodash security issue --- README.md | 18 ++++++------------ factory_settings.ini | 2 +- interface/config-overrides.js | 2 +- interface/package-lock.json | 6 +++--- interface/package.json | 4 ++-- lib/framework/ESP8266React.cpp | 23 ++++++++++++++--------- lib/framework/ESP8266React.h | 7 ++++++- lib/framework/ESPFS.h | 7 +++++++ lib/framework/FactoryResetService.cpp | 7 +++++-- lib/framework/SystemStatus.cpp | 6 +++--- lib/framework/SystemStatus.h | 3 +-- platformio.ini | 1 + src/main.cpp | 12 ++---------- 13 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 lib/framework/ESPFS.h diff --git a/README.md b/README.md index 9b3354b..793dfcb 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,13 @@ platformio run -t upload The interface has been configured with create-react-app and react-app-rewired so the build can customized for the target device. The large artefacts are gzipped and source maps and service worker are excluded from the production build. This reduces the production build to around ~150k, which easily fits on the device. -The interface will be automatically built by PlatformIO before it builds the firmware. The project can be configured to serve the interface from either PROGMEM or SPIFFS as your project requires. The default configuration is to serve the content from PROGMEM, serving from SPIFFS requires an additional upload step which is documented below. +The interface will be automatically built by PlatformIO before it builds the firmware. The project can be configured to serve the interface from either PROGMEM or the filesystem as your project requires. The default configuration is to serve the content from PROGMEM, serving from the filesystem requires an additional upload step which is [documented below](#serving-the-interface-from-the-filesystem). #### Serving the interface from PROGMEM By default, the project is configured to serve the interface from PROGMEM. -> **Tip**: You do not need to upload a file system image unless you configure the framework to [serve the interface from SPIFFS](#serving-the-interface-from-spiffs). +> **Tip**: You do not need to upload a file system image unless you configure the framework to [serve the interface from the filesystem](#serving-the-interface-from-the-filesystem). The interface will consume ~150k of program space which can be problematic if you already have a large binary artefact or if you have added large dependencies to the interface. The ESP32 binaries are fairly large in there simplest form so the addition of the interface resources requires us to use special partitioning for the ESP32. @@ -96,9 +96,9 @@ platform = espressif32 board = node32s ``` -#### Serving the interface from SPIFFS +#### Serving the interface from the filesystem -If you choose to serve the interface from SPIFFS you will need to change the default configuration and upload the file system image manually. +If you choose to serve the interface from the filesystem you will need to change the default configuration and upload the file system image manually. Disable `-D PROGMEM_WWW build` flag in ['platformio.ini'](platformio.ini) and re-build the firmware. The build process will now copy the compiled interface to the `data/` directory and it may be uploaded to the device by pressing the "Upload File System image" button: @@ -340,7 +340,7 @@ The following code creates the web server and esp8266React framework: ```cpp AsyncWebServer server(80); -ESP8266React esp8266React(&server, &SPIFFS); +ESP8266React esp8266React(&server); ``` Now in the `setup()` function the initialization is performed: @@ -350,13 +350,6 @@ void setup() { // start serial and filesystem Serial.begin(SERIAL_BAUD_RATE); - // start the file system (must be done before starting the framework) -#ifdef ESP32 - SPIFFS.begin(true); -#elif defined(ESP8266) - SPIFFS.begin(); -#endif - // start the framework and demo project esp8266React.begin(); @@ -615,6 +608,7 @@ The framework supplies access to various features via getter functions: SettingsService | Description ---------------------------- | ---------------------------------------------- +getFS() | The filesystem used by the framework getSecurityManager() | The security manager - detailed above getSecuritySettingsService() | Configures the users and other security settings getWiFiSettingsService() | Configures and manages the WiFi network connection diff --git a/factory_settings.ini b/factory_settings.ini index 2d38dee..f0887fa 100644 --- a/factory_settings.ini +++ b/factory_settings.ini @@ -38,7 +38,7 @@ build_flags = -D FACTORY_MQTT_PASSWORD=\"\" ; if unspecified the devices hardware ID will be used ;-D FACTORY_MQTT_CLIENT_ID=\"esp-react\" - -D FACTORY_MQTT_KEEP_ALIVE=16 + -D FACTORY_MQTT_KEEP_ALIVE=60 -D FACTORY_MQTT_CLEAN_SESSION=true -D FACTORY_MQTT_MAX_TOPIC_LENGTH=128 diff --git a/interface/config-overrides.js b/interface/config-overrides.js index a80d407..61cd113 100644 --- a/interface/config-overrides.js +++ b/interface/config-overrides.js @@ -9,7 +9,7 @@ const fs = require('fs'); module.exports = function override(config, env) { if (env === "production") { - // rename the ouput file, we need it's path to be short, for SPIFFS + // rename the ouput file, we need it's path to be short, for embedded FS config.output.filename = 'js/[id].[chunkhash:4].js'; config.output.chunkFilename = 'js/[id].[chunkhash:4].js'; diff --git a/interface/package-lock.json b/interface/package-lock.json index 80f5755..3a85785 100644 --- a/interface/package-lock.json +++ b/interface/package-lock.json @@ -8257,9 +8257,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" }, "lodash._reinterpolate": { "version": "3.0.0", diff --git a/interface/package.json b/interface/package.json index eb4c77c..54a7cd0 100644 --- a/interface/package.json +++ b/interface/package.json @@ -6,7 +6,7 @@ "@material-ui/core": "^4.9.8", "@material-ui/icons": "^4.9.1", "@types/jwt-decode": "^2.2.1", - "@types/lodash": "^4.14.149", + "@types/lodash": "^4.14.157", "@types/node": "^12.12.32", "@types/react": "^16.9.27", "@types/react-dom": "^16.9.5", @@ -15,7 +15,7 @@ "@types/react-router-dom": "^5.1.3", "compression-webpack-plugin": "^4.0.0", "jwt-decode": "^2.2.0", - "lodash": "^4.17.15", + "lodash": "^4.17.19", "mime-types": "^2.1.25", "moment": "^2.26.0", "notistack": "^0.9.17", diff --git a/lib/framework/ESP8266React.cpp b/lib/framework/ESP8266React.cpp index b4f8f4c..59be381 100644 --- a/lib/framework/ESP8266React.cpp +++ b/lib/framework/ESP8266React.cpp @@ -1,32 +1,32 @@ #include -ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) : +ESP8266React::ESP8266React(AsyncWebServer* server) : _featureService(server), - _securitySettingsService(server, fs), - _wifiSettingsService(server, fs, &_securitySettingsService), + _securitySettingsService(server, &ESPFS), + _wifiSettingsService(server, &ESPFS, &_securitySettingsService), _wifiScanner(server, &_securitySettingsService), _wifiStatus(server, &_securitySettingsService), - _apSettingsService(server, fs, &_securitySettingsService), + _apSettingsService(server, &ESPFS, &_securitySettingsService), _apStatus(server, &_securitySettingsService, &_apSettingsService), #if FT_ENABLED(FT_NTP) - _ntpSettingsService(server, fs, &_securitySettingsService), + _ntpSettingsService(server, &ESPFS, &_securitySettingsService), _ntpStatus(server, &_securitySettingsService), #endif #if FT_ENABLED(FT_OTA) - _otaSettingsService(server, fs, &_securitySettingsService), + _otaSettingsService(server, &ESPFS, &_securitySettingsService), #endif #if FT_ENABLED(FT_UPLOAD_FIRMWARE) _uploadFirmwareService(server, &_securitySettingsService), #endif #if FT_ENABLED(FT_MQTT) - _mqttSettingsService(server, fs, &_securitySettingsService), + _mqttSettingsService(server, &ESPFS, &_securitySettingsService), _mqttStatus(server, &_mqttSettingsService, &_securitySettingsService), #endif #if FT_ENABLED(FT_SECURITY) _authenticationService(server, &_securitySettingsService), #endif _restartService(server, &_securitySettingsService), - _factoryResetService(server, fs, &_securitySettingsService), + _factoryResetService(server, &ESPFS, &_securitySettingsService), _systemStatus(server, &_securitySettingsService) { #ifdef PROGMEM_WWW // Serve static resources from PROGMEM @@ -63,7 +63,7 @@ ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) : // OPTIONS get a straight up 200 response server->onNotFound([](AsyncWebServerRequest* request) { if (request->method() == HTTP_GET) { - request->send(SPIFFS, "/www/index.html"); + request->send(ESPFS, "/www/index.html"); } else if (request->method() == HTTP_OPTIONS) { request->send(200); } else { @@ -81,6 +81,11 @@ ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) : } void ESP8266React::begin() { +#ifdef ESP32 + ESPFS.begin(true); +#elif defined(ESP8266) + ESPFS.begin(); +#endif _wifiSettingsService.begin(); _apSettingsService.begin(); #if FT_ENABLED(FT_NTP) diff --git a/lib/framework/ESP8266React.h b/lib/framework/ESP8266React.h index 1bb23a7..b1daf8a 100644 --- a/lib/framework/ESP8266React.h +++ b/lib/framework/ESP8266React.h @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef PROGMEM_WWW #include @@ -35,11 +36,15 @@ class ESP8266React { public: - ESP8266React(AsyncWebServer* server, FS* fs); + ESP8266React(AsyncWebServer* server); void begin(); void loop(); + FS* getFS() { + return &ESPFS; + } + SecurityManager* getSecurityManager() { return &_securitySettingsService; } diff --git a/lib/framework/ESPFS.h b/lib/framework/ESPFS.h new file mode 100644 index 0000000..7585f02 --- /dev/null +++ b/lib/framework/ESPFS.h @@ -0,0 +1,7 @@ +#ifdef ESP32 +#include +#define ESPFS SPIFFS +#elif defined(ESP8266) +#include +#define ESPFS LittleFS +#endif diff --git a/lib/framework/FactoryResetService.cpp b/lib/framework/FactoryResetService.cpp index c3ecd9e..9742207 100644 --- a/lib/framework/FactoryResetService.cpp +++ b/lib/framework/FactoryResetService.cpp @@ -15,7 +15,7 @@ void FactoryResetService::handleRequest(AsyncWebServerRequest* request) { } /** - * Delete function assumes that all files are stored flat, within the config directory + * Delete function assumes that all files are stored flat, within the config directory. */ void FactoryResetService::factoryReset() { #ifdef ESP32 @@ -27,7 +27,10 @@ void FactoryResetService::factoryReset() { #elif defined(ESP8266) Dir configDirectory = fs->openDir(FS_CONFIG_DIRECTORY); while (configDirectory.next()) { - fs->remove(configDirectory.fileName()); + String path = FS_CONFIG_DIRECTORY; + path.concat("/"); + path.concat(configDirectory.fileName()); + fs->remove(path); } #endif RestartService::restartNow(); diff --git a/lib/framework/SystemStatus.cpp b/lib/framework/SystemStatus.cpp index f879051..d146374 100644 --- a/lib/framework/SystemStatus.cpp +++ b/lib/framework/SystemStatus.cpp @@ -31,11 +31,11 @@ void SystemStatus::systemStatus(AsyncWebServerRequest* request) { // TODO - Ideally this class will take an *FS and extract the file system information from there. // ESP8266 and ESP32 do not have feature parity in FS.h which currently makes that difficult. #ifdef ESP32 - root["fs_total"] = SPIFFS.totalBytes(); - root["fs_used"] = SPIFFS.usedBytes(); + root["fs_total"] = ESPFS.totalBytes(); + root["fs_used"] = ESPFS.usedBytes(); #elif defined(ESP8266) FSInfo fs_info; - SPIFFS.info(fs_info); + ESPFS.info(fs_info); root["fs_total"] = fs_info.totalBytes; root["fs_used"] = fs_info.usedBytes; #endif diff --git a/lib/framework/SystemStatus.h b/lib/framework/SystemStatus.h index 7c1243b..3175ec4 100644 --- a/lib/framework/SystemStatus.h +++ b/lib/framework/SystemStatus.h @@ -4,17 +4,16 @@ #ifdef ESP32 #include #include -#include #elif defined(ESP8266) #include #include -#include #endif #include #include #include #include +#include #define MAX_ESP_STATUS_SIZE 1024 #define SYSTEM_STATUS_SERVICE_PATH "/rest/systemStatus" diff --git a/platformio.ini b/platformio.ini index e68aed3..174353c 100644 --- a/platformio.ini +++ b/platformio.ini @@ -40,6 +40,7 @@ lib_deps = platform = espressif8266 board = esp12e board_build.f_cpu = 160000000L +board_build.filesystem = littlefs [env:node32s] ; Comment out min_spiffs.csv setting if disabling PROGMEM_WWW with ESP32 diff --git a/src/main.cpp b/src/main.cpp index 978d74b..0b1081a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,14 +1,13 @@ #include #include #include -#include #define SERIAL_BAUD_RATE 115200 AsyncWebServer server(80); -ESP8266React esp8266React(&server, &SPIFFS); +ESP8266React esp8266React(&server); LightMqttSettingsService lightMqttSettingsService = - LightMqttSettingsService(&server, &SPIFFS, esp8266React.getSecurityManager()); + LightMqttSettingsService(&server, esp8266React.getFS(), esp8266React.getSecurityManager()); LightStateService lightStateService = LightStateService(&server, esp8266React.getSecurityManager(), esp8266React.getMqttClient(), @@ -18,13 +17,6 @@ void setup() { // start serial and filesystem Serial.begin(SERIAL_BAUD_RATE); - // start the file system (must be done before starting the framework) -#ifdef ESP32 - SPIFFS.begin(true); -#elif defined(ESP8266) - SPIFFS.begin(); -#endif - // start the framework and demo project esp8266React.begin();