Configurable Flash Write Interval (#2119)

Co-authored-by: Alex <33379584+alexyao2015@users.noreply.github.com>
Co-authored-by: Otto winter <otto@otto-winter.com>

CODEOWNERS

@@ -104,6 +104,7 @@ esphome/components/pn532/* @OttoWinter @jesserockz
 esphome/components/pn532_i2c/* @OttoWinter @jesserockz
 esphome/components/pn532_spi/* @OttoWinter @jesserockz
 esphome/components/power_supply/* @esphome/core
+esphome/components/preferences/* @esphome/core
 esphome/components/pulse_meter/* @stevebaxter
 esphome/components/pvvx_mithermometer/* @pasiz
 esphome/components/rc522/* @glmnet

esphome/components/esp32/__init__.py

@@ -34,6 +34,7 @@ from .gpio import esp32_pin_to_code  # noqa
 
 _LOGGER = logging.getLogger(__name__)
 CODEOWNERS = ["@esphome/core"]
+AUTO_LOAD = ["preferences"]
 
 
 def set_core_data(config):

esphome/components/esp32/preferences.cpp

@@ -4,30 +4,53 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 #include <nvs_flash.h>
+#include <cstring>
+#include <vector>
+#include <string>
 
 namespace esphome {
 namespace esp32 {
 
 static const char *const TAG = "esp32.preferences";
 
+struct NVSData {
+  std::string key;
+  std::vector<uint8_t> data;
+};
+
+static std::vector<NVSData> s_pending_save;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+
 class ESP32PreferenceBackend : public ESPPreferenceBackend {
  public:
   std::string key;
   uint32_t nvs_handle;
   bool save(const uint8_t *data, size_t len) override {
-    esp_err_t err = nvs_set_blob(nvs_handle, key.c_str(), data, len);
+    // try find in pending saves and update that
-    if (err != 0) {
+    for (auto &obj : s_pending_save) {
-      ESP_LOGV(TAG, "nvs_set_blob('%s', len=%u) failed: %s", key.c_str(), len, esp_err_to_name(err));
+      if (obj.key == key) {
-      return false;
+        obj.data.assign(data, data + len);
-    }
+        return true;
-    err = nvs_commit(nvs_handle);
+      }
-    if (err != 0) {
-      ESP_LOGV(TAG, "nvs_commit('%s', len=%u) failed: %s", key.c_str(), len, esp_err_to_name(err));
-      return false;
     }
+    NVSData save{};
+    save.key = key;
+    save.data.assign(data, data + len);
+    s_pending_save.emplace_back(save);
     return true;
   }
   bool load(uint8_t *data, size_t len) override {
+    // try find in pending saves and load from that
+    for (auto &obj : s_pending_save) {
+      if (obj.key == key) {
+        if (obj.data.size() != len) {
+          // size mismatch
+          return false;
+        }
+        memcpy(data, obj.data.data(), len);
+        return true;
+      }
+    }
+
     size_t actual_len;
     esp_err_t err = nvs_get_blob(nvs_handle, key.c_str(), nullptr, &actual_len);
     if (err != 0) {
@@ -82,6 +105,37 @@ class ESP32Preferences : public ESPPreferences {
 
     return ESPPreferenceObject(pref);
   }
+
+  bool sync() override {
+    if (s_pending_save.empty())
+      return true;
+
+    ESP_LOGD(TAG, "Saving preferences to flash...");
+    // goal try write all pending saves even if one fails
+    bool any_failed = false;
+
+    // go through vector from back to front (makes erase easier/more efficient)
+    for (ssize_t i = s_pending_save.size() - 1; i >= 0; i--) {
+      const auto &save = s_pending_save[i];
+      esp_err_t err = nvs_set_blob(nvs_handle, save.key.c_str(), save.data.data(), save.data.size());
+      if (err != 0) {
+        ESP_LOGV(TAG, "nvs_set_blob('%s', len=%u) failed: %s", save.key.c_str(), save.data.size(),
+                 esp_err_to_name(err));
+        any_failed = true;
+        continue;
+      }
+      s_pending_save.erase(s_pending_save.begin() + i);
+    }
+
+    // note: commit on esp-idf currently is a no-op, nvs_set_blob always writes
+    esp_err_t err = nvs_commit(nvs_handle);
+    if (err != 0) {
+      ESP_LOGV(TAG, "nvs_commit() failed: %s", esp_err_to_name(err));
+      return false;
+    }
+
+    return !any_failed;
+  }
 };
 
 void setup_preferences() {

esphome/components/esp8266/__init__.py

@@ -23,6 +23,7 @@ from .gpio import esp8266_pin_to_code  # noqa
 
 CODEOWNERS = ["@esphome/core"]
 _LOGGER = logging.getLogger(__name__)
+AUTO_LOAD = ["preferences"]
 
 
 def set_core_data(config):

esphome/components/esp8266/preferences.cpp

@@ -73,33 +73,7 @@ template<class It> uint32_t calculate_crc(It first, It last, uint32_t type) {
   return crc;
 }
 
-static bool safe_flash() {
+static bool save_to_flash(size_t offset, const uint32_t *data, size_t len) {
-  if (!s_flash_dirty)
-    return true;
-
-  ESP_LOGVV(TAG, "Saving preferences to flash...");
-  SpiFlashOpResult erase_res, write_res = SPI_FLASH_RESULT_OK;
-  {
-    InterruptLock lock;
-    erase_res = spi_flash_erase_sector(get_esp8266_flash_sector());
-    if (erase_res == SPI_FLASH_RESULT_OK) {
-      write_res = spi_flash_write(get_esp8266_flash_address(), s_flash_storage, ESP8266_FLASH_STORAGE_SIZE * 4);
-    }
-  }
-  if (erase_res != SPI_FLASH_RESULT_OK) {
-    ESP_LOGV(TAG, "Erase ESP8266 flash failed!");
-    return false;
-  }
-  if (write_res != SPI_FLASH_RESULT_OK) {
-    ESP_LOGV(TAG, "Write ESP8266 flash failed!");
-    return false;
-  }
-
-  s_flash_dirty = false;
-  return true;
-}
-
-static bool safe_to_flash(size_t offset, const uint32_t *data, size_t len) {
   for (uint32_t i = 0; i < len; i++) {
     uint32_t j = offset + i;
     if (j >= ESP8266_FLASH_STORAGE_SIZE)
@@ -110,7 +84,7 @@ static bool safe_to_flash(size_t offset, const uint32_t *data, size_t len) {
       s_flash_dirty = true;
     *ptr = v;
   }
-  return safe_flash();
+  return true;
 }
 
 static bool load_from_flash(size_t offset, uint32_t *data, size_t len) {
@@ -123,7 +97,7 @@ static bool load_from_flash(size_t offset, uint32_t *data, size_t len) {
   return true;
 }
 
-static bool safe_to_rtc(size_t offset, const uint32_t *data, size_t len) {
+static bool save_to_rtc(size_t offset, const uint32_t *data, size_t len) {
   for (uint32_t i = 0; i < len; i++)
     if (!esp_rtc_user_mem_write(offset + i, data[i]))
       return false;
@@ -154,9 +128,9 @@ class ESP8266PreferenceBackend : public ESPPreferenceBackend {
     buffer[buffer.size() - 1] = calculate_crc(buffer.begin(), buffer.end() - 1, type);
 
     if (in_flash) {
-      return safe_to_flash(offset, buffer.data(), buffer.size());
+      return save_to_flash(offset, buffer.data(), buffer.size());
     } else {
-      return safe_to_rtc(offset, buffer.data(), buffer.size());
+      return save_to_rtc(offset, buffer.data(), buffer.size());
     }
   }
   bool load(uint8_t *data, size_t len) override {
@@ -245,6 +219,34 @@ class ESP8266Preferences : public ESPPreferences {
     return make_preference(length, type, false);
 #endif
   }
+
+  bool sync() override {
+    if (!s_flash_dirty)
+      return true;
+    if (s_prevent_write)
+      return false;
+
+    ESP_LOGD(TAG, "Saving preferences to flash...");
+    SpiFlashOpResult erase_res, write_res = SPI_FLASH_RESULT_OK;
+    {
+      InterruptLock lock;
+      erase_res = spi_flash_erase_sector(get_esp8266_flash_sector());
+      if (erase_res == SPI_FLASH_RESULT_OK) {
+        write_res = spi_flash_write(get_esp8266_flash_address(), s_flash_storage, ESP8266_FLASH_STORAGE_SIZE * 4);
+      }
+    }
+    if (erase_res != SPI_FLASH_RESULT_OK) {
+      ESP_LOGV(TAG, "Erase ESP8266 flash failed!");
+      return false;
+    }
+    if (write_res != SPI_FLASH_RESULT_OK) {
+      ESP_LOGV(TAG, "Write ESP8266 flash failed!");
+      return false;
+    }
+
+    s_flash_dirty = false;
+    return true;
+  }
 };
 
 void setup_preferences() {

esphome/components/ota/ota_component.cpp

@@ -548,7 +548,10 @@ bool OTAComponent::should_enter_safe_mode(uint8_t num_attempts, uint32_t enable_
     return false;
   }
 }
-void OTAComponent::write_rtc_(uint32_t val) { this->rtc_.save(&val); }
+void OTAComponent::write_rtc_(uint32_t val) {
+  this->rtc_.save(&val);
+  global_preferences->sync();
+}
 uint32_t OTAComponent::read_rtc_() {
   uint32_t val;
   if (!this->rtc_.load(&val))

esphome/components/preferences/__init__.py

@@ -0,0 +1,24 @@
+from esphome.const import CONF_ID
+import esphome.codegen as cg
+import esphome.config_validation as cv
+
+CODEOWNERS = ["@esphome/core"]
+
+preferences_ns = cg.esphome_ns.namespace("preferences")
+IntervalSyncer = preferences_ns.class_("IntervalSyncer", cg.Component)
+
+CONF_FLASH_WRITE_INTERVAL = "flash_write_interval"
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(IntervalSyncer),
+        cv.Optional(
+            CONF_FLASH_WRITE_INTERVAL, default="60s"
+        ): cv.positive_time_period_milliseconds,
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    cg.add(var.set_write_interval(config[CONF_FLASH_WRITE_INTERVAL]))
+    await cg.register_component(var, config)

esphome/components/preferences/syncer.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "esphome/core/preferences.h"
+#include "esphome/core/component.h"
+
+namespace esphome {
+namespace preferences {
+
+class IntervalSyncer : public Component {
+ public:
+  void set_write_interval(uint32_t write_interval) { write_interval_ = write_interval; }
+  void setup() override {
+    set_interval(write_interval_, []() { global_preferences->sync(); });
+  }
+  void on_shutdown() override { global_preferences->sync(); }
+  float get_setup_priority() const override { return setup_priority::BUS; }
+
+ protected:
+  uint32_t write_interval_;
+};
+
+}  // namespace preferences
+}  // namespace esphome

esphome/components/wifi/wifi_component.cpp

@@ -232,6 +232,8 @@ void WiFiComponent::save_wifi_sta(const std::string &ssid, const std::string &pa
   strncpy(save.ssid, ssid.c_str(), sizeof(save.ssid));
   strncpy(save.password, password.c_str(), sizeof(save.password));
   this->pref_.save(&save);
+  // ensure it's written immediately
+  global_preferences->sync();
 
   WiFiAP sta{};
   sta.set_ssid(ssid);

esphome/core/preferences.h

@@ -37,6 +37,14 @@ class ESPPreferences {
  public:
   virtual ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash) = 0;
   virtual ESPPreferenceObject make_preference(size_t length, uint32_t type) = 0;
+
+  /**
+   * Commit pending writes to flash.
+   *
+   * @return true if write is successful.
+   */
+  virtual bool sync() = 0;
+
 #ifndef USE_ESP8266
   template<typename T, typename std::enable_if<std::is_trivially_copyable<T>::value, bool>::type = true>
 #else