web server esp idf suppport (#3500)

* initial web_server_idf implementation

* initial web_server_idf implementation

* fix lint errors

* fix lint errors

* add captive_portal support

* fix lint errors

* fix lint errors

* add url decode

* Increase the max supported size of headers section in HTTP request

* add ota support

* add mulipart form data support (ota required)

* make linter happy

* make linter happy

* make linter happy

* fix review marks

* add DefaultHeaders support

* add DefaultHeaders support

* unify file names

* using std::isnan

* parse multipart requests only when ota enabled

* parse multipart requests only when ota enabled

* parse multipart requests only when ota enabled

* parse multipart requests only when ota enabled

* parse multipart requests only when ota enabled

* drop multipart request support

* drop multipart request support

* drop multipart request support

* OTA is disabled by default

* fail when OTA enabled on IDF framework

* changing file permissions to remove execute bit

* return back PGM_P and strncpy_P macro

* temp web_server fix to be compat with 2022.12

* fix config handling w/o web_server

* fix compilation with "local"

* fully remove all idf ota

* merge with esphome 2023.6

* add core/hal to web_server_base

* Update esphome/components/web_server_base/__init__.py

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

* Update __init__.py

* Update __init__.py

---------

Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

CODEOWNERS

@@ -312,6 +312,7 @@ esphome/components/version/* @esphome/core
 esphome/components/voice_assistant/* @jesserockz
 esphome/components/wake_on_lan/* @willwill2will54
 esphome/components/web_server_base/* @OttoWinter
+esphome/components/web_server_idf/* @dentra
 esphome/components/whirlpool/* @glmnet
 esphome/components/whynter/* @aeonsablaze
 esphome/components/wiegand/* @ssieb

esphome/components/captive_portal/__init__.py

@@ -21,7 +21,6 @@ CONFIG_SCHEMA = cv.All(
             ),
         }
     ).extend(cv.COMPONENT_SCHEMA),
-    cv.only_with_arduino,
     cv.only_on(["esp32", "esp8266"]),
 )
 
@@ -34,8 +33,9 @@ async def to_code(config):
     await cg.register_component(var, config)
     cg.add_define("USE_CAPTIVE_PORTAL")
 
-    if CORE.is_esp32:
+    if CORE.using_arduino:
-        cg.add_library("DNSServer", None)
+        if CORE.is_esp32:
-        cg.add_library("WiFi", None)
+            cg.add_library("DNSServer", None)
-    if CORE.is_esp8266:
+            cg.add_library("WiFi", None)
-        cg.add_library("DNSServer", None)
+        if CORE.is_esp8266:
+            cg.add_library("DNSServer", None)

esphome/components/captive_portal/captive_portal.cpp

@@ -1,5 +1,3 @@
-#ifdef USE_ARDUINO
-
 #include "captive_portal.h"
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
@@ -46,10 +44,12 @@ void CaptivePortal::start() {
     this->base_->add_ota_handler();
   }
 
+#ifdef USE_ARDUINO
   this->dns_server_ = make_unique<DNSServer>();
   this->dns_server_->setErrorReplyCode(DNSReplyCode::NoError);
   network::IPAddress ip = wifi::global_wifi_component->wifi_soft_ap_ip();
   this->dns_server_->start(53, "*", (uint32_t) ip);
+#endif
 
   this->base_->get_server()->onNotFound([this](AsyncWebServerRequest *req) {
     if (!this->active_ || req->host().c_str() == wifi::global_wifi_component->wifi_soft_ap_ip().str()) {
@@ -67,7 +67,7 @@ void CaptivePortal::start() {
 
 void CaptivePortal::handleRequest(AsyncWebServerRequest *req) {
   if (req->url() == "/") {
-    AsyncWebServerResponse *response = req->beginResponse_P(200, "text/html", INDEX_GZ, sizeof(INDEX_GZ));
+    auto *response = req->beginResponse_P(200, "text/html", INDEX_GZ, sizeof(INDEX_GZ));
     response->addHeader("Content-Encoding", "gzip");
     req->send(response);
     return;
@@ -91,5 +91,3 @@ CaptivePortal *global_captive_portal = nullptr;  // NOLINT(cppcoreguidelines-avo
 
 }  // namespace captive_portal
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/captive_portal/captive_portal.h

@@ -1,9 +1,9 @@
 #pragma once
 
-#ifdef USE_ARDUINO
-
 #include <memory>
+#ifdef USE_ARDUINO
 #include <DNSServer.h>
+#endif
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
@@ -18,18 +18,22 @@ class CaptivePortal : public AsyncWebHandler, public Component {
   CaptivePortal(web_server_base::WebServerBase *base);
   void setup() override;
   void dump_config() override;
+#ifdef USE_ARDUINO
   void loop() override {
     if (this->dns_server_ != nullptr)
       this->dns_server_->processNextRequest();
   }
+#endif
   float get_setup_priority() const override;
   void start();
   bool is_active() const { return this->active_; }
   void end() {
     this->active_ = false;
     this->base_->deinit();
+#ifdef USE_ARDUINO
     this->dns_server_->stop();
     this->dns_server_ = nullptr;
+#endif
   }
 
   bool canHandle(AsyncWebServerRequest *request) override {
@@ -58,12 +62,12 @@ class CaptivePortal : public AsyncWebHandler, public Component {
   web_server_base::WebServerBase *base_;
   bool initialized_{false};
   bool active_{false};
+#ifdef USE_ARDUINO
   std::unique_ptr<DNSServer> dns_server_{nullptr};
+#endif
 };
 
 extern CaptivePortal *global_captive_portal;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 }  // namespace captive_portal
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server/__init__.py

@@ -47,6 +47,12 @@ def validate_local(config):
     return config
 
 
+def validate_ota(config):
+    if CORE.using_esp_idf and config[CONF_OTA]:
+        raise cv.Invalid("Enabling 'ota' is not supported for IDF framework yet")
+    return config
+
+
 CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
@@ -71,15 +77,17 @@ CONFIG_SCHEMA = cv.All(
                 web_server_base.WebServerBase
             ),
             cv.Optional(CONF_INCLUDE_INTERNAL, default=False): cv.boolean,
-            cv.Optional(CONF_OTA, default=True): cv.boolean,
+            cv.SplitDefault(
+                CONF_OTA, esp8266=True, esp32_arduino=True, esp32_idf=False
+            ): cv.boolean,
             cv.Optional(CONF_LOG, default=True): cv.boolean,
             cv.Optional(CONF_LOCAL): cv.boolean,
         }
     ).extend(cv.COMPONENT_SCHEMA),
-    cv.only_with_arduino,
     cv.only_on(["esp32", "esp8266"]),
     default_url,
     validate_local,
+    validate_ota,
 )
 
 

esphome/components/web_server/list_entities.cpp

@@ -1,5 +1,3 @@
-#ifdef USE_ARDUINO
-
 #include "list_entities.h"
 #include "esphome/core/application.h"
 #include "esphome/core/log.h"
@@ -103,5 +101,3 @@ bool ListEntitiesIterator::on_alarm_control_panel(alarm_control_panel::AlarmCont
 
 }  // namespace web_server
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server/list_entities.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#ifdef USE_ARDUINO
-
 #include "esphome/core/component.h"
 #include "esphome/core/component_iterator.h"
 #include "esphome/core/defines.h"
@@ -59,5 +57,3 @@ class ListEntitiesIterator : public ComponentIterator {
 
 }  // namespace web_server
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server/web_server.cpp

@@ -1,5 +1,3 @@
-#ifdef USE_ARDUINO
-
 #include "web_server.h"
 
 #include "esphome/components/json/json_util.h"
@@ -9,7 +7,9 @@
 #include "esphome/core/log.h"
 #include "esphome/core/util.h"
 
+#ifdef USE_ARDUINO
 #include "StreamString.h"
+#endif
 
 #include <cstdlib>
 
@@ -181,7 +181,7 @@ void WebServer::handle_index_request(AsyncWebServerRequest *request) {
   stream->print(F("<link rel=\"stylesheet\" href=\"/0.css\">"));
 #endif
   if (strlen(this->css_url_) > 0) {
-    stream->print(F("<link rel=\"stylesheet\" href=\""));
+    stream->print(F(R"(<link rel="stylesheet" href=")"));
     stream->print(this->css_url_);
     stream->print(F("\">"));
   }
@@ -381,7 +381,7 @@ void WebServer::handle_sensor_request(AsyncWebServerRequest *request, const UrlM
 std::string WebServer::sensor_json(sensor::Sensor *obj, float value, JsonDetail start_config) {
   return json::build_json([obj, value, start_config](JsonObject root) {
     std::string state;
-    if (isnan(value)) {
+    if (std::isnan(value)) {
       state = "NA";
     } else {
       state = value_accuracy_to_string(value, obj->get_accuracy_decimals());
@@ -524,11 +524,8 @@ void WebServer::handle_fan_request(AsyncWebServerRequest *request, const UrlMatc
       request->send(200);
     } else if (match.method == "turn_on") {
       auto call = obj->turn_on();
-      if (request->hasParam("speed")) {
-        String speed = request->getParam("speed")->value();
-      }
       if (request->hasParam("speed_level")) {
-        String speed_level = request->getParam("speed_level")->value();
+        auto speed_level = request->getParam("speed_level")->value();
         auto val = parse_number<int>(speed_level.c_str());
         if (!val.has_value()) {
           ESP_LOGW(TAG, "Can't convert '%s' to number!", speed_level.c_str());
@@ -537,7 +534,7 @@ void WebServer::handle_fan_request(AsyncWebServerRequest *request, const UrlMatc
         call.set_speed(*val);
       }
       if (request->hasParam("oscillation")) {
-        String speed = request->getParam("oscillation")->value();
+        auto speed = request->getParam("oscillation")->value();
         auto val = parse_on_off(speed.c_str());
         switch (val) {
           case PARSE_ON:
@@ -585,29 +582,54 @@ void WebServer::handle_light_request(AsyncWebServerRequest *request, const UrlMa
       request->send(200);
     } else if (match.method == "turn_on") {
       auto call = obj->turn_on();
-      if (request->hasParam("brightness"))
+      if (request->hasParam("brightness")) {
-        call.set_brightness(request->getParam("brightness")->value().toFloat() / 255.0f);
+        auto brightness = parse_number<float>(request->getParam("brightness")->value().c_str());
-      if (request->hasParam("r"))
+        if (brightness.has_value()) {
-        call.set_red(request->getParam("r")->value().toFloat() / 255.0f);
+          call.set_brightness(*brightness / 255.0f);
-      if (request->hasParam("g"))
+        }
-        call.set_green(request->getParam("g")->value().toFloat() / 255.0f);
+      }
-      if (request->hasParam("b"))
+      if (request->hasParam("r")) {
-        call.set_blue(request->getParam("b")->value().toFloat() / 255.0f);
+        auto r = parse_number<float>(request->getParam("r")->value().c_str());
-      if (request->hasParam("white_value"))
+        if (r.has_value()) {
-        call.set_white(request->getParam("white_value")->value().toFloat() / 255.0f);
+          call.set_red(*r / 255.0f);
-      if (request->hasParam("color_temp"))
+        }
-        call.set_color_temperature(request->getParam("color_temp")->value().toFloat());
+      }
-
+      if (request->hasParam("g")) {
+        auto g = parse_number<float>(request->getParam("g")->value().c_str());
+        if (g.has_value()) {
+          call.set_green(*g / 255.0f);
+        }
+      }
+      if (request->hasParam("b")) {
+        auto b = parse_number<float>(request->getParam("b")->value().c_str());
+        if (b.has_value()) {
+          call.set_blue(*b / 255.0f);
+        }
+      }
+      if (request->hasParam("white_value")) {
+        auto white_value = parse_number<float>(request->getParam("white_value")->value().c_str());
+        if (white_value.has_value()) {
+          call.set_white(*white_value / 255.0f);
+        }
+      }
+      if (request->hasParam("color_temp")) {
+        auto color_temp = parse_number<float>(request->getParam("color_temp")->value().c_str());
+        if (color_temp.has_value()) {
+          call.set_color_temperature(*color_temp);
+        }
+      }
       if (request->hasParam("flash")) {
-        float length_s = request->getParam("flash")->value().toFloat();
+        auto flash = parse_number<uint32_t>(request->getParam("flash")->value().c_str());
-        call.set_flash_length(static_cast<uint32_t>(length_s * 1000));
+        if (flash.has_value()) {
+          call.set_flash_length(*flash * 1000);
+        }
       }
-
       if (request->hasParam("transition")) {
-        float length_s = request->getParam("transition")->value().toFloat();
+        auto transition = parse_number<uint32_t>(request->getParam("transition")->value().c_str());
-        call.set_transition_length(static_cast<uint32_t>(length_s * 1000));
+        if (transition.has_value()) {
+          call.set_transition_length(*transition * 1000);
+        }
       }
-
       if (request->hasParam("effect")) {
         const char *effect = request->getParam("effect")->value().c_str();
         call.set_effect(effect);
@@ -618,8 +640,10 @@ void WebServer::handle_light_request(AsyncWebServerRequest *request, const UrlMa
     } else if (match.method == "turn_off") {
       auto call = obj->turn_off();
       if (request->hasParam("transition")) {
-        auto length = (uint32_t) request->getParam("transition")->value().toFloat() * 1000;
+        auto transition = parse_number<uint32_t>(request->getParam("transition")->value().c_str());
-        call.set_transition_length(length);
+        if (transition.has_value()) {
+          call.set_transition_length(*transition * 1000);
+        }
       }
       this->schedule_([call]() mutable { call.perform(); });
       request->send(200);
@@ -681,10 +705,18 @@ void WebServer::handle_cover_request(AsyncWebServerRequest *request, const UrlMa
       return;
     }
 
-    if (request->hasParam("position"))
+    if (request->hasParam("position")) {
-      call.set_position(request->getParam("position")->value().toFloat());
+      auto position = parse_number<float>(request->getParam("position")->value().c_str());
-    if (request->hasParam("tilt"))
+      if (position.has_value()) {
-      call.set_tilt(request->getParam("tilt")->value().toFloat());
+        call.set_position(*position);
+      }
+    }
+    if (request->hasParam("tilt")) {
+      auto tilt = parse_number<float>(request->getParam("tilt")->value().c_str());
+      if (tilt.has_value()) {
+        call.set_tilt(*tilt);
+      }
+    }
 
     this->schedule_([call]() mutable { call.perform(); });
     request->send(200);
@@ -725,10 +757,9 @@ void WebServer::handle_number_request(AsyncWebServerRequest *request, const UrlM
 
     auto call = obj->make_call();
     if (request->hasParam("value")) {
-      String value = request->getParam("value")->value();
+      auto value = parse_number<float>(request->getParam("value")->value().c_str());
-      optional<float> value_f = parse_number<float>(value.c_str());
+      if (value.has_value())
-      if (value_f.has_value())
+        call.set_value(*value);
-        call.set_value(*value_f);
     }
 
     this->schedule_([call]() mutable { call.perform(); });
@@ -747,7 +778,7 @@ std::string WebServer::number_json(number::Number *obj, float value, JsonDetail
       root["step"] = obj->traits.get_step();
       root["mode"] = (int) obj->traits.get_mode();
     }
-    if (isnan(value)) {
+    if (std::isnan(value)) {
       root["value"] = "\"NaN\"";
       root["state"] = "NA";
     } else {
@@ -784,7 +815,7 @@ void WebServer::handle_select_request(AsyncWebServerRequest *request, const UrlM
     auto call = obj->make_call();
 
     if (request->hasParam("option")) {
-      String option = request->getParam("option")->value();
+      auto option = request->getParam("option")->value();
       call.set_option(option.c_str());  // NOLINT(clang-diagnostic-deprecated-declarations)
     }
 
@@ -834,29 +865,26 @@ void WebServer::handle_climate_request(AsyncWebServerRequest *request, const Url
     auto call = obj->make_call();
 
     if (request->hasParam("mode")) {
-      String mode = request->getParam("mode")->value();
+      auto mode = request->getParam("mode")->value();
       call.set_mode(mode.c_str());
     }
 
     if (request->hasParam("target_temperature_high")) {
-      String value = request->getParam("target_temperature_high")->value();
+      auto target_temperature_high = parse_number<float>(request->getParam("target_temperature_high")->value().c_str());
-      optional<float> value_f = parse_number<float>(value.c_str());
+      if (target_temperature_high.has_value())
-      if (value_f.has_value())
+        call.set_target_temperature_high(*target_temperature_high);
-        call.set_target_temperature_high(*value_f);
     }
 
     if (request->hasParam("target_temperature_low")) {
-      String value = request->getParam("target_temperature_low")->value();
+      auto target_temperature_low = parse_number<float>(request->getParam("target_temperature_low")->value().c_str());
-      optional<float> value_f = parse_number<float>(value.c_str());
+      if (target_temperature_low.has_value())
-      if (value_f.has_value())
+        call.set_target_temperature_low(*target_temperature_low);
-        call.set_target_temperature_low(*value_f);
     }
 
     if (request->hasParam("target_temperature")) {
-      String value = request->getParam("target_temperature")->value();
+      auto target_temperature = parse_number<float>(request->getParam("target_temperature")->value().c_str());
-      optional<float> value_f = parse_number<float>(value.c_str());
+      if (target_temperature.has_value())
-      if (value_f.has_value())
+        call.set_target_temperature(*target_temperature);
-        call.set_target_temperature(*value_f);
     }
 
     this->schedule_([call]() mutable { call.perform(); });
@@ -1231,5 +1259,3 @@ void WebServer::schedule_(std::function<void()> &&f) {
 
 }  // namespace web_server
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server/web_server.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#ifdef USE_ARDUINO
-
 #include "list_entities.h"
 
 #include "esphome/components/web_server_base/web_server_base.h"
@@ -291,5 +289,3 @@ class WebServer : public Controller, public Component, public AsyncWebHandler {
 
 }  // namespace web_server
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server_base/__init__.py

@@ -5,7 +5,15 @@ from esphome.core import coroutine_with_priority, CORE
 
 CODEOWNERS = ["@OttoWinter"]
 DEPENDENCIES = ["network"]
-AUTO_LOAD = ["async_tcp"]
+
+
+def AUTO_LOAD():
+    if CORE.using_arduino:
+        return ["async_tcp"]
+    if CORE.using_esp_idf:
+        return ["web_server_idf"]
+    return []
+
 
 web_server_base_ns = cg.esphome_ns.namespace("web_server_base")
 WebServerBase = web_server_base_ns.class_("WebServerBase", cg.Component)
@@ -23,9 +31,10 @@ async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
 
-    if CORE.is_esp32:
+    if CORE.using_arduino:
-        cg.add_library("WiFi", None)
+        if CORE.is_esp32:
-        cg.add_library("FS", None)
+            cg.add_library("WiFi", None)
-        cg.add_library("Update", None)
+            cg.add_library("FS", None)
-    # https://github.com/esphome/ESPAsyncWebServer/blob/master/library.json
+            cg.add_library("Update", None)
-    cg.add_library("esphome/ESPAsyncWebServer-esphome", "2.1.0")
+        # https://github.com/esphome/ESPAsyncWebServer/blob/master/library.json
+        cg.add_library("esphome/ESPAsyncWebServer-esphome", "2.1.0")

esphome/components/web_server_base/web_server_base.cpp

@@ -1,16 +1,17 @@
-#ifdef USE_ARDUINO
-
 #include "web_server_base.h"
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
-#include <StreamString.h>
+#include "esphome/core/helpers.h"
 
+#ifdef USE_ARDUINO
+#include <StreamString.h>
 #ifdef USE_ESP32
 #include <Update.h>
 #endif
 #ifdef USE_ESP8266
 #include <Updater.h>
 #endif
+#endif
 
 namespace esphome {
 namespace web_server_base {
@@ -24,18 +25,22 @@ void WebServerBase::add_handler(AsyncWebHandler *handler) {
     handler = new internal::AuthMiddlewareHandler(handler, &credentials_);
   }
   this->handlers_.push_back(handler);
-  if (this->server_ != nullptr)
+  if (this->server_ != nullptr) {
     this->server_->addHandler(handler);
+  }
 }
 
 void report_ota_error() {
+#ifdef USE_ARDUINO
   StreamString ss;
   Update.printError(ss);
   ESP_LOGW(TAG, "OTA Update failed! Error: %s", ss.c_str());
+#endif
 }
 
 void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const String &filename, size_t index,
                                      uint8_t *data, size_t len, bool final) {
+#ifdef USE_ARDUINO
   bool success;
   if (index == 0) {
     ESP_LOGI(TAG, "OTA Update Start: %s", filename.c_str());
@@ -45,9 +50,10 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
     // NOLINTNEXTLINE(readability-static-accessed-through-instance)
     success = Update.begin((ESP.getFreeSketchSpace() - 0x1000) & 0xFFFFF000);
 #endif
-#ifdef USE_ESP32
+#ifdef USE_ESP32_FRAMEWORK_ARDUINO
-    if (Update.isRunning())
+    if (Update.isRunning()) {
       Update.abort();
+    }
     success = Update.begin(UPDATE_SIZE_UNKNOWN, U_FLASH);
 #endif
     if (!success) {
@@ -85,8 +91,10 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
       report_ota_error();
     }
   }
+#endif
 }
 void OTARequestHandler::handleRequest(AsyncWebServerRequest *request) {
+#ifdef USE_ARDUINO
   AsyncWebServerResponse *response;
   if (!Update.hasError()) {
     response = request->beginResponse(200, "text/plain", "Update Successful!");
@@ -98,10 +106,13 @@ void OTARequestHandler::handleRequest(AsyncWebServerRequest *request) {
   }
   response->addHeader("Connection", "close");
   request->send(response);
+#endif
 }
 
 void WebServerBase::add_ota_handler() {
+#ifdef USE_ARDUINO
   this->add_handler(new OTARequestHandler(this));  // NOLINT
+#endif
 }
 float WebServerBase::get_setup_priority() const {
   // Before WiFi (captive portal)
@@ -110,5 +121,3 @@ float WebServerBase::get_setup_priority() const {
 
 }  // namespace web_server_base
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server_base/web_server_base.h

@@ -1,14 +1,17 @@
 #pragma once
 
-#ifdef USE_ARDUINO
-
 #include <memory>
 #include <utility>
 #include <vector>
 
 #include "esphome/core/component.h"
 
+#ifdef USE_ARDUINO
 #include <ESPAsyncWebServer.h>
+#elif USE_ESP_IDF
+#include "esphome/core/hal.h"
+#include "esphome/components/web_server_idf/web_server_idf.h"
+#endif
 
 namespace esphome {
 namespace web_server_base {
@@ -141,5 +144,3 @@ class OTARequestHandler : public AsyncWebHandler {
 
 }  // namespace web_server_base
 }  // namespace esphome
-
-#endif  // USE_ARDUINO

esphome/components/web_server_idf/__init__.py

@@ -0,0 +1,14 @@
+import esphome.config_validation as cv
+from esphome.components.esp32 import add_idf_sdkconfig_option
+
+CODEOWNERS = ["@dentra"]
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema({}),
+    cv.only_with_esp_idf,
+)
+
+
+async def to_code(config):
+    # Increase the maximum supported size of headers section in HTTP request packet to be processed by the server
+    add_idf_sdkconfig_option("CONFIG_HTTPD_MAX_REQ_HDR_LEN", 1024)

esphome/components/web_server_idf/web_server_idf.cpp

@@ -0,0 +1,374 @@
+#ifdef USE_ESP_IDF
+
+#include <cstdarg>
+
+#include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
+
+#include "esp_tls_crypto.h"
+
+#include "web_server_idf.h"
+
+namespace esphome {
+namespace web_server_idf {
+
+#ifndef HTTPD_409
+#define HTTPD_409 "409 Conflict"
+#endif
+
+#define CRLF_STR "\r\n"
+#define CRLF_LEN (sizeof(CRLF_STR) - 1)
+
+static const char *const TAG = "web_server_idf";
+
+void AsyncWebServer::end() {
+  if (this->server_) {
+    httpd_stop(this->server_);
+    this->server_ = nullptr;
+  }
+}
+
+void AsyncWebServer::begin() {
+  if (this->server_) {
+    this->end();
+  }
+  httpd_config_t config = HTTPD_DEFAULT_CONFIG();
+  config.server_port = this->port_;
+  config.uri_match_fn = [](const char * /*unused*/, const char * /*unused*/, size_t /*unused*/) { return true; };
+  if (httpd_start(&this->server_, &config) == ESP_OK) {
+    const httpd_uri_t handler_get = {
+        .uri = "",
+        .method = HTTP_GET,
+        .handler = AsyncWebServer::request_handler,
+        .user_ctx = this,
+    };
+    httpd_register_uri_handler(this->server_, &handler_get);
+
+    const httpd_uri_t handler_post = {
+        .uri = "",
+        .method = HTTP_POST,
+        .handler = AsyncWebServer::request_handler,
+        .user_ctx = this,
+    };
+    httpd_register_uri_handler(this->server_, &handler_post);
+  }
+}
+
+esp_err_t AsyncWebServer::request_handler(httpd_req_t *r) {
+  ESP_LOGV(TAG, "Enter AsyncWebServer::request_handler. method=%u, uri=%s", r->method, r->uri);
+  AsyncWebServerRequest req(r);
+  auto *server = static_cast<AsyncWebServer *>(r->user_ctx);
+  for (auto *handler : server->handlers_) {
+    if (handler->canHandle(&req)) {
+      // At now process only basic requests.
+      // OTA requires multipart request support and handleUpload for it
+      handler->handleRequest(&req);
+      return ESP_OK;
+    }
+  }
+  if (server->on_not_found_) {
+    server->on_not_found_(&req);
+    return ESP_OK;
+  }
+  return ESP_ERR_NOT_FOUND;
+}
+
+AsyncWebServerRequest::~AsyncWebServerRequest() {
+  delete this->rsp_;
+  for (const auto &pair : this->params_) {
+    delete pair.second;  // NOLINT(cppcoreguidelines-owning-memory)
+  }
+}
+
+optional<std::string> AsyncWebServerRequest::get_header(const char *name) const {
+  size_t buf_len = httpd_req_get_hdr_value_len(*this, name);
+  if (buf_len == 0) {
+    return {};
+  }
+  auto buf = std::unique_ptr<char[]>(new char[++buf_len]);
+  if (!buf) {
+    ESP_LOGE(TAG, "No enough memory for get header %s", name);
+    return {};
+  }
+  if (httpd_req_get_hdr_value_str(*this, name, buf.get(), buf_len) != ESP_OK) {
+    return {};
+  }
+  return {buf.get()};
+}
+
+std::string AsyncWebServerRequest::url() const {
+  auto *str = strchr(this->req_->uri, '?');
+  if (str == nullptr) {
+    return this->req_->uri;
+  }
+  return std::string(this->req_->uri, str - this->req_->uri);
+}
+
+std::string AsyncWebServerRequest::host() const { return this->get_header("Host").value(); }
+
+void AsyncWebServerRequest::send(AsyncWebServerResponse *response) {
+  httpd_resp_send(*this, response->get_content_data(), response->get_content_size());
+}
+
+void AsyncWebServerRequest::send(int code, const char *content_type, const char *content) {
+  this->init_response_(nullptr, code, content_type);
+  if (content) {
+    httpd_resp_send(*this, content, HTTPD_RESP_USE_STRLEN);
+  } else {
+    httpd_resp_send(*this, nullptr, 0);
+  }
+}
+
+void AsyncWebServerRequest::redirect(const std::string &url) {
+  httpd_resp_set_status(*this, "302 Found");
+  httpd_resp_set_hdr(*this, "Location", url.c_str());
+  httpd_resp_send(*this, nullptr, 0);
+}
+
+void AsyncWebServerRequest::init_response_(AsyncWebServerResponse *rsp, int code, const char *content_type) {
+  httpd_resp_set_status(*this, code == 200   ? HTTPD_200
+                               : code == 404 ? HTTPD_404
+                               : code == 409 ? HTTPD_409
+                                             : to_string(code).c_str());
+
+  if (content_type && *content_type) {
+    httpd_resp_set_type(*this, content_type);
+  }
+  httpd_resp_set_hdr(*this, "Accept-Ranges", "none");
+
+  for (const auto &pair : DefaultHeaders::Instance().headers_) {
+    httpd_resp_set_hdr(*this, pair.first.c_str(), pair.second.c_str());
+  }
+
+  delete this->rsp_;
+  this->rsp_ = rsp;
+}
+
+bool AsyncWebServerRequest::authenticate(const char *username, const char *password) const {
+  if (username == nullptr || password == nullptr || *username == 0) {
+    return true;
+  }
+  auto auth = this->get_header("Authorization");
+  if (!auth.has_value()) {
+    return false;
+  }
+
+  auto *auth_str = auth.value().c_str();
+
+  const auto auth_prefix_len = sizeof("Basic ") - 1;
+  if (strncmp("Basic ", auth_str, auth_prefix_len) != 0) {
+    ESP_LOGW(TAG, "Only Basic authorization supported yet");
+    return false;
+  }
+
+  std::string user_info;
+  user_info += username;
+  user_info += ':';
+  user_info += password;
+
+  size_t n = 0, out;
+  esp_crypto_base64_encode(nullptr, 0, &n, reinterpret_cast<const uint8_t *>(user_info.c_str()), user_info.size());
+
+  auto digest = std::unique_ptr<char[]>(new char[n + 1]);
+  esp_crypto_base64_encode(reinterpret_cast<uint8_t *>(digest.get()), n, &out,
+                           reinterpret_cast<const uint8_t *>(user_info.c_str()), user_info.size());
+
+  return strncmp(digest.get(), auth_str + auth_prefix_len, auth.value().size() - auth_prefix_len) == 0;
+}
+
+void AsyncWebServerRequest::requestAuthentication(const char *realm) const {
+  httpd_resp_set_hdr(*this, "Connection", "keep-alive");
+  auto auth_val = str_sprintf("Basic realm=\"%s\"", realm ? realm : "Login Required");
+  httpd_resp_set_hdr(*this, "WWW-Authenticate", auth_val.c_str());
+  httpd_resp_send_err(*this, HTTPD_401_UNAUTHORIZED, nullptr);
+}
+
+static std::string url_decode(const std::string &in) {
+  std::string out;
+  out.reserve(in.size());
+  for (std::size_t i = 0; i < in.size(); ++i) {
+    if (in[i] == '%') {
+      ++i;
+      if (i + 1 < in.size()) {
+        auto c = parse_hex<uint8_t>(&in[i], 2);
+        if (c.has_value()) {
+          out += static_cast<char>(*c);
+          ++i;
+        } else {
+          out += '%';
+          out += in[i++];
+          out += in[i];
+        }
+      } else {
+        out += '%';
+        out += in[i];
+      }
+    } else if (in[i] == '+') {
+      out += ' ';
+    } else {
+      out += in[i];
+    }
+  }
+  return out;
+}
+
+AsyncWebParameter *AsyncWebServerRequest::getParam(const std::string &name) {
+  auto find = this->params_.find(name);
+  if (find != this->params_.end()) {
+    return find->second;
+  }
+
+  auto query_len = httpd_req_get_url_query_len(this->req_);
+  if (query_len == 0) {
+    return nullptr;
+  }
+
+  auto query_str = std::unique_ptr<char[]>(new char[++query_len]);
+  if (!query_str) {
+    ESP_LOGE(TAG, "No enough memory for get query param");
+    return nullptr;
+  }
+
+  auto res = httpd_req_get_url_query_str(*this, query_str.get(), query_len);
+  if (res != ESP_OK) {
+    ESP_LOGW(TAG, "Can't get query for request: %s", esp_err_to_name(res));
+    return nullptr;
+  }
+
+  auto query_val = std::unique_ptr<char[]>(new char[query_len]);
+  if (!query_val) {
+    ESP_LOGE(TAG, "No enough memory for get query param value");
+    return nullptr;
+  }
+
+  res = httpd_query_key_value(query_str.get(), name.c_str(), query_val.get(), query_len);
+  if (res != ESP_OK) {
+    this->params_.insert({name, nullptr});
+    return nullptr;
+  }
+  query_str.release();
+  auto decoded = url_decode(query_val.get());
+  query_val.release();
+  auto *param = new AsyncWebParameter(decoded);  // NOLINT(cppcoreguidelines-owning-memory)
+  this->params_.insert(std::make_pair(name, param));
+  return param;
+}
+
+void AsyncWebServerResponse::addHeader(const char *name, const char *value) {
+  httpd_resp_set_hdr(*this->req_, name, value);
+}
+
+void AsyncResponseStream::print(float value) { this->print(to_string(value)); }
+
+void AsyncResponseStream::printf(const char *fmt, ...) {
+  std::string str;
+  va_list args;
+
+  va_start(args, fmt);
+  size_t length = vsnprintf(nullptr, 0, fmt, args);
+  va_end(args);
+
+  str.resize(length);
+  va_start(args, fmt);
+  vsnprintf(&str[0], length + 1, fmt, args);
+  va_end(args);
+
+  this->print(str);
+}
+
+AsyncEventSource::~AsyncEventSource() {
+  for (auto *ses : this->sessions_) {
+    delete ses;  // NOLINT(cppcoreguidelines-owning-memory)
+  }
+}
+
+void AsyncEventSource::handleRequest(AsyncWebServerRequest *request) {
+  auto *rsp = new AsyncEventSourceResponse(request, this);  // NOLINT(cppcoreguidelines-owning-memory)
+  if (this->on_connect_) {
+    this->on_connect_(rsp);
+  }
+  this->sessions_.insert(rsp);
+}
+
+void AsyncEventSource::send(const char *message, const char *event, uint32_t id, uint32_t reconnect) {
+  for (auto *ses : this->sessions_) {
+    ses->send(message, event, id, reconnect);
+  }
+}
+
+AsyncEventSourceResponse::AsyncEventSourceResponse(const AsyncWebServerRequest *request, AsyncEventSource *server)
+    : server_(server) {
+  httpd_req_t *req = *request;
+
+  httpd_resp_set_status(req, HTTPD_200);
+  httpd_resp_set_type(req, "text/event-stream");
+  httpd_resp_set_hdr(req, "Cache-Control", "no-cache");
+  httpd_resp_set_hdr(req, "Connection", "keep-alive");
+
+  httpd_resp_send_chunk(req, CRLF_STR, CRLF_LEN);
+
+  req->sess_ctx = this;
+  req->free_ctx = AsyncEventSourceResponse::destroy;
+
+  this->hd_ = req->handle;
+  this->fd_ = httpd_req_to_sockfd(req);
+}
+
+void AsyncEventSourceResponse::destroy(void *ptr) {
+  auto *rsp = static_cast<AsyncEventSourceResponse *>(ptr);
+  rsp->server_->sessions_.erase(rsp);
+  delete rsp;  // NOLINT(cppcoreguidelines-owning-memory)
+}
+
+void AsyncEventSourceResponse::send(const char *message, const char *event, uint32_t id, uint32_t reconnect) {
+  if (this->fd_ == 0) {
+    return;
+  }
+
+  std::string ev;
+
+  if (reconnect) {
+    ev.append("retry: ", sizeof("retry: ") - 1);
+    ev.append(to_string(reconnect));
+    ev.append(CRLF_STR, CRLF_LEN);
+  }
+
+  if (id) {
+    ev.append("id: ", sizeof("id: ") - 1);
+    ev.append(to_string(id));
+    ev.append(CRLF_STR, CRLF_LEN);
+  }
+
+  if (event && *event) {
+    ev.append("event: ", sizeof("event: ") - 1);
+    ev.append(event);
+    ev.append(CRLF_STR, CRLF_LEN);
+  }
+
+  if (message && *message) {
+    ev.append("data: ", sizeof("data: ") - 1);
+    ev.append(message);
+    ev.append(CRLF_STR, CRLF_LEN);
+  }
+
+  if (ev.empty()) {
+    return;
+  }
+
+  ev.append(CRLF_STR, CRLF_LEN);
+
+  // Sending chunked content prelude
+  auto cs = str_snprintf("%x" CRLF_STR, 4 * sizeof(ev.size()) + CRLF_LEN, ev.size());
+  httpd_socket_send(this->hd_, this->fd_, cs.c_str(), cs.size(), 0);
+
+  // Sendiing content chunk
+  httpd_socket_send(this->hd_, this->fd_, ev.c_str(), ev.size(), 0);
+
+  // Indicate end of chunk
+  httpd_socket_send(this->hd_, this->fd_, CRLF_STR, CRLF_LEN, 0);
+}
+
+}  // namespace web_server_idf
+}  // namespace esphome
+
+#endif  // !defined(USE_ESP_IDF)

esphome/components/web_server_idf/web_server_idf.h

@@ -0,0 +1,277 @@
+#pragma once
+#ifdef USE_ESP_IDF
+
+#include <esp_http_server.h>
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <map>
+#include <set>
+
+namespace esphome {
+namespace web_server_idf {
+
+#define F(string_literal) (string_literal)
+#define PGM_P const char *
+#define strncpy_P strncpy
+
+using String = std::string;
+
+class AsyncWebParameter {
+ public:
+  AsyncWebParameter(std::string value) : value_(std::move(value)) {}
+  const std::string &value() const { return this->value_; }
+
+ protected:
+  std::string value_;
+};
+
+class AsyncWebServerRequest;
+
+class AsyncWebServerResponse {
+ public:
+  AsyncWebServerResponse(const AsyncWebServerRequest *req) : req_(req) {}
+  virtual ~AsyncWebServerResponse() {}
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void addHeader(const char *name, const char *value);
+
+  virtual const char *get_content_data() const = 0;
+  virtual size_t get_content_size() const = 0;
+
+ protected:
+  const AsyncWebServerRequest *req_;
+};
+
+class AsyncWebServerResponseEmpty : public AsyncWebServerResponse {
+ public:
+  AsyncWebServerResponseEmpty(const AsyncWebServerRequest *req) : AsyncWebServerResponse(req) {}
+
+  const char *get_content_data() const override { return nullptr; };
+  size_t get_content_size() const override { return 0; };
+};
+
+class AsyncWebServerResponseContent : public AsyncWebServerResponse {
+ public:
+  AsyncWebServerResponseContent(const AsyncWebServerRequest *req, std::string content)
+      : AsyncWebServerResponse(req), content_(std::move(content)) {}
+
+  const char *get_content_data() const override { return this->content_.c_str(); };
+  size_t get_content_size() const override { return this->content_.size(); };
+
+ protected:
+  std::string content_;
+};
+
+class AsyncResponseStream : public AsyncWebServerResponse {
+ public:
+  AsyncResponseStream(const AsyncWebServerRequest *req) : AsyncWebServerResponse(req) {}
+
+  const char *get_content_data() const override { return this->content_.c_str(); };
+  size_t get_content_size() const override { return this->content_.size(); };
+
+  void print(const char *str) { this->content_.append(str); }
+  void print(const std::string &str) { this->content_.append(str); }
+  void print(float value);
+  void printf(const char *fmt, ...) __attribute__((format(printf, 2, 3)));
+
+ protected:
+  std::string content_;
+};
+
+class AsyncWebServerResponseProgmem : public AsyncWebServerResponse {
+ public:
+  AsyncWebServerResponseProgmem(const AsyncWebServerRequest *req, const uint8_t *data, const size_t size)
+      : AsyncWebServerResponse(req), data_(data), size_(size) {}
+
+  const char *get_content_data() const override { return reinterpret_cast<const char *>(this->data_); };
+  size_t get_content_size() const override { return this->size_; };
+
+ protected:
+  const uint8_t *data_;
+  const size_t size_;
+};
+
+class AsyncWebServerRequest {
+  // FIXME friend class AsyncWebServerResponse;
+  friend class AsyncWebServer;
+
+ public:
+  ~AsyncWebServerRequest();
+
+  http_method method() const { return static_cast<http_method>(this->req_->method); }
+  std::string url() const;
+  std::string host() const;
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  size_t contentLength() const { return this->req_->content_len; }
+
+  bool authenticate(const char *username, const char *password) const;
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void requestAuthentication(const char *realm = nullptr) const;
+
+  void redirect(const std::string &url);
+
+  void send(AsyncWebServerResponse *response);
+  void send(int code, const char *content_type = nullptr, const char *content = nullptr);
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncWebServerResponse *beginResponse(int code, const char *content_type) {
+    auto *res = new AsyncWebServerResponseEmpty(this);  // NOLINT(cppcoreguidelines-owning-memory)
+    this->init_response_(res, 200, content_type);
+    return res;
+  }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncWebServerResponse *beginResponse(int code, const char *content_type, const std::string &content) {
+    auto *res = new AsyncWebServerResponseContent(this, content);  // NOLINT(cppcoreguidelines-owning-memory)
+    this->init_response_(res, code, content_type);
+    return res;
+  }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncWebServerResponse *beginResponse_P(int code, const char *content_type, const uint8_t *data,
+                                          const size_t data_size) {
+    auto *res = new AsyncWebServerResponseProgmem(this, data, data_size);  // NOLINT(cppcoreguidelines-owning-memory)
+    this->init_response_(res, code, content_type);
+    return res;
+  }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncResponseStream *beginResponseStream(const char *content_type) {
+    auto *res = new AsyncResponseStream(this);  // NOLINT(cppcoreguidelines-owning-memory)
+    this->init_response_(res, 200, content_type);
+    return res;
+  }
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  bool hasParam(const std::string &name) { return this->getParam(name) != nullptr; }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncWebParameter *getParam(const std::string &name);
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  bool hasArg(const char *name) { return this->hasParam(name); }
+  std::string arg(const std::string &name) {
+    auto *param = this->getParam(name);
+    if (param) {
+      return param->value();
+    }
+    return {};
+  }
+
+  operator httpd_req_t *() const { return this->req_; }
+  optional<std::string> get_header(const char *name) const;
+
+ protected:
+  httpd_req_t *req_;
+  AsyncWebServerResponse *rsp_{};
+  std::map<std::string, AsyncWebParameter *> params_;
+  AsyncWebServerRequest(httpd_req_t *req) : req_(req) {}
+  void init_response_(AsyncWebServerResponse *rsp, int code, const char *content_type);
+};
+
+class AsyncWebHandler;
+
+class AsyncWebServer {
+ public:
+  AsyncWebServer(uint16_t port) : port_(port){};
+  ~AsyncWebServer() { this->end(); }
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void onNotFound(std::function<void(AsyncWebServerRequest *request)> fn) { on_not_found_ = std::move(fn); }
+
+  void begin();
+  void end();
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  AsyncWebHandler &addHandler(AsyncWebHandler *handler) {
+    this->handlers_.push_back(handler);
+    return *handler;
+  }
+
+ protected:
+  uint16_t port_{};
+  httpd_handle_t server_{};
+  static esp_err_t request_handler(httpd_req_t *r);
+  std::vector<AsyncWebHandler *> handlers_;
+  std::function<void(AsyncWebServerRequest *request)> on_not_found_{};
+};
+
+class AsyncWebHandler {
+ public:
+  virtual ~AsyncWebHandler() {}
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  virtual bool canHandle(AsyncWebServerRequest *request) { return false; }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  virtual void handleRequest(AsyncWebServerRequest *request) {}
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  virtual void handleUpload(AsyncWebServerRequest *request, const std::string &filename, size_t index, uint8_t *data,
+                            size_t len, bool final) {}
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  virtual void handleBody(AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total) {}
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  virtual bool isRequestHandlerTrivial() { return true; }
+};
+
+class AsyncEventSource;
+
+class AsyncEventSourceResponse {
+  friend class AsyncEventSource;
+
+ public:
+  void send(const char *message, const char *event = nullptr, uint32_t id = 0, uint32_t reconnect = 0);
+
+ protected:
+  AsyncEventSourceResponse(const AsyncWebServerRequest *request, AsyncEventSource *server);
+  static void destroy(void *p);
+  AsyncEventSource *server_;
+  httpd_handle_t hd_{};
+  int fd_{};
+};
+
+using AsyncEventSourceClient = AsyncEventSourceResponse;
+
+class AsyncEventSource : public AsyncWebHandler {
+  friend class AsyncEventSourceResponse;
+  using connect_handler_t = std::function<void(AsyncEventSourceClient *)>;
+
+ public:
+  AsyncEventSource(std::string url) : url_(std::move(url)) {}
+  ~AsyncEventSource() override;
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  bool canHandle(AsyncWebServerRequest *request) override {
+    return request->method() == HTTP_GET && request->url() == this->url_;
+  }
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void handleRequest(AsyncWebServerRequest *request) override;
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void onConnect(connect_handler_t cb) { this->on_connect_ = std::move(cb); }
+
+  void send(const char *message, const char *event = nullptr, uint32_t id = 0, uint32_t reconnect = 0);
+
+ protected:
+  std::string url_;
+  std::set<AsyncEventSourceResponse *> sessions_;
+  connect_handler_t on_connect_{};
+};
+
+class DefaultHeaders {
+  friend class AsyncWebServerRequest;
+
+ public:
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  void addHeader(const char *name, const char *value) { this->headers_.emplace_back(name, value); }
+
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  static DefaultHeaders &Instance() {
+    static DefaultHeaders instance;
+    return instance;
+  }
+
+ protected:
+  std::vector<std::pair<std::string, std::string>> headers_;
+};
+
+}  // namespace web_server_idf
+}  // namespace esphome
+
+using namespace esphome::web_server_idf;  // NOLINT(google-global-names-in-headers)
+
+#endif  // !defined(USE_ESP_IDF)

script/build_codeowners.py

@@ -7,6 +7,7 @@ from collections import defaultdict
 from esphome.helpers import write_file_if_changed
 from esphome.config import get_component, get_platform
 from esphome.core import CORE
+from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -38,6 +39,7 @@ parts = [BASE]
 
 # Fake some directory so that get_component works
 CORE.config_path = str(root)
+CORE.data[KEY_CORE] = {KEY_TARGET_FRAMEWORK: None}
 
 codeowners = defaultdict(list)