From 4ef2e284969cffe756727e85c961ce247b7f9ca5 Mon Sep 17 00:00:00 2001
From: Patrick ZAJDA <patrick@zajda.fr>
Date: Sun, 5 Nov 2023 21:27:14 +0100
Subject: [PATCH 01/41] Add basic shell autocompletion using argcomplete
 (#5618)

---
 esphome/__main__.py | 4 ++++
 requirements.txt    | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/esphome/__main__.py b/esphome/__main__.py
index cf540f58ba..a253fc78a0 100644
--- a/esphome/__main__.py
+++ b/esphome/__main__.py
@@ -1,3 +1,4 @@
+# PYTHON_ARGCOMPLETE_OK
 import argparse
 import functools
 import logging
@@ -7,6 +8,8 @@ import sys
 import time
 from datetime import datetime
 
+import argcomplete
+
 from esphome import const, writer, yaml_util
 import esphome.codegen as cg
 from esphome.config import iter_components, read_config, strip_default_ids
@@ -966,6 +969,7 @@ def parse_args(argv):
     # Finally, run the new-style parser again with the possibly swapped arguments,
     # and let it error out if the command is unparsable.
     parser.set_defaults(deprecated_argv_suggestion=deprecated_argv_suggestion)
+    argcomplete.autocomplete(parser)
     return parser.parse_args(arguments)
 
 
diff --git a/requirements.txt b/requirements.txt
index b9b6708ff2..5c57f3e933 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,3 +19,6 @@ kconfiglib==13.7.1
 
 # esp-idf >= 5.0 requires this
 pyparsing >= 3.0
+
+# For autocompletion
+argcomplete>=2.0.0

From 4ca9aefc43c189583d0f99d2023a7ea9e11fc36b Mon Sep 17 00:00:00 2001
From: Pavlo Dudnytskyi <paveldudn@gmail.com>
Date: Sun, 5 Nov 2023 21:28:50 +0100
Subject: [PATCH 02/41] Fixed int variables for user defined service in case of
 ESP32-C3 (#5675)

---
 esphome/components/api/user_services.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/esphome/components/api/user_services.cpp b/esphome/components/api/user_services.cpp
index 49618f5467..7e73722a92 100644
--- a/esphome/components/api/user_services.cpp
+++ b/esphome/components/api/user_services.cpp
@@ -5,7 +5,7 @@ namespace esphome {
 namespace api {
 
 template<> bool get_execute_arg_value<bool>(const ExecuteServiceArgument &arg) { return arg.bool_; }
-template<> int get_execute_arg_value<int>(const ExecuteServiceArgument &arg) {
+template<> int32_t get_execute_arg_value<int32_t>(const ExecuteServiceArgument &arg) {
   if (arg.legacy_int != 0)
     return arg.legacy_int;
   return arg.int_;
@@ -26,11 +26,13 @@ template<> std::vector<std::string> get_execute_arg_value<std::vector<std::strin
 }
 
 template<> enums::ServiceArgType to_service_arg_type<bool>() { return enums::SERVICE_ARG_TYPE_BOOL; }
-template<> enums::ServiceArgType to_service_arg_type<int>() { return enums::SERVICE_ARG_TYPE_INT; }
+template<> enums::ServiceArgType to_service_arg_type<int32_t>() { return enums::SERVICE_ARG_TYPE_INT; }
 template<> enums::ServiceArgType to_service_arg_type<float>() { return enums::SERVICE_ARG_TYPE_FLOAT; }
 template<> enums::ServiceArgType to_service_arg_type<std::string>() { return enums::SERVICE_ARG_TYPE_STRING; }
 template<> enums::ServiceArgType to_service_arg_type<std::vector<bool>>() { return enums::SERVICE_ARG_TYPE_BOOL_ARRAY; }
-template<> enums::ServiceArgType to_service_arg_type<std::vector<int>>() { return enums::SERVICE_ARG_TYPE_INT_ARRAY; }
+template<> enums::ServiceArgType to_service_arg_type<std::vector<int32_t>>() {
+  return enums::SERVICE_ARG_TYPE_INT_ARRAY;
+}
 template<> enums::ServiceArgType to_service_arg_type<std::vector<float>>() {
   return enums::SERVICE_ARG_TYPE_FLOAT_ARRAY;
 }

From 3ee85d7516747d946b31649de0234104f0594244 Mon Sep 17 00:00:00 2001
From: micw <michael@wyraz.de>
Date: Sun, 5 Nov 2023 22:27:11 +0100
Subject: [PATCH 03/41] Add callback for raw sml messages (#5668)

---
 esphome/components/sml/__init__.py  | 32 ++++++++++++++++++++++++++++-
 esphome/components/sml/automation.h | 19 +++++++++++++++++
 esphome/components/sml/constants.h  |  4 +++-
 esphome/components/sml/sml.cpp      | 20 ++++++++++++++----
 esphome/components/sml/sml.h        |  4 ++++
 5 files changed, 73 insertions(+), 6 deletions(-)
 create mode 100644 esphome/components/sml/automation.h

diff --git a/esphome/components/sml/__init__.py b/esphome/components/sml/__init__.py
index f3b6dd95ef..8bcfb69a45 100644
--- a/esphome/components/sml/__init__.py
+++ b/esphome/components/sml/__init__.py
@@ -1,9 +1,10 @@
 import re
 
+from esphome import automation
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.components import uart
-from esphome.const import CONF_ID
+from esphome.const import CONF_ID, CONF_TRIGGER_ID
 
 CODEOWNERS = ["@alengwenus"]
 
@@ -16,10 +17,26 @@ MULTI_CONF = True
 CONF_SML_ID = "sml_id"
 CONF_OBIS_CODE = "obis_code"
 CONF_SERVER_ID = "server_id"
+CONF_ON_DATA = "on_data"
+
+sml_ns = cg.esphome_ns.namespace("sml")
+
+DataTrigger = sml_ns.class_(
+    "DataTrigger",
+    automation.Trigger.template(
+        cg.std_vector.template(cg.uint8).operator("ref"), cg.bool_
+    ),
+)
+
 
 CONFIG_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.declare_id(Sml),
+        cv.Optional(CONF_ON_DATA): automation.validate_automation(
+            {
+                cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(DataTrigger),
+            }
+        ),
     }
 ).extend(uart.UART_DEVICE_SCHEMA)
 
@@ -28,6 +45,19 @@ async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
     await uart.register_uart_device(var, config)
+    for conf in config.get(CONF_ON_DATA, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(
+            trigger,
+            [
+                (
+                    cg.std_vector.template(cg.uint8).operator("ref").operator("const"),
+                    "bytes",
+                ),
+                (cg.bool_, "valid"),
+            ],
+            conf,
+        )
 
 
 def obis_code(value):
diff --git a/esphome/components/sml/automation.h b/esphome/components/sml/automation.h
new file mode 100644
index 0000000000..d51063065d
--- /dev/null
+++ b/esphome/components/sml/automation.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "esphome/core/automation.h"
+#include "sml.h"
+
+#include <vector>
+
+namespace esphome {
+namespace sml {
+
+class DataTrigger : public Trigger<const std::vector<uint8_t> &, bool> {
+ public:
+  explicit DataTrigger(Sml *sml) {
+    sml->add_on_data_callback([this](const std::vector<uint8_t> &data, bool valid) { this->trigger(data, valid); });
+  }
+};
+
+}  // namespace sml
+}  // namespace esphome
diff --git a/esphome/components/sml/constants.h b/esphome/components/sml/constants.h
index 08a124ccad..d6761d4bb7 100644
--- a/esphome/components/sml/constants.h
+++ b/esphome/components/sml/constants.h
@@ -18,8 +18,10 @@ enum SmlType : uint8_t {
 enum SmlMessageType : uint16_t { SML_PUBLIC_OPEN_RES = 0x0101, SML_GET_LIST_RES = 0x701 };
 
 // masks with two-bit mapping 0x1b -> 0b01; 0x01 -> 0b10; 0x1a -> 0b11
-const uint16_t START_MASK = 0x55aa;  // 0x1b 1b 1b 1b 1b 01 01 01 01
+const uint16_t START_MASK = 0x55aa;  // 0x1b 1b 1b 1b 01 01 01 01
 const uint16_t END_MASK = 0x0157;    // 0x1b 1b 1b 1b 1a
 
+const std::vector<uint8_t> START_SEQ = {0x1b, 0x1b, 0x1b, 0x1b, 0x01, 0x01, 0x01, 0x01};
+
 }  // namespace sml
 }  // namespace esphome
diff --git a/esphome/components/sml/sml.cpp b/esphome/components/sml/sml.cpp
index 921623d4fd..bac13be923 100644
--- a/esphome/components/sml/sml.cpp
+++ b/esphome/components/sml/sml.cpp
@@ -35,16 +35,24 @@ void Sml::loop() {
       case START_BYTES_DETECTED: {
         this->record_ = true;
         this->sml_data_.clear();
+        // add start sequence (for callbacks)
+        this->sml_data_.insert(this->sml_data_.begin(), START_SEQ.begin(), START_SEQ.end());
         break;
       };
       case END_BYTES_DETECTED: {
         if (this->record_) {
           this->record_ = false;
 
-          if (!check_sml_data(this->sml_data_))
+          bool valid = check_sml_data(this->sml_data_);
+
+          // call callbacks
+          this->data_callbacks_.call(this->sml_data_, valid);
+
+          if (!valid)
             break;
 
-          // remove footer bytes
+          // remove start/end sequence
+          this->sml_data_.erase(this->sml_data_.begin(), this->sml_data_.begin() + START_SEQ.size());
           this->sml_data_.resize(this->sml_data_.size() - 8);
           this->process_sml_file_(this->sml_data_);
         }
@@ -54,6 +62,10 @@ void Sml::loop() {
   }
 }
 
+void Sml::add_on_data_callback(std::function<void(std::vector<uint8_t>, bool)> &&callback) {
+  this->data_callbacks_.add(std::move(callback));
+}
+
 void Sml::process_sml_file_(const bytes &sml_data) {
   SmlFile sml_file = SmlFile(sml_data);
   std::vector<ObisInfo> obis_info = sml_file.get_obis_info();
@@ -100,14 +112,14 @@ bool check_sml_data(const bytes &buffer) {
   }
 
   uint16_t crc_received = (buffer.at(buffer.size() - 2) << 8) | buffer.at(buffer.size() - 1);
-  uint16_t crc_calculated = crc16(buffer.data(), buffer.size() - 2, 0x6e23, 0x8408, true, true);
+  uint16_t crc_calculated = crc16(buffer.data() + 8, buffer.size() - 10, 0x6e23, 0x8408, true, true);
   crc_calculated = (crc_calculated >> 8) | (crc_calculated << 8);
   if (crc_received == crc_calculated) {
     ESP_LOGV(TAG, "Checksum verification successful with CRC16/X25.");
     return true;
   }
 
-  crc_calculated = crc16(buffer.data(), buffer.size() - 2, 0xed50, 0x8408);
+  crc_calculated = crc16(buffer.data() + 8, buffer.size() - 10, 0xed50, 0x8408);
   if (crc_received == crc_calculated) {
     ESP_LOGV(TAG, "Checksum verification successful with CRC16/KERMIT.");
     return true;
diff --git a/esphome/components/sml/sml.h b/esphome/components/sml/sml.h
index ebc8b17d7f..b0c932ca95 100644
--- a/esphome/components/sml/sml.h
+++ b/esphome/components/sml/sml.h
@@ -3,6 +3,7 @@
 #include <string>
 #include <vector>
 #include "esphome/core/component.h"
+#include "esphome/core/helpers.h"
 #include "esphome/components/uart/uart.h"
 #include "sml_parser.h"
 
@@ -23,6 +24,7 @@ class Sml : public Component, public uart::UARTDevice {
   void loop() override;
   void dump_config() override;
   std::vector<SmlListener *> sml_listeners_{};
+  void add_on_data_callback(std::function<void(std::vector<uint8_t>, bool)> &&callback);
 
  protected:
   void process_sml_file_(const bytes &sml_data);
@@ -35,6 +37,8 @@ class Sml : public Component, public uart::UARTDevice {
   bool record_ = false;
   uint16_t incoming_mask_ = 0;
   bytes sml_data_;
+
+  CallbackManager<void(const std::vector<uint8_t> &, bool)> data_callbacks_{};
 };
 
 bool check_sml_data(const bytes &buffer);

From e0cee472c3bfc29c87b6c0e52aa1f428c3c3c8ff Mon Sep 17 00:00:00 2001
From: DAVe3283 <DAVe3283+GitHub@gmail.com>
Date: Sun, 5 Nov 2023 16:13:01 -0700
Subject: [PATCH 04/41] Fix compile with latest esp-idf on esp32c6 (#5677)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/aht10/aht10.cpp            |  5 ++-
 esphome/components/atm90e32/atm90e32.cpp      | 11 +++--
 esphome/components/bl0939/bl0939.cpp          |  8 ++--
 esphome/components/bl0940/bl0940.cpp          |  7 ++--
 esphome/components/bl0942/bl0942.cpp          |  5 ++-
 esphome/components/bmp3xx/bmp3xx.cpp          |  6 ++-
 esphome/components/canbus/canbus.cpp          |  4 +-
 esphome/components/cd74hc4067/cd74hc4067.cpp  |  3 +-
 esphome/components/cse7761/cse7761.cpp        |  2 +-
 esphome/components/cse7766/cse7766.cpp        |  6 ++-
 .../ethernet/ethernet_component.cpp           |  3 +-
 .../fingerprint_grow/fingerprint_grow.cpp     |  3 +-
 esphome/components/nfc/ndef_message.cpp       |  3 +-
 esphome/components/pid/pid_autotuner.cpp      |  7 ++--
 esphome/components/pzem004t/pzem004t.cpp      |  3 +-
 .../remote_base/drayton_protocol.cpp          | 16 +++++---
 .../components/remote_base/remote_base.cpp    | 10 ++---
 .../remote_receiver/remote_receiver_esp32.cpp | 12 ++++--
 esphome/components/rf_bridge/rf_bridge.cpp    | 11 +++--
 esphome/components/servo/servo.cpp            |  5 ++-
 esphome/components/sgp4x/sgp4x.cpp            |  2 +-
 esphome/components/spi/spi_arduino.cpp        |  2 +-
 esphome/components/sprinkler/sprinkler.cpp    | 40 ++++++++++---------
 .../status_led/light/status_led_light.cpp     |  3 +-
 .../thermostat/thermostat_climate.cpp         | 27 ++++++++-----
 .../components/tof10120/tof10120_sensor.cpp   |  3 +-
 .../components/tuya/sensor/tuya_sensor.cpp    |  3 +-
 esphome/components/vbus/vbus.cpp              |  5 ++-
 esphome/components/wireguard/wireguard.cpp    |  5 ++-
 29 files changed, 132 insertions(+), 88 deletions(-)

diff --git a/esphome/components/aht10/aht10.cpp b/esphome/components/aht10/aht10.cpp
index ad1a68498b..1ca06b458a 100644
--- a/esphome/components/aht10/aht10.cpp
+++ b/esphome/components/aht10/aht10.cpp
@@ -15,6 +15,7 @@
 #include "aht10.h"
 #include "esphome/core/log.h"
 #include "esphome/core/hal.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace aht10 {
@@ -72,7 +73,7 @@ void AHT10Component::update() {
     delay_ms = AHT10_HUMIDITY_DELAY;
   bool success = false;
   for (int i = 0; i < AHT10_ATTEMPTS; ++i) {
-    ESP_LOGVV(TAG, "Attempt %d at %6u", i, millis());
+    ESP_LOGVV(TAG, "Attempt %d at %6" PRIu32, i, millis());
     delay(delay_ms);
     if (this->read(data, 6) != i2c::ERROR_OK) {
       ESP_LOGD(TAG, "Communication with AHT10 failed, waiting...");
@@ -96,7 +97,7 @@ void AHT10Component::update() {
       }
     } else {
       // data is valid, we can break the loop
-      ESP_LOGVV(TAG, "Answer at %6u", millis());
+      ESP_LOGVV(TAG, "Answer at %6" PRIu32, millis());
       success = true;
       break;
     }
diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp
index e4b8448da6..e38fd3866a 100644
--- a/esphome/components/atm90e32/atm90e32.cpp
+++ b/esphome/components/atm90e32/atm90e32.cpp
@@ -1,6 +1,7 @@
 #include "atm90e32.h"
 #include "atm90e32_reg.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace atm90e32 {
@@ -173,7 +174,7 @@ uint16_t ATM90E32Component::read16_(uint16_t a_register) {
   this->disable();
 
   output = (uint16_t(data[0] & 0xFF) << 8) | (data[1] & 0xFF);
-  ESP_LOGVV(TAG, "read16_ 0x%04X output 0x%04X", a_register, output);
+  ESP_LOGVV(TAG, "read16_ 0x%04" PRIX16 " output 0x%04" PRIX16, a_register, output);
   return output;
 }
 
@@ -182,8 +183,10 @@ int ATM90E32Component::read32_(uint16_t addr_h, uint16_t addr_l) {
   uint16_t val_l = this->read16_(addr_l);
   int32_t val = (val_h << 16) | val_l;
 
-  ESP_LOGVV(TAG, "read32_ addr_h 0x%04X val_h 0x%04X addr_l 0x%04X val_l 0x%04X = %d", addr_h, val_h, addr_l, val_l,
-            val);
+  ESP_LOGVV(TAG,
+            "read32_ addr_h 0x%04" PRIX16 " val_h 0x%04" PRIX16 " addr_l 0x%04" PRIX16 " val_l 0x%04" PRIX16
+            " = %" PRId32,
+            addr_h, val_h, addr_l, val_l, val);
 
   return val;
 }
@@ -192,7 +195,7 @@ void ATM90E32Component::write16_(uint16_t a_register, uint16_t val) {
   uint8_t addrh = (a_register >> 8) & 0x03;
   uint8_t addrl = (a_register & 0xFF);
 
-  ESP_LOGVV(TAG, "write16_ 0x%04X val 0x%04X", a_register, val);
+  ESP_LOGVV(TAG, "write16_ 0x%04" PRIX16 " val 0x%04" PRIX16, a_register, val);
   this->enable();
   delayMicroseconds(10);
   this->write_byte(addrh);
diff --git a/esphome/components/bl0939/bl0939.cpp b/esphome/components/bl0939/bl0939.cpp
index 0575507c46..7428e48740 100644
--- a/esphome/components/bl0939/bl0939.cpp
+++ b/esphome/components/bl0939/bl0939.cpp
@@ -1,5 +1,6 @@
 #include "bl0939.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace bl0939 {
@@ -80,7 +81,7 @@ void BL0939::setup() {
 void BL0939::received_package_(const DataPacket *data) const {
   // Bad header
   if (data->frame_header != BL0939_PACKET_HEADER) {
-    ESP_LOGI("bl0939", "Invalid data. Header mismatch: %d", data->frame_header);
+    ESP_LOGI(TAG, "Invalid data. Header mismatch: %d", data->frame_header);
     return;
   }
 
@@ -120,8 +121,9 @@ void BL0939::received_package_(const DataPacket *data) const {
     energy_sensor_sum_->publish_state(total_energy_consumption);
   }
 
-  ESP_LOGV("bl0939", "BL0939: U %fV, I1 %fA, I2 %fA, P1 %fW, P2 %fW, CntA %d, CntB %d, ∫P1 %fkWh, ∫P2 %fkWh", v_rms,
-           ia_rms, ib_rms, a_watt, b_watt, cfa_cnt, cfb_cnt, a_energy_consumption, b_energy_consumption);
+  ESP_LOGV(TAG,
+           "BL0939: U %fV, I1 %fA, I2 %fA, P1 %fW, P2 %fW, CntA %" PRId32 ", CntB %" PRId32 ", ∫P1 %fkWh, ∫P2 %fkWh",
+           v_rms, ia_rms, ib_rms, a_watt, b_watt, cfa_cnt, cfb_cnt, a_energy_consumption, b_energy_consumption);
 }
 
 void BL0939::dump_config() {  // NOLINT(readability-function-cognitive-complexity)
diff --git a/esphome/components/bl0940/bl0940.cpp b/esphome/components/bl0940/bl0940.cpp
index ed193b23f3..24990d5482 100644
--- a/esphome/components/bl0940/bl0940.cpp
+++ b/esphome/components/bl0940/bl0940.cpp
@@ -1,5 +1,6 @@
 #include "bl0940.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace bl0940 {
@@ -77,7 +78,7 @@ float BL0940::update_temp_(sensor::Sensor *sensor, ube16_t temperature) const {
   float converted_temp = ((float) 170 / 448) * (tb / 2 - 32) - 45;
   if (sensor != nullptr) {
     if (sensor->has_state() && std::abs(converted_temp - sensor->get_state()) > max_temperature_diff_) {
-      ESP_LOGD("bl0940", "Invalid temperature change. Sensor: '%s', Old temperature: %f, New temperature: %f",
+      ESP_LOGD(TAG, "Invalid temperature change. Sensor: '%s', Old temperature: %f, New temperature: %f",
                sensor->get_name().c_str(), sensor->get_state(), converted_temp);
       return 0.0f;
     }
@@ -89,7 +90,7 @@ float BL0940::update_temp_(sensor::Sensor *sensor, ube16_t temperature) const {
 void BL0940::received_package_(const DataPacket *data) const {
   // Bad header
   if (data->frame_header != BL0940_PACKET_HEADER) {
-    ESP_LOGI("bl0940", "Invalid data. Header mismatch: %d", data->frame_header);
+    ESP_LOGI(TAG, "Invalid data. Header mismatch: %d", data->frame_header);
     return;
   }
 
@@ -115,7 +116,7 @@ void BL0940::received_package_(const DataPacket *data) const {
     energy_sensor_->publish_state(total_energy_consumption);
   }
 
-  ESP_LOGV("bl0940", "BL0940: U %fV, I %fA, P %fW, Cnt %d, ∫P %fkWh, T1 %f°C, T2 %f°C", v_rms, i_rms, watt, cf_cnt,
+  ESP_LOGV(TAG, "BL0940: U %fV, I %fA, P %fW, Cnt %" PRId32 ", ∫P %fkWh, T1 %f°C, T2 %f°C", v_rms, i_rms, watt, cf_cnt,
            total_energy_consumption, tps1, tps2);
 }
 
diff --git a/esphome/components/bl0942/bl0942.cpp b/esphome/components/bl0942/bl0942.cpp
index e6d18a82a7..38b1c89036 100644
--- a/esphome/components/bl0942/bl0942.cpp
+++ b/esphome/components/bl0942/bl0942.cpp
@@ -1,5 +1,6 @@
 #include "bl0942.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace bl0942 {
@@ -104,8 +105,8 @@ void BL0942::received_package_(DataPacket *data) {
     frequency_sensor_->publish_state(frequency);
   }
 
-  ESP_LOGV(TAG, "BL0942: U %fV, I %fA, P %fW, Cnt %d, ∫P %fkWh, frequency %f°Hz, status 0x%08X", v_rms, i_rms, watt,
-           cf_cnt, total_energy_consumption, frequency, data->status);
+  ESP_LOGV(TAG, "BL0942: U %fV, I %fA, P %fW, Cnt %" PRId32 ", ∫P %fkWh, frequency %fHz, status 0x%08X", v_rms, i_rms,
+           watt, cf_cnt, total_energy_consumption, frequency, data->status);
 }
 
 void BL0942::dump_config() {  // NOLINT(readability-function-cognitive-complexity)
diff --git a/esphome/components/bmp3xx/bmp3xx.cpp b/esphome/components/bmp3xx/bmp3xx.cpp
index 410b7a3173..de28fd76ff 100644
--- a/esphome/components/bmp3xx/bmp3xx.cpp
+++ b/esphome/components/bmp3xx/bmp3xx.cpp
@@ -8,6 +8,7 @@
 #include "bmp3xx.h"
 #include "esphome/core/log.h"
 #include "esphome/core/hal.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace bmp3xx {
@@ -198,8 +199,9 @@ void BMP3XXComponent::update() {
     return;
   }
 
-  ESP_LOGVV(TAG, "measurement time %d", uint32_t(ceilf(meas_time)));
-  this->set_timeout("data", uint32_t(ceilf(meas_time)), [this]() {
+  const uint32_t meas_timeout = uint32_t(ceilf(meas_time));
+  ESP_LOGVV(TAG, "measurement time %" PRIu32, meas_timeout);
+  this->set_timeout("data", meas_timeout, [this]() {
     float temperature = 0.0f;
     float pressure = 0.0f;
     if (this->pressure_sensor_ != nullptr) {
diff --git a/esphome/components/canbus/canbus.cpp b/esphome/components/canbus/canbus.cpp
index 8cffebdaa0..73b86cba87 100644
--- a/esphome/components/canbus/canbus.cpp
+++ b/esphome/components/canbus/canbus.cpp
@@ -51,9 +51,9 @@ void Canbus::send_data(uint32_t can_id, bool use_extended_id, bool remote_transm
 
 void Canbus::add_trigger(CanbusTrigger *trigger) {
   if (trigger->use_extended_id_) {
-    ESP_LOGVV(TAG, "add trigger for extended canid=0x%08x", trigger->can_id_);
+    ESP_LOGVV(TAG, "add trigger for extended canid=0x%08" PRIx32, trigger->can_id_);
   } else {
-    ESP_LOGVV(TAG, "add trigger for std canid=0x%03x", trigger->can_id_);
+    ESP_LOGVV(TAG, "add trigger for std canid=0x%03" PRIx32, trigger->can_id_);
   }
   this->triggers_.push_back(trigger);
 };
diff --git a/esphome/components/cd74hc4067/cd74hc4067.cpp b/esphome/components/cd74hc4067/cd74hc4067.cpp
index ea789c2d8c..f7629df3ca 100644
--- a/esphome/components/cd74hc4067/cd74hc4067.cpp
+++ b/esphome/components/cd74hc4067/cd74hc4067.cpp
@@ -1,5 +1,6 @@
 #include "cd74hc4067.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace cd74hc4067 {
@@ -27,7 +28,7 @@ void CD74HC4067Component::dump_config() {
   LOG_PIN("  S1 Pin: ", this->pin_s1_);
   LOG_PIN("  S2 Pin: ", this->pin_s2_);
   LOG_PIN("  S3 Pin: ", this->pin_s3_);
-  ESP_LOGCONFIG(TAG, "switch delay: %d", this->switch_delay_);
+  ESP_LOGCONFIG(TAG, "switch delay: %" PRIu32, this->switch_delay_);
 }
 
 void CD74HC4067Component::activate_pin(uint8_t pin) {
diff --git a/esphome/components/cse7761/cse7761.cpp b/esphome/components/cse7761/cse7761.cpp
index 3b8364f0bc..337996f6ef 100644
--- a/esphome/components/cse7761/cse7761.cpp
+++ b/esphome/components/cse7761/cse7761.cpp
@@ -217,7 +217,7 @@ void CSE7761Component::get_data_() {
     this->voltage_sensor_->publish_state(voltage);
   }
 
-  for (uint32_t channel = 0; channel < 2; channel++) {
+  for (uint8_t channel = 0; channel < 2; channel++) {
     // Active power = PowerPA * PowerPAC * 1000 / 0x80000000
     float active_power = (float) this->data_.active_power[channel] / this->coefficient_by_unit_(POWER_PAC);  // W
     float amps = (float) this->data_.current_rms[channel] / this->coefficient_by_unit_(RMS_IAC);             // A
diff --git a/esphome/components/cse7766/cse7766.cpp b/esphome/components/cse7766/cse7766.cpp
index f232f35ea6..60132fd98f 100644
--- a/esphome/components/cse7766/cse7766.cpp
+++ b/esphome/components/cse7766/cse7766.cpp
@@ -1,5 +1,6 @@
 #include "cse7766.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace cse7766 {
@@ -162,7 +163,7 @@ void CSE7766Component::update() {
     if (counts != 0) {
       const auto avg = acc / counts;
 
-      ESP_LOGV(TAG, "Got %s_acc=%.2f %s_counts=%d %s=%.1f", name, acc, name, counts, name, avg);
+      ESP_LOGV(TAG, "Got %s_acc=%.2f %s_counts=%" PRIu32 " %s=%.1f", name, acc, name, counts, name, avg);
 
       if (sensor != nullptr) {
         sensor->publish_state(avg);
@@ -178,7 +179,8 @@ void CSE7766Component::update() {
   publish_state("power", this->power_sensor_, this->power_acc_, this->power_counts_);
 
   if (this->energy_total_counts_ != 0) {
-    ESP_LOGV(TAG, "Got energy_total=%.2f energy_total_counts=%d", this->energy_total_, this->energy_total_counts_);
+    ESP_LOGV(TAG, "Got energy_total=%.2f energy_total_counts=%" PRIu32, this->energy_total_,
+             this->energy_total_counts_);
 
     if (this->energy_sensor_ != nullptr) {
       this->energy_sensor_->publish_state(this->energy_total_);
diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp
index 03fdc49338..50d5855e6a 100644
--- a/esphome/components/ethernet/ethernet_component.cpp
+++ b/esphome/components/ethernet/ethernet_component.cpp
@@ -5,6 +5,7 @@
 
 #ifdef USE_ESP32
 
+#include <cinttypes>
 #include <lwip/dns.h>
 #include "esp_event.h"
 
@@ -275,7 +276,7 @@ void EthernetComponent::got_ip_event_handler(void *arg, esp_event_base_t event_b
 #if ENABLE_IPV6
 void EthernetComponent::got_ip6_event_handler(void *arg, esp_event_base_t event_base, int32_t event_id,
                                               void *event_data) {
-  ESP_LOGV(TAG, "[Ethernet event] ETH Got IP6 (num=%d)", event_id);
+  ESP_LOGV(TAG, "[Ethernet event] ETH Got IP6 (num=%" PRId32 ")", event_id);
   global_eth_component->got_ipv6_ = true;
   global_eth_component->ipv6_count_ += 1;
 }
diff --git a/esphome/components/fingerprint_grow/fingerprint_grow.cpp b/esphome/components/fingerprint_grow/fingerprint_grow.cpp
index 4043f32dcb..8729e0f4bf 100644
--- a/esphome/components/fingerprint_grow/fingerprint_grow.cpp
+++ b/esphome/components/fingerprint_grow/fingerprint_grow.cpp
@@ -1,5 +1,6 @@
 #include "fingerprint_grow.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace fingerprint_grow {
@@ -204,7 +205,7 @@ bool FingerprintGrowComponent::check_password_() {
 }
 
 bool FingerprintGrowComponent::set_password_() {
-  ESP_LOGI(TAG, "Setting new password: %d", this->new_password_);
+  ESP_LOGI(TAG, "Setting new password: %" PRIu32, this->new_password_);
   this->data_ = {SET_PASSWORD, (uint8_t) (this->new_password_ >> 24), (uint8_t) (this->new_password_ >> 16),
                  (uint8_t) (this->new_password_ >> 8), (uint8_t) (this->new_password_ & 0xFF)};
   if (this->send_command_() == OK) {
diff --git a/esphome/components/nfc/ndef_message.cpp b/esphome/components/nfc/ndef_message.cpp
index 461856d377..e7304445c5 100644
--- a/esphome/components/nfc/ndef_message.cpp
+++ b/esphome/components/nfc/ndef_message.cpp
@@ -1,4 +1,5 @@
 #include "ndef_message.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace nfc {
@@ -32,7 +33,7 @@ NdefMessage::NdefMessage(std::vector<uint8_t> &data) {
       id_length = data[index++];
     }
 
-    ESP_LOGVV(TAG, "Lengths: type=%d, payload=%d, id=%d", type_length, payload_length, id_length);
+    ESP_LOGVV(TAG, "Lengths: type=%d, payload=%" PRIu32 ", id=%d", type_length, payload_length, id_length);
 
     std::string type_str(data.begin() + index, data.begin() + index + type_length);
 
diff --git a/esphome/components/pid/pid_autotuner.cpp b/esphome/components/pid/pid_autotuner.cpp
index 1b3ddebcc5..28d16e17ab 100644
--- a/esphome/components/pid/pid_autotuner.cpp
+++ b/esphome/components/pid/pid_autotuner.cpp
@@ -1,5 +1,6 @@
 #include "pid_autotuner.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 #ifndef M_PI
 #define M_PI 3.1415926535897932384626433
@@ -108,7 +109,7 @@ PIDAutotuner::PIDAutotuneResult PIDAutotuner::update(float setpoint, float proce
     }
     uint32_t phase = this->relay_function_.phase_count;
     ESP_LOGVV(TAG, "%s: >", this->id_.c_str());
-    ESP_LOGVV(TAG, "  Phase %u, enough=%u", phase, enough_data_phase_);
+    ESP_LOGVV(TAG, "  Phase %" PRIu32 ", enough=%" PRIu32, phase, enough_data_phase_);
 
     if (this->enough_data_phase_ == 0) {
       this->enough_data_phase_ = phase;
@@ -186,8 +187,8 @@ void PIDAutotuner::dump_config() {
     ESP_LOGD(TAG, "  Autotune is still running!");
     ESP_LOGD(TAG, "  Status: Trying to reach %.2f °C", setpoint_ - relay_function_.current_target_error());
     ESP_LOGD(TAG, "  Stats so far:");
-    ESP_LOGD(TAG, "    Phases: %u", relay_function_.phase_count);
-    ESP_LOGD(TAG, "    Detected %u zero-crossings", frequency_detector_.zerocrossing_intervals.size());  // NOLINT
+    ESP_LOGD(TAG, "    Phases: %" PRIu32, relay_function_.phase_count);
+    ESP_LOGD(TAG, "    Detected %zu zero-crossings", frequency_detector_.zerocrossing_intervals.size());
     ESP_LOGD(TAG, "    Current Phase Min: %.2f, Max: %.2f", amplitude_detector_.phase_min,
              amplitude_detector_.phase_max);
   }
diff --git a/esphome/components/pzem004t/pzem004t.cpp b/esphome/components/pzem004t/pzem004t.cpp
index e5418765bd..35b66b03f2 100644
--- a/esphome/components/pzem004t/pzem004t.cpp
+++ b/esphome/components/pzem004t/pzem004t.cpp
@@ -1,5 +1,6 @@
 #include "pzem004t.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace pzem004t {
@@ -75,7 +76,7 @@ void PZEM004T::loop() {
         uint32_t energy = (uint32_t(resp[1]) << 16) | (uint32_t(resp[2]) << 8) | (uint32_t(resp[3]));
         if (this->energy_sensor_ != nullptr)
           this->energy_sensor_->publish_state(energy);
-        ESP_LOGD(TAG, "Got Energy %u Wh", energy);
+        ESP_LOGD(TAG, "Got Energy %" PRIu32 " Wh", energy);
         this->write_state_(DONE);
         break;
       }
diff --git a/esphome/components/remote_base/drayton_protocol.cpp b/esphome/components/remote_base/drayton_protocol.cpp
index 12a553455f..fb5f37b470 100644
--- a/esphome/components/remote_base/drayton_protocol.cpp
+++ b/esphome/components/remote_base/drayton_protocol.cpp
@@ -138,9 +138,12 @@ optional<DraytonData> DraytonProtocol::decode(RemoteReceiveData src) {
     return {};
   }
 
-  ESP_LOGVV(TAG, "Decode Drayton: %d, %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", src.size(),
-            src.peek(0), src.peek(1), src.peek(2), src.peek(3), src.peek(4), src.peek(5), src.peek(6), src.peek(7),
-            src.peek(8), src.peek(9), src.peek(10), src.peek(11), src.peek(12), src.peek(13), src.peek(14),
+  ESP_LOGVV(TAG,
+            "Decode Drayton: %" PRId32 ", %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
+            " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
+            " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 "",
+            src.size(), src.peek(0), src.peek(1), src.peek(2), src.peek(3), src.peek(4), src.peek(5), src.peek(6),
+            src.peek(7), src.peek(8), src.peek(9), src.peek(10), src.peek(11), src.peek(12), src.peek(13), src.peek(14),
             src.peek(15), src.peek(16), src.peek(17), src.peek(18), src.peek(19));
 
   // If first preamble item is a space, skip it
@@ -150,7 +153,8 @@ optional<DraytonData> DraytonProtocol::decode(RemoteReceiveData src) {
 
   // Look for sync pulse, after. If sucessful index points to space of sync symbol
   for (uint16_t preamble = 0; preamble <= NBITS_PREAMBLE * 2; preamble += 2) {
-    ESP_LOGVV(TAG, "Decode Drayton: preamble %d  %d %d", preamble, src.peek(preamble), src.peek(preamble + 1));
+    ESP_LOGVV(TAG, "Decode Drayton: preamble %d  %" PRId32 " %" PRId32, preamble, src.peek(preamble),
+              src.peek(preamble + 1));
     if (src.peek_mark(2 * BIT_TIME_US, preamble) &&
         (src.peek_space(2 * BIT_TIME_US, preamble + 1) || src.peek_space(3 * BIT_TIME_US, preamble + 1))) {
       src.advance(preamble + 1);
@@ -177,7 +181,7 @@ optional<DraytonData> DraytonProtocol::decode(RemoteReceiveData src) {
   // if there is no transition at the start of the bit period, then the transition in the middle of
   // the previous bit period.
   while (--bit >= 1) {
-    ESP_LOGVV(TAG, "Decode Drayton: Data, %2d %08x", bit, out_data);
+    ESP_LOGVV(TAG, "Decode Drayton: Data, %2d %08" PRIx32, bit, out_data);
     if ((src.expect_space(BIT_TIME_US) || src.expect_space(2 * BIT_TIME_US)) &&
         (src.expect_mark(BIT_TIME_US) || src.peek_mark(2 * BIT_TIME_US))) {
       out_data |= 0 << bit;
@@ -185,7 +189,7 @@ optional<DraytonData> DraytonProtocol::decode(RemoteReceiveData src) {
                (src.expect_space(BIT_TIME_US) || src.peek_space(2 * BIT_TIME_US))) {
       out_data |= 1 << bit;
     } else {
-      ESP_LOGVV(TAG, "Decode Drayton: Fail 2, %2d %08x", bit, out_data);
+      ESP_LOGVV(TAG, "Decode Drayton: Fail 2, %2d %08" PRIx32, bit, out_data);
       return {};
     }
   }
diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp
index 7fe5e47ee7..40c699e8ea 100644
--- a/esphome/components/remote_base/remote_base.cpp
+++ b/esphome/components/remote_base/remote_base.cpp
@@ -125,7 +125,7 @@ void RemoteTransmitterBase::send_(uint32_t send_times, uint32_t send_wait) {
   const auto &vec = this->temp_.get_data();
   char buffer[256];
   uint32_t buffer_offset = 0;
-  buffer_offset += sprintf(buffer, "Sending times=%u wait=%ums: ", send_times, send_wait);
+  buffer_offset += sprintf(buffer, "Sending times=%" PRIu32 " wait=%" PRIu32 "ms: ", send_times, send_wait);
 
   for (size_t i = 0; i < vec.size(); i++) {
     const int32_t value = vec[i];
@@ -133,9 +133,9 @@ void RemoteTransmitterBase::send_(uint32_t send_times, uint32_t send_wait) {
     int written;
 
     if (i + 1 < vec.size()) {
-      written = snprintf(buffer + buffer_offset, remaining_length, "%d, ", value);
+      written = snprintf(buffer + buffer_offset, remaining_length, "%" PRId32 ", ", value);
     } else {
-      written = snprintf(buffer + buffer_offset, remaining_length, "%d", value);
+      written = snprintf(buffer + buffer_offset, remaining_length, "%" PRId32, value);
     }
 
     if (written < 0 || written >= int(remaining_length)) {
@@ -145,9 +145,9 @@ void RemoteTransmitterBase::send_(uint32_t send_times, uint32_t send_wait) {
       buffer_offset = 0;
       written = sprintf(buffer, "  ");
       if (i + 1 < vec.size()) {
-        written += sprintf(buffer + written, "%d, ", value);
+        written += sprintf(buffer + written, "%" PRId32 ", ", value);
       } else {
-        written += sprintf(buffer + written, "%d", value);
+        written += sprintf(buffer + written, "%" PRId32, value);
       }
     }
 
diff --git a/esphome/components/remote_receiver/remote_receiver_esp32.cpp b/esphome/components/remote_receiver/remote_receiver_esp32.cpp
index 1ebb5a5955..d19ab695e1 100644
--- a/esphome/components/remote_receiver/remote_receiver_esp32.cpp
+++ b/esphome/components/remote_receiver/remote_receiver_esp32.cpp
@@ -92,14 +92,18 @@ void RemoteReceiverComponent::decode_rmt_(rmt_item32_t *item, size_t len) {
   ESP_LOGVV(TAG, "START:");
   for (size_t i = 0; i < item_count; i++) {
     if (item[i].level0) {
-      ESP_LOGVV(TAG, "%u A: ON %uus (%u ticks)", i, this->to_microseconds_(item[i].duration0), item[i].duration0);
+      ESP_LOGVV(TAG, "%zu A: ON %" PRIu32 "us (%u ticks)", i, this->to_microseconds_(item[i].duration0),
+                item[i].duration0);
     } else {
-      ESP_LOGVV(TAG, "%u A: OFF %uus (%u ticks)", i, this->to_microseconds_(item[i].duration0), item[i].duration0);
+      ESP_LOGVV(TAG, "%zu A: OFF %" PRIu32 "us (%u ticks)", i, this->to_microseconds_(item[i].duration0),
+                item[i].duration0);
     }
     if (item[i].level1) {
-      ESP_LOGVV(TAG, "%u B: ON %uus (%u ticks)", i, this->to_microseconds_(item[i].duration1), item[i].duration1);
+      ESP_LOGVV(TAG, "%zu B: ON %" PRIu32 "us (%u ticks)", i, this->to_microseconds_(item[i].duration1),
+                item[i].duration1);
     } else {
-      ESP_LOGVV(TAG, "%u B: OFF %uus (%u ticks)", i, this->to_microseconds_(item[i].duration1), item[i].duration1);
+      ESP_LOGVV(TAG, "%zu B: OFF %" PRIu32 "us (%u ticks)", i, this->to_microseconds_(item[i].duration1),
+                item[i].duration1);
     }
   }
   ESP_LOGVV(TAG, "\n");
diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp
index c34b3d2dc4..3b3e00a416 100644
--- a/esphome/components/rf_bridge/rf_bridge.cpp
+++ b/esphome/components/rf_bridge/rf_bridge.cpp
@@ -1,5 +1,6 @@
 #include "rf_bridge.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 #include <cstring>
 
 namespace esphome {
@@ -53,8 +54,10 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) {
         ESP_LOGD(TAG, "Learning success");
       }
 
-      ESP_LOGI(TAG, "Received RFBridge Code: sync=0x%04X low=0x%04X high=0x%04X code=0x%06X", data.sync, data.low,
-               data.high, data.code);
+      ESP_LOGI(TAG,
+               "Received RFBridge Code: sync=0x%04" PRIX16 " low=0x%04" PRIX16 " high=0x%04" PRIX16
+               " code=0x%06" PRIX32,
+               data.sync, data.low, data.high, data.code);
       this->data_callback_.call(data);
       break;
     }
@@ -144,8 +147,8 @@ void RFBridgeComponent::loop() {
 }
 
 void RFBridgeComponent::send_code(RFBridgeData data) {
-  ESP_LOGD(TAG, "Sending code: sync=0x%04X low=0x%04X high=0x%04X code=0x%06X", data.sync, data.low, data.high,
-           data.code);
+  ESP_LOGD(TAG, "Sending code: sync=0x%04" PRIX16 " low=0x%04" PRIX16 " high=0x%04" PRIX16 " code=0x%06" PRIX32,
+           data.sync, data.low, data.high, data.code);
   this->write(RF_CODE_START);
   this->write(RF_CODE_RFOUT);
   this->write((data.sync >> 8) & 0xFF);
diff --git a/esphome/components/servo/servo.cpp b/esphome/components/servo/servo.cpp
index 78fc45c679..666c017dea 100644
--- a/esphome/components/servo/servo.cpp
+++ b/esphome/components/servo/servo.cpp
@@ -1,6 +1,7 @@
 #include "servo.h"
 #include "esphome/core/log.h"
 #include "esphome/core/hal.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace servo {
@@ -14,8 +15,8 @@ void Servo::dump_config() {
   ESP_LOGCONFIG(TAG, "  Idle Level: %.1f%%", this->idle_level_ * 100.0f);
   ESP_LOGCONFIG(TAG, "  Min Level: %.1f%%", this->min_level_ * 100.0f);
   ESP_LOGCONFIG(TAG, "  Max Level: %.1f%%", this->max_level_ * 100.0f);
-  ESP_LOGCONFIG(TAG, "  auto detach time: %d ms", this->auto_detach_time_);
-  ESP_LOGCONFIG(TAG, "  run duration: %d ms", this->transition_length_);
+  ESP_LOGCONFIG(TAG, "  auto detach time: %" PRIu32 " ms", this->auto_detach_time_);
+  ESP_LOGCONFIG(TAG, "  run duration: %" PRIu32 " ms", this->transition_length_);
 }
 
 void Servo::loop() {
diff --git a/esphome/components/sgp4x/sgp4x.cpp b/esphome/components/sgp4x/sgp4x.cpp
index 561d41e225..7e474b9371 100644
--- a/esphome/components/sgp4x/sgp4x.cpp
+++ b/esphome/components/sgp4x/sgp4x.cpp
@@ -166,7 +166,7 @@ bool SGP4xComponent::measure_gas_indices_(int32_t &voc, int32_t &nox) {
   if (nox_sensor_) {
     nox = nox_algorithm_.process(nox_sraw);
   }
-  ESP_LOGV(TAG, "VOC = %d, NOx = %d", voc, nox);
+  ESP_LOGV(TAG, "VOC = %" PRId32 ", NOx = %" PRId32, voc, nox);
   // Store baselines after defined interval or if the difference between current and stored baseline becomes too
   // much
   if (this->store_baseline_ && this->seconds_since_last_store_ > SHORTEST_BASELINE_STORE_INTERVAL) {
diff --git a/esphome/components/spi/spi_arduino.cpp b/esphome/components/spi/spi_arduino.cpp
index 2e6b2d6064..4628486550 100644
--- a/esphome/components/spi/spi_arduino.cpp
+++ b/esphome/components/spi/spi_arduino.cpp
@@ -18,7 +18,7 @@ class SPIDelegateHw : public SPIDelegate {
 #elif defined(ESP8266)
     // Arduino ESP8266 library has mangled values for SPI modes :-(
     auto mode = (this->mode_ & 0x01) + ((this->mode_ & 0x02) << 3);
-    ESP_LOGV(TAG, "8266 mangled SPI mode 0x%X", mode);
+    ESP_LOGVV(TAG, "8266 mangled SPI mode 0x%X", mode);
     SPISettings const settings(this->data_rate_, this->bit_order_, mode);
 #else
     SPISettings const settings(this->data_rate_, this->bit_order_, this->mode_);
diff --git a/esphome/components/sprinkler/sprinkler.cpp b/esphome/components/sprinkler/sprinkler.cpp
index 6900c9461b..982d9add1a 100644
--- a/esphome/components/sprinkler/sprinkler.cpp
+++ b/esphome/components/sprinkler/sprinkler.cpp
@@ -4,6 +4,7 @@
 #include "esphome/core/application.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 #include <utility>
 
 namespace esphome {
@@ -875,7 +876,7 @@ void Sprinkler::queue_valve(optional<size_t> valve_number, optional<uint32_t> ru
     if (this->is_a_valid_valve(valve_number.value()) && (this->queued_valves_.size() < this->max_queue_size_)) {
       SprinklerQueueItem item{valve_number.value(), run_duration.value()};
       this->queued_valves_.insert(this->queued_valves_.begin(), item);
-      ESP_LOGD(TAG, "Valve %u placed into queue with run duration of %u seconds", valve_number.value_or(0),
+      ESP_LOGD(TAG, "Valve %zu placed into queue with run duration of %" PRIu32 " seconds", valve_number.value_or(0),
                run_duration.value_or(0));
     }
   }
@@ -954,7 +955,7 @@ void Sprinkler::pause() {
   this->paused_valve_ = this->active_valve();
   this->resume_duration_ = this->time_remaining_active_valve();
   this->shutdown(false);
-  ESP_LOGD(TAG, "Paused valve %u with %u seconds remaining", this->paused_valve_.value_or(0),
+  ESP_LOGD(TAG, "Paused valve %zu with %" PRIu32 " seconds remaining", this->paused_valve_.value_or(0),
            this->resume_duration_.value_or(0));
 }
 
@@ -967,7 +968,7 @@ void Sprinkler::resume() {
   if (this->paused_valve_.has_value() && (this->resume_duration_.has_value())) {
     // Resume only if valve has not been completed yet
     if (!this->valve_cycle_complete_(this->paused_valve_.value())) {
-      ESP_LOGD(TAG, "Resuming valve %u with %u seconds remaining", this->paused_valve_.value_or(0),
+      ESP_LOGD(TAG, "Resuming valve %zu with %" PRIu32 " seconds remaining", this->paused_valve_.value_or(0),
                this->resume_duration_.value_or(0));
       this->fsm_request_(this->paused_valve_.value(), this->resume_duration_.value());
     }
@@ -1389,7 +1390,8 @@ void Sprinkler::load_next_valve_run_request_(const optional<size_t> first_valve)
       this->next_req_.set_run_duration(
           this->valve_run_duration_adjusted(this->next_valve_number_in_cycle_(first_valve).value_or(0)));
     } else if ((this->repeat_count_++ < this->repeat().value_or(0))) {
-      ESP_LOGD(TAG, "Repeating - starting cycle %u of %u", this->repeat_count_ + 1, this->repeat().value_or(0) + 1);
+      ESP_LOGD(TAG, "Repeating - starting cycle %" PRIu32 " of %" PRIu32, this->repeat_count_ + 1,
+               this->repeat().value_or(0) + 1);
       // if there are repeats remaining and no more valves were left in the cycle, start a new cycle
       this->prep_full_cycle_();
       if (this->next_valve_number_in_cycle_().has_value()) {  // this should always succeed here, but just in case...
@@ -1420,7 +1422,7 @@ void Sprinkler::start_valve_(SprinklerValveRunRequest *req) {
   for (auto &vo : this->valve_op_) {  // find the first available SprinklerValveOperator, load it and start it up
     if (vo.state() == IDLE) {
       auto run_duration = req->run_duration() ? req->run_duration() : this->valve_run_duration_adjusted(req->valve());
-      ESP_LOGD(TAG, "%s is starting valve %u for %u seconds, cycle %u of %u",
+      ESP_LOGD(TAG, "%s is starting valve %zu for %" PRIu32 " seconds, cycle %" PRIu32 " of %" PRIu32,
                this->req_as_str_(req->request_is_from()).c_str(), req->valve(), run_duration, this->repeat_count_ + 1,
                this->repeat().value_or(0) + 1);
       req->set_valve_operator(&vo);
@@ -1645,7 +1647,7 @@ void Sprinkler::start_timer_(const SprinklerTimerIndex timer_index) {
     this->timer_[timer_index].start_time = millis();
     this->timer_[timer_index].active = true;
   }
-  ESP_LOGVV(TAG, "Timer %u started for %u sec", static_cast<size_t>(timer_index),
+  ESP_LOGVV(TAG, "Timer %zu started for %" PRIu32 " sec", static_cast<size_t>(timer_index),
             this->timer_duration_(timer_index) / 1000);
 }
 
@@ -1684,48 +1686,48 @@ void Sprinkler::sm_timer_callback_() {
 void Sprinkler::dump_config() {
   ESP_LOGCONFIG(TAG, "Sprinkler Controller -- %s", this->name_.c_str());
   if (this->manual_selection_delay_.has_value()) {
-    ESP_LOGCONFIG(TAG, "  Manual Selection Delay: %u seconds", this->manual_selection_delay_.value_or(0));
+    ESP_LOGCONFIG(TAG, "  Manual Selection Delay: %" PRIu32 " seconds", this->manual_selection_delay_.value_or(0));
   }
   if (this->repeat().has_value()) {
-    ESP_LOGCONFIG(TAG, "  Repeat Cycles: %u times", this->repeat().value_or(0));
+    ESP_LOGCONFIG(TAG, "  Repeat Cycles: %" PRIu32 " times", this->repeat().value_or(0));
   }
   if (this->start_delay_) {
     if (this->start_delay_is_valve_delay_) {
-      ESP_LOGCONFIG(TAG, "  Pump Start Valve Delay: %u seconds", this->start_delay_);
+      ESP_LOGCONFIG(TAG, "  Pump Start Valve Delay: %" PRIu32 " seconds", this->start_delay_);
     } else {
-      ESP_LOGCONFIG(TAG, "  Pump Start Pump Delay: %u seconds", this->start_delay_);
+      ESP_LOGCONFIG(TAG, "  Pump Start Pump Delay: %" PRIu32 " seconds", this->start_delay_);
     }
   }
   if (this->stop_delay_) {
     if (this->stop_delay_is_valve_delay_) {
-      ESP_LOGCONFIG(TAG, "  Pump Stop Valve Delay: %u seconds", this->stop_delay_);
+      ESP_LOGCONFIG(TAG, "  Pump Stop Valve Delay: %" PRIu32 " seconds", this->stop_delay_);
     } else {
-      ESP_LOGCONFIG(TAG, "  Pump Stop Pump Delay: %u seconds", this->stop_delay_);
+      ESP_LOGCONFIG(TAG, "  Pump Stop Pump Delay: %" PRIu32 " seconds", this->stop_delay_);
     }
   }
   if (this->switching_delay_.has_value()) {
     if (this->valve_overlap_) {
-      ESP_LOGCONFIG(TAG, "  Valve Overlap: %u seconds", this->switching_delay_.value_or(0));
+      ESP_LOGCONFIG(TAG, "  Valve Overlap: %" PRIu32 " seconds", this->switching_delay_.value_or(0));
     } else {
-      ESP_LOGCONFIG(TAG, "  Valve Open Delay: %u seconds", this->switching_delay_.value_or(0));
+      ESP_LOGCONFIG(TAG, "  Valve Open Delay: %" PRIu32 " seconds", this->switching_delay_.value_or(0));
       ESP_LOGCONFIG(TAG, "  Pump Switch Off During Valve Open Delay: %s",
                     YESNO(this->pump_switch_off_during_valve_open_delay_));
     }
   }
   for (size_t valve_number = 0; valve_number < this->number_of_valves(); valve_number++) {
-    ESP_LOGCONFIG(TAG, "  Valve %u:", valve_number);
+    ESP_LOGCONFIG(TAG, "  Valve %zu:", valve_number);
     ESP_LOGCONFIG(TAG, "    Name: %s", this->valve_name(valve_number));
-    ESP_LOGCONFIG(TAG, "    Run Duration: %u seconds", this->valve_run_duration(valve_number));
+    ESP_LOGCONFIG(TAG, "    Run Duration: %" PRIu32 " seconds", this->valve_run_duration(valve_number));
     if (this->valve_[valve_number].valve_switch.pulse_duration()) {
-      ESP_LOGCONFIG(TAG, "    Pulse Duration: %u milliseconds",
+      ESP_LOGCONFIG(TAG, "    Pulse Duration: %" PRIu32 " milliseconds",
                     this->valve_[valve_number].valve_switch.pulse_duration());
     }
   }
   if (!this->pump_.empty()) {
-    ESP_LOGCONFIG(TAG, "  Total number of pumps: %u", this->pump_.size());
+    ESP_LOGCONFIG(TAG, "  Total number of pumps: %zu", this->pump_.size());
   }
   if (!this->valve_.empty()) {
-    ESP_LOGCONFIG(TAG, "  Total number of valves: %u", this->valve_.size());
+    ESP_LOGCONFIG(TAG, "  Total number of valves: %zu", this->valve_.size());
   }
 }
 
diff --git a/esphome/components/status_led/light/status_led_light.cpp b/esphome/components/status_led/light/status_led_light.cpp
index b47d1f5bd0..549024c4df 100644
--- a/esphome/components/status_led/light/status_led_light.cpp
+++ b/esphome/components/status_led/light/status_led_light.cpp
@@ -1,6 +1,7 @@
 #include "status_led_light.h"
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace status_led {
@@ -11,7 +12,7 @@ void StatusLEDLightOutput::loop() {
   uint32_t new_state = App.get_app_state() & STATUS_LED_MASK;
 
   if (new_state != this->last_app_state_) {
-    ESP_LOGV(TAG, "New app state 0x%08X", new_state);
+    ESP_LOGV(TAG, "New app state 0x%08" PRIX32, new_state);
   }
 
   if ((new_state & STATUS_LED_ERROR) != 0u) {
diff --git a/esphome/components/thermostat/thermostat_climate.cpp b/esphome/components/thermostat/thermostat_climate.cpp
index 386e13dc37..73b061b07c 100644
--- a/esphome/components/thermostat/thermostat_climate.cpp
+++ b/esphome/components/thermostat/thermostat_climate.cpp
@@ -1,5 +1,6 @@
 #include "thermostat_climate.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace thermostat {
@@ -1283,11 +1284,13 @@ void ThermostatClimate::dump_config() {
     ESP_LOGCONFIG(TAG, "    Overrun: %.1f°C", this->cooling_overrun_);
     if ((this->supplemental_cool_delta_ > 0) || (this->timer_duration_(thermostat::TIMER_COOLING_MAX_RUN_TIME) > 0)) {
       ESP_LOGCONFIG(TAG, "    Supplemental Delta: %.1f°C", this->supplemental_cool_delta_);
-      ESP_LOGCONFIG(TAG, "    Maximum Run Time: %us",
+      ESP_LOGCONFIG(TAG, "    Maximum Run Time: %" PRIu32 "s",
                     this->timer_duration_(thermostat::TIMER_COOLING_MAX_RUN_TIME) / 1000);
     }
-    ESP_LOGCONFIG(TAG, "    Minimum Off Time: %us", this->timer_duration_(thermostat::TIMER_COOLING_OFF) / 1000);
-    ESP_LOGCONFIG(TAG, "    Minimum Run Time: %us", this->timer_duration_(thermostat::TIMER_COOLING_ON) / 1000);
+    ESP_LOGCONFIG(TAG, "    Minimum Off Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_COOLING_OFF) / 1000);
+    ESP_LOGCONFIG(TAG, "    Minimum Run Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_COOLING_ON) / 1000);
   }
   if (this->supports_heat_) {
     ESP_LOGCONFIG(TAG, "  Heating Parameters:");
@@ -1295,24 +1298,28 @@ void ThermostatClimate::dump_config() {
     ESP_LOGCONFIG(TAG, "    Overrun: %.1f°C", this->heating_overrun_);
     if ((this->supplemental_heat_delta_ > 0) || (this->timer_duration_(thermostat::TIMER_HEATING_MAX_RUN_TIME) > 0)) {
       ESP_LOGCONFIG(TAG, "    Supplemental Delta: %.1f°C", this->supplemental_heat_delta_);
-      ESP_LOGCONFIG(TAG, "    Maximum Run Time: %us",
+      ESP_LOGCONFIG(TAG, "    Maximum Run Time: %" PRIu32 "s",
                     this->timer_duration_(thermostat::TIMER_HEATING_MAX_RUN_TIME) / 1000);
     }
-    ESP_LOGCONFIG(TAG, "    Minimum Off Time: %us", this->timer_duration_(thermostat::TIMER_HEATING_OFF) / 1000);
-    ESP_LOGCONFIG(TAG, "    Minimum Run Time: %us", this->timer_duration_(thermostat::TIMER_HEATING_ON) / 1000);
+    ESP_LOGCONFIG(TAG, "    Minimum Off Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_HEATING_OFF) / 1000);
+    ESP_LOGCONFIG(TAG, "    Minimum Run Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_HEATING_ON) / 1000);
   }
   if (this->supports_fan_only_) {
-    ESP_LOGCONFIG(TAG, "  Fanning Minimum Off Time: %us", this->timer_duration_(thermostat::TIMER_FANNING_OFF) / 1000);
-    ESP_LOGCONFIG(TAG, "  Fanning Minimum Run Time: %us", this->timer_duration_(thermostat::TIMER_FANNING_ON) / 1000);
+    ESP_LOGCONFIG(TAG, "  Fanning Minimum Off Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_FANNING_OFF) / 1000);
+    ESP_LOGCONFIG(TAG, "  Fanning Minimum Run Time: %" PRIu32 "s",
+                  this->timer_duration_(thermostat::TIMER_FANNING_ON) / 1000);
   }
   if (this->supports_fan_mode_on_ || this->supports_fan_mode_off_ || this->supports_fan_mode_auto_ ||
       this->supports_fan_mode_low_ || this->supports_fan_mode_medium_ || this->supports_fan_mode_high_ ||
       this->supports_fan_mode_middle_ || this->supports_fan_mode_focus_ || this->supports_fan_mode_diffuse_ ||
       this->supports_fan_mode_quiet_) {
-    ESP_LOGCONFIG(TAG, "  Minimum Fan Mode Switching Time: %us",
+    ESP_LOGCONFIG(TAG, "  Minimum Fan Mode Switching Time: %" PRIu32 "s",
                   this->timer_duration_(thermostat::TIMER_FAN_MODE) / 1000);
   }
-  ESP_LOGCONFIG(TAG, "  Minimum Idle Time: %us", this->timer_[thermostat::TIMER_IDLE_ON].time / 1000);
+  ESP_LOGCONFIG(TAG, "  Minimum Idle Time: %" PRIu32 "s", this->timer_[thermostat::TIMER_IDLE_ON].time / 1000);
   ESP_LOGCONFIG(TAG, "  Supports AUTO: %s", YESNO(this->supports_auto_));
   ESP_LOGCONFIG(TAG, "  Supports HEAT/COOL: %s", YESNO(this->supports_heat_cool_));
   ESP_LOGCONFIG(TAG, "  Supports COOL: %s", YESNO(this->supports_cool_));
diff --git a/esphome/components/tof10120/tof10120_sensor.cpp b/esphome/components/tof10120/tof10120_sensor.cpp
index 5cd086938e..32cd604be9 100644
--- a/esphome/components/tof10120/tof10120_sensor.cpp
+++ b/esphome/components/tof10120/tof10120_sensor.cpp
@@ -1,6 +1,7 @@
 #include "tof10120_sensor.h"
 #include "esphome/core/log.h"
 #include "esphome/core/hal.h"
+#include <cinttypes>
 
 // Very basic support for TOF10120 distance sensor
 
@@ -44,7 +45,7 @@ void TOF10120Sensor::update() {
   }
 
   uint32_t distance_mm = (data[0] << 8) | data[1];
-  ESP_LOGI(TAG, "Data read: %dmm", distance_mm);
+  ESP_LOGI(TAG, "Data read: %" PRIu32 "mm", distance_mm);
 
   if (distance_mm == TOF10120_OUT_OF_RANGE_VALUE) {
     ESP_LOGW(TAG, "Distance measurement out of range");
diff --git a/esphome/components/tuya/sensor/tuya_sensor.cpp b/esphome/components/tuya/sensor/tuya_sensor.cpp
index 1e39c1bc35..673471a6ce 100644
--- a/esphome/components/tuya/sensor/tuya_sensor.cpp
+++ b/esphome/components/tuya/sensor/tuya_sensor.cpp
@@ -1,5 +1,6 @@
 #include "esphome/core/log.h"
 #include "tuya_sensor.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace tuya {
@@ -18,7 +19,7 @@ void TuyaSensor::setup() {
       ESP_LOGV(TAG, "MCU reported sensor %u is: %u", datapoint.id, datapoint.value_enum);
       this->publish_state(datapoint.value_enum);
     } else if (datapoint.type == TuyaDatapointType::BITMASK) {
-      ESP_LOGV(TAG, "MCU reported sensor %u is: %x", datapoint.id, datapoint.value_bitmask);
+      ESP_LOGV(TAG, "MCU reported sensor %u is: %" PRIx32, datapoint.id, datapoint.value_bitmask);
       this->publish_state(datapoint.value_bitmask);
     }
   });
diff --git a/esphome/components/vbus/vbus.cpp b/esphome/components/vbus/vbus.cpp
index c9758891cc..e474dcfe17 100644
--- a/esphome/components/vbus/vbus.cpp
+++ b/esphome/components/vbus/vbus.cpp
@@ -1,6 +1,7 @@
 #include "vbus.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
+#include <cinttypes>
 
 namespace esphome {
 namespace vbus {
@@ -64,8 +65,8 @@ void VBus::loop() {
         uint16_t id = (this->buffer_[8] << 8) + this->buffer_[7];
         uint32_t value =
             (this->buffer_[12] << 24) + (this->buffer_[11] << 16) + (this->buffer_[10] << 8) + this->buffer_[9];
-        ESP_LOGV(TAG, "P1 C%04x %04x->%04x: %04x %04x (%d)", this->command_, this->source_, this->dest_, id, value,
-                 value);
+        ESP_LOGV(TAG, "P1 C%04x %04x->%04x: %04x %04" PRIx32 " (%" PRIu32 ")", this->command_, this->source_,
+                 this->dest_, id, value, value);
       } else if ((this->protocol_ == 0x10) && (this->buffer_.size() == 9)) {
         if (!checksum(this->buffer_.data(), 0, 9)) {
           ESP_LOGE(TAG, "P1 checksum failed");
diff --git a/esphome/components/wireguard/wireguard.cpp b/esphome/components/wireguard/wireguard.cpp
index 3cd4409dda..f89a5ebbad 100644
--- a/esphome/components/wireguard/wireguard.cpp
+++ b/esphome/components/wireguard/wireguard.cpp
@@ -2,6 +2,7 @@
 
 #ifdef USE_ESP32
 
+#include <cinttypes>
 #include <ctime>
 #include <functional>
 
@@ -144,7 +145,7 @@ void Wireguard::dump_config() {
   }
   ESP_LOGCONFIG(TAG, "  Peer Persistent Keepalive: %d%s", this->keepalive_,
                 (this->keepalive_ > 0 ? "s" : " (DISABLED)"));
-  ESP_LOGCONFIG(TAG, "  Reboot Timeout: %d%s", (this->reboot_timeout_ / 1000),
+  ESP_LOGCONFIG(TAG, "  Reboot Timeout: %" PRIu32 "%s", (this->reboot_timeout_ / 1000),
                 (this->reboot_timeout_ != 0 ? "s" : " (DISABLED)"));
   // be careful: if proceed_allowed_ is true, require connection is false
   ESP_LOGCONFIG(TAG, "  Require Connection to Proceed: %s", (this->proceed_allowed_ ? "NO" : "YES"));
@@ -287,7 +288,7 @@ void resume_wdt() {
 #if ESP_IDF_VERSION_MAJOR >= 5
   wdtc.timeout_ms = CONFIG_ESP_TASK_WDT_TIMEOUT_S * 1000;
   esp_task_wdt_reconfigure(&wdtc);
-  ESP_LOGV(TAG, "wdt resumed with %d ms timeout", wdtc.timeout_ms);
+  ESP_LOGV(TAG, "wdt resumed with %" PRIu32 " ms timeout", wdtc.timeout_ms);
 #else
   esp_task_wdt_init(CONFIG_ESP_TASK_WDT_TIMEOUT_S, false);
   ESP_LOGV(TAG, "wdt resumed with %d seconds timeout", CONFIG_ESP_TASK_WDT_TIMEOUT_S);

From ff8b9040975ce71dfd8eab7679619fe7859aff19 Mon Sep 17 00:00:00 2001
From: kahrendt <kahrendt@gmail.com>
Date: Sun, 5 Nov 2023 20:19:03 -0500
Subject: [PATCH 05/41] Null topic_prefix disables MQTT publishing/subscription
 unless topic is explicitly configured (#5644)

---
 esphome/components/mqtt/__init__.py        | 60 +++++++++++++---------
 esphome/components/mqtt/mqtt_component.cpp | 33 ++++++++----
 tests/test4.yaml                           | 27 ++++++----
 3 files changed, 77 insertions(+), 43 deletions(-)

diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py
index 6d0846edad..baf32097fa 100644
--- a/esphome/components/mqtt/__init__.py
+++ b/esphome/components/mqtt/__init__.py
@@ -133,33 +133,47 @@ def validate_config(value):
     # Populate default fields
     out = value.copy()
     topic_prefix = value[CONF_TOPIC_PREFIX]
+    # If the topic prefix is not null and these messages are not configured, then set them to the default
+    # If the topic prefix is null and these messages are not configured, then set them to null
     if CONF_BIRTH_MESSAGE not in value:
-        out[CONF_BIRTH_MESSAGE] = {
-            CONF_TOPIC: f"{topic_prefix}/status",
-            CONF_PAYLOAD: "online",
-            CONF_QOS: 0,
-            CONF_RETAIN: True,
-        }
+        if topic_prefix != "":
+            out[CONF_BIRTH_MESSAGE] = {
+                CONF_TOPIC: f"{topic_prefix}/status",
+                CONF_PAYLOAD: "online",
+                CONF_QOS: 0,
+                CONF_RETAIN: True,
+            }
+        else:
+            out[CONF_BIRTH_MESSAGE] = {}
     if CONF_WILL_MESSAGE not in value:
-        out[CONF_WILL_MESSAGE] = {
-            CONF_TOPIC: f"{topic_prefix}/status",
-            CONF_PAYLOAD: "offline",
-            CONF_QOS: 0,
-            CONF_RETAIN: True,
-        }
+        if topic_prefix != "":
+            out[CONF_WILL_MESSAGE] = {
+                CONF_TOPIC: f"{topic_prefix}/status",
+                CONF_PAYLOAD: "offline",
+                CONF_QOS: 0,
+                CONF_RETAIN: True,
+            }
+        else:
+            out[CONF_WILL_MESSAGE] = {}
     if CONF_SHUTDOWN_MESSAGE not in value:
-        out[CONF_SHUTDOWN_MESSAGE] = {
-            CONF_TOPIC: f"{topic_prefix}/status",
-            CONF_PAYLOAD: "offline",
-            CONF_QOS: 0,
-            CONF_RETAIN: True,
-        }
+        if topic_prefix != "":
+            out[CONF_SHUTDOWN_MESSAGE] = {
+                CONF_TOPIC: f"{topic_prefix}/status",
+                CONF_PAYLOAD: "offline",
+                CONF_QOS: 0,
+                CONF_RETAIN: True,
+            }
+        else:
+            out[CONF_SHUTDOWN_MESSAGE] = {}
     if CONF_LOG_TOPIC not in value:
-        out[CONF_LOG_TOPIC] = {
-            CONF_TOPIC: f"{topic_prefix}/debug",
-            CONF_QOS: 0,
-            CONF_RETAIN: True,
-        }
+        if topic_prefix != "":
+            out[CONF_LOG_TOPIC] = {
+                CONF_TOPIC: f"{topic_prefix}/debug",
+                CONF_QOS: 0,
+                CONF_RETAIN: True,
+            }
+        else:
+            out[CONF_LOG_TOPIC] = {}
     return out
 
 
diff --git a/esphome/components/mqtt/mqtt_component.cpp b/esphome/components/mqtt/mqtt_component.cpp
index 95dc082e84..f9f8c850e9 100644
--- a/esphome/components/mqtt/mqtt_component.cpp
+++ b/esphome/components/mqtt/mqtt_component.cpp
@@ -23,8 +23,13 @@ std::string MQTTComponent::get_discovery_topic_(const MQTTDiscoveryInfo &discove
 }
 
 std::string MQTTComponent::get_default_topic_for_(const std::string &suffix) const {
-  return global_mqtt_client->get_topic_prefix() + "/" + this->component_type() + "/" + this->get_default_object_id_() +
-         "/" + suffix;
+  const std::string &topic_prefix = global_mqtt_client->get_topic_prefix();
+  if (topic_prefix.empty()) {
+    // If the topic_prefix is null, the default topic should be null
+    return "";
+  }
+
+  return topic_prefix + "/" + this->component_type() + "/" + this->get_default_object_id_() + "/" + suffix;
 }
 
 std::string MQTTComponent::get_state_topic_() const {
@@ -245,17 +250,25 @@ std::string MQTTComponent::friendly_name() const { return this->get_entity()->ge
 std::string MQTTComponent::get_icon() const { return this->get_entity()->get_icon(); }
 bool MQTTComponent::is_disabled_by_default() const { return this->get_entity()->is_disabled_by_default(); }
 bool MQTTComponent::is_internal() {
-  if ((this->get_state_topic_().empty()) || (this->get_command_topic_().empty())) {
-    // If both state_topic and command_topic are empty, then the entity is internal to mqtt
+  if (this->has_custom_state_topic_) {
+    // If the custom state_topic is null, return true as it is internal and should not publish
+    // else, return false, as it is explicitly set to a topic, so it is not internal and should publish
+    return this->get_state_topic_().empty();
+  }
+
+  if (this->has_custom_command_topic_) {
+    // If the custom command_topic is null, return true as it is internal and should not publish
+    // else, return false, as it is explicitly set to a topic, so it is not internal and should publish
+    return this->get_command_topic_().empty();
+  }
+
+  // No custom topics have been set
+  if (this->get_default_topic_for_("").empty()) {
+    // If the default topic prefix is null, then the component, by default, is internal and should not publish
     return true;
   }
 
-  if (this->has_custom_state_topic_ || this->has_custom_command_topic_) {
-    // If a custom state_topic or command_topic is set, then the entity is not internal to mqtt
-    return false;
-  }
-
-  // Use ESPHome's entity internal state
+  // Use ESPHome's component internal state if topic_prefix is not null with no custom state_topic or command_topic
   return this->get_entity()->is_internal();
 }
 
diff --git a/tests/test4.yaml b/tests/test4.yaml
index 3d0ed2f658..a5e5b05e8b 100644
--- a/tests/test4.yaml
+++ b/tests/test4.yaml
@@ -24,6 +24,13 @@ ethernet:
 network:
   enable_ipv6: true
 
+mqtt:
+  broker: test.mosquitto.org
+  port: 1883
+  discovery: true
+  discovery_prefix: homeassistant
+  topic_prefix:
+
 api:
 
 i2c:
@@ -32,15 +39,15 @@ i2c:
   scan: false
 
 spi:
-- id: spi_id_1
-  clk_pin: GPIO21
-  mosi_pin: GPIO22
-  miso_pin: GPIO23
-  interface: hardware
-- id: spi_id_2
-  clk_pin: GPIO32
-  mosi_pin: GPIO33
-  interface: hardware
+  - id: spi_id_1
+    clk_pin: GPIO21
+    mosi_pin: GPIO22
+    miso_pin: GPIO23
+    interface: hardware
+  - id: spi_id_2
+    clk_pin: GPIO32
+    mosi_pin: GPIO33
+    interface: hardware
 
 uart:
   - id: uart115200
@@ -281,6 +288,7 @@ sensor:
     id: a01nyub_sensor
     name: "a01nyub Distance"
     uart_id: uart9600
+    state_topic: "esphome/sensor/a01nyub_sensor/state"
 
 #
 # platform sensor.apds9960 requires component apds9960
@@ -764,7 +772,6 @@ speaker:
     i2s_dout_pin: GPIO25
     mode: mono
 
-
 voice_assistant:
   microphone: mic_id_external
   speaker: speaker_id

From d5aeb32ca67e89c07bed3f5191918b3a539ae228 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rodrigo=20Mart=C3=ADn?= <contact@rodrigomartin.dev>
Date: Mon, 6 Nov 2023 03:54:39 +0100
Subject: [PATCH 06/41] feat: Add ESP32 BLE enable/disable automations (#5616)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 CODEOWNERS                                    |   4 +-
 esphome/components/esp32_ble/__init__.py      |  27 ++-
 esphome/components/esp32_ble/ble.cpp          | 162 ++++++++++++++++--
 esphome/components/esp32_ble/ble.h            |  58 ++++++-
 .../components/esp32_ble_server/__init__.py   |   3 +-
 .../esp32_ble_server/ble_characteristic.cpp   |  12 ++
 .../esp32_ble_server/ble_characteristic.h     |   2 +
 .../esp32_ble_server/ble_server.cpp           |  91 ++++++----
 .../components/esp32_ble_server/ble_server.h  |  32 ++--
 .../esp32_ble_server/ble_service.cpp          |  41 ++++-
 .../components/esp32_ble_server/ble_service.h |   9 +-
 .../components/esp32_ble_tracker/__init__.py  |   1 +
 .../esp32_ble_tracker/esp32_ble_tracker.cpp   |  60 +++++--
 .../esp32_ble_tracker/esp32_ble_tracker.h     |   9 +-
 .../esp32_improv/esp32_improv_component.cpp   |  30 ++--
 .../esp32_improv/esp32_improv_component.h     |   2 +-
 tests/test11.5.yaml                           |   7 +
 17 files changed, 446 insertions(+), 104 deletions(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index b639088900..a48d0ab2e0 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -89,9 +89,9 @@ esphome/components/ektf2232/* @jesserockz
 esphome/components/emc2101/* @ellull
 esphome/components/ens210/* @itn3rd77
 esphome/components/esp32/* @esphome/core
-esphome/components/esp32_ble/* @jesserockz
+esphome/components/esp32_ble/* @Rapsssito @jesserockz
 esphome/components/esp32_ble_client/* @jesserockz
-esphome/components/esp32_ble_server/* @clydebarrow @jesserockz
+esphome/components/esp32_ble_server/* @Rapsssito @clydebarrow @jesserockz
 esphome/components/esp32_camera_web_server/* @ayufan
 esphome/components/esp32_can/* @Sympatron
 esphome/components/esp32_improv/* @jesserockz
diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py
index b4cb595da0..57a7341505 100644
--- a/esphome/components/esp32_ble/__init__.py
+++ b/esphome/components/esp32_ble/__init__.py
@@ -1,15 +1,17 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
+from esphome import automation
 from esphome.const import CONF_ID
 from esphome.core import CORE
 from esphome.components.esp32 import add_idf_sdkconfig_option, get_esp32_variant, const
 
 DEPENDENCIES = ["esp32"]
-CODEOWNERS = ["@jesserockz"]
+CODEOWNERS = ["@jesserockz", "@Rapsssito"]
 CONFLICTS_WITH = ["esp32_ble_beacon"]
 
 CONF_BLE_ID = "ble_id"
 CONF_IO_CAPABILITY = "io_capability"
+CONF_ENABLE_ON_BOOT = "enable_on_boot"
 
 NO_BLUETOOTH_VARIANTS = [const.VARIANT_ESP32S2]
 
@@ -20,6 +22,10 @@ GAPEventHandler = esp32_ble_ns.class_("GAPEventHandler")
 GATTcEventHandler = esp32_ble_ns.class_("GATTcEventHandler")
 GATTsEventHandler = esp32_ble_ns.class_("GATTsEventHandler")
 
+BLEEnabledCondition = esp32_ble_ns.class_("BLEEnabledCondition", automation.Condition)
+BLEEnableAction = esp32_ble_ns.class_("BLEEnableAction", automation.Action)
+BLEDisableAction = esp32_ble_ns.class_("BLEDisableAction", automation.Action)
+
 IoCapability = esp32_ble_ns.enum("IoCapability")
 IO_CAPABILITY = {
     "none": IoCapability.IO_CAP_NONE,
@@ -35,6 +41,7 @@ CONFIG_SCHEMA = cv.Schema(
         cv.Optional(CONF_IO_CAPABILITY, default="none"): cv.enum(
             IO_CAPABILITY, lower=True
         ),
+        cv.Optional(CONF_ENABLE_ON_BOOT, default=True): cv.boolean,
     }
 ).extend(cv.COMPONENT_SCHEMA)
 
@@ -50,9 +57,25 @@ FINAL_VALIDATE_SCHEMA = validate_variant
 
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
-    await cg.register_component(var, config)
+    cg.add(var.set_enable_on_boot(config[CONF_ENABLE_ON_BOOT]))
     cg.add(var.set_io_capability(config[CONF_IO_CAPABILITY]))
+    await cg.register_component(var, config)
 
     if CORE.using_esp_idf:
         add_idf_sdkconfig_option("CONFIG_BT_ENABLED", True)
         add_idf_sdkconfig_option("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True)
+
+
+@automation.register_condition("ble.enabled", BLEEnabledCondition, cv.Schema({}))
+async def ble_enabled_to_code(config, condition_id, template_arg, args):
+    return cg.new_Pvariable(condition_id, template_arg)
+
+
+@automation.register_action("ble.enable", BLEEnableAction, cv.Schema({}))
+async def ble_enable_to_code(config, action_id, template_arg, args):
+    return cg.new_Pvariable(action_id, template_arg)
+
+
+@automation.register_action("ble.disable", BLEDisableAction, cv.Schema({}))
+async def ble_disable_to_code(config, action_id, template_arg, args):
+    return cg.new_Pvariable(action_id, template_arg)
diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp
index 6c9124447a..3797f3221e 100644
--- a/esphome/components/esp32_ble/ble.cpp
+++ b/esphome/components/esp32_ble/ble.cpp
@@ -5,8 +5,8 @@
 #include "esphome/core/log.h"
 
 #include <esp_bt.h>
-#include <esp_bt_main.h>
 #include <esp_bt_device.h>
+#include <esp_bt_main.h>
 #include <esp_gap_ble_api.h>
 #include <freertos/FreeRTOS.h>
 #include <freertos/FreeRTOSConfig.h>
@@ -26,30 +26,85 @@ void ESP32BLE::setup() {
   global_ble = this;
   ESP_LOGCONFIG(TAG, "Setting up BLE...");
 
-  if (!ble_setup_()) {
-    ESP_LOGE(TAG, "BLE could not be set up");
+  if (!ble_pre_setup_()) {
+    ESP_LOGE(TAG, "BLE could not be prepared for configuration");
     this->mark_failed();
     return;
   }
 
-#ifdef USE_ESP32_BLE_SERVER
-  this->advertising_ = new BLEAdvertising();  // NOLINT(cppcoreguidelines-owning-memory)
-
-  this->advertising_->set_scan_response(true);
-  this->advertising_->set_min_preferred_interval(0x06);
-  this->advertising_->start();
-#endif  // USE_ESP32_BLE_SERVER
-
-  ESP_LOGD(TAG, "BLE setup complete");
+  this->state_ = BLE_COMPONENT_STATE_DISABLED;
+  if (this->enable_on_boot_) {
+    this->enable();
+  }
 }
 
-bool ESP32BLE::ble_setup_() {
+void ESP32BLE::enable() {
+  if (this->state_ != BLE_COMPONENT_STATE_DISABLED)
+    return;
+
+  this->state_ = BLE_COMPONENT_STATE_ENABLE;
+}
+
+void ESP32BLE::disable() {
+  if (this->state_ == BLE_COMPONENT_STATE_DISABLED)
+    return;
+
+  this->state_ = BLE_COMPONENT_STATE_DISABLE;
+}
+
+bool ESP32BLE::is_active() { return this->state_ == BLE_COMPONENT_STATE_ACTIVE; }
+
+void ESP32BLE::advertising_start() {
+  this->advertising_init_();
+  if (!this->is_active())
+    return;
+  this->advertising_->start();
+}
+
+void ESP32BLE::advertising_set_service_data(const std::vector<uint8_t> &data) {
+  this->advertising_init_();
+  this->advertising_->set_service_data(data);
+  this->advertising_start();
+}
+
+void ESP32BLE::advertising_set_manufacturer_data(const std::vector<uint8_t> &data) {
+  this->advertising_init_();
+  this->advertising_->set_manufacturer_data(data);
+  this->advertising_start();
+}
+
+void ESP32BLE::advertising_add_service_uuid(ESPBTUUID uuid) {
+  this->advertising_init_();
+  this->advertising_->add_service_uuid(uuid);
+  this->advertising_start();
+}
+
+void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) {
+  this->advertising_init_();
+  this->advertising_->remove_service_uuid(uuid);
+  this->advertising_start();
+}
+
+bool ESP32BLE::ble_pre_setup_() {
   esp_err_t err = nvs_flash_init();
   if (err != ESP_OK) {
     ESP_LOGE(TAG, "nvs_flash_init failed: %d", err);
     return false;
   }
+  return true;
+}
 
+void ESP32BLE::advertising_init_() {
+  if (this->advertising_ != nullptr)
+    return;
+  this->advertising_ = new BLEAdvertising();  // NOLINT(cppcoreguidelines-owning-memory)
+
+  this->advertising_->set_scan_response(true);
+  this->advertising_->set_min_preferred_interval(0x06);
+}
+
+bool ESP32BLE::ble_setup_() {
+  esp_err_t err;
 #ifdef USE_ARDUINO
   if (!btStart()) {
     ESP_LOGE(TAG, "btStart failed: %d", esp_bt_controller_get_status());
@@ -146,7 +201,88 @@ bool ESP32BLE::ble_setup_() {
   return true;
 }
 
+bool ESP32BLE::ble_dismantle_() {
+  esp_err_t err = esp_bluedroid_disable();
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_bluedroid_disable failed: %d", err);
+    return false;
+  }
+  err = esp_bluedroid_deinit();
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_bluedroid_deinit failed: %d", err);
+    return false;
+  }
+
+#ifdef USE_ARDUINO
+  if (!btStop()) {
+    ESP_LOGE(TAG, "btStop failed: %d", esp_bt_controller_get_status());
+    return false;
+  }
+#else
+  if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_IDLE) {
+    // stop bt controller
+    if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_ENABLED) {
+      err = esp_bt_controller_disable();
+      if (err != ESP_OK) {
+        ESP_LOGE(TAG, "esp_bt_controller_disable failed: %s", esp_err_to_name(err));
+        return false;
+      }
+      while (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_ENABLED)
+        ;
+    }
+    if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_INITED) {
+      err = esp_bt_controller_deinit();
+      if (err != ESP_OK) {
+        ESP_LOGE(TAG, "esp_bt_controller_deinit failed: %s", esp_err_to_name(err));
+        return false;
+      }
+    }
+    if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_IDLE) {
+      ESP_LOGE(TAG, "esp bt controller disable failed");
+      return false;
+    }
+  }
+#endif
+  return true;
+}
+
 void ESP32BLE::loop() {
+  switch (this->state_) {
+    case BLE_COMPONENT_STATE_OFF:
+    case BLE_COMPONENT_STATE_DISABLED:
+      return;
+    case BLE_COMPONENT_STATE_DISABLE: {
+      ESP_LOGD(TAG, "Disabling BLE...");
+
+      for (auto *ble_event_handler : this->ble_status_event_handlers_) {
+        ble_event_handler->ble_before_disabled_event_handler();
+      }
+
+      if (!ble_dismantle_()) {
+        ESP_LOGE(TAG, "BLE could not be dismantled");
+        this->mark_failed();
+        return;
+      }
+      this->state_ = BLE_COMPONENT_STATE_DISABLED;
+      return;
+    }
+    case BLE_COMPONENT_STATE_ENABLE: {
+      ESP_LOGD(TAG, "Enabling BLE...");
+      this->state_ = BLE_COMPONENT_STATE_OFF;
+
+      if (!ble_setup_()) {
+        ESP_LOGE(TAG, "BLE could not be set up");
+        this->mark_failed();
+        return;
+      }
+
+      this->state_ = BLE_COMPONENT_STATE_ACTIVE;
+      return;
+    }
+    case BLE_COMPONENT_STATE_ACTIVE:
+      break;
+  }
+
   BLEEvent *ble_event = this->ble_events_.pop();
   while (ble_event != nullptr) {
     switch (ble_event->type_) {
diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h
index cde17da425..023960d6e4 100644
--- a/esphome/components/esp32_ble/ble.h
+++ b/esphome/components/esp32_ble/ble.h
@@ -1,19 +1,21 @@
 #pragma once
 
 #include "ble_advertising.h"
+#include "ble_uuid.h"
 
+#include "esphome/core/automation.h"
 #include "esphome/core/component.h"
 #include "esphome/core/defines.h"
 #include "esphome/core/helpers.h"
 
-#include "queue.h"
 #include "ble_event.h"
+#include "queue.h"
 
 #ifdef USE_ESP32
 
 #include <esp_gap_ble_api.h>
-#include <esp_gatts_api.h>
 #include <esp_gattc_api.h>
+#include <esp_gatts_api.h>
 
 namespace esphome {
 namespace esp32_ble {
@@ -35,6 +37,19 @@ enum IoCapability {
   IO_CAP_KBDISP = ESP_IO_CAP_KBDISP,
 };
 
+enum BLEComponentState {
+  /** Nothing has been initialized yet. */
+  BLE_COMPONENT_STATE_OFF = 0,
+  /** BLE should be disabled on next loop. */
+  BLE_COMPONENT_STATE_DISABLE,
+  /** BLE is disabled. */
+  BLE_COMPONENT_STATE_DISABLED,
+  /** BLE should be enabled on next loop. */
+  BLE_COMPONENT_STATE_ENABLE,
+  /** BLE is active. */
+  BLE_COMPONENT_STATE_ACTIVE,
+};
+
 class GAPEventHandler {
  public:
   virtual void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) = 0;
@@ -52,20 +67,36 @@ class GATTsEventHandler {
                                    esp_ble_gatts_cb_param_t *param) = 0;
 };
 
+class BLEStatusEventHandler {
+ public:
+  virtual void ble_before_disabled_event_handler() = 0;
+};
+
 class ESP32BLE : public Component {
  public:
   void set_io_capability(IoCapability io_capability) { this->io_cap_ = (esp_ble_io_cap_t) io_capability; }
 
+  void enable();
+  void disable();
+  bool is_active();
   void setup() override;
   void loop() override;
   void dump_config() override;
   float get_setup_priority() const override;
 
-  BLEAdvertising *get_advertising() { return this->advertising_; }
+  void advertising_start();
+  void advertising_set_service_data(const std::vector<uint8_t> &data);
+  void advertising_set_manufacturer_data(const std::vector<uint8_t> &data);
+  void advertising_add_service_uuid(ESPBTUUID uuid);
+  void advertising_remove_service_uuid(ESPBTUUID uuid);
 
   void register_gap_event_handler(GAPEventHandler *handler) { this->gap_event_handlers_.push_back(handler); }
   void register_gattc_event_handler(GATTcEventHandler *handler) { this->gattc_event_handlers_.push_back(handler); }
   void register_gatts_event_handler(GATTsEventHandler *handler) { this->gatts_event_handlers_.push_back(handler); }
+  void register_ble_status_event_handler(BLEStatusEventHandler *handler) {
+    this->ble_status_event_handlers_.push_back(handler);
+  }
+  void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; }
 
  protected:
   static void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if, esp_ble_gatts_cb_param_t *param);
@@ -77,19 +108,40 @@ class ESP32BLE : public Component {
   void real_gap_event_handler_(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param);
 
   bool ble_setup_();
+  bool ble_dismantle_();
+  bool ble_pre_setup_();
+  void advertising_init_();
 
   std::vector<GAPEventHandler *> gap_event_handlers_;
   std::vector<GATTcEventHandler *> gattc_event_handlers_;
   std::vector<GATTsEventHandler *> gatts_event_handlers_;
+  std::vector<BLEStatusEventHandler *> ble_status_event_handlers_;
+  BLEComponentState state_{BLE_COMPONENT_STATE_OFF};
 
   Queue<BLEEvent> ble_events_;
   BLEAdvertising *advertising_;
   esp_ble_io_cap_t io_cap_{ESP_IO_CAP_NONE};
+  bool enable_on_boot_;
 };
 
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
 extern ESP32BLE *global_ble;
 
+template<typename... Ts> class BLEEnabledCondition : public Condition<Ts...> {
+ public:
+  bool check(Ts... x) override { return global_ble->is_active(); }
+};
+
+template<typename... Ts> class BLEEnableAction : public Action<Ts...> {
+ public:
+  void play(Ts... x) override { global_ble->enable(); }
+};
+
+template<typename... Ts> class BLEDisableAction : public Action<Ts...> {
+ public:
+  void play(Ts... x) override { global_ble->disable(); }
+};
+
 }  // namespace esp32_ble
 }  // namespace esphome
 
diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py
index 5e1ad71501..f53c9450f4 100644
--- a/esphome/components/esp32_ble_server/__init__.py
+++ b/esphome/components/esp32_ble_server/__init__.py
@@ -6,7 +6,7 @@ from esphome.core import CORE
 from esphome.components.esp32 import add_idf_sdkconfig_option
 
 AUTO_LOAD = ["esp32_ble"]
-CODEOWNERS = ["@jesserockz", "@clydebarrow"]
+CODEOWNERS = ["@jesserockz", "@clydebarrow", "@Rapsssito"]
 CONFLICTS_WITH = ["esp32_ble_beacon"]
 DEPENDENCIES = ["esp32"]
 
@@ -41,6 +41,7 @@ async def to_code(config):
 
     parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID])
     cg.add(parent.register_gatts_event_handler(var))
+    cg.add(parent.register_ble_status_event_handler(var))
     cg.add(var.set_parent(parent))
 
     cg.add(var.set_manufacturer(config[CONF_MANUFACTURER]))
diff --git a/esphome/components/esp32_ble_server/ble_characteristic.cpp b/esphome/components/esp32_ble_server/ble_characteristic.cpp
index 15a51f6ede..6ff7d615f9 100644
--- a/esphome/components/esp32_ble_server/ble_characteristic.cpp
+++ b/esphome/components/esp32_ble_server/ble_characteristic.cpp
@@ -11,6 +11,13 @@ namespace esp32_ble_server {
 
 static const char *const TAG = "esp32_ble_server.characteristic";
 
+BLECharacteristic::~BLECharacteristic() {
+  for (auto *descriptor : this->descriptors_) {
+    delete descriptor;  // NOLINT(cppcoreguidelines-owning-memory)
+  }
+  vSemaphoreDelete(this->set_value_lock_);
+}
+
 BLECharacteristic::BLECharacteristic(const ESPBTUUID uuid, uint32_t properties) : uuid_(uuid) {
   this->set_value_lock_ = xSemaphoreCreateBinary();
   xSemaphoreGive(this->set_value_lock_);
@@ -98,6 +105,11 @@ void BLECharacteristic::notify(bool notification) {
 
 void BLECharacteristic::add_descriptor(BLEDescriptor *descriptor) { this->descriptors_.push_back(descriptor); }
 
+void BLECharacteristic::remove_descriptor(BLEDescriptor *descriptor) {
+  this->descriptors_.erase(std::remove(this->descriptors_.begin(), this->descriptors_.end(), descriptor),
+                           this->descriptors_.end());
+}
+
 void BLECharacteristic::do_create(BLEService *service) {
   this->service_ = service;
   esp_attr_control_t control;
diff --git a/esphome/components/esp32_ble_server/ble_characteristic.h b/esphome/components/esp32_ble_server/ble_characteristic.h
index d7af3a934a..8837c796a5 100644
--- a/esphome/components/esp32_ble_server/ble_characteristic.h
+++ b/esphome/components/esp32_ble_server/ble_characteristic.h
@@ -25,6 +25,7 @@ class BLEService;
 class BLECharacteristic {
  public:
   BLECharacteristic(ESPBTUUID uuid, uint32_t properties);
+  ~BLECharacteristic();
 
   void set_value(const uint8_t *data, size_t length);
   void set_value(std::vector<uint8_t> value);
@@ -52,6 +53,7 @@ class BLECharacteristic {
   void on_write(const std::function<void(const std::vector<uint8_t> &)> &&func) { this->on_write_ = func; }
 
   void add_descriptor(BLEDescriptor *descriptor);
+  void remove_descriptor(BLEDescriptor *descriptor);
 
   BLEService *get_service() { return this->service_; }
   ESPBTUUID get_uuid() { return this->uuid_; }
diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp
index ca244aba95..338413f64e 100644
--- a/esphome/components/esp32_ble_server/ble_server.cpp
+++ b/esphome/components/esp32_ble_server/ble_server.cpp
@@ -30,13 +30,13 @@ void BLEServer::setup() {
     ESP_LOGE(TAG, "BLE Server was marked failed by ESP32BLE");
     return;
   }
-
-  ESP_LOGD(TAG, "Setting up BLE Server...");
-
   global_ble_server = this;
 }
 
 void BLEServer::loop() {
+  if (!this->parent_->is_active()) {
+    return;
+  }
   switch (this->state_) {
     case RUNNING:
       return;
@@ -53,10 +53,16 @@ void BLEServer::loop() {
     }
     case REGISTERING: {
       if (this->registered_) {
-        this->device_information_service_ = this->create_service(DEVICE_INFORMATION_SERVICE_UUID);
-
-        this->create_device_characteristics_();
-
+        // Create all services previously created
+        for (auto &pair : this->services_) {
+          pair.second->do_create(this);
+        }
+        if (this->device_information_service_ == nullptr) {
+          this->create_service(ESPBTUUID::from_uint16(DEVICE_INFORMATION_SERVICE_UUID));
+          this->device_information_service_ =
+              this->get_service(ESPBTUUID::from_uint16(DEVICE_INFORMATION_SERVICE_UUID));
+          this->create_device_characteristics_();
+        }
         this->state_ = STARTING_SERVICE;
       }
       break;
@@ -67,7 +73,6 @@ void BLEServer::loop() {
       }
       if (this->device_information_service_->is_running()) {
         this->state_ = RUNNING;
-        this->can_proceed_ = true;
         this->restart_advertising_();
         ESP_LOGD(TAG, "BLE server setup successfully");
       } else if (!this->device_information_service_->is_starting()) {
@@ -78,10 +83,13 @@ void BLEServer::loop() {
   }
 }
 
+bool BLEServer::is_running() { return this->parent_->is_active() && this->state_ == RUNNING; }
+
+bool BLEServer::can_proceed() { return this->is_running() || !this->parent_->is_active(); }
+
 void BLEServer::restart_advertising_() {
-  if (this->state_ == RUNNING) {
-    esp32_ble::global_ble->get_advertising()->set_manufacturer_data(this->manufacturer_data_);
-    esp32_ble::global_ble->get_advertising()->start();
+  if (this->is_running()) {
+    this->parent_->advertising_set_manufacturer_data(this->manufacturer_data_);
   }
 }
 
@@ -107,24 +115,36 @@ bool BLEServer::create_device_characteristics_() {
   return true;
 }
 
-std::shared_ptr<BLEService> BLEServer::create_service(const uint8_t *uuid, bool advertise) {
-  return this->create_service(ESPBTUUID::from_raw(uuid), advertise);
-}
-std::shared_ptr<BLEService> BLEServer::create_service(uint16_t uuid, bool advertise) {
-  return this->create_service(ESPBTUUID::from_uint16(uuid), advertise);
-}
-std::shared_ptr<BLEService> BLEServer::create_service(const std::string &uuid, bool advertise) {
-  return this->create_service(ESPBTUUID::from_raw(uuid), advertise);
-}
-std::shared_ptr<BLEService> BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles,
-                                                      uint8_t inst_id) {
-  ESP_LOGV(TAG, "Creating service - %s", uuid.to_string().c_str());
-  std::shared_ptr<BLEService> service = std::make_shared<BLEService>(uuid, num_handles, inst_id);
-  this->services_.emplace_back(service);
-  if (advertise) {
-    esp32_ble::global_ble->get_advertising()->add_service_uuid(uuid);
+void BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles, uint8_t inst_id) {
+  ESP_LOGV(TAG, "Creating BLE service - %s", uuid.to_string().c_str());
+  // If the service already exists, do nothing
+  BLEService *service = this->get_service(uuid);
+  if (service != nullptr) {
+    ESP_LOGW(TAG, "BLE service %s already exists", uuid.to_string().c_str());
+    return;
   }
+  service = new BLEService(uuid, num_handles, inst_id, advertise);  // NOLINT(cppcoreguidelines-owning-memory)
+  this->services_.emplace(uuid.to_string(), service);
   service->do_create(this);
+}
+
+void BLEServer::remove_service(ESPBTUUID uuid) {
+  ESP_LOGV(TAG, "Removing BLE service - %s", uuid.to_string().c_str());
+  BLEService *service = this->get_service(uuid);
+  if (service == nullptr) {
+    ESP_LOGW(TAG, "BLE service %s not found", uuid.to_string().c_str());
+    return;
+  }
+  service->do_delete();
+  delete service;  // NOLINT(cppcoreguidelines-owning-memory)
+  this->services_.erase(uuid.to_string());
+}
+
+BLEService *BLEServer::get_service(ESPBTUUID uuid) {
+  BLEService *service = nullptr;
+  if (this->services_.count(uuid.to_string()) > 0) {
+    service = this->services_.at(uuid.to_string());
+  }
   return service;
 }
 
@@ -144,7 +164,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga
       ESP_LOGD(TAG, "BLE Client disconnected");
       if (this->remove_client_(param->disconnect.conn_id))
         this->connected_clients_--;
-      esp32_ble::global_ble->get_advertising()->start();
+      this->parent_->advertising_start();
       for (auto *component : this->service_components_) {
         component->on_client_disconnect();
       }
@@ -159,11 +179,22 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga
       break;
   }
 
-  for (const auto &service : this->services_) {
-    service->gatts_event_handler(event, gatts_if, param);
+  for (const auto &pair : this->services_) {
+    pair.second->gatts_event_handler(event, gatts_if, param);
   }
 }
 
+void BLEServer::ble_before_disabled_event_handler() {
+  // Delete all clients
+  this->clients_.clear();
+  // Delete all services
+  for (auto &pair : this->services_) {
+    pair.second->do_delete();
+  }
+  this->registered_ = false;
+  this->state_ = INIT;
+}
+
 float BLEServer::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH + 10; }
 
 void BLEServer::dump_config() { ESP_LOGCONFIG(TAG, "ESP32 BLE Server:"); }
diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h
index 14c88be1a2..e379e67296 100644
--- a/esphome/components/esp32_ble_server/ble_server.h
+++ b/esphome/components/esp32_ble_server/ble_server.h
@@ -11,9 +11,9 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
 
-#include <map>
 #include <memory>
 #include <vector>
+#include <unordered_map>
 
 #ifdef USE_ESP32
 
@@ -33,15 +33,16 @@ class BLEServiceComponent {
   virtual void stop();
 };
 
-class BLEServer : public Component, public GATTsEventHandler, public Parented<ESP32BLE> {
+class BLEServer : public Component, public GATTsEventHandler, public BLEStatusEventHandler, public Parented<ESP32BLE> {
  public:
   void setup() override;
   void loop() override;
   void dump_config() override;
   float get_setup_priority() const override;
-  bool can_proceed() override { return this->can_proceed_; }
+  bool can_proceed() override;
 
   void teardown();
+  bool is_running();
 
   void set_manufacturer(const std::string &manufacturer) { this->manufacturer_ = manufacturer; }
   void set_model(const std::string &model) { this->model_ = model; }
@@ -50,32 +51,28 @@ class BLEServer : public Component, public GATTsEventHandler, public Parented<ES
     this->restart_advertising_();
   }
 
-  std::shared_ptr<BLEService> create_service(const uint8_t *uuid, bool advertise = false);
-  std::shared_ptr<BLEService> create_service(uint16_t uuid, bool advertise = false);
-  std::shared_ptr<BLEService> create_service(const std::string &uuid, bool advertise = false);
-  std::shared_ptr<BLEService> create_service(ESPBTUUID uuid, bool advertise = false, uint16_t num_handles = 15,
-                                             uint8_t inst_id = 0);
+  void create_service(ESPBTUUID uuid, bool advertise = false, uint16_t num_handles = 15, uint8_t inst_id = 0);
+  void remove_service(ESPBTUUID uuid);
+  BLEService *get_service(ESPBTUUID uuid);
 
   esp_gatt_if_t get_gatts_if() { return this->gatts_if_; }
   uint32_t get_connected_client_count() { return this->connected_clients_; }
-  const std::map<uint16_t, void *> &get_clients() { return this->clients_; }
+  const std::unordered_map<uint16_t, void *> &get_clients() { return this->clients_; }
 
   void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if,
                            esp_ble_gatts_cb_param_t *param) override;
 
+  void ble_before_disabled_event_handler() override;
+
   void register_service_component(BLEServiceComponent *component) { this->service_components_.push_back(component); }
 
  protected:
   bool create_device_characteristics_();
   void restart_advertising_();
 
-  void add_client_(uint16_t conn_id, void *client) {
-    this->clients_.insert(std::pair<uint16_t, void *>(conn_id, client));
-  }
+  void add_client_(uint16_t conn_id, void *client) { this->clients_.emplace(conn_id, client); }
   bool remove_client_(uint16_t conn_id) { return this->clients_.erase(conn_id) > 0; }
 
-  bool can_proceed_{false};
-
   std::string manufacturer_;
   optional<std::string> model_;
   std::vector<uint8_t> manufacturer_data_;
@@ -83,10 +80,9 @@ class BLEServer : public Component, public GATTsEventHandler, public Parented<ES
   bool registered_{false};
 
   uint32_t connected_clients_{0};
-  std::map<uint16_t, void *> clients_;
-
-  std::vector<std::shared_ptr<BLEService>> services_;
-  std::shared_ptr<BLEService> device_information_service_;
+  std::unordered_map<uint16_t, void *> clients_;
+  std::unordered_map<std::string, BLEService *> services_;
+  BLEService *device_information_service_;
 
   std::vector<BLEServiceComponent *> service_components_;
 
diff --git a/esphome/components/esp32_ble_server/ble_service.cpp b/esphome/components/esp32_ble_server/ble_service.cpp
index e5aaebc137..368f03fb52 100644
--- a/esphome/components/esp32_ble_server/ble_service.cpp
+++ b/esphome/components/esp32_ble_server/ble_service.cpp
@@ -9,8 +9,8 @@ namespace esp32_ble_server {
 
 static const char *const TAG = "esp32_ble_server.service";
 
-BLEService::BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id)
-    : uuid_(uuid), num_handles_(num_handles), inst_id_(inst_id) {}
+BLEService::BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id, bool advertise)
+    : uuid_(uuid), num_handles_(num_handles), inst_id_(inst_id), advertise_(advertise) {}
 
 BLEService::~BLEService() {
   for (auto &chr : this->characteristics_)
@@ -58,6 +58,20 @@ void BLEService::do_create(BLEServer *server) {
   this->init_state_ = CREATING;
 }
 
+void BLEService::do_delete() {
+  if (this->init_state_ == DELETING || this->init_state_ == DELETED)
+    return;
+  this->init_state_ = DELETING;
+  this->created_characteristic_count_ = 0;
+  this->last_created_characteristic_ = nullptr;
+  this->stop_();
+  esp_err_t err = esp_ble_gatts_delete_service(this->handle_);
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_ble_gatts_delete_service failed: %d", err);
+    return;
+  }
+}
+
 bool BLEService::do_create_characteristics_() {
   if (this->created_characteristic_count_ >= this->characteristics_.size() &&
       (this->last_created_characteristic_ == nullptr || this->last_created_characteristic_->is_created()))
@@ -75,24 +89,34 @@ bool BLEService::do_create_characteristics_() {
 void BLEService::start() {
   if (this->do_create_characteristics_())
     return;
+  should_start_ = true;
 
   esp_err_t err = esp_ble_gatts_start_service(this->handle_);
   if (err != ESP_OK) {
     ESP_LOGE(TAG, "esp_ble_gatts_start_service failed: %d", err);
     return;
   }
+  if (this->advertise_)
+    esp32_ble::global_ble->advertising_add_service_uuid(this->uuid_);
   this->running_state_ = STARTING;
 }
 
 void BLEService::stop() {
+  should_start_ = false;
+  this->stop_();
+}
+
+void BLEService::stop_() {
+  if (this->running_state_ == STOPPING || this->running_state_ == STOPPED)
+    return;
+  this->running_state_ = STOPPING;
   esp_err_t err = esp_ble_gatts_stop_service(this->handle_);
   if (err != ESP_OK) {
     ESP_LOGE(TAG, "esp_ble_gatts_stop_service failed: %d", err);
     return;
   }
-  esp32_ble::global_ble->get_advertising()->remove_service_uuid(this->uuid_);
-  esp32_ble::global_ble->get_advertising()->start();
-  this->running_state_ = STOPPING;
+  if (this->advertise_)
+    esp32_ble::global_ble->advertising_remove_service_uuid(this->uuid_);
 }
 
 bool BLEService::is_created() { return this->init_state_ == CREATED; }
@@ -116,9 +140,16 @@ void BLEService::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t g
           this->inst_id_ == param->create.service_id.id.inst_id) {
         this->handle_ = param->create.service_handle;
         this->init_state_ = CREATED;
+        if (this->should_start_)
+          this->start();
       }
       break;
     }
+    case ESP_GATTS_DELETE_EVT:
+      if (param->del.service_handle == this->handle_) {
+        this->init_state_ = DELETED;
+      }
+      break;
     case ESP_GATTS_START_EVT: {
       if (param->start.service_handle == this->handle_) {
         this->running_state_ = RUNNING;
diff --git a/esphome/components/esp32_ble_server/ble_service.h b/esphome/components/esp32_ble_server/ble_service.h
index 93b4217517..5e5883b6bf 100644
--- a/esphome/components/esp32_ble_server/ble_service.h
+++ b/esphome/components/esp32_ble_server/ble_service.h
@@ -22,7 +22,7 @@ using namespace esp32_ble;
 
 class BLEService {
  public:
-  BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id);
+  BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id, bool advertise);
   ~BLEService();
   BLECharacteristic *get_characteristic(ESPBTUUID uuid);
   BLECharacteristic *get_characteristic(uint16_t uuid);
@@ -38,6 +38,7 @@ class BLEService {
   BLEServer *get_server() { return this->server_; }
 
   void do_create(BLEServer *server);
+  void do_delete();
   void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if, esp_ble_gatts_cb_param_t *param);
 
   void start();
@@ -48,6 +49,7 @@ class BLEService {
 
   bool is_running() { return this->running_state_ == RUNNING; }
   bool is_starting() { return this->running_state_ == STARTING; }
+  bool is_deleted() { return this->init_state_ == DELETED; }
 
  protected:
   std::vector<BLECharacteristic *> characteristics_;
@@ -58,8 +60,11 @@ class BLEService {
   uint16_t num_handles_;
   uint16_t handle_{0xFFFF};
   uint8_t inst_id_;
+  bool advertise_{false};
+  bool should_start_{false};
 
   bool do_create_characteristics_();
+  void stop_();
 
   enum InitState : uint8_t {
     FAILED = 0x00,
@@ -67,6 +72,8 @@ class BLEService {
     CREATING,
     CREATING_DEPENDENTS,
     CREATED,
+    DELETING,
+    DELETED,
   } init_state_{INIT};
 
   enum RunningState : uint8_t {
diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py
index 8ba77c7db7..2ead59c025 100644
--- a/esphome/components/esp32_ble_tracker/__init__.py
+++ b/esphome/components/esp32_ble_tracker/__init__.py
@@ -212,6 +212,7 @@ async def to_code(config):
     parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID])
     cg.add(parent.register_gap_event_handler(var))
     cg.add(parent.register_gattc_event_handler(var))
+    cg.add(parent.register_ble_status_event_handler(var))
     cg.add(var.set_parent(parent))
 
     params = config[CONF_SCAN_PARAMETERS]
diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp
index f67f29477d..a5bbd85b47 100644
--- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp
+++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp
@@ -64,17 +64,19 @@ void ESP32BLETracker::setup() {
     }
   });
 #endif
-
-  if (this->scan_continuous_) {
-    if (xSemaphoreTake(this->scan_end_lock_, 0L)) {
-      this->start_scan_(true);
-    } else {
-      ESP_LOGW(TAG, "Cannot start scan!");
-    }
-  }
 }
 
 void ESP32BLETracker::loop() {
+  if (!this->parent_->is_active()) {
+    this->ble_was_disabled_ = true;
+    return;
+  } else if (this->ble_was_disabled_) {
+    this->ble_was_disabled_ = false;
+    // If the BLE stack was disabled, we need to start the scan again.
+    if (this->scan_continuous_) {
+      this->start_scan();
+    }
+  }
   int connecting = 0;
   int discovered = 0;
   int searching = 0;
@@ -182,8 +184,7 @@ void ESP32BLETracker::loop() {
         xSemaphoreGive(this->scan_end_lock_);
       } else {
         ESP_LOGD(TAG, "Stopping scan after failure...");
-        esp_ble_gap_stop_scanning();
-        this->cancel_timeout("scan");
+        this->stop_scan_();
       }
       if (this->scan_start_failed_) {
         ESP_LOGE(TAG, "Scan start failed: %d", this->scan_start_failed_);
@@ -212,8 +213,7 @@ void ESP32BLETracker::loop() {
           client->set_state(ClientState::READY_TO_CONNECT);
         } else {
           ESP_LOGD(TAG, "Pausing scan to make connection...");
-          esp_ble_gap_stop_scanning();
-          this->cancel_timeout("scan");
+          this->stop_scan_();
         }
         break;
       }
@@ -232,11 +232,31 @@ void ESP32BLETracker::start_scan() {
 void ESP32BLETracker::stop_scan() {
   ESP_LOGD(TAG, "Stopping scan.");
   this->scan_continuous_ = false;
-  esp_ble_gap_stop_scanning();
+  this->stop_scan_();
+}
+
+void ESP32BLETracker::ble_before_disabled_event_handler() {
+  this->stop_scan_();
+  xSemaphoreGive(this->scan_end_lock_);
+}
+
+void ESP32BLETracker::stop_scan_() {
   this->cancel_timeout("scan");
+  if (this->scanner_idle_) {
+    return;
+  }
+  esp_err_t err = esp_ble_gap_stop_scanning();
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_ble_gap_stop_scanning failed: %d", err);
+    return;
+  }
 }
 
 void ESP32BLETracker::start_scan_(bool first) {
+  if (!this->parent_->is_active()) {
+    ESP_LOGW(TAG, "Cannot start scan while ESP32BLE is disabled.");
+    return;
+  }
   // The lock must be held when calling this function.
   if (xSemaphoreTake(this->scan_end_lock_, 0L)) {
     ESP_LOGE(TAG, "start_scan called without holding scan_end_lock_");
@@ -249,15 +269,23 @@ void ESP32BLETracker::start_scan_(bool first) {
       listener->on_scan_end();
   }
   this->already_discovered_.clear();
-  this->scanner_idle_ = false;
   this->scan_params_.scan_type = this->scan_active_ ? BLE_SCAN_TYPE_ACTIVE : BLE_SCAN_TYPE_PASSIVE;
   this->scan_params_.own_addr_type = BLE_ADDR_TYPE_PUBLIC;
   this->scan_params_.scan_filter_policy = BLE_SCAN_FILTER_ALLOW_ALL;
   this->scan_params_.scan_interval = this->scan_interval_;
   this->scan_params_.scan_window = this->scan_window_;
 
-  esp_ble_gap_set_scan_params(&this->scan_params_);
-  esp_ble_gap_start_scanning(this->scan_duration_);
+  esp_err_t err = esp_ble_gap_set_scan_params(&this->scan_params_);
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_ble_gap_set_scan_params failed: %d", err);
+    return;
+  }
+  err = esp_ble_gap_start_scanning(this->scan_duration_);
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "esp_ble_gap_start_scanning failed: %d", err);
+    return;
+  }
+  this->scanner_idle_ = false;
 
   this->set_timeout("scan", this->scan_duration_ * 2000, []() {
     ESP_LOGE(TAG, "ESP-IDF BLE scan never terminated, rebooting to restore BLE stack...");
diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h
index 6efdded3ff..0d986804ce 100644
--- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h
+++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h
@@ -177,7 +177,11 @@ class ESPBTClient : public ESPBTDeviceListener {
   ClientState state_;
 };
 
-class ESP32BLETracker : public Component, public GAPEventHandler, public GATTcEventHandler, public Parented<ESP32BLE> {
+class ESP32BLETracker : public Component,
+                        public GAPEventHandler,
+                        public GATTcEventHandler,
+                        public BLEStatusEventHandler,
+                        public Parented<ESP32BLE> {
  public:
   void set_scan_duration(uint32_t scan_duration) { scan_duration_ = scan_duration; }
   void set_scan_interval(uint32_t scan_interval) { scan_interval_ = scan_interval; }
@@ -204,8 +208,10 @@ class ESP32BLETracker : public Component, public GAPEventHandler, public GATTcEv
   void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if,
                            esp_ble_gattc_cb_param_t *param) override;
   void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) override;
+  void ble_before_disabled_event_handler() override;
 
  protected:
+  void stop_scan_();
   /// Start a single scan by setting up the parameters and doing some esp-idf calls.
   void start_scan_(bool first);
   /// Called when a scan ends
@@ -236,6 +242,7 @@ class ESP32BLETracker : public Component, public GAPEventHandler, public GATTcEv
   bool scan_continuous_;
   bool scan_active_;
   bool scanner_idle_;
+  bool ble_was_disabled_{true};
   bool raw_advertisements_{false};
   bool parse_advertisements_{false};
   SemaphoreHandle_t scan_result_lock_;
diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp
index 19340c3dd8..90e69e1cfa 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.cpp
+++ b/esphome/components/esp32_improv/esp32_improv_component.cpp
@@ -16,9 +16,6 @@ static const char *const ESPHOME_MY_LINK = "https://my.home-assistant.io/redirec
 ESP32ImprovComponent::ESP32ImprovComponent() { global_improv_component = this; }
 
 void ESP32ImprovComponent::setup() {
-  this->service_ = global_ble_server->create_service(improv::SERVICE_UUID, true);
-  this->setup_characteristics();
-
 #ifdef USE_BINARY_SENSOR
   if (this->authorizer_ != nullptr) {
     this->authorizer_->add_on_state_callback([this](bool state) {
@@ -70,6 +67,19 @@ void ESP32ImprovComponent::setup_characteristics() {
 }
 
 void ESP32ImprovComponent::loop() {
+  if (!global_ble_server->is_running()) {
+    this->state_ = improv::STATE_STOPPED;
+    this->incoming_data_.clear();
+    return;
+  }
+  if (this->service_ == nullptr) {
+    // Setup the service
+    ESP_LOGD(TAG, "Creating Improv service");
+    global_ble_server->create_service(ESPBTUUID::from_raw(improv::SERVICE_UUID), true);
+    this->service_ = global_ble_server->get_service(ESPBTUUID::from_raw(improv::SERVICE_UUID));
+    this->setup_characteristics();
+  }
+
   if (!this->incoming_data_.empty())
     this->process_incoming_data_();
   uint32_t now = millis();
@@ -80,11 +90,10 @@ void ESP32ImprovComponent::loop() {
 
       if (this->service_->is_created() && this->should_start_ && this->setup_complete_) {
         if (this->service_->is_running()) {
-          esp32_ble::global_ble->get_advertising()->start();
+          esp32_ble::global_ble->advertising_start();
 
           this->set_state_(improv::STATE_AWAITING_AUTHORIZATION);
           this->set_error_(improv::ERROR_NONE);
-          this->should_start_ = false;
           ESP_LOGD(TAG, "Service started!");
         } else {
           this->service_->start();
@@ -138,10 +147,7 @@ void ESP32ImprovComponent::loop() {
 #endif
         std::vector<uint8_t> data = improv::build_rpc_response(improv::WIFI_SETTINGS, urls);
         this->send_response_(data);
-        this->set_timeout("end-service", 1000, [this] {
-          this->service_->stop();
-          this->set_state_(improv::STATE_STOPPED);
-        });
+        this->stop();
       }
       break;
     }
@@ -206,8 +212,7 @@ void ESP32ImprovComponent::set_state_(improv::State state) {
   service_data[6] = 0x00;  // Reserved
   service_data[7] = 0x00;  // Reserved
 
-  esp32_ble::global_ble->get_advertising()->set_service_data(service_data);
-  esp32_ble::global_ble->get_advertising()->start();
+  esp32_ble::global_ble->advertising_set_service_data(service_data);
 }
 
 void ESP32ImprovComponent::set_error_(improv::Error error) {
@@ -237,7 +242,10 @@ void ESP32ImprovComponent::start() {
 }
 
 void ESP32ImprovComponent::stop() {
+  this->should_start_ = false;
   this->set_timeout("end-service", 1000, [this] {
+    if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr)
+      return;
     this->service_->stop();
     this->set_state_(improv::STATE_STOPPED);
   });
diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h
index 00c6cf885a..3ed377a6ad 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.h
+++ b/esphome/components/esp32_improv/esp32_improv_component.h
@@ -68,7 +68,7 @@ class ESP32ImprovComponent : public Component, public BLEServiceComponent {
   std::vector<uint8_t> incoming_data_;
   wifi::WiFiAP connecting_sta_;
 
-  std::shared_ptr<BLEService> service_;
+  BLEService *service_ = nullptr;
   BLECharacteristic *status_;
   BLECharacteristic *error_;
   BLECharacteristic *rpc_;
diff --git a/tests/test11.5.yaml b/tests/test11.5.yaml
index 06985611e7..9b25b9f273 100644
--- a/tests/test11.5.yaml
+++ b/tests/test11.5.yaml
@@ -104,6 +104,12 @@ binary_sensor:
     on_release:
       then:
         - switch.turn_off: Led0
+        - if:
+            condition: ble.enabled
+            then:
+              - ble.disable:
+            else:
+              - ble.enable:
 
   - platform: tm1638
     id: Button1
@@ -273,6 +279,7 @@ output:
 demo:
 
 esp32_ble:
+  enable_on_boot: false
 
 esp32_ble_server:
   manufacturer: ESPHome

From 84bbf02bde3cd9ef7ec24a97e8d372feb60463ec Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Mon, 6 Nov 2023 16:41:45 +1300
Subject: [PATCH 07/41] ble_client rssi sensor fix when not connected (#5632)

---
 .../ble_client/sensor/ble_rssi_sensor.cpp          | 14 +++++++++++---
 .../components/ble_client/sensor/ble_rssi_sensor.h |  6 +++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/esphome/components/ble_client/sensor/ble_rssi_sensor.cpp b/esphome/components/ble_client/sensor/ble_rssi_sensor.cpp
index 13e51ed5b3..a36e191e32 100644
--- a/esphome/components/ble_client/sensor/ble_rssi_sensor.cpp
+++ b/esphome/components/ble_client/sensor/ble_rssi_sensor.cpp
@@ -1,8 +1,8 @@
 #include "ble_rssi_sensor.h"
-#include "esphome/core/log.h"
+#include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h"
 #include "esphome/core/application.h"
 #include "esphome/core/helpers.h"
-#include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h"
+#include "esphome/core/log.h"
 
 #ifdef USE_ESP32
 
@@ -37,6 +37,10 @@ void BLEClientRSSISensor::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga
     }
     case ESP_GATTC_SEARCH_CMPL_EVT:
       this->node_state = espbt::ClientState::ESTABLISHED;
+      if (this->should_update_) {
+        this->should_update_ = false;
+        this->get_rssi_();
+      }
       break;
     default:
       break;
@@ -50,6 +54,7 @@ void BLEClientRSSISensor::gap_event_handler(esp_gap_ble_cb_event_t event, esp_bl
       if (param->read_rssi_cmpl.status == ESP_BT_STATUS_SUCCESS) {
         int8_t rssi = param->read_rssi_cmpl.rssi;
         ESP_LOGI(TAG, "ESP_GAP_BLE_READ_RSSI_COMPLETE_EVT RSSI: %d", rssi);
+        this->status_clear_warning();
         this->publish_state(rssi);
       }
       break;
@@ -61,9 +66,12 @@ void BLEClientRSSISensor::gap_event_handler(esp_gap_ble_cb_event_t event, esp_bl
 void BLEClientRSSISensor::update() {
   if (this->node_state != espbt::ClientState::ESTABLISHED) {
     ESP_LOGW(TAG, "[%s] Cannot poll, not connected", this->get_name().c_str());
+    this->should_update_ = true;
     return;
   }
-
+  this->get_rssi_();
+}
+void BLEClientRSSISensor::get_rssi_() {
   ESP_LOGV(TAG, "requesting rssi from %s", this->parent()->address_str().c_str());
   auto status = esp_ble_gap_read_rssi(this->parent()->get_remote_bda());
   if (status != ESP_OK) {
diff --git a/esphome/components/ble_client/sensor/ble_rssi_sensor.h b/esphome/components/ble_client/sensor/ble_rssi_sensor.h
index 028df83832..5dd3fc7af9 100644
--- a/esphome/components/ble_client/sensor/ble_rssi_sensor.h
+++ b/esphome/components/ble_client/sensor/ble_rssi_sensor.h
@@ -1,9 +1,9 @@
 #pragma once
 
-#include "esphome/core/component.h"
 #include "esphome/components/ble_client/ble_client.h"
 #include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h"
 #include "esphome/components/sensor/sensor.h"
+#include "esphome/core/component.h"
 
 #ifdef USE_ESP32
 #include <esp_gattc_api.h>
@@ -24,6 +24,10 @@ class BLEClientRSSISensor : public sensor::Sensor, public PollingComponent, publ
 
   void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if,
                            esp_ble_gattc_cb_param_t *param) override;
+
+ protected:
+  void get_rssi_();
+  bool should_update_{false};
 };
 
 }  // namespace ble_client

From b978985aa19df5874b190d56f5df4764964f51ca Mon Sep 17 00:00:00 2001
From: marshn <marshn@spa-technology.co.uk>
Date: Mon, 6 Nov 2023 19:30:23 +0000
Subject: [PATCH 08/41] Add Byron Doorbell RF protocol (#4718)

---
 esphome/components/remote_base/__init__.py    |  49 +++++++
 .../remote_base/byronsx_protocol.cpp          | 134 ++++++++++++++++++
 .../components/remote_base/byronsx_protocol.h |  46 ++++++
 3 files changed, 229 insertions(+)
 create mode 100644 esphome/components/remote_base/byronsx_protocol.cpp
 create mode 100644 esphome/components/remote_base/byronsx_protocol.h

diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py
index fb9d5e56a6..25dedd71d8 100644
--- a/esphome/components/remote_base/__init__.py
+++ b/esphome/components/remote_base/__init__.py
@@ -242,6 +242,55 @@ async def build_dumpers(config):
     return dumpers
 
 
+# ByronSX
+(
+    ByronSXData,
+    ByronSXBinarySensor,
+    ByronSXTrigger,
+    ByronSXAction,
+    ByronSXDumper,
+) = declare_protocol("ByronSX")
+BYRONSX_SCHEMA = cv.Schema(
+    {
+        cv.Required(CONF_ADDRESS): cv.All(cv.hex_int, cv.Range(min=0, max=0xFF)),
+        cv.Optional(CONF_COMMAND, default=0x10): cv.All(
+            cv.hex_int, cv.one_of(1, 2, 3, 5, 6, 9, 0xD, 0xE, 0x10, int=True)
+        ),
+    }
+)
+
+
+@register_binary_sensor("byronsx", ByronSXBinarySensor, BYRONSX_SCHEMA)
+def byronsx_binary_sensor(var, config):
+    cg.add(
+        var.set_data(
+            cg.StructInitializer(
+                ByronSXData,
+                ("address", config[CONF_ADDRESS]),
+                ("command", config[CONF_COMMAND]),
+            )
+        )
+    )
+
+
+@register_trigger("byronsx", ByronSXTrigger, ByronSXData)
+def byronsx_trigger(var, config):
+    pass
+
+
+@register_dumper("byronsx", ByronSXDumper)
+def byronsx_dumper(var, config):
+    pass
+
+
+@register_action("byronsx", ByronSXAction, BYRONSX_SCHEMA)
+async def byronsx_action(var, config, args):
+    template_ = await cg.templatable(config[CONF_ADDRESS], args, cg.uint8)
+    cg.add(var.set_address(template_))
+    template_ = await cg.templatable(config[CONF_COMMAND], args, cg.uint8)
+    cg.add(var.set_command(template_))
+
+
 # CanalSat
 (
     CanalSatData,
diff --git a/esphome/components/remote_base/byronsx_protocol.cpp b/esphome/components/remote_base/byronsx_protocol.cpp
new file mode 100644
index 0000000000..3096283b30
--- /dev/null
+++ b/esphome/components/remote_base/byronsx_protocol.cpp
@@ -0,0 +1,134 @@
+#include "byronsx_protocol.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace remote_base {
+
+static const char *const TAG = "remote.byronsx";
+
+static const uint32_t BIT_TIME_US = 333;
+static const uint8_t NBITS_ADDRESS = 8;
+static const uint8_t NBITS_COMMAND = 4;
+static const uint8_t NBITS_START_BIT = 1;
+static const uint8_t NBITS_DATA = NBITS_ADDRESS + NBITS_COMMAND /*+ NBITS_COMMAND*/;
+
+/*
+ByronSX Protocol
+Each transmitted packet appears to consist of thirteen bits of PWM encoded
+data. Each bit period of aprox 1ms consists of a transmitter OFF period
+followed by a transmitter ON period. The 'on' and 'off' periods are either
+short (approx 332us) or long (664us).
+
+A short 'off' followed by a long 'on' represents a '1' bit.
+A long 'off' followed by a short 'on' represents a '0' bit.
+
+A the beginning of each packet is and initial 'off' period of approx 5.6ms
+followed by a short 'on'.
+
+The data payload consists of twelve bits which appear to be an eight bit
+address floowied by a 4 bit chime number.
+
+SAAAAAAAACCCC
+
+Whese:
+S = the initial short start pulse
+A = The eight address bits
+C - The four chime bits
+
+--------------------
+
+I have also used 'RFLink' software (RLink Firmware Version: 1.1 Revision: 48)
+to capture these packets, eg:
+
+20;19;Byron;ID=004f;SWITCH=02;CMD=ON;CHIME=02;
+
+This module transmits and interprets packets in the same way as RFLink.
+
+marshn
+
+*/
+
+void ByronSXProtocol::encode(RemoteTransmitData *dst, const ByronSXData &data) {
+  uint32_t out_data = 0x0;
+
+  ESP_LOGD(TAG, "Send ByronSX: address=%04x command=%03x", data.address, data.command);
+
+  out_data = data.address;
+  out_data <<= NBITS_COMMAND;
+  out_data |= data.command;
+
+  ESP_LOGV(TAG, "Send ByronSX: out_data %03x", out_data);
+
+  // Initial Mark start bit
+  dst->mark(1 * BIT_TIME_US);
+
+  for (uint32_t mask = 1UL << (NBITS_DATA - 1); mask != 0; mask >>= 1) {
+    if (out_data & mask) {
+      dst->space(2 * BIT_TIME_US);
+      dst->mark(1 * BIT_TIME_US);
+    } else {
+      dst->space(1 * BIT_TIME_US);
+      dst->mark(2 * BIT_TIME_US);
+    }
+  }
+  // final space at end of packet
+  dst->space(17 * BIT_TIME_US);
+}
+
+optional<ByronSXData> ByronSXProtocol::decode(RemoteReceiveData src) {
+  ByronSXData out{
+      .address = 0,
+      .command = 0,
+  };
+
+  if (src.size() != (NBITS_DATA + NBITS_START_BIT) * 2) {
+    return {};
+  }
+
+  // Skip start bit
+  if (!src.expect_mark(BIT_TIME_US)) {
+    return {};
+  }
+
+  ESP_LOGVV(TAG, "%3d: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", src.size(), src.peek(0),
+            src.peek(1), src.peek(2), src.peek(3), src.peek(4), src.peek(5), src.peek(6), src.peek(7), src.peek(8),
+            src.peek(9), src.peek(10), src.peek(11), src.peek(12), src.peek(13), src.peek(14), src.peek(15),
+            src.peek(16), src.peek(17), src.peek(18), src.peek(19));
+
+  ESP_LOGVV(TAG, "     %d %d %d %d %d %d", src.peek(20), src.peek(21), src.peek(22), src.peek(23), src.peek(24),
+            src.peek(25));
+
+  // Read data bits
+  uint32_t out_data = 0;
+  int8_t bit = NBITS_DATA;
+  while (--bit >= 0) {
+    if (src.expect_space(2 * BIT_TIME_US) && src.expect_mark(BIT_TIME_US)) {
+      out_data |= 1 << bit;
+    } else if (src.expect_space(BIT_TIME_US) && src.expect_mark(2 * BIT_TIME_US)) {
+      out_data |= 0 << bit;
+    } else {
+      ESP_LOGV(TAG, "Decode ByronSX: Fail 2, %2d %08x", bit, out_data);
+      return {};
+    }
+    ESP_LOGVV(TAG, "Decode ByronSX: Data, %2d %08x", bit, out_data);
+  }
+
+  // last bit followed by a long space
+  if (!src.peek_space_at_least(BIT_TIME_US)) {
+    ESP_LOGV(TAG, "Decode ByronSX: Fail 4 ");
+    return {};
+  }
+
+  out.command = (uint8_t) (out_data & 0xF);
+  out_data >>= NBITS_COMMAND;
+  out.address = (uint16_t) (out_data & 0xFF);
+
+  return out;
+}
+
+void ByronSXProtocol::dump(const ByronSXData &data) {
+  ESP_LOGD(TAG, "Received ByronSX: address=0x%08X, command=0x%02x", data.address, data.command);
+}
+
+}  // namespace remote_base
+}  // namespace esphome
diff --git a/esphome/components/remote_base/byronsx_protocol.h b/esphome/components/remote_base/byronsx_protocol.h
new file mode 100644
index 0000000000..5d23237ab1
--- /dev/null
+++ b/esphome/components/remote_base/byronsx_protocol.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "remote_base.h"
+
+namespace esphome {
+namespace remote_base {
+
+struct ByronSXData {
+  uint8_t address;
+  uint8_t command;
+
+  bool operator==(const ByronSXData &rhs) const {
+    // Treat 0x10 as a special, wildcard command/chime
+    // This allows us to match on just the address if wanted.
+    if (address != rhs.address) {
+      return false;
+    }
+    return (rhs.command == 0x10 || command == rhs.command);
+  }
+};
+
+class ByronSXProtocol : public RemoteProtocol<ByronSXData> {
+ public:
+  void encode(RemoteTransmitData *dst, const ByronSXData &data) override;
+  optional<ByronSXData> decode(RemoteReceiveData src) override;
+  void dump(const ByronSXData &data) override;
+};
+
+DECLARE_REMOTE_PROTOCOL(ByronSX)
+
+template<typename... Ts> class ByronSXAction : public RemoteTransmitterActionBase<Ts...> {
+ public:
+  TEMPLATABLE_VALUE(uint8_t, address)
+  TEMPLATABLE_VALUE(uint8_t, command)
+
+  void encode(RemoteTransmitData *dst, Ts... x) override {
+    ByronSXData data{};
+    data.address = this->address_.value(x...);
+    data.command = this->command_.value(x...);
+    ByronSXProtocol().encode(dst, data);
+  }
+};
+
+}  // namespace remote_base
+}  // namespace esphome

From fce59819f5627dcf32267b3da20ead54cc8acd69 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Mon, 6 Nov 2023 16:07:59 -0600
Subject: [PATCH 09/41] Refactor dashboard zeroconf support (#5681)

---
 esphome/dashboard/dashboard.py | 109 ++++++++++++++++------
 esphome/zeroconf.py            | 162 ++++++++-------------------------
 2 files changed, 123 insertions(+), 148 deletions(-)

diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py
index ce8976cb0f..f6eb079430 100644
--- a/esphome/dashboard/dashboard.py
+++ b/esphome/dashboard/dashboard.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import base64
 import binascii
 import codecs
@@ -15,7 +17,6 @@ import shutil
 import subprocess
 import threading
 from pathlib import Path
-from typing import Optional
 
 import tornado
 import tornado.concurrent
@@ -42,7 +43,13 @@ from esphome.storage_json import (
     trash_storage_path,
 )
 from esphome.util import get_serial_ports, shlex_quote
-from esphome.zeroconf import DashboardImportDiscovery, DashboardStatus, EsphomeZeroconf
+from esphome.zeroconf import (
+    ESPHOME_SERVICE_TYPE,
+    DashboardBrowser,
+    DashboardImportDiscovery,
+    DashboardStatus,
+    EsphomeZeroconf,
+)
 
 from .util import friendly_name_slugify, password_hash
 
@@ -517,6 +524,8 @@ class ImportRequestHandler(BaseHandler):
                 network,
                 encryption,
             )
+            # Make sure the device gets marked online right away
+            PING_REQUEST.set()
         except FileExistsError:
             self.set_status(500)
             self.write("File already exists")
@@ -542,13 +551,11 @@ class DownloadListRequestHandler(BaseHandler):
             self.send_error(404)
             return
 
-        from esphome.components.esp32 import (
-            get_download_types as esp32_types,
-            VARIANTS as ESP32_VARIANTS,
-        )
+        from esphome.components.esp32 import VARIANTS as ESP32_VARIANTS
+        from esphome.components.esp32 import get_download_types as esp32_types
         from esphome.components.esp8266 import get_download_types as esp8266_types
-        from esphome.components.rp2040 import get_download_types as rp2040_types
         from esphome.components.libretiny import get_download_types as libretiny_types
+        from esphome.components.rp2040 import get_download_types as rp2040_types
 
         downloads = []
         platform = storage_json.target_platform.lower()
@@ -661,12 +668,21 @@ class DashboardEntry:
         self._storage = None
         self._loaded_storage = False
 
+    def __repr__(self):
+        return (
+            f"DashboardEntry({self.path} "
+            f"address={self.address} "
+            f"web_port={self.web_port} "
+            f"name={self.name} "
+            f"no_mdns={self.no_mdns})"
+        )
+
     @property
     def filename(self):
         return os.path.basename(self.path)
 
     @property
-    def storage(self) -> Optional[StorageJSON]:
+    def storage(self) -> StorageJSON | None:
         if not self._loaded_storage:
             self._storage = StorageJSON.load(ext_storage_path(self.filename))
             self._loaded_storage = True
@@ -831,10 +847,10 @@ class PrometheusServiceDiscoveryHandler(BaseHandler):
 class BoardsRequestHandler(BaseHandler):
     @authenticated
     def get(self, platform: str):
+        from esphome.components.bk72xx.boards import BOARDS as BK72XX_BOARDS
         from esphome.components.esp32.boards import BOARDS as ESP32_BOARDS
         from esphome.components.esp8266.boards import BOARDS as ESP8266_BOARDS
         from esphome.components.rp2040.boards import BOARDS as RP2040_BOARDS
-        from esphome.components.bk72xx.boards import BOARDS as BK72XX_BOARDS
         from esphome.components.rtl87xx.boards import BOARDS as RTL87XX_BOARDS
 
         platform_to_boards = {
@@ -865,35 +881,76 @@ class BoardsRequestHandler(BaseHandler):
 
 
 class MDNSStatusThread(threading.Thread):
+    def __init__(self):
+        """Initialize the MDNSStatusThread."""
+        super().__init__()
+        # This is the current mdns state for each host (True, False, None)
+        self.host_mdns_state: dict[str, bool | None] = {}
+        # This is the hostnames to filenames mapping
+        self.host_name_to_filename: dict[str, str] = {}
+        # This is a set of host names to track (i.e no_mdns = false)
+        self.host_name_with_mdns_enabled: set[set] = set()
+        self._refresh_hosts()
+
+    def _refresh_hosts(self):
+        """Refresh the hosts to track."""
+        entries = _list_dashboard_entries()
+        host_name_with_mdns_enabled = self.host_name_with_mdns_enabled
+        host_mdns_state = self.host_mdns_state
+        host_name_to_filename = self.host_name_to_filename
+
+        for entry in entries:
+            name = entry.name
+            # If no_mdns is set, remove it from the set
+            if entry.no_mdns:
+                host_name_with_mdns_enabled.discard(name)
+                continue
+
+            # We are tracking this host
+            host_name_with_mdns_enabled.add(name)
+            filename = entry.filename
+
+            # If we just adopted/imported this host, we likely
+            # already have a state for it, so we should make sure
+            # to set it so the dashboard shows it as online
+            if name in host_mdns_state:
+                PING_RESULT[filename] = host_mdns_state[name]
+
+            # Make sure the mapping is up to date
+            # so when we get an mdns update we can map it back
+            # to the filename
+            host_name_to_filename[name] = filename
+
     def run(self):
         global IMPORT_RESULT
 
         zc = EsphomeZeroconf()
+        host_mdns_state = self.host_mdns_state
+        host_name_to_filename = self.host_name_to_filename
+        host_name_with_mdns_enabled = self.host_name_with_mdns_enabled
 
-        def on_update(dat):
-            for key, b in dat.items():
-                PING_RESULT[key] = b
+        def on_update(dat: dict[str, bool | None]) -> None:
+            """Update the global PING_RESULT dict."""
+            for name, result in dat.items():
+                host_mdns_state[name] = result
+                if name in host_name_with_mdns_enabled:
+                    filename = host_name_to_filename[name]
+                    PING_RESULT[filename] = result
 
-        stat = DashboardStatus(zc, on_update)
-        imports = DashboardImportDiscovery(zc)
+        self._refresh_hosts()
+        stat = DashboardStatus(on_update)
+        imports = DashboardImportDiscovery()
+        browser = DashboardBrowser(
+            zc, ESPHOME_SERVICE_TYPE, [stat.browser_callback, imports.browser_callback]
+        )
 
-        stat.start()
         while not STOP_EVENT.is_set():
-            entries = _list_dashboard_entries()
-            hosts = {}
-            for entry in entries:
-                if entry.no_mdns is not True:
-                    hosts[entry.filename] = f"{entry.name}.local."
-
-            stat.request_query(hosts)
+            self._refresh_hosts()
             IMPORT_RESULT = imports.import_state
-
             PING_REQUEST.wait()
             PING_REQUEST.clear()
 
-        stat.stop()
-        stat.join()
-        imports.cancel()
+        browser.cancel()
         zc.close()
 
 
diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py
index 14dd740a96..d20111ce20 100644
--- a/esphome/zeroconf.py
+++ b/esphome/zeroconf.py
@@ -1,130 +1,49 @@
+from __future__ import annotations
+
 import logging
-import socket
-import threading
-import time
 from dataclasses import dataclass
-from typing import Optional
+from typing import Callable
 
 from zeroconf import (
-    DNSAddress,
-    DNSOutgoing,
-    DNSQuestion,
-    RecordUpdate,
-    RecordUpdateListener,
+    IPVersion,
     ServiceBrowser,
+    ServiceInfo,
     ServiceStateChange,
     Zeroconf,
-    current_time_millis,
 )
 
 from esphome.storage_json import StorageJSON, ext_storage_path
 
-_CLASS_IN = 1
-_FLAGS_QR_QUERY = 0x0000  # query
-_TYPE_A = 1
 _LOGGER = logging.getLogger(__name__)
 
 
-class HostResolver(RecordUpdateListener):
+class HostResolver(ServiceInfo):
     """Resolve a host name to an IP address."""
 
-    def __init__(self, name: str):
-        self.name = name
-        self.address: Optional[bytes] = None
-
-    def async_update_records(
-        self, zc: Zeroconf, now: float, records: list[RecordUpdate]
-    ) -> None:
-        """Update multiple records in one shot.
-
-        This will run in zeroconf's event loop thread so it
-        must be thread-safe.
-        """
-        for record_update in records:
-            record, _ = record_update
-            if record is None:
-                continue
-            if record.type == _TYPE_A:
-                assert isinstance(record, DNSAddress)
-                if record.name == self.name:
-                    self.address = record.address
-
-    def request(self, zc: Zeroconf, timeout: float) -> bool:
-        now = time.time()
-        delay = 0.2
-        next_ = now + delay
-        last = now + timeout
-
-        try:
-            zc.add_listener(self, None)
-            while self.address is None:
-                if last <= now:
-                    # Timeout
-                    return False
-                if next_ <= now:
-                    out = DNSOutgoing(_FLAGS_QR_QUERY)
-                    out.add_question(DNSQuestion(self.name, _TYPE_A, _CLASS_IN))
-                    zc.send(out)
-                    next_ = now + delay
-                    delay *= 2
-
-                time.sleep(min(next_, last) - now)
-                now = time.time()
-        finally:
-            zc.remove_listener(self)
-
-        return True
+    @property
+    def _is_complete(self) -> bool:
+        """The ServiceInfo has all expected properties."""
+        return bool(self._ipv4_addresses)
 
 
-class DashboardStatus(threading.Thread):
-    PING_AFTER = 15 * 1000  # Send new mDNS request after 15 seconds
-    OFFLINE_AFTER = PING_AFTER * 2  # Offline if no mDNS response after 30 seconds
-
-    def __init__(self, zc: Zeroconf, on_update) -> None:
-        threading.Thread.__init__(self)
-        self.zc = zc
-        self.query_hosts: set[str] = set()
-        self.key_to_host: dict[str, str] = {}
-        self.stop_event = threading.Event()
-        self.query_event = threading.Event()
+class DashboardStatus:
+    def __init__(self, on_update: Callable[[dict[str, bool | None], []]]) -> None:
+        """Initialize the dashboard status."""
         self.on_update = on_update
 
-    def request_query(self, hosts: dict[str, str]) -> None:
-        self.query_hosts = set(hosts.values())
-        self.key_to_host = hosts
-        self.query_event.set()
-
-    def stop(self) -> None:
-        self.stop_event.set()
-        self.query_event.set()
-
-    def host_status(self, key: str) -> bool:
-        entries = self.zc.cache.entries_with_name(key)
-        if not entries:
-            return False
-        now = current_time_millis()
-
-        return any(
-            (entry.created + DashboardStatus.OFFLINE_AFTER) >= now for entry in entries
-        )
-
-    def run(self) -> None:
-        while not self.stop_event.is_set():
-            self.on_update(
-                {key: self.host_status(host) for key, host in self.key_to_host.items()}
-            )
-            now = current_time_millis()
-            for host in self.query_hosts:
-                entries = self.zc.cache.entries_with_name(host)
-                if not entries or all(
-                    (entry.created + DashboardStatus.PING_AFTER) <= now
-                    for entry in entries
-                ):
-                    out = DNSOutgoing(_FLAGS_QR_QUERY)
-                    out.add_question(DNSQuestion(host, _TYPE_A, _CLASS_IN))
-                    self.zc.send(out)
-            self.query_event.wait()
-            self.query_event.clear()
+    def browser_callback(
+        self,
+        zeroconf: Zeroconf,
+        service_type: str,
+        name: str,
+        state_change: ServiceStateChange,
+    ) -> None:
+        """Handle a service update."""
+        short_name = name.partition(".")[0]
+        if state_change == ServiceStateChange.Removed:
+            self.on_update({short_name: False})
+        elif state_change in (ServiceStateChange.Updated, ServiceStateChange.Added):
+            self.on_update({short_name: True})
 
 
 ESPHOME_SERVICE_TYPE = "_esphomelib._tcp.local."
@@ -138,7 +57,7 @@ TXT_RECORD_VERSION = b"version"
 
 @dataclass
 class DiscoveredImport:
-    friendly_name: Optional[str]
+    friendly_name: str | None
     device_name: str
     package_import_url: str
     project_name: str
@@ -146,15 +65,15 @@ class DiscoveredImport:
     network: str
 
 
+class DashboardBrowser(ServiceBrowser):
+    """A class to browse for ESPHome nodes."""
+
+
 class DashboardImportDiscovery:
-    def __init__(self, zc: Zeroconf) -> None:
-        self.zc = zc
-        self.service_browser = ServiceBrowser(
-            self.zc, ESPHOME_SERVICE_TYPE, [self._on_update]
-        )
+    def __init__(self) -> None:
         self.import_state: dict[str, DiscoveredImport] = {}
 
-    def _on_update(
+    def browser_callback(
         self,
         zeroconf: Zeroconf,
         service_type: str,
@@ -167,8 +86,6 @@ class DashboardImportDiscovery:
             name,
             state_change,
         )
-        if service_type != ESPHOME_SERVICE_TYPE:
-            return
         if state_change == ServiceStateChange.Removed:
             self.import_state.pop(name, None)
             return
@@ -212,9 +129,6 @@ class DashboardImportDiscovery:
             network=network,
         )
 
-    def cancel(self) -> None:
-        self.service_browser.cancel()
-
     def update_device_mdns(self, node_name: str, version: str):
         storage_path = ext_storage_path(node_name + ".yaml")
         storage_json = StorageJSON.load(storage_path)
@@ -234,7 +148,11 @@ class DashboardImportDiscovery:
 
 class EsphomeZeroconf(Zeroconf):
     def resolve_host(self, host: str, timeout=3.0):
-        info = HostResolver(host)
-        if info.request(self, timeout):
-            return socket.inet_ntoa(info.address)
+        """Resolve a host name to an IP address."""
+        name = host.partition(".")[0]
+        info = HostResolver(f"{name}.{ESPHOME_SERVICE_TYPE}", ESPHOME_SERVICE_TYPE)
+        if (info.load_from_cache(self) or info.request(self, timeout * 1000)) and (
+            addresses := info.ip_addresses_by_version(IPVersion.V4Only)
+        ):
+            return str(addresses[0])
         return None

From 43b36ac3c7ac6843ddd34605fa31a047590bb0c2 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 7 Nov 2023 12:38:38 +1300
Subject: [PATCH 10/41] Allow static assets to be cached if not in debug mode
 (#5684)

---
 esphome/dashboard/dashboard.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py
index f6eb079430..b416b00e60 100644
--- a/esphome/dashboard/dashboard.py
+++ b/esphome/dashboard/dashboard.py
@@ -1328,9 +1328,11 @@ def make_app(debug=get_bool_env(ENV_DEV)):
             if "favicon.ico" in path:
                 self.set_header("Cache-Control", "max-age=84600, public")
             else:
-                self.set_header(
-                    "Cache-Control", "no-store, no-cache, must-revalidate, max-age=0"
-                )
+                if debug:
+                    self.set_header(
+                        "Cache-Control",
+                        "no-store, no-cache, must-revalidate, max-age=0",
+                    )
 
     app_settings = {
         "debug": debug,

From d141e1cd6725e5c6a34052b4c097be52eebbbd29 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 7 Nov 2023 12:38:52 +1300
Subject: [PATCH 11/41] Remove extra code in old sgp40 (#5685)

---
 esphome/components/sgp40/sensor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/sgp40/sensor.py b/esphome/components/sgp40/sensor.py
index cb4231c168..ad9de6fe24 100644
--- a/esphome/components/sgp40/sensor.py
+++ b/esphome/components/sgp40/sensor.py
@@ -2,7 +2,7 @@ import esphome.config_validation as cv
 
 CODEOWNERS = ["@SenexCrenshaw"]
 
-CONFIG_SCHEMA = CONFIG_SCHEMA = cv.invalid(
+CONFIG_SCHEMA = cv.invalid(
     "SGP40 is deprecated.\nPlease use the SGP4x platform instead.\nSGP4x supports both SPG40 and SGP41.\n"
     " See https://esphome.io/components/sensor/sgp4x.html"
 )

From dd0270207f1b356e7f275cfd8b2d8bff53451f8c Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 7 Nov 2023 12:39:47 +1300
Subject: [PATCH 12/41] Allow pulse light effect to have separate on and off
 transition lengths (#5659)

---
 esphome/components/light/base_light_effects.h | 10 ++++---
 esphome/components/light/effects.py           | 30 ++++++++++++++++---
 tests/test1.yaml                              | 14 +++++++++
 3 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/esphome/components/light/base_light_effects.h b/esphome/components/light/base_light_effects.h
index 9211bba7c9..d126e4960c 100644
--- a/esphome/components/light/base_light_effects.h
+++ b/esphome/components/light/base_light_effects.h
@@ -3,8 +3,8 @@
 #include <utility>
 #include <vector>
 
-#include "light_effect.h"
 #include "esphome/core/automation.h"
+#include "light_effect.h"
 
 namespace esphome {
 namespace light {
@@ -27,8 +27,8 @@ class PulseLightEffect : public LightEffect {
     auto call = this->state_->turn_on();
     float out = this->on_ ? this->max_brightness : this->min_brightness;
     call.set_brightness_if_supported(out);
+    call.set_transition_length_if_supported(this->on_ ? this->transition_on_length_ : this->transition_off_length_);
     this->on_ = !this->on_;
-    call.set_transition_length_if_supported(this->transition_length_);
     // don't tell HA every change
     call.set_publish(false);
     call.set_save(false);
@@ -37,7 +37,8 @@ class PulseLightEffect : public LightEffect {
     this->last_color_change_ = now;
   }
 
-  void set_transition_length(uint32_t transition_length) { this->transition_length_ = transition_length; }
+  void set_transition_on_length(uint32_t transition_length) { this->transition_on_length_ = transition_length; }
+  void set_transition_off_length(uint32_t transition_length) { this->transition_off_length_ = transition_length; }
 
   void set_update_interval(uint32_t update_interval) { this->update_interval_ = update_interval; }
 
@@ -49,7 +50,8 @@ class PulseLightEffect : public LightEffect {
  protected:
   bool on_ = false;
   uint32_t last_color_change_{0};
-  uint32_t transition_length_{};
+  uint32_t transition_on_length_{};
+  uint32_t transition_off_length_{};
   uint32_t update_interval_{};
   float min_brightness{0.0};
   float max_brightness{1.0};
diff --git a/esphome/components/light/effects.py b/esphome/components/light/effects.py
index c694d6f50c..5212e90938 100644
--- a/esphome/components/light/effects.py
+++ b/esphome/components/light/effects.py
@@ -76,6 +76,8 @@ CONF_ADDRESSABLE_RANDOM_TWINKLE = "addressable_random_twinkle"
 CONF_ADDRESSABLE_FIREWORKS = "addressable_fireworks"
 CONF_ADDRESSABLE_FLICKER = "addressable_flicker"
 CONF_AUTOMATION = "automation"
+CONF_ON_LENGTH = "on_length"
+CONF_OFF_LENGTH = "off_length"
 
 BINARY_EFFECTS = []
 MONOCHROMATIC_EFFECTS = []
@@ -170,9 +172,15 @@ async def automation_effect_to_code(config, effect_id):
     PulseLightEffect,
     "Pulse",
     {
-        cv.Optional(
-            CONF_TRANSITION_LENGTH, default="1s"
-        ): cv.positive_time_period_milliseconds,
+        cv.Optional(CONF_TRANSITION_LENGTH, default="1s"): cv.Any(
+            cv.positive_time_period_milliseconds,
+            cv.Schema(
+                {
+                    cv.Required(CONF_ON_LENGTH): cv.positive_time_period_milliseconds,
+                    cv.Required(CONF_OFF_LENGTH): cv.positive_time_period_milliseconds,
+                }
+            ),
+        ),
         cv.Optional(
             CONF_UPDATE_INTERVAL, default="1s"
         ): cv.positive_time_period_milliseconds,
@@ -182,7 +190,21 @@ async def automation_effect_to_code(config, effect_id):
 )
 async def pulse_effect_to_code(config, effect_id):
     effect = cg.new_Pvariable(effect_id, config[CONF_NAME])
-    cg.add(effect.set_transition_length(config[CONF_TRANSITION_LENGTH]))
+    if isinstance(config[CONF_TRANSITION_LENGTH], dict):
+        cg.add(
+            effect.set_transition_on_length(
+                config[CONF_TRANSITION_LENGTH][CONF_ON_LENGTH]
+            )
+        )
+        cg.add(
+            effect.set_transition_off_length(
+                config[CONF_TRANSITION_LENGTH][CONF_OFF_LENGTH]
+            )
+        )
+    else:
+        transition_length = config[CONF_TRANSITION_LENGTH]
+        cg.add(effect.set_transition_on_length(transition_length))
+        cg.add(effect.set_transition_off_length(transition_length))
     cg.add(effect.set_update_interval(config[CONF_UPDATE_INTERVAL]))
     cg.add(
         effect.set_min_max_brightness(
diff --git a/tests/test1.yaml b/tests/test1.yaml
index 7bd6dcb41d..64d1d36f52 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -2179,6 +2179,20 @@ light:
             state += 1;
             if (state == 4)
               state = 0;
+      - pulse:
+          transition_length: 10s
+          update_interval: 20s
+          min_brightness: 10%
+          max_brightness: 90%
+      - pulse:
+          name: pulse2
+          transition_length:
+            on_length: 10s
+            off_length: 5s
+          update_interval: 15s
+          min_brightness: 10%
+          max_brightness: 90%
+
   - platform: rgb
     name: Living Room Lights
     id: ${roomname}_lights

From 972c18a7ca2dd36fc4118152fff6007f760df873 Mon Sep 17 00:00:00 2001
From: Greg Cormier <gcormier@gmail.com>
Date: Mon, 6 Nov 2023 18:46:30 -0500
Subject: [PATCH 13/41] Add differential pressure sensor support for CFSensor
 XGZP68xxD devices (#5562)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 CODEOWNERS                               |  1 +
 esphome/components/xgzp68xx/__init__.py  |  1 +
 esphome/components/xgzp68xx/sensor.py    | 62 ++++++++++++++++
 esphome/components/xgzp68xx/xgzp68xx.cpp | 91 ++++++++++++++++++++++++
 esphome/components/xgzp68xx/xgzp68xx.h   | 27 +++++++
 tests/test1.yaml                         |  8 +++
 6 files changed, 190 insertions(+)
 create mode 100644 esphome/components/xgzp68xx/__init__.py
 create mode 100644 esphome/components/xgzp68xx/sensor.py
 create mode 100644 esphome/components/xgzp68xx/xgzp68xx.cpp
 create mode 100644 esphome/components/xgzp68xx/xgzp68xx.h

diff --git a/CODEOWNERS b/CODEOWNERS
index a48d0ab2e0..067b886320 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -349,6 +349,7 @@ esphome/components/wiegand/* @ssieb
 esphome/components/wireguard/* @droscy @lhoracek @thomas0bernard
 esphome/components/wl_134/* @hobbypunk90
 esphome/components/x9c/* @EtienneMD
+esphome/components/xgzp68xx/* @gcormier
 esphome/components/xiaomi_lywsd03mmc/* @ahpohl
 esphome/components/xiaomi_mhoc303/* @drug123
 esphome/components/xiaomi_mhoc401/* @vevsvevs
diff --git a/esphome/components/xgzp68xx/__init__.py b/esphome/components/xgzp68xx/__init__.py
new file mode 100644
index 0000000000..122ffaf6ed
--- /dev/null
+++ b/esphome/components/xgzp68xx/__init__.py
@@ -0,0 +1 @@
+CODEOWNERS = ["@gcormier"]
diff --git a/esphome/components/xgzp68xx/sensor.py b/esphome/components/xgzp68xx/sensor.py
new file mode 100644
index 0000000000..3e381aaa61
--- /dev/null
+++ b/esphome/components/xgzp68xx/sensor.py
@@ -0,0 +1,62 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import i2c, sensor
+from esphome.const import (
+    DEVICE_CLASS_PRESSURE,
+    CONF_ID,
+    DEVICE_CLASS_TEMPERATURE,
+    STATE_CLASS_MEASUREMENT,
+    UNIT_PASCAL,
+    UNIT_CELSIUS,
+    CONF_TEMPERATURE,
+    CONF_PRESSURE,
+)
+
+DEPENDENCIES = ["i2c"]
+CODEOWNERS = ["@gcormier"]
+
+CONF_K_VALUE = "k_value"
+
+xgzp68xx_ns = cg.esphome_ns.namespace("xgzp68xx")
+XGZP68XXComponent = xgzp68xx_ns.class_(
+    "XGZP68XXComponent", cg.PollingComponent, i2c.I2CDevice
+)
+
+CONFIG_SCHEMA = (
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(XGZP68XXComponent),
+            cv.Optional(CONF_PRESSURE): sensor.sensor_schema(
+                unit_of_measurement=UNIT_PASCAL,
+                accuracy_decimals=1,
+                device_class=DEVICE_CLASS_PRESSURE,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Optional(CONF_TEMPERATURE): sensor.sensor_schema(
+                unit_of_measurement=UNIT_CELSIUS,
+                accuracy_decimals=1,
+                device_class=DEVICE_CLASS_TEMPERATURE,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Optional(CONF_K_VALUE, default=4096): cv.uint16_t,
+        }
+    )
+    .extend(cv.polling_component_schema("60s"))
+    .extend(i2c.i2c_device_schema(0x6D))
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+    await i2c.register_i2c_device(var, config)
+
+    if temperature_config := config.get(CONF_TEMPERATURE):
+        sens = await sensor.new_sensor(temperature_config)
+        cg.add(var.set_temperature_sensor(sens))
+
+    if pressure_config := config.get(CONF_PRESSURE):
+        sens = await sensor.new_sensor(pressure_config)
+        cg.add(var.set_pressure_sensor(sens))
+
+    cg.add(var.set_k_value(config[CONF_K_VALUE]))
diff --git a/esphome/components/xgzp68xx/xgzp68xx.cpp b/esphome/components/xgzp68xx/xgzp68xx.cpp
new file mode 100644
index 0000000000..ea3583c3c5
--- /dev/null
+++ b/esphome/components/xgzp68xx/xgzp68xx.cpp
@@ -0,0 +1,91 @@
+#include "xgzp68xx.h"
+#include "esphome/core/log.h"
+#include "esphome/core/hal.h"
+#include "esphome/core/helpers.h"
+#include "esphome/components/i2c/i2c.h"
+
+namespace esphome {
+namespace xgzp68xx {
+
+static const char *const TAG = "xgzp68xx.sensor";
+
+static const uint8_t CMD_ADDRESS = 0x30;
+static const uint8_t SYSCONFIG_ADDRESS = 0xA5;
+static const uint8_t PCONFIG_ADDRESS = 0xA6;
+static const uint8_t READ_COMMAND = 0x0A;
+
+void XGZP68XXComponent::update() {
+  // Request temp + pressure acquisition
+  this->write_register(0x30, &READ_COMMAND, 1);
+
+  // Wait 20mS per datasheet
+  this->set_timeout("measurement", 20, [this]() {
+    uint8_t data[5];
+    uint32_t pressure_raw;
+    uint16_t temperature_raw;
+    float pressure_in_pa, temperature;
+    int success;
+
+    // Read the sensor data
+    success = this->read_register(0x06, data, 5);
+    if (success != 0) {
+      ESP_LOGE(TAG, "Failed to read sensor data! Error code: %i", success);
+      return;
+    }
+
+    pressure_raw = encode_uint24(data[0], data[1], data[2]);
+    temperature_raw = encode_uint16(data[3], data[4]);
+
+    // Convert the pressure data to hPa
+    ESP_LOGV(TAG, "Got raw pressure=%d, raw temperature=%d ", pressure_raw, temperature_raw);
+    ESP_LOGV(TAG, "K value is %d ", this->k_value_);
+
+    // The most significant bit of both pressure and temperature will be 1 to indicate a negative value.
+    // This is directly from the datasheet, and the calculations below will handle this.
+    if (pressure_raw > pow(2, 23)) {
+      // Negative pressure
+      pressure_in_pa = (pressure_raw - pow(2, 24)) / (float) (this->k_value_);
+    } else {
+      // Positive pressure
+      pressure_in_pa = pressure_raw / (float) (this->k_value_);
+    }
+
+    if (temperature_raw > pow(2, 15)) {
+      // Negative temperature
+      temperature = (float) (temperature_raw - pow(2, 16)) / 256.0f;
+    } else {
+      // Positive temperature
+      temperature = (float) temperature_raw / 256.0f;
+    }
+
+    if (this->pressure_sensor_ != nullptr)
+      this->pressure_sensor_->publish_state(pressure_in_pa);
+
+    if (this->temperature_sensor_ != nullptr)
+      this->temperature_sensor_->publish_state(temperature);
+  });  // end of set_timeout
+}
+
+void XGZP68XXComponent::setup() {
+  ESP_LOGD(TAG, "Setting up XGZP68xx...");
+  uint8_t config;
+
+  // Display some sample bits to confirm we are talking to the sensor
+  this->read_register(SYSCONFIG_ADDRESS, &config, 1);
+  ESP_LOGCONFIG(TAG, "Gain value is %d", (config >> 3) & 0b111);
+  ESP_LOGCONFIG(TAG, "XGZP68xx started!");
+}
+
+void XGZP68XXComponent::dump_config() {
+  ESP_LOGCONFIG(TAG, "XGZP68xx");
+  LOG_SENSOR("  ", "Temperature: ", this->temperature_sensor_);
+  LOG_SENSOR("  ", "Pressure: ", this->pressure_sensor_);
+  LOG_I2C_DEVICE(this);
+  if (this->is_failed()) {
+    ESP_LOGE(TAG, "  Connection with XGZP68xx failed!");
+  }
+  LOG_UPDATE_INTERVAL(this);
+}
+
+}  // namespace xgzp68xx
+}  // namespace esphome
diff --git a/esphome/components/xgzp68xx/xgzp68xx.h b/esphome/components/xgzp68xx/xgzp68xx.h
new file mode 100644
index 0000000000..1bb7304b15
--- /dev/null
+++ b/esphome/components/xgzp68xx/xgzp68xx.h
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/components/sensor/sensor.h"
+#include "esphome/components/i2c/i2c.h"
+
+namespace esphome {
+namespace xgzp68xx {
+
+class XGZP68XXComponent : public PollingComponent, public sensor::Sensor, public i2c::I2CDevice {
+ public:
+  SUB_SENSOR(temperature)
+  SUB_SENSOR(pressure)
+  void set_k_value(uint16_t k_value) { this->k_value_ = k_value; }
+
+  void update() override;
+  void setup() override;
+  void dump_config() override;
+
+ protected:
+  /// Internal method to read the pressure from the component after it has been scheduled.
+  void read_pressure_();
+  uint16_t k_value_;
+};
+
+}  // namespace xgzp68xx
+}  // namespace esphome
diff --git a/tests/test1.yaml b/tests/test1.yaml
index 64d1d36f52..f653960c40 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -351,6 +351,14 @@ dfrobot_sen0395:
     uart_id: dfrobot_mmwave_uart
 
 sensor:
+  - platform: xgzp68xx
+    i2c_id: i2c_bus
+    temperature:
+      name: Pressure Temperature
+    pressure:
+      name: Differential pressure
+    k_value: 4096
+
   - platform: pmwcs3
     i2c_id: i2c_bus
     e25:

From 7ac9caa169d2375a22f1e87d842571da479acb6e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 7 Nov 2023 12:48:37 +1300
Subject: [PATCH 14/41] Bump zeroconf from 0.119.0 to 0.120.0 (#5682)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 5c57f3e933..2747258883 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,7 @@ esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20230904.0
 aioesphomeapi==18.2.1
-zeroconf==0.119.0
+zeroconf==0.120.0
 
 # esp-idf requires this, but doesn't bundle it by default
 # https://github.com/espressif/esp-idf/blob/220590d599e134d7a5e7f1e683cc4550349ffbf8/requirements.txt#L24

From defe8ac97b580fede9584f50c8ee6feacb05d4b2 Mon Sep 17 00:00:00 2001
From: Angel Nunez Mencias <git@angelnu.com>
Date: Tue, 7 Nov 2023 01:17:29 +0100
Subject: [PATCH 15/41] Add spi support for ade7953 (#5439)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 CODEOWNERS                                    |   3 +
 esphome/components/ade7953/__init__.py        |   1 +
 esphome/components/ade7953/ade7953.cpp        |  53 -----
 esphome/components/ade7953/ade7953.h          |  97 ---------
 esphome/components/ade7953/sensor.py          |  91 +-------
 esphome/components/ade7953_base/__init__.py   | 196 ++++++++++++++++++
 .../components/ade7953_base/ade7953_base.cpp  | 129 ++++++++++++
 .../components/ade7953_base/ade7953_base.h    | 121 +++++++++++
 esphome/components/ade7953_i2c/__init__.py    |   1 +
 .../components/ade7953_i2c/ade7953_i2c.cpp    |  80 +++++++
 esphome/components/ade7953_i2c/ade7953_i2c.h  |  28 +++
 esphome/components/ade7953_i2c/sensor.py      |  27 +++
 esphome/components/ade7953_spi/__init__.py    |   1 +
 .../components/ade7953_spi/ade7953_spi.cpp    |  81 ++++++++
 esphome/components/ade7953_spi/ade7953_spi.h  |  32 +++
 esphome/components/ade7953_spi/sensor.py      |  27 +++
 tests/test11.5.yaml                           |  63 ++++++
 tests/test3.1.yaml                            |  44 +++-
 18 files changed, 834 insertions(+), 241 deletions(-)
 delete mode 100644 esphome/components/ade7953/ade7953.cpp
 delete mode 100644 esphome/components/ade7953/ade7953.h
 create mode 100644 esphome/components/ade7953_base/__init__.py
 create mode 100644 esphome/components/ade7953_base/ade7953_base.cpp
 create mode 100644 esphome/components/ade7953_base/ade7953_base.h
 create mode 100644 esphome/components/ade7953_i2c/__init__.py
 create mode 100644 esphome/components/ade7953_i2c/ade7953_i2c.cpp
 create mode 100644 esphome/components/ade7953_i2c/ade7953_i2c.h
 create mode 100644 esphome/components/ade7953_i2c/sensor.py
 create mode 100644 esphome/components/ade7953_spi/__init__.py
 create mode 100644 esphome/components/ade7953_spi/ade7953_spi.cpp
 create mode 100644 esphome/components/ade7953_spi/ade7953_spi.h
 create mode 100644 esphome/components/ade7953_spi/sensor.py

diff --git a/CODEOWNERS b/CODEOWNERS
index 067b886320..2dcef6c514 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -17,6 +17,9 @@ esphome/components/ac_dimmer/* @glmnet
 esphome/components/adc/* @esphome/core
 esphome/components/adc128s102/* @DeerMaximum
 esphome/components/addressable_light/* @justfalter
+esphome/components/ade7953/* @angelnu
+esphome/components/ade7953_i2c/* @angelnu
+esphome/components/ade7953_spi/* @angelnu
 esphome/components/airthings_ble/* @jeromelaban
 esphome/components/airthings_wave_base/* @jeromelaban @kpfleming @ncareau
 esphome/components/airthings_wave_mini/* @ncareau
diff --git a/esphome/components/ade7953/__init__.py b/esphome/components/ade7953/__init__.py
index e69de29bb2..d3078a0b67 100644
--- a/esphome/components/ade7953/__init__.py
+++ b/esphome/components/ade7953/__init__.py
@@ -0,0 +1 @@
+CODEOWNERS = ["@angelnu"]
diff --git a/esphome/components/ade7953/ade7953.cpp b/esphome/components/ade7953/ade7953.cpp
deleted file mode 100644
index 2c61fc6a44..0000000000
--- a/esphome/components/ade7953/ade7953.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-#include "ade7953.h"
-#include "esphome/core/log.h"
-
-namespace esphome {
-namespace ade7953 {
-
-static const char *const TAG = "ade7953";
-
-void ADE7953::dump_config() {
-  ESP_LOGCONFIG(TAG, "ADE7953:");
-  LOG_PIN("  IRQ Pin: ", irq_pin_);
-  LOG_I2C_DEVICE(this);
-  LOG_UPDATE_INTERVAL(this);
-  LOG_SENSOR("  ", "Voltage Sensor", this->voltage_sensor_);
-  LOG_SENSOR("  ", "Current A Sensor", this->current_a_sensor_);
-  LOG_SENSOR("  ", "Current B Sensor", this->current_b_sensor_);
-  LOG_SENSOR("  ", "Active Power A Sensor", this->active_power_a_sensor_);
-  LOG_SENSOR("  ", "Active Power B Sensor", this->active_power_b_sensor_);
-}
-
-#define ADE_PUBLISH_(name, val, factor) \
-  if (err == i2c::ERROR_OK && this->name##_sensor_) { \
-    float value = (val) / (factor); \
-    this->name##_sensor_->publish_state(value); \
-  }
-#define ADE_PUBLISH(name, val, factor) ADE_PUBLISH_(name, val, factor)
-
-void ADE7953::update() {
-  if (!this->is_setup_)
-    return;
-
-  uint32_t val;
-  i2c::ErrorCode err = ade_read_32_(0x0312, &val);
-  ADE_PUBLISH(active_power_a, (int32_t) val, 154.0f);
-  err = ade_read_32_(0x0313, &val);
-  ADE_PUBLISH(active_power_b, (int32_t) val, 154.0f);
-  err = ade_read_32_(0x031A, &val);
-  ADE_PUBLISH(current_a, (uint32_t) val, 100000.0f);
-  err = ade_read_32_(0x031B, &val);
-  ADE_PUBLISH(current_b, (uint32_t) val, 100000.0f);
-  err = ade_read_32_(0x031C, &val);
-  ADE_PUBLISH(voltage, (uint32_t) val, 26000.0f);
-
-  //    auto apparent_power_a = this->ade_read_<int32_t>(0x0310);
-  //    auto apparent_power_b = this->ade_read_<int32_t>(0x0311);
-  //    auto reactive_power_a = this->ade_read_<int32_t>(0x0314);
-  //    auto reactive_power_b = this->ade_read_<int32_t>(0x0315);
-  //    auto power_factor_a = this->ade_read_<int16_t>(0x010A);
-  //    auto power_factor_b = this->ade_read_<int16_t>(0x010B);
-}
-
-}  // namespace ade7953
-}  // namespace esphome
diff --git a/esphome/components/ade7953/ade7953.h b/esphome/components/ade7953/ade7953.h
deleted file mode 100644
index c0c1cc4db8..0000000000
--- a/esphome/components/ade7953/ade7953.h
+++ /dev/null
@@ -1,97 +0,0 @@
-#pragma once
-
-#include "esphome/core/component.h"
-#include "esphome/core/hal.h"
-#include "esphome/components/i2c/i2c.h"
-#include "esphome/components/sensor/sensor.h"
-
-#include <vector>
-
-namespace esphome {
-namespace ade7953 {
-
-class ADE7953 : public i2c::I2CDevice, public PollingComponent {
- public:
-  void set_irq_pin(InternalGPIOPin *irq_pin) { irq_pin_ = irq_pin; }
-  void set_voltage_sensor(sensor::Sensor *voltage_sensor) { voltage_sensor_ = voltage_sensor; }
-  void set_current_a_sensor(sensor::Sensor *current_a_sensor) { current_a_sensor_ = current_a_sensor; }
-  void set_current_b_sensor(sensor::Sensor *current_b_sensor) { current_b_sensor_ = current_b_sensor; }
-  void set_active_power_a_sensor(sensor::Sensor *active_power_a_sensor) {
-    active_power_a_sensor_ = active_power_a_sensor;
-  }
-  void set_active_power_b_sensor(sensor::Sensor *active_power_b_sensor) {
-    active_power_b_sensor_ = active_power_b_sensor;
-  }
-
-  void setup() override {
-    if (this->irq_pin_ != nullptr) {
-      this->irq_pin_->setup();
-    }
-    this->set_timeout(100, [this]() {
-      this->ade_write_8_(0x0010, 0x04);
-      this->ade_write_8_(0x00FE, 0xAD);
-      this->ade_write_16_(0x0120, 0x0030);
-      this->is_setup_ = true;
-    });
-  }
-
-  void dump_config() override;
-
-  void update() override;
-
- protected:
-  i2c::ErrorCode ade_write_8_(uint16_t reg, uint8_t value) {
-    std::vector<uint8_t> data;
-    data.push_back(reg >> 8);
-    data.push_back(reg >> 0);
-    data.push_back(value);
-    return write(data.data(), data.size());
-  }
-  i2c::ErrorCode ade_write_16_(uint16_t reg, uint16_t value) {
-    std::vector<uint8_t> data;
-    data.push_back(reg >> 8);
-    data.push_back(reg >> 0);
-    data.push_back(value >> 8);
-    data.push_back(value >> 0);
-    return write(data.data(), data.size());
-  }
-  i2c::ErrorCode ade_write_32_(uint16_t reg, uint32_t value) {
-    std::vector<uint8_t> data;
-    data.push_back(reg >> 8);
-    data.push_back(reg >> 0);
-    data.push_back(value >> 24);
-    data.push_back(value >> 16);
-    data.push_back(value >> 8);
-    data.push_back(value >> 0);
-    return write(data.data(), data.size());
-  }
-  i2c::ErrorCode ade_read_32_(uint16_t reg, uint32_t *value) {
-    uint8_t reg_data[2];
-    reg_data[0] = reg >> 8;
-    reg_data[1] = reg >> 0;
-    i2c::ErrorCode err = write(reg_data, 2);
-    if (err != i2c::ERROR_OK)
-      return err;
-    uint8_t recv[4];
-    err = read(recv, 4);
-    if (err != i2c::ERROR_OK)
-      return err;
-    *value = 0;
-    *value |= ((uint32_t) recv[0]) << 24;
-    *value |= ((uint32_t) recv[1]) << 16;
-    *value |= ((uint32_t) recv[2]) << 8;
-    *value |= ((uint32_t) recv[3]);
-    return i2c::ERROR_OK;
-  }
-
-  InternalGPIOPin *irq_pin_{nullptr};
-  bool is_setup_{false};
-  sensor::Sensor *voltage_sensor_{nullptr};
-  sensor::Sensor *current_a_sensor_{nullptr};
-  sensor::Sensor *current_b_sensor_{nullptr};
-  sensor::Sensor *active_power_a_sensor_{nullptr};
-  sensor::Sensor *active_power_b_sensor_{nullptr};
-};
-
-}  // namespace ade7953
-}  // namespace esphome
diff --git a/esphome/components/ade7953/sensor.py b/esphome/components/ade7953/sensor.py
index 8a43baf475..0caa2ef454 100644
--- a/esphome/components/ade7953/sensor.py
+++ b/esphome/components/ade7953/sensor.py
@@ -1,90 +1,5 @@
-import esphome.codegen as cg
 import esphome.config_validation as cv
-from esphome.components import sensor, i2c
-from esphome import pins
-from esphome.const import (
-    CONF_ID,
-    CONF_IRQ_PIN,
-    CONF_VOLTAGE,
-    DEVICE_CLASS_CURRENT,
-    DEVICE_CLASS_POWER,
-    DEVICE_CLASS_VOLTAGE,
-    STATE_CLASS_MEASUREMENT,
-    UNIT_AMPERE,
-    UNIT_VOLT,
-    UNIT_WATT,
+
+CONFIG_SCHEMA = CONFIG_SCHEMA = cv.invalid(
+    "The ade7953 sensor component has been renamed to ade7953_i2c."
 )
-
-DEPENDENCIES = ["i2c"]
-
-ade7953_ns = cg.esphome_ns.namespace("ade7953")
-ADE7953 = ade7953_ns.class_("ADE7953", cg.PollingComponent, i2c.I2CDevice)
-
-CONF_CURRENT_A = "current_a"
-CONF_CURRENT_B = "current_b"
-CONF_ACTIVE_POWER_A = "active_power_a"
-CONF_ACTIVE_POWER_B = "active_power_b"
-
-CONFIG_SCHEMA = (
-    cv.Schema(
-        {
-            cv.GenerateID(): cv.declare_id(ADE7953),
-            cv.Optional(CONF_IRQ_PIN): pins.internal_gpio_input_pin_schema,
-            cv.Optional(CONF_VOLTAGE): sensor.sensor_schema(
-                unit_of_measurement=UNIT_VOLT,
-                accuracy_decimals=1,
-                device_class=DEVICE_CLASS_VOLTAGE,
-                state_class=STATE_CLASS_MEASUREMENT,
-            ),
-            cv.Optional(CONF_CURRENT_A): sensor.sensor_schema(
-                unit_of_measurement=UNIT_AMPERE,
-                accuracy_decimals=2,
-                device_class=DEVICE_CLASS_CURRENT,
-                state_class=STATE_CLASS_MEASUREMENT,
-            ),
-            cv.Optional(CONF_CURRENT_B): sensor.sensor_schema(
-                unit_of_measurement=UNIT_AMPERE,
-                accuracy_decimals=2,
-                device_class=DEVICE_CLASS_CURRENT,
-                state_class=STATE_CLASS_MEASUREMENT,
-            ),
-            cv.Optional(CONF_ACTIVE_POWER_A): sensor.sensor_schema(
-                unit_of_measurement=UNIT_WATT,
-                accuracy_decimals=1,
-                device_class=DEVICE_CLASS_POWER,
-                state_class=STATE_CLASS_MEASUREMENT,
-            ),
-            cv.Optional(CONF_ACTIVE_POWER_B): sensor.sensor_schema(
-                unit_of_measurement=UNIT_WATT,
-                accuracy_decimals=1,
-                device_class=DEVICE_CLASS_POWER,
-                state_class=STATE_CLASS_MEASUREMENT,
-            ),
-        }
-    )
-    .extend(cv.polling_component_schema("60s"))
-    .extend(i2c.i2c_device_schema(0x38))
-)
-
-
-async def to_code(config):
-    var = cg.new_Pvariable(config[CONF_ID])
-    await cg.register_component(var, config)
-    await i2c.register_i2c_device(var, config)
-
-    if irq_pin_config := config.get(CONF_IRQ_PIN):
-        irq_pin = await cg.gpio_pin_expression(irq_pin_config)
-        cg.add(var.set_irq_pin(irq_pin))
-
-    for key in [
-        CONF_VOLTAGE,
-        CONF_CURRENT_A,
-        CONF_CURRENT_B,
-        CONF_ACTIVE_POWER_A,
-        CONF_ACTIVE_POWER_B,
-    ]:
-        if key not in config:
-            continue
-        conf = config[key]
-        sens = await sensor.new_sensor(conf)
-        cg.add(getattr(var, f"set_{key}_sensor")(sens))
diff --git a/esphome/components/ade7953_base/__init__.py b/esphome/components/ade7953_base/__init__.py
new file mode 100644
index 0000000000..d4c18f8ffd
--- /dev/null
+++ b/esphome/components/ade7953_base/__init__.py
@@ -0,0 +1,196 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import sensor
+from esphome import pins
+from esphome.const import (
+    CONF_IRQ_PIN,
+    CONF_VOLTAGE,
+    CONF_FREQUENCY,
+    DEVICE_CLASS_CURRENT,
+    DEVICE_CLASS_APPARENT_POWER,
+    DEVICE_CLASS_POWER,
+    DEVICE_CLASS_REACTIVE_POWER,
+    DEVICE_CLASS_POWER_FACTOR,
+    DEVICE_CLASS_VOLTAGE,
+    DEVICE_CLASS_FREQUENCY,
+    STATE_CLASS_MEASUREMENT,
+    UNIT_VOLT,
+    UNIT_HERTZ,
+    UNIT_AMPERE,
+    UNIT_VOLT_AMPS,
+    UNIT_WATT,
+    UNIT_VOLT_AMPS_REACTIVE,
+    UNIT_PERCENT,
+)
+
+CONF_CURRENT_A = "current_a"
+CONF_CURRENT_B = "current_b"
+CONF_ACTIVE_POWER_A = "active_power_a"
+CONF_ACTIVE_POWER_B = "active_power_b"
+CONF_APPARENT_POWER_A = "apparent_power_a"
+CONF_APPARENT_POWER_B = "apparent_power_b"
+CONF_REACTIVE_POWER_A = "reactive_power_a"
+CONF_REACTIVE_POWER_B = "reactive_power_b"
+CONF_POWER_FACTOR_A = "power_factor_a"
+CONF_POWER_FACTOR_B = "power_factor_b"
+CONF_VOLTAGE_PGA_GAIN = "voltage_pga_gain"
+CONF_CURRENT_PGA_GAIN_A = "current_pga_gain_a"
+CONF_CURRENT_PGA_GAIN_B = "current_pga_gain_b"
+CONF_VOLTAGE_GAIN = "voltage_gain"
+CONF_CURRENT_GAIN_A = "current_gain_a"
+CONF_CURRENT_GAIN_B = "current_gain_b"
+CONF_ACTIVE_POWER_GAIN_A = "active_power_gain_a"
+CONF_ACTIVE_POWER_GAIN_B = "active_power_gain_b"
+PGA_GAINS = {
+    "1x": 0b000,
+    "2x": 0b001,
+    "4x": 0b010,
+    "8x": 0b011,
+    "16x": 0b100,
+    "22x": 0b101,
+}
+
+ade7953_base_ns = cg.esphome_ns.namespace("ade7953_base")
+ADE7953 = ade7953_base_ns.class_("ADE7953", cg.PollingComponent)
+
+ADE7953_CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.Optional(CONF_IRQ_PIN): pins.internal_gpio_input_pin_schema,
+        cv.Optional(CONF_VOLTAGE): sensor.sensor_schema(
+            unit_of_measurement=UNIT_VOLT,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_VOLTAGE,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_FREQUENCY): sensor.sensor_schema(
+            unit_of_measurement=UNIT_HERTZ,
+            accuracy_decimals=2,
+            device_class=DEVICE_CLASS_FREQUENCY,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_CURRENT_A): sensor.sensor_schema(
+            unit_of_measurement=UNIT_AMPERE,
+            accuracy_decimals=2,
+            device_class=DEVICE_CLASS_CURRENT,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_CURRENT_B): sensor.sensor_schema(
+            unit_of_measurement=UNIT_AMPERE,
+            accuracy_decimals=2,
+            device_class=DEVICE_CLASS_CURRENT,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_ACTIVE_POWER_A): sensor.sensor_schema(
+            unit_of_measurement=UNIT_WATT,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_ACTIVE_POWER_B): sensor.sensor_schema(
+            unit_of_measurement=UNIT_WATT,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_APPARENT_POWER_A): sensor.sensor_schema(
+            unit_of_measurement=UNIT_VOLT_AMPS,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_APPARENT_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_APPARENT_POWER_B): sensor.sensor_schema(
+            unit_of_measurement=UNIT_VOLT_AMPS,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_APPARENT_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_REACTIVE_POWER_A): sensor.sensor_schema(
+            unit_of_measurement=UNIT_VOLT_AMPS_REACTIVE,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_REACTIVE_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_REACTIVE_POWER_B): sensor.sensor_schema(
+            unit_of_measurement=UNIT_VOLT_AMPS_REACTIVE,
+            accuracy_decimals=1,
+            device_class=DEVICE_CLASS_REACTIVE_POWER,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_POWER_FACTOR_A): sensor.sensor_schema(
+            unit_of_measurement=UNIT_PERCENT,
+            accuracy_decimals=2,
+            device_class=DEVICE_CLASS_POWER_FACTOR,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(CONF_POWER_FACTOR_B): sensor.sensor_schema(
+            unit_of_measurement=UNIT_PERCENT,
+            accuracy_decimals=2,
+            device_class=DEVICE_CLASS_POWER_FACTOR,
+            state_class=STATE_CLASS_MEASUREMENT,
+        ),
+        cv.Optional(
+            CONF_VOLTAGE_PGA_GAIN,
+            default="1x",
+        ): cv.one_of(*PGA_GAINS, lower=True),
+        cv.Optional(
+            CONF_CURRENT_PGA_GAIN_A,
+            default="1x",
+        ): cv.one_of(*PGA_GAINS, lower=True),
+        cv.Optional(
+            CONF_CURRENT_PGA_GAIN_B,
+            default="1x",
+        ): cv.one_of(*PGA_GAINS, lower=True),
+        cv.Optional(CONF_VOLTAGE_GAIN, default=0x400000): cv.hex_int_range(
+            min=0x100000, max=0x800000
+        ),
+        cv.Optional(CONF_CURRENT_GAIN_A, default=0x400000): cv.hex_int_range(
+            min=0x100000, max=0x800000
+        ),
+        cv.Optional(CONF_CURRENT_GAIN_B, default=0x400000): cv.hex_int_range(
+            min=0x100000, max=0x800000
+        ),
+        cv.Optional(CONF_ACTIVE_POWER_GAIN_A, default=0x400000): cv.hex_int_range(
+            min=0x100000, max=0x800000
+        ),
+        cv.Optional(CONF_ACTIVE_POWER_GAIN_B, default=0x400000): cv.hex_int_range(
+            min=0x100000, max=0x800000
+        ),
+    }
+).extend(cv.polling_component_schema("60s"))
+
+
+async def register_ade7953(var, config):
+    await cg.register_component(var, config)
+
+    if irq_pin_config := config.get(CONF_IRQ_PIN):
+        irq_pin = await cg.gpio_pin_expression(irq_pin_config)
+        cg.add(var.set_irq_pin(irq_pin))
+
+    cg.add(var.set_pga_v(PGA_GAINS[config.get(CONF_VOLTAGE_PGA_GAIN)]))
+    cg.add(var.set_pga_ia(PGA_GAINS[config.get(CONF_CURRENT_PGA_GAIN_A)]))
+    cg.add(var.set_pga_ib(PGA_GAINS[config.get(CONF_CURRENT_PGA_GAIN_B)]))
+    cg.add(var.set_vgain(config.get(CONF_VOLTAGE_GAIN)))
+    cg.add(var.set_aigain(config.get(CONF_CURRENT_GAIN_A)))
+    cg.add(var.set_bigain(config.get(CONF_CURRENT_GAIN_B)))
+    cg.add(var.set_awgain(config.get(CONF_ACTIVE_POWER_GAIN_A)))
+    cg.add(var.set_bwgain(config.get(CONF_ACTIVE_POWER_GAIN_B)))
+
+    for key in [
+        CONF_VOLTAGE,
+        CONF_FREQUENCY,
+        CONF_CURRENT_A,
+        CONF_CURRENT_B,
+        CONF_POWER_FACTOR_A,
+        CONF_POWER_FACTOR_B,
+        CONF_APPARENT_POWER_A,
+        CONF_APPARENT_POWER_B,
+        CONF_ACTIVE_POWER_A,
+        CONF_ACTIVE_POWER_B,
+        CONF_REACTIVE_POWER_A,
+        CONF_REACTIVE_POWER_B,
+    ]:
+        if key not in config:
+            continue
+        conf = config[key]
+        sens = await sensor.new_sensor(conf)
+        cg.add(getattr(var, f"set_{key}_sensor")(sens))
diff --git a/esphome/components/ade7953_base/ade7953_base.cpp b/esphome/components/ade7953_base/ade7953_base.cpp
new file mode 100644
index 0000000000..af6fe0a5df
--- /dev/null
+++ b/esphome/components/ade7953_base/ade7953_base.cpp
@@ -0,0 +1,129 @@
+#include "ade7953_base.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace ade7953_base {
+
+static const char *const TAG = "ade7953";
+
+void ADE7953::setup() {
+  if (this->irq_pin_ != nullptr) {
+    this->irq_pin_->setup();
+  }
+
+  // The chip might take up to 100ms to initialise
+  this->set_timeout(100, [this]() {
+    // this->ade_write_8(0x0010, 0x04);
+    this->ade_write_8(0x00FE, 0xAD);
+    this->ade_write_16(0x0120, 0x0030);
+    // Set gains
+    this->ade_write_8(PGA_V_8, pga_v_);
+    this->ade_write_8(PGA_IA_8, pga_ia_);
+    this->ade_write_8(PGA_IB_8, pga_ib_);
+    this->ade_write_32(AVGAIN_32, vgain_);
+    this->ade_write_32(AIGAIN_32, aigain_);
+    this->ade_write_32(BIGAIN_32, bigain_);
+    this->ade_write_32(AWGAIN_32, awgain_);
+    this->ade_write_32(BWGAIN_32, bwgain_);
+    // Read back gains for debugging
+    this->ade_read_8(PGA_V_8, &pga_v_);
+    this->ade_read_8(PGA_IA_8, &pga_ia_);
+    this->ade_read_8(PGA_IB_8, &pga_ib_);
+    this->ade_read_32(AVGAIN_32, &vgain_);
+    this->ade_read_32(AIGAIN_32, &aigain_);
+    this->ade_read_32(BIGAIN_32, &bigain_);
+    this->ade_read_32(AWGAIN_32, &awgain_);
+    this->ade_read_32(BWGAIN_32, &bwgain_);
+    this->is_setup_ = true;
+  });
+}
+
+void ADE7953::dump_config() {
+  LOG_PIN("  IRQ Pin: ", irq_pin_);
+  LOG_UPDATE_INTERVAL(this);
+  LOG_SENSOR("  ", "Voltage Sensor", this->voltage_sensor_);
+  LOG_SENSOR("  ", "Current A Sensor", this->current_a_sensor_);
+  LOG_SENSOR("  ", "Current B Sensor", this->current_b_sensor_);
+  LOG_SENSOR("  ", "Power Factor A Sensor", this->power_factor_a_sensor_);
+  LOG_SENSOR("  ", "Power Factor B Sensor", this->power_factor_b_sensor_);
+  LOG_SENSOR("  ", "Apparent Power A Sensor", this->apparent_power_a_sensor_);
+  LOG_SENSOR("  ", "Apparent Power B Sensor", this->apparent_power_b_sensor_);
+  LOG_SENSOR("  ", "Active Power A Sensor", this->active_power_a_sensor_);
+  LOG_SENSOR("  ", "Active Power B Sensor", this->active_power_b_sensor_);
+  LOG_SENSOR("  ", "Rective Power A Sensor", this->reactive_power_a_sensor_);
+  LOG_SENSOR("  ", "Reactive Power B Sensor", this->reactive_power_b_sensor_);
+  ESP_LOGCONFIG(TAG, "  PGA_V_8: 0x%X", pga_v_);
+  ESP_LOGCONFIG(TAG, "  PGA_IA_8: 0x%X", pga_ia_);
+  ESP_LOGCONFIG(TAG, "  PGA_IB_8: 0x%X", pga_ib_);
+  ESP_LOGCONFIG(TAG, "  VGAIN_32: 0x%08jX", (uintmax_t) vgain_);
+  ESP_LOGCONFIG(TAG, "  AIGAIN_32: 0x%08jX", (uintmax_t) aigain_);
+  ESP_LOGCONFIG(TAG, "  BIGAIN_32: 0x%08jX", (uintmax_t) bigain_);
+  ESP_LOGCONFIG(TAG, "  AWGAIN_32: 0x%08jX", (uintmax_t) awgain_);
+  ESP_LOGCONFIG(TAG, "  BWGAIN_32: 0x%08jX", (uintmax_t) bwgain_);
+}
+
+#define ADE_PUBLISH_(name, val, factor) \
+  if (err == 0 && this->name##_sensor_) { \
+    float value = (val) / (factor); \
+    this->name##_sensor_->publish_state(value); \
+  }
+#define ADE_PUBLISH(name, val, factor) ADE_PUBLISH_(name, val, factor)
+
+void ADE7953::update() {
+  if (!this->is_setup_)
+    return;
+
+  bool err;
+
+  uint32_t interrupts_a = 0;
+  uint32_t interrupts_b = 0;
+  if (this->irq_pin_ != nullptr) {
+    // Read and reset interrupts
+    this->ade_read_32(0x032E, &interrupts_a);
+    this->ade_read_32(0x0331, &interrupts_b);
+  }
+
+  uint32_t val;
+  uint16_t val_16;
+
+  // Power factor
+  err = this->ade_read_16(0x010A, &val_16);
+  ADE_PUBLISH(power_factor_a, (int16_t) val_16, (0x7FFF / 100.0f));
+  err = this->ade_read_16(0x010B, &val_16);
+  ADE_PUBLISH(power_factor_b, (int16_t) val_16, (0x7FFF / 100.0f));
+
+  // Apparent power
+  err = this->ade_read_32(0x0310, &val);
+  ADE_PUBLISH(apparent_power_a, (int32_t) val, 154.0f);
+  err = this->ade_read_32(0x0311, &val);
+  ADE_PUBLISH(apparent_power_b, (int32_t) val, 154.0f);
+
+  // Active power
+  err = this->ade_read_32(0x0312, &val);
+  ADE_PUBLISH(active_power_a, (int32_t) val, 154.0f);
+  err = this->ade_read_32(0x0313, &val);
+  ADE_PUBLISH(active_power_b, (int32_t) val, 154.0f);
+
+  // Reactive power
+  err = this->ade_read_32(0x0314, &val);
+  ADE_PUBLISH(reactive_power_a, (int32_t) val, 154.0f);
+  err = this->ade_read_32(0x0315, &val);
+  ADE_PUBLISH(reactive_power_b, (int32_t) val, 154.0f);
+
+  // Current
+  err = this->ade_read_32(0x031A, &val);
+  ADE_PUBLISH(current_a, (uint32_t) val, 100000.0f);
+  err = this->ade_read_32(0x031B, &val);
+  ADE_PUBLISH(current_b, (uint32_t) val, 100000.0f);
+
+  // Voltage
+  err = this->ade_read_32(0x031C, &val);
+  ADE_PUBLISH(voltage, (uint32_t) val, 26000.0f);
+
+  // Frequency
+  err = this->ade_read_16(0x010E, &val_16);
+  ADE_PUBLISH(frequency, 223750.0f, 1 + val_16);
+}
+
+}  // namespace ade7953_base
+}  // namespace esphome
diff --git a/esphome/components/ade7953_base/ade7953_base.h b/esphome/components/ade7953_base/ade7953_base.h
new file mode 100644
index 0000000000..f57a8aa1df
--- /dev/null
+++ b/esphome/components/ade7953_base/ade7953_base.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+#include "esphome/components/sensor/sensor.h"
+
+#include <vector>
+
+namespace esphome {
+namespace ade7953_base {
+
+static const uint8_t PGA_V_8 =
+    0x007;  // PGA_V,  (R/W) Default: 0x00, Unsigned, Voltage channel gain configuration (Bits[2:0])
+static const uint8_t PGA_IA_8 =
+    0x008;  // PGA_IA, (R/W) Default: 0x00, Unsigned, Current Channel A gain configuration (Bits[2:0])
+static const uint8_t PGA_IB_8 =
+    0x009;  // PGA_IB, (R/W) Default: 0x00, Unsigned, Current Channel B gain configuration (Bits[2:0])
+
+static const uint32_t AIGAIN_32 =
+    0x380;  // AIGAIN, (R/W)   Default: 0x400000, Unsigned,Current channel gain (Current Channel A)(32 bit)
+static const uint32_t AVGAIN_32 = 0x381;  // AVGAIN, (R/W)   Default: 0x400000, Unsigned,Voltage channel gain(32 bit)
+static const uint32_t AWGAIN_32 =
+    0x382;  // AWGAIN, (R/W)   Default: 0x400000, Unsigned,Active power gain (Current Channel A)(32 bit)
+static const uint32_t AVARGAIN_32 =
+    0x383;  // AVARGAIN, (R/W) Default: 0x400000, Unsigned, Reactive power gain (Current Channel A)(32 bit)
+static const uint32_t AVAGAIN_32 =
+    0x384;  // AVAGAIN, (R/W)  Default: 0x400000, Unsigned,Apparent power gain (Current Channel A)(32 bit)
+
+static const uint32_t BIGAIN_32 =
+    0x38C;  // BIGAIN, (R/W)   Default: 0x400000, Unsigned,Current channel gain (Current Channel B)(32 bit)
+static const uint32_t BVGAIN_32 = 0x38D;  // BVGAIN, (R/W)   Default: 0x400000, Unsigned,Voltage channel gain(32 bit)
+static const uint32_t BWGAIN_32 =
+    0x38E;  // BWGAIN, (R/W)   Default: 0x400000, Unsigned,Active power gain (Current Channel B)(32 bit)
+static const uint32_t BVARGAIN_32 =
+    0x38F;  // BVARGAIN, (R/W) Default: 0x400000, Unsigned, Reactive power gain (Current Channel B)(32 bit)
+static const uint32_t BVAGAIN_32 =
+    0x390;  // BVAGAIN, (R/W)  Default: 0x400000, Unsigned,Apparent power gain (Current Channel B)(32 bit)
+
+class ADE7953 : public PollingComponent, public sensor::Sensor {
+ public:
+  void set_irq_pin(InternalGPIOPin *irq_pin) { irq_pin_ = irq_pin; }
+
+  // Set PGA input gains: 0 1x, 1 2x, 0b10 4x
+  void set_pga_v(uint8_t pga_v) { pga_v_ = pga_v; }
+  void set_pga_ia(uint8_t pga_ia) { pga_ia_ = pga_ia; }
+  void set_pga_ib(uint8_t pga_ib) { pga_ib_ = pga_ib; }
+
+  // Set input gains
+  void set_vgain(uint32_t vgain) { vgain_ = vgain; }
+  void set_aigain(uint32_t aigain) { aigain_ = aigain; }
+  void set_bigain(uint32_t bigain) { bigain_ = bigain; }
+  void set_awgain(uint32_t awgain) { awgain_ = awgain; }
+  void set_bwgain(uint32_t bwgain) { bwgain_ = bwgain; }
+
+  void set_voltage_sensor(sensor::Sensor *voltage_sensor) { voltage_sensor_ = voltage_sensor; }
+  void set_frequency_sensor(sensor::Sensor *frequency_sensor) { frequency_sensor_ = frequency_sensor; }
+
+  void set_power_factor_a_sensor(sensor::Sensor *power_factor_a) { power_factor_a_sensor_ = power_factor_a; }
+  void set_power_factor_b_sensor(sensor::Sensor *power_factor_b) { power_factor_b_sensor_ = power_factor_b; }
+
+  void set_current_a_sensor(sensor::Sensor *current_a_sensor) { current_a_sensor_ = current_a_sensor; }
+  void set_current_b_sensor(sensor::Sensor *current_b_sensor) { current_b_sensor_ = current_b_sensor; }
+
+  void set_apparent_power_a_sensor(sensor::Sensor *apparent_power_a) { apparent_power_a_sensor_ = apparent_power_a; }
+  void set_apparent_power_b_sensor(sensor::Sensor *apparent_power_b) { apparent_power_b_sensor_ = apparent_power_b; }
+
+  void set_active_power_a_sensor(sensor::Sensor *active_power_a_sensor) {
+    active_power_a_sensor_ = active_power_a_sensor;
+  }
+  void set_active_power_b_sensor(sensor::Sensor *active_power_b_sensor) {
+    active_power_b_sensor_ = active_power_b_sensor;
+  }
+
+  void set_reactive_power_a_sensor(sensor::Sensor *reactive_power_a) { reactive_power_a_sensor_ = reactive_power_a; }
+  void set_reactive_power_b_sensor(sensor::Sensor *reactive_power_b) { reactive_power_b_sensor_ = reactive_power_b; }
+
+  void setup() override;
+
+  void dump_config() override;
+
+  void update() override;
+
+ protected:
+  InternalGPIOPin *irq_pin_{nullptr};
+  bool is_setup_{false};
+  sensor::Sensor *voltage_sensor_{nullptr};
+  sensor::Sensor *frequency_sensor_{nullptr};
+  sensor::Sensor *current_a_sensor_{nullptr};
+  sensor::Sensor *current_b_sensor_{nullptr};
+  sensor::Sensor *apparent_power_a_sensor_{nullptr};
+  sensor::Sensor *apparent_power_b_sensor_{nullptr};
+  sensor::Sensor *active_power_a_sensor_{nullptr};
+  sensor::Sensor *active_power_b_sensor_{nullptr};
+  sensor::Sensor *reactive_power_a_sensor_{nullptr};
+  sensor::Sensor *reactive_power_b_sensor_{nullptr};
+  sensor::Sensor *power_factor_a_sensor_{nullptr};
+  sensor::Sensor *power_factor_b_sensor_{nullptr};
+  uint8_t pga_v_;
+  uint8_t pga_ia_;
+  uint8_t pga_ib_;
+  uint32_t vgain_;
+  uint32_t aigain_;
+  uint32_t bigain_;
+  uint32_t awgain_;
+  uint32_t bwgain_;
+
+  virtual bool ade_write_8(uint16_t reg, uint8_t value) = 0;
+
+  virtual bool ade_write_16(uint16_t reg, uint16_t value) = 0;
+
+  virtual bool ade_write_32(uint16_t reg, uint32_t value) = 0;
+
+  virtual bool ade_read_8(uint16_t reg, uint8_t *value) = 0;
+
+  virtual bool ade_read_16(uint16_t reg, uint16_t *value) = 0;
+
+  virtual bool ade_read_32(uint16_t reg, uint32_t *value) = 0;
+};
+
+}  // namespace ade7953_base
+}  // namespace esphome
diff --git a/esphome/components/ade7953_i2c/__init__.py b/esphome/components/ade7953_i2c/__init__.py
new file mode 100644
index 0000000000..d3078a0b67
--- /dev/null
+++ b/esphome/components/ade7953_i2c/__init__.py
@@ -0,0 +1 @@
+CODEOWNERS = ["@angelnu"]
diff --git a/esphome/components/ade7953_i2c/ade7953_i2c.cpp b/esphome/components/ade7953_i2c/ade7953_i2c.cpp
new file mode 100644
index 0000000000..572337428a
--- /dev/null
+++ b/esphome/components/ade7953_i2c/ade7953_i2c.cpp
@@ -0,0 +1,80 @@
+#include "ade7953_i2c.h"
+#include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
+
+namespace esphome {
+namespace ade7953_i2c {
+
+static const char *const TAG = "ade7953";
+
+void AdE7953I2c::dump_config() {
+  ESP_LOGCONFIG(TAG, "ADE7953_i2c:");
+  LOG_I2C_DEVICE(this);
+  ade7953_base::ADE7953::dump_config();
+}
+bool AdE7953I2c::ade_write_8(uint16_t reg, uint8_t value) {
+  std::vector<uint8_t> data(3);
+  data.push_back(reg >> 8);
+  data.push_back(reg >> 0);
+  data.push_back(value);
+  return this->write(data.data(), data.size()) != i2c::ERROR_OK;
+}
+bool AdE7953I2c::ade_write_16(uint16_t reg, uint16_t value) {
+  std::vector<uint8_t> data(4);
+  data.push_back(reg >> 8);
+  data.push_back(reg >> 0);
+  data.push_back(value >> 8);
+  data.push_back(value >> 0);
+  return this->write(data.data(), data.size()) != i2c::ERROR_OK;
+}
+bool AdE7953I2c::ade_write_32(uint16_t reg, uint32_t value) {
+  std::vector<uint8_t> data(6);
+  data.push_back(reg >> 8);
+  data.push_back(reg >> 0);
+  data.push_back(value >> 24);
+  data.push_back(value >> 16);
+  data.push_back(value >> 8);
+  data.push_back(value >> 0);
+  return this->write(data.data(), data.size()) != i2c::ERROR_OK;
+}
+bool AdE7953I2c::ade_read_8(uint16_t reg, uint8_t *value) {
+  uint8_t reg_data[2];
+  reg_data[0] = reg >> 8;
+  reg_data[1] = reg >> 0;
+  i2c::ErrorCode err = this->write(reg_data, 2);
+  if (err != i2c::ERROR_OK)
+    return true;
+  err = this->read(value, 1);
+  return (err != i2c::ERROR_OK);
+}
+bool AdE7953I2c::ade_read_16(uint16_t reg, uint16_t *value) {
+  uint8_t reg_data[2];
+  reg_data[0] = reg >> 8;
+  reg_data[1] = reg >> 0;
+  i2c::ErrorCode err = this->write(reg_data, 2);
+  if (err != i2c::ERROR_OK)
+    return true;
+  uint8_t recv[2];
+  err = this->read(recv, 2);
+  if (err != i2c::ERROR_OK)
+    return true;
+  *value = encode_uint16(recv[0], recv[1]);
+  return false;
+}
+bool AdE7953I2c::ade_read_32(uint16_t reg, uint32_t *value) {
+  uint8_t reg_data[2];
+  reg_data[0] = reg >> 8;
+  reg_data[1] = reg >> 0;
+  i2c::ErrorCode err = this->write(reg_data, 2);
+  if (err != i2c::ERROR_OK)
+    return true;
+  uint8_t recv[4];
+  err = this->read(recv, 4);
+  if (err != i2c::ERROR_OK)
+    return true;
+  *value = encode_uint32(recv[0], recv[1], recv[2], recv[3]);
+  return false;
+}
+
+}  // namespace ade7953_i2c
+}  // namespace esphome
diff --git a/esphome/components/ade7953_i2c/ade7953_i2c.h b/esphome/components/ade7953_i2c/ade7953_i2c.h
new file mode 100644
index 0000000000..65dc30dddb
--- /dev/null
+++ b/esphome/components/ade7953_i2c/ade7953_i2c.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+#include "esphome/components/i2c/i2c.h"
+#include "esphome/components/sensor/sensor.h"
+#include "esphome/components/ade7953_base/ade7953_base.h"
+
+#include <vector>
+
+namespace esphome {
+namespace ade7953_i2c {
+
+class AdE7953I2c : public ade7953_base::ADE7953, public i2c::I2CDevice {
+ public:
+  void dump_config() override;
+
+ protected:
+  bool ade_write_8(uint16_t reg, uint8_t value) override;
+  bool ade_write_16(uint16_t reg, uint16_t value) override;
+  bool ade_write_32(uint16_t reg, uint32_t value) override;
+  bool ade_read_8(uint16_t reg, uint8_t *value) override;
+  bool ade_read_16(uint16_t reg, uint16_t *value) override;
+  bool ade_read_32(uint16_t reg, uint32_t *value) override;
+};
+
+}  // namespace ade7953_i2c
+}  // namespace esphome
diff --git a/esphome/components/ade7953_i2c/sensor.py b/esphome/components/ade7953_i2c/sensor.py
new file mode 100644
index 0000000000..e52a44eced
--- /dev/null
+++ b/esphome/components/ade7953_i2c/sensor.py
@@ -0,0 +1,27 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import i2c, ade7953_base
+from esphome.const import CONF_ID
+
+
+DEPENDENCIES = ["i2c"]
+AUTO_LOAD = ["ade7953_base"]
+
+ade7953_ns = cg.esphome_ns.namespace("ade7953_i2c")
+ADE7953 = ade7953_ns.class_("AdE7953I2c", ade7953_base.ADE7953, i2c.I2CDevice)
+
+CONFIG_SCHEMA = (
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(ADE7953),
+        }
+    )
+    .extend(ade7953_base.ADE7953_CONFIG_SCHEMA)
+    .extend(i2c.i2c_device_schema(0x38))
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await i2c.register_i2c_device(var, config)
+    await ade7953_base.register_ade7953(var, config)
diff --git a/esphome/components/ade7953_spi/__init__.py b/esphome/components/ade7953_spi/__init__.py
new file mode 100644
index 0000000000..d3078a0b67
--- /dev/null
+++ b/esphome/components/ade7953_spi/__init__.py
@@ -0,0 +1 @@
+CODEOWNERS = ["@angelnu"]
diff --git a/esphome/components/ade7953_spi/ade7953_spi.cpp b/esphome/components/ade7953_spi/ade7953_spi.cpp
new file mode 100644
index 0000000000..cfd5d71d0a
--- /dev/null
+++ b/esphome/components/ade7953_spi/ade7953_spi.cpp
@@ -0,0 +1,81 @@
+#include "ade7953_spi.h"
+#include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
+
+namespace esphome {
+namespace ade7953_spi {
+
+static const char *const TAG = "ade7953";
+
+void AdE7953Spi::setup() {
+  this->spi_setup();
+  ade7953_base::ADE7953::setup();
+}
+
+void AdE7953Spi::dump_config() {
+  ESP_LOGCONFIG(TAG, "ADE7953_spi:");
+  LOG_PIN("  CS Pin: ", this->cs_);
+  ade7953_base::ADE7953::dump_config();
+}
+
+bool AdE7953Spi::ade_write_8(uint16_t reg, uint8_t value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0);
+  this->transfer_byte(value);
+  this->disable();
+  return false;
+}
+
+bool AdE7953Spi::ade_write_16(uint16_t reg, uint16_t value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0);
+  this->write_byte16(value);
+  this->disable();
+  return false;
+}
+
+bool AdE7953Spi::ade_write_32(uint16_t reg, uint32_t value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0);
+  this->write_byte16(value >> 16);
+  this->write_byte16(value & 0xFFFF);
+  this->disable();
+  return false;
+}
+
+bool AdE7953Spi::ade_read_8(uint16_t reg, uint8_t *value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0x80);
+  *value = this->read_byte();
+  this->disable();
+  return false;
+}
+
+bool AdE7953Spi::ade_read_16(uint16_t reg, uint16_t *value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0x80);
+  uint8_t recv[2];
+  this->read_array(recv, 4);
+  *value = encode_uint16(recv[0], recv[1]);
+  this->disable();
+  return false;
+}
+
+bool AdE7953Spi::ade_read_32(uint16_t reg, uint32_t *value) {
+  this->enable();
+  this->write_byte16(reg);
+  this->transfer_byte(0x80);
+  uint8_t recv[4];
+  this->read_array(recv, 4);
+  *value = encode_uint32(recv[0], recv[1], recv[2], recv[3]);
+  this->disable();
+  return false;
+}
+
+}  // namespace ade7953_spi
+}  // namespace esphome
diff --git a/esphome/components/ade7953_spi/ade7953_spi.h b/esphome/components/ade7953_spi/ade7953_spi.h
new file mode 100644
index 0000000000..d96852b9bb
--- /dev/null
+++ b/esphome/components/ade7953_spi/ade7953_spi.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+#include "esphome/components/spi/spi.h"
+#include "esphome/components/sensor/sensor.h"
+#include "esphome/components/ade7953_base/ade7953_base.h"
+
+#include <vector>
+
+namespace esphome {
+namespace ade7953_spi {
+
+class AdE7953Spi : public ade7953_base::ADE7953,
+                   public spi::SPIDevice<spi::BIT_ORDER_MSB_FIRST, spi::CLOCK_POLARITY_HIGH, spi::CLOCK_PHASE_LEADING,
+                                         spi::DATA_RATE_1MHZ> {
+ public:
+  void setup() override;
+
+  void dump_config() override;
+
+ protected:
+  bool ade_write_8(uint16_t reg, uint8_t value) override;
+  bool ade_write_16(uint16_t reg, uint16_t value) override;
+  bool ade_write_32(uint16_t reg, uint32_t value) override;
+  bool ade_read_8(uint16_t reg, uint8_t *value) override;
+  bool ade_read_16(uint16_t reg, uint16_t *value) override;
+  bool ade_read_32(uint16_t reg, uint32_t *value) override;
+};
+
+}  // namespace ade7953_spi
+}  // namespace esphome
diff --git a/esphome/components/ade7953_spi/sensor.py b/esphome/components/ade7953_spi/sensor.py
new file mode 100644
index 0000000000..5f9682c711
--- /dev/null
+++ b/esphome/components/ade7953_spi/sensor.py
@@ -0,0 +1,27 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import spi, ade7953_base
+from esphome.const import CONF_ID
+
+
+DEPENDENCIES = ["spi"]
+AUTO_LOAD = ["ade7953_base"]
+
+ade7953_ns = cg.esphome_ns.namespace("ade7953_spi")
+ADE7953 = ade7953_ns.class_("AdE7953Spi", ade7953_base.ADE7953, spi.SPIDevice)
+
+CONFIG_SCHEMA = (
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(ADE7953),
+        }
+    )
+    .extend(ade7953_base.ADE7953_CONFIG_SCHEMA)
+    .extend(spi.spi_device_schema())
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await spi.register_spi_device(var, config)
+    await ade7953_base.register_ade7953(var, config)
diff --git a/tests/test11.5.yaml b/tests/test11.5.yaml
index 9b25b9f273..685487e871 100644
--- a/tests/test11.5.yaml
+++ b/tests/test11.5.yaml
@@ -51,6 +51,15 @@ uart:
 i2c:
   frequency: 100khz
 
+spi:
+  - id: spi_1
+    clk_pin: 12
+    mosi_pin: 13
+    miso_pin: 14
+  - id: spi_2
+    clk_pin: 32
+    mosi_pin: 33
+
 modbus:
   uart_id: uart_1
   flow_control_pin: 5
@@ -574,6 +583,60 @@ sensor:
     temperature:
       name: Kuntze temperature
 
+  - platform: ade7953_i2c
+    irq_pin: 16
+    voltage:
+      name: ADE7953 Voltage
+    current_a:
+      name: ADE7953 Current A
+    current_b:
+      name: ADE7953 Current B
+    power_factor_a:
+      name: "ADE7953 Power Factor A"
+    power_factor_b:
+      name: "ADE7953 Power Factor B"
+    apparent_power_a:
+      name: "ADE7953 Apparent Power A"
+    apparent_power_b:
+      name: "ADE7953 Apparent Power B"
+    active_power_a:
+      name: ADE7953 Active Power A
+    active_power_b:
+      name: ADE7953 Active Power B
+    reactive_power_a:
+      name: "ADE7953 Reactive Power A"
+    reactive_power_b:
+      name: "ADE7953 Reactive Power B"
+    update_interval: 1s
+
+  - platform: ade7953_spi
+    spi_id: spi_1
+    cs_pin: 04
+    irq_pin: 16
+    voltage:
+      name: ADE7953 Voltage
+    current_a:
+      name: ADE7953 Current A
+    current_b:
+      name: ADE7953 Current B
+    power_factor_a:
+      name: "ADE7953 Power Factor A"
+    power_factor_b:
+      name: "ADE7953 Power Factor B"
+    apparent_power_a:
+      name: "ADE7953 Apparent Power A"
+    apparent_power_b:
+      name: "ADE7953 Apparent Power B"
+    active_power_a:
+      name: ADE7953 Active Power A
+    active_power_b:
+      name: ADE7953 Active Power B
+    reactive_power_a:
+      name: "ADE7953 Reactive Power A"
+    reactive_power_b:
+      name: "ADE7953 Reactive Power B"
+    update_interval: 1s
+
 script:
   - id: automation_test
     then:
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 8884479f61..151e53fd62 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -169,7 +169,7 @@ sensor:
       - id: custom_sensor
         name: Custom Sensor
 
-  - platform: ade7953
+  - platform: ade7953_i2c
     irq_pin: GPIO16
     voltage:
       name: ADE7953 Voltage
@@ -180,12 +180,50 @@ sensor:
     current_b:
       name: ADE7953 Current B
       id: ade7953_current_b
+    power_factor_a:
+      name: "ADE7953 Power Factor A"
+    power_factor_b:
+      name: "ADE7953 Power Factor B"
+    apparent_power_a:
+      name: "ADE7953 Apparent Power A"
+    apparent_power_b:
+      name: "ADE7953 Apparent Power B"
     active_power_a:
       name: ADE7953 Active Power A
-      id: ade7953_active_power_a
     active_power_b:
       name: ADE7953 Active Power B
-      id: ade7953_active_power_b
+    reactive_power_a:
+      name: "ADE7953 Reactive Power A"
+    reactive_power_b:
+      name: "ADE7953 Reactive Power B"
+    update_interval: 1s
+
+  - platform: ade7953_spi
+    cs_pin: GPIO04
+    irq_pin: GPIO16
+    voltage:
+      name: ADE7953 Voltage
+    current_a:
+      name: ADE7953 Current A
+    current_b:
+      name: ADE7953 Current B
+    power_factor_a:
+      name: "ADE7953 Power Factor A"
+    power_factor_b:
+      name: "ADE7953 Power Factor B"
+    apparent_power_a:
+      name: "ADE7953 Apparent Power A"
+    apparent_power_b:
+      name: "ADE7953 Apparent Power B"
+    active_power_a:
+      name: ADE7953 Active Power A
+    active_power_b:
+      name: ADE7953 Active Power B
+    reactive_power_a:
+      name: "ADE7953 Reactive Power A"
+    reactive_power_b:
+      name: "ADE7953 Reactive Power B"
+    update_interval: 1s
 
   - platform: tmp102
     name: TMP102 Temperature

From a8a9c6192da1cc48f3488861e82b16bf084f1ebf Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Tue, 7 Nov 2023 01:48:15 +0100
Subject: [PATCH 16/41] Remove page jump on Nextion startup (#5673)

---
 esphome/components/nextion/base_component.py  |  1 +
 esphome/components/nextion/display.py         |  5 ++++
 esphome/components/nextion/nextion.cpp        | 10 ++++++-
 esphome/components/nextion/nextion.h          | 28 +++++++++++++++++++
 .../components/nextion/nextion_commands.cpp   |  3 ++
 5 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/esphome/components/nextion/base_component.py b/esphome/components/nextion/base_component.py
index b2a857c888..f1c3a1d227 100644
--- a/esphome/components/nextion/base_component.py
+++ b/esphome/components/nextion/base_component.py
@@ -21,6 +21,7 @@ CONF_ON_SETUP = "on_setup"
 CONF_ON_PAGE = "on_page"
 CONF_TOUCH_SLEEP_TIMEOUT = "touch_sleep_timeout"
 CONF_WAKE_UP_PAGE = "wake_up_page"
+CONF_START_UP_PAGE = "start_up_page"
 CONF_AUTO_WAKE_ON_TOUCH = "auto_wake_on_touch"
 CONF_WAVE_MAX_LENGTH = "wave_max_length"
 CONF_BACKGROUND_COLOR = "background_color"
diff --git a/esphome/components/nextion/display.py b/esphome/components/nextion/display.py
index 72f56bd6f3..afb64ceeea 100644
--- a/esphome/components/nextion/display.py
+++ b/esphome/components/nextion/display.py
@@ -18,6 +18,7 @@ from .base_component import (
     CONF_TFT_URL,
     CONF_TOUCH_SLEEP_TIMEOUT,
     CONF_WAKE_UP_PAGE,
+    CONF_START_UP_PAGE,
     CONF_AUTO_WAKE_ON_TOUCH,
 )
 
@@ -59,6 +60,7 @@ CONFIG_SCHEMA = (
             ),
             cv.Optional(CONF_TOUCH_SLEEP_TIMEOUT): cv.int_range(min=3, max=65535),
             cv.Optional(CONF_WAKE_UP_PAGE): cv.positive_int,
+            cv.Optional(CONF_START_UP_PAGE): cv.positive_int,
             cv.Optional(CONF_AUTO_WAKE_ON_TOUCH, default=True): cv.boolean,
         }
     )
@@ -95,6 +97,9 @@ async def to_code(config):
     if CONF_WAKE_UP_PAGE in config:
         cg.add(var.set_wake_up_page_internal(config[CONF_WAKE_UP_PAGE]))
 
+    if CONF_START_UP_PAGE in config:
+        cg.add(var.set_start_up_page_internal(config[CONF_START_UP_PAGE]))
+
     if CONF_AUTO_WAKE_ON_TOUCH in config:
         cg.add(var.set_auto_wake_on_touch_internal(config[CONF_AUTO_WAKE_ON_TOUCH]))
 
diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp
index 78c07be890..76bdb283f6 100644
--- a/esphome/components/nextion/nextion.cpp
+++ b/esphome/components/nextion/nextion.cpp
@@ -134,6 +134,10 @@ void Nextion::dump_config() {
   if (this->wake_up_page_ != -1) {
     ESP_LOGCONFIG(TAG, "  Wake Up Page :       %d", this->wake_up_page_);
   }
+
+  if (this->start_up_page_ != -1) {
+    ESP_LOGCONFIG(TAG, "  Start Up Page :      %d", this->start_up_page_);
+  }
 }
 
 float Nextion::get_setup_priority() const { return setup_priority::DATA; }
@@ -231,7 +235,11 @@ void Nextion::loop() {
     this->send_command_("bkcmd=3");  // Always, returns 0x00 to 0x23 result of serial command.
 
     this->set_backlight_brightness(this->brightness_);
-    this->goto_page("0");
+
+    // Check if a startup page has been set and send the command
+    if (this->start_up_page_ != -1) {
+      this->goto_page(this->start_up_page_);
+    }
 
     this->set_auto_wake_on_touch(this->auto_wake_on_touch_);
 
diff --git a/esphome/components/nextion/nextion.h b/esphome/components/nextion/nextion.h
index 92ff3fe235..7b5641b711 100644
--- a/esphome/components/nextion/nextion.h
+++ b/esphome/components/nextion/nextion.h
@@ -334,6 +334,18 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
    * Switches to the page named `main`. Pages are named in the Nextion Editor.
    */
   void goto_page(const char *page);
+  /**
+   * Show the page with a given id.
+   * @param page The id of the page.
+   *
+   * Example:
+   * ```cpp
+   * it.goto_page(2);
+   * ```
+   *
+   * Switches to the page named `main`. Pages are named in the Nextion Editor.
+   */
+  void goto_page(uint8_t page);
   /**
    * Hide a component.
    * @param component The component name.
@@ -605,6 +617,20 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
    * The display will wake up to page 2.
    */
   void set_wake_up_page(uint8_t page_id = 255);
+  /**
+   * Sets which page Nextion loads when connecting to ESPHome.
+   * @param page_id The page id, from 0 to the lage page in Nextion. Set 255 (not set to any existing page) to
+   * wakes up to current page.
+   *
+   * Example:
+   * ```cpp
+   * it.set_start_up_page(2);
+   * ```
+   *
+   * The display will go to page 2 when it establishes a connection to ESPHome.
+   */
+  void set_start_up_page(uint8_t page_id = 255);
+
   /**
    * Sets if Nextion should auto-wake from sleep when touch press occurs.
    * @param auto_wake True or false. When auto_wake is true and Nextion is in sleep mode,
@@ -736,6 +762,7 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
     this->touch_sleep_timeout_ = touch_sleep_timeout;
   }
   void set_wake_up_page_internal(uint8_t wake_up_page) { this->wake_up_page_ = wake_up_page; }
+  void set_start_up_page_internal(uint8_t start_up_page) { this->start_up_page_ = start_up_page; }
   void set_auto_wake_on_touch_internal(bool auto_wake_on_touch) { this->auto_wake_on_touch_ = auto_wake_on_touch; }
 
  protected:
@@ -758,6 +785,7 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
   bool is_updating_ = false;
   uint32_t touch_sleep_timeout_ = 0;
   int wake_up_page_ = -1;
+  int start_up_page_ = -1;
   bool auto_wake_on_touch_ = true;
 
   /**
diff --git a/esphome/components/nextion/nextion_commands.cpp b/esphome/components/nextion/nextion_commands.cpp
index c4caf29287..a3157c731a 100644
--- a/esphome/components/nextion/nextion_commands.cpp
+++ b/esphome/components/nextion/nextion_commands.cpp
@@ -13,6 +13,8 @@ void Nextion::set_wake_up_page(uint8_t page_id) {
   this->add_no_result_to_queue_with_set_internal_("wake_up_page", "wup", page_id, true);
 }
 
+void Nextion::set_start_up_page(uint8_t page_id) { this->start_up_page_ = page_id; }
+
 void Nextion::set_touch_sleep_timeout(uint16_t timeout) {
   if (timeout < 3) {
     ESP_LOGD(TAG, "Sleep timeout out of bounds, range 3-65535");
@@ -124,6 +126,7 @@ void Nextion::set_component_text_printf(const char *component, const char *forma
 
 // General Nextion
 void Nextion::goto_page(const char *page) { this->add_no_result_to_queue_with_printf_("goto_page", "page %s", page); }
+void Nextion::goto_page(uint8_t page) { this->add_no_result_to_queue_with_printf_("goto_page", "page %i", page); }
 
 void Nextion::set_backlight_brightness(float brightness) {
   if (brightness < 0 || brightness > 1.0) {

From 708ed8f38a6f9c0e71f903af9c9314b7f0a89310 Mon Sep 17 00:00:00 2001
From: Daniel Baulig <me@danielbaulig.de>
Date: Mon, 6 Nov 2023 16:59:03 -0800
Subject: [PATCH 17/41] [web_server] Adds the ability to handle Private Network
 Access preflight requests (#5669)

---
 esphome/components/web_server/__init__.py     |  4 ++
 esphome/components/web_server/web_server.cpp  | 37 +++++++++++++++++++
 esphome/components/web_server/web_server.h    |  5 +++
 .../web_server_idf/web_server_idf.cpp         | 14 +++++++
 .../web_server_idf/web_server_idf.h           |  3 ++
 esphome/const.py                              |  1 +
 6 files changed, 64 insertions(+)

diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py
index b8698438e2..2708b5d06e 100644
--- a/esphome/components/web_server/__init__.py
+++ b/esphome/components/web_server/__init__.py
@@ -9,6 +9,7 @@ from esphome.const import (
     CONF_ID,
     CONF_JS_INCLUDE,
     CONF_JS_URL,
+    CONF_ENABLE_PRIVATE_NETWORK_ACCESS,
     CONF_PORT,
     CONF_AUTH,
     CONF_USERNAME,
@@ -68,6 +69,7 @@ CONFIG_SCHEMA = cv.All(
             cv.Optional(CONF_CSS_INCLUDE): cv.file_,
             cv.Optional(CONF_JS_URL): cv.string,
             cv.Optional(CONF_JS_INCLUDE): cv.file_,
+            cv.Optional(CONF_ENABLE_PRIVATE_NETWORK_ACCESS, default=True): cv.boolean,
             cv.Optional(CONF_AUTH): cv.Schema(
                 {
                     cv.Required(CONF_USERNAME): cv.All(
@@ -158,6 +160,8 @@ async def to_code(config):
         cg.add(var.set_js_url(config[CONF_JS_URL]))
     cg.add(var.set_allow_ota(config[CONF_OTA]))
     cg.add(var.set_expose_log(config[CONF_LOG]))
+    if config[CONF_ENABLE_PRIVATE_NETWORK_ACCESS]:
+        cg.add_define("USE_WEBSERVER_PRIVATE_NETWORK_ACCESS")
     if CONF_AUTH in config:
         cg.add(paren.set_auth_username(config[CONF_AUTH][CONF_USERNAME]))
         cg.add(paren.set_auth_password(config[CONF_AUTH][CONF_PASSWORD]))
diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp
index ccc86e5e53..0d72e274cd 100644
--- a/esphome/components/web_server/web_server.cpp
+++ b/esphome/components/web_server/web_server.cpp
@@ -34,6 +34,13 @@ namespace web_server {
 
 static const char *const TAG = "web_server";
 
+#ifdef USE_WEBSERVER_PRIVATE_NETWORK_ACCESS
+static const char *const HEADER_PNA_NAME = "Private-Network-Access-Name";
+static const char *const HEADER_PNA_ID = "Private-Network-Access-ID";
+static const char *const HEADER_CORS_REQ_PNA = "Access-Control-Request-Private-Network";
+static const char *const HEADER_CORS_ALLOW_PNA = "Access-Control-Allow-Private-Network";
+#endif
+
 #if USE_WEBSERVER_VERSION == 1
 void write_row(AsyncResponseStream *stream, EntityBase *obj, const std::string &klass, const std::string &action,
                const std::function<void(AsyncResponseStream &stream, EntityBase *obj)> &action_func = nullptr) {
@@ -359,6 +366,17 @@ void WebServer::handle_index_request(AsyncWebServerRequest *request) {
 }
 #endif
 
+#ifdef USE_WEBSERVER_PRIVATE_NETWORK_ACCESS
+void WebServer::handle_pna_cors_request(AsyncWebServerRequest *request) {
+  AsyncWebServerResponse *response = request->beginResponse(200, "");
+  response->addHeader(HEADER_CORS_ALLOW_PNA, "true");
+  response->addHeader(HEADER_PNA_NAME, App.get_name().c_str());
+  std::string mac = get_mac_address_pretty();
+  response->addHeader(HEADER_PNA_ID, mac.c_str());
+  request->send(response);
+}
+#endif
+
 #ifdef USE_WEBSERVER_CSS_INCLUDE
 void WebServer::handle_css_request(AsyncWebServerRequest *request) {
   AsyncWebServerResponse *response =
@@ -1145,6 +1163,18 @@ bool WebServer::canHandle(AsyncWebServerRequest *request) {
     return true;
 #endif
 
+#ifdef USE_WEBSERVER_PRIVATE_NETWORK_ACCESS
+  if (request->method() == HTTP_OPTIONS && request->hasHeader(HEADER_CORS_REQ_PNA)) {
+#ifdef USE_ARDUINO
+    // Header needs to be added to interesting header list for it to not be
+    // nuked by the time we handle the request later.
+    // Only required in Arduino framework.
+    request->addInterestingHeader(HEADER_CORS_REQ_PNA);
+#endif
+    return true;
+  }
+#endif
+
   UrlMatch match = match_url(request->url().c_str(), true);
   if (!match.valid)
     return false;
@@ -1240,6 +1270,13 @@ void WebServer::handleRequest(AsyncWebServerRequest *request) {
   }
 #endif
 
+#ifdef USE_WEBSERVER_PRIVATE_NETWORK_ACCESS
+  if (request->method() == HTTP_OPTIONS && request->hasHeader(HEADER_CORS_REQ_PNA)) {
+    this->handle_pna_cors_request(request);
+    return;
+  }
+#endif
+
   UrlMatch match = match_url(request->url().c_str());
 #ifdef USE_SENSOR
   if (match.domain == "sensor") {
diff --git a/esphome/components/web_server/web_server.h b/esphome/components/web_server/web_server.h
index 45b99d4eba..465e231984 100644
--- a/esphome/components/web_server/web_server.h
+++ b/esphome/components/web_server/web_server.h
@@ -130,6 +130,11 @@ class WebServer : public Controller, public Component, public AsyncWebHandler {
   void handle_js_request(AsyncWebServerRequest *request);
 #endif
 
+#ifdef USE_WEBSERVER_PRIVATE_NETWORK_ACCESS
+  // Handle Private Network Access CORS OPTIONS request
+  void handle_pna_cors_request(AsyncWebServerRequest *request);
+#endif
+
 #ifdef USE_SENSOR
   void on_sensor_update(sensor::Sensor *obj, float state) override;
   /// Handle a sensor request under '/sensor/<id>'.
diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp
index 444e682460..8e67f3f169 100644
--- a/esphome/components/web_server_idf/web_server_idf.cpp
+++ b/esphome/components/web_server_idf/web_server_idf.cpp
@@ -51,6 +51,14 @@ void AsyncWebServer::begin() {
         .user_ctx = this,
     };
     httpd_register_uri_handler(this->server_, &handler_post);
+
+    const httpd_uri_t handler_options = {
+        .uri = "",
+        .method = HTTP_OPTIONS,
+        .handler = AsyncWebServer::request_handler,
+        .user_ctx = this,
+    };
+    httpd_register_uri_handler(this->server_, &handler_options);
   }
 }
 
@@ -80,6 +88,8 @@ AsyncWebServerRequest::~AsyncWebServerRequest() {
   }
 }
 
+bool AsyncWebServerRequest::hasHeader(const char *name) const { return httpd_req_get_hdr_value_len(*this, name); }
+
 optional<std::string> AsyncWebServerRequest::get_header(const char *name) const {
   size_t buf_len = httpd_req_get_hdr_value_len(*this, name);
   if (buf_len == 0) {
@@ -305,6 +315,10 @@ AsyncEventSourceResponse::AsyncEventSourceResponse(const AsyncWebServerRequest *
   httpd_resp_set_hdr(req, "Cache-Control", "no-cache");
   httpd_resp_set_hdr(req, "Connection", "keep-alive");
 
+  for (const auto &pair : DefaultHeaders::Instance().headers_) {
+    httpd_resp_set_hdr(req, pair.first.c_str(), pair.second.c_str());
+  }
+
   httpd_resp_send_chunk(req, CRLF_STR, CRLF_LEN);
 
   req->sess_ctx = this;
diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h
index f3cecca16f..bc64e5231e 100644
--- a/esphome/components/web_server_idf/web_server_idf.h
+++ b/esphome/components/web_server_idf/web_server_idf.h
@@ -157,6 +157,8 @@ class AsyncWebServerRequest {
 
   operator httpd_req_t *() const { return this->req_; }
   optional<std::string> get_header(const char *name) const;
+  // NOLINTNEXTLINE(readability-identifier-naming)
+  bool hasHeader(const char *name) const;
 
  protected:
   httpd_req_t *req_;
@@ -254,6 +256,7 @@ class AsyncEventSource : public AsyncWebHandler {
 
 class DefaultHeaders {
   friend class AsyncWebServerRequest;
+  friend class AsyncEventSourceResponse;
 
  public:
   // NOLINTNEXTLINE(readability-identifier-naming)
diff --git a/esphome/const.py b/esphome/const.py
index 9457958863..c2fa9951ff 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -227,6 +227,7 @@ CONF_ELSE = "else"
 CONF_ENABLE_BTM = "enable_btm"
 CONF_ENABLE_IPV6 = "enable_ipv6"
 CONF_ENABLE_PIN = "enable_pin"
+CONF_ENABLE_PRIVATE_NETWORK_ACCESS = "enable_private_network_access"
 CONF_ENABLE_RRM = "enable_rrm"
 CONF_ENABLE_TIME = "enable_time"
 CONF_ENERGY = "energy"

From 888c298d7e73d5d3a670e640a0bcb0b0c40d9830 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:59:26 +1300
Subject: [PATCH 18/41] Update esphome-dashboard to version 20231107.0 (#5686)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2747258883..4e9a1c3b0e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@ pyserial==3.5
 platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
-esphome-dashboard==20230904.0
+esphome-dashboard==20231107.0
 aioesphomeapi==18.2.1
 zeroconf==0.120.0
 

From 31fec2d6920f677f18b07b5280359850648891fc Mon Sep 17 00:00:00 2001
From: Charles Johnson <32775248+ChemicalXandco@users.noreply.github.com>
Date: Tue, 7 Nov 2023 05:17:13 +0000
Subject: [PATCH 19/41] add wifi.on_connect and wifi.on_disconnect triggers
 (#3639)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/wifi/__init__.py        | 20 +++++++++++++++++++-
 esphome/components/wifi/wifi_component.cpp | 10 ++++++++++
 esphome/components/wifi/wifi_component.h   | 15 +++++++++++----
 tests/test1.yaml                           |  4 ++++
 4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py
index 86ce53b804..c42835f169 100644
--- a/esphome/components/wifi/__init__.py
+++ b/esphome/components/wifi/__init__.py
@@ -32,6 +32,8 @@ from esphome.const import (
     CONF_KEY,
     CONF_USERNAME,
     CONF_EAP,
+    CONF_ON_CONNECT,
+    CONF_ON_DISCONNECT,
 )
 from esphome.core import CORE, HexInt, coroutine_with_priority
 from esphome.components.esp32 import add_idf_sdkconfig_option, get_esp32_variant, const
@@ -306,6 +308,10 @@ CONFIG_SCHEMA = cv.All(
                 "new mdns component instead."
             ),
             cv.Optional(CONF_ENABLE_ON_BOOT, default=True): cv.boolean,
+            cv.Optional(CONF_ON_CONNECT): automation.validate_automation(single=True),
+            cv.Optional(CONF_ON_DISCONNECT): automation.validate_automation(
+                single=True
+            ),
         }
     ),
     _validate,
@@ -425,9 +431,21 @@ async def to_code(config):
 
     cg.add_define("USE_WIFI")
 
-    # Register at end for OTA safe mode
+    # must register before OTA safe mode check
     await cg.register_component(var, config)
 
+    await cg.past_safe_mode()
+
+    if on_connect_config := config.get(CONF_ON_CONNECT):
+        await automation.build_automation(
+            var.get_connect_trigger(), [], on_connect_config
+        )
+
+    if on_disconnect_config := config.get(CONF_ON_DISCONNECT):
+        await automation.build_automation(
+            var.get_disconnect_trigger(), [], on_disconnect_config
+        )
+
 
 @automation.register_condition("wifi.connected", WiFiConnectedCondition, cv.Schema({}))
 async def wifi_connected_to_code(config, condition_id, template_arg, args):
diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp
index b08f20de21..95d430de4f 100644
--- a/esphome/components/wifi/wifi_component.cpp
+++ b/esphome/components/wifi/wifi_component.cpp
@@ -138,9 +138,19 @@ void WiFiComponent::loop() {
           ESP_LOGW(TAG, "WiFi Connection lost... Reconnecting...");
           this->state_ = WIFI_COMPONENT_STATE_STA_CONNECTING;
           this->retry_connect();
+
+          if (this->handled_connected_state_) {
+            this->disconnect_trigger_->trigger();
+            this->handled_connected_state_ = false;
+          }
         } else {
           this->status_clear_warning();
           this->last_connected_ = now;
+
+          if (!this->handled_connected_state_) {
+            this->connect_trigger_->trigger();
+            this->handled_connected_state_ = true;
+          }
         }
         break;
       }
diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h
index b418a5b353..3ee69bb5de 100644
--- a/esphome/components/wifi/wifi_component.h
+++ b/esphome/components/wifi/wifi_component.h
@@ -1,18 +1,18 @@
 #pragma once
 
+#include "esphome/components/network/ip_address.h"
+#include "esphome/core/automation.h"
 #include "esphome/core/component.h"
 #include "esphome/core/defines.h"
-#include "esphome/core/automation.h"
 #include "esphome/core/helpers.h"
-#include "esphome/components/network/ip_address.h"
 
 #include <string>
 #include <vector>
 
 #ifdef USE_ESP32_FRAMEWORK_ARDUINO
-#include <esp_wifi.h>
-#include <WiFiType.h>
 #include <WiFi.h>
+#include <WiFiType.h>
+#include <esp_wifi.h>
 #endif
 
 #ifdef USE_LIBRETINY
@@ -294,6 +294,9 @@ class WiFiComponent : public Component {
 
   void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; }
 
+  Trigger<> *get_connect_trigger() const { return this->connect_trigger_; };
+  Trigger<> *get_disconnect_trigger() const { return this->disconnect_trigger_; };
+
  protected:
   static std::string format_mac_addr(const uint8_t mac[6]);
   void setup_ap_config_();
@@ -354,6 +357,7 @@ class WiFiComponent : public Component {
   bool has_ap_{false};
   WiFiAP ap_;
   WiFiComponentState state_{WIFI_COMPONENT_STATE_OFF};
+  bool handled_connected_state_{false};
   uint32_t action_started_;
   uint8_t num_retried_{0};
   uint32_t last_connected_{0};
@@ -373,6 +377,9 @@ class WiFiComponent : public Component {
   bool rrm_{false};
 #endif
   bool enable_on_boot_;
+
+  Trigger<> *connect_trigger_{new Trigger<>()};
+  Trigger<> *disconnect_trigger_{new Trigger<>()};
 };
 
 extern WiFiComponent *global_wifi_component;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
diff --git a/tests/test1.yaml b/tests/test1.yaml
index f653960c40..6d1136eb1b 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -73,6 +73,10 @@ wifi:
   domain: .local
   reboot_timeout: 120s
   power_save_mode: light
+  on_connect:
+    - light.turn_on: ${roomname}_lights
+  on_disconnect:
+    - light.turn_off: ${roomname}_lights
 
 network:
   enable_ipv6: true

From 78e3ce7718677e56d04823684f7cf20a006d6817 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 7 Nov 2023 00:04:55 -0600
Subject: [PATCH 20/41] Implement a memory cache for dashboard entries to avoid
 frequent disk reads (#5687)

---
 esphome/dashboard/dashboard.py | 92 ++++++++++++++++++++++++++++++++--
 1 file changed, 87 insertions(+), 5 deletions(-)

diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py
index b416b00e60..119094de28 100644
--- a/esphome/dashboard/dashboard.py
+++ b/esphome/dashboard/dashboard.py
@@ -67,6 +67,9 @@ class DashboardSettings:
         self.on_ha_addon = False
         self.cookie_secret = None
         self.absolute_config_dir = None
+        self._entry_cache: dict[
+            str, tuple[tuple[int, int, float, int], DashboardEntry]
+        ] = {}
 
     def parse_args(self, args):
         self.on_ha_addon = args.ha_addon
@@ -121,9 +124,70 @@ class DashboardSettings:
         Path(joined_path).resolve().relative_to(self.absolute_config_dir)
         return joined_path
 
-    def list_yaml_files(self):
+    def list_yaml_files(self) -> list[str]:
         return util.list_yaml_files([self.config_dir])
 
+    def entries(self) -> list[DashboardEntry]:
+        """Fetch all dashboard entries, thread-safe."""
+        path_to_cache_key: dict[str, tuple[int, int, float, int]] = {}
+        #
+        # The cache key is (inode, device, mtime, size)
+        # which allows us to avoid locking since it ensures
+        # every iteration of this call will always return the newest
+        # items from disk at the cost of a stat() call on each
+        # file which is much faster than reading the file
+        # for the cache hit case which is the common case.
+        #
+        # Because there is no lock the cache may
+        # get built more than once but that's fine as its still
+        # thread-safe and results in orders of magnitude less
+        # reads from disk than if we did not cache at all and
+        # does not have a lock contention issue.
+        #
+        for file in self.list_yaml_files():
+            try:
+                # Prefer the json storage path if it exists
+                stat = os.stat(ext_storage_path(os.path.basename(file)))
+            except OSError:
+                try:
+                    # Fallback to the yaml file if the storage
+                    # file does not exist or could not be generated
+                    stat = os.stat(file)
+                except OSError:
+                    # File was deleted, ignore
+                    continue
+            path_to_cache_key[file] = (
+                stat.st_ino,
+                stat.st_dev,
+                stat.st_mtime,
+                stat.st_size,
+            )
+
+        entry_cache = self._entry_cache
+
+        # Remove entries that no longer exist
+        removed: list[str] = []
+        for file in entry_cache:
+            if file not in path_to_cache_key:
+                removed.append(file)
+
+        for file in removed:
+            entry_cache.pop(file)
+
+        dashboard_entries: list[DashboardEntry] = []
+        for file, cache_key in path_to_cache_key.items():
+            if cached_entry := entry_cache.get(file):
+                entry_key, dashboard_entry = cached_entry
+                if entry_key == cache_key:
+                    dashboard_entries.append(dashboard_entry)
+                    continue
+
+            dashboard_entry = DashboardEntry(file)
+            dashboard_entries.append(dashboard_entry)
+            entry_cache[file] = (cache_key, dashboard_entry)
+
+        return dashboard_entries
+
 
 settings = DashboardSettings()
 
@@ -657,18 +721,26 @@ class EsphomeVersionHandler(BaseHandler):
         self.finish()
 
 
-def _list_dashboard_entries():
-    files = settings.list_yaml_files()
-    return [DashboardEntry(file) for file in files]
+def _list_dashboard_entries() -> list[DashboardEntry]:
+    return settings.entries()
 
 
 class DashboardEntry:
-    def __init__(self, path):
+    """Represents a single dashboard entry.
+
+    This class is thread-safe and read-only.
+    """
+
+    __slots__ = ("path", "_storage", "_loaded_storage")
+
+    def __init__(self, path: str) -> None:
+        """Initialize the DashboardEntry."""
         self.path = path
         self._storage = None
         self._loaded_storage = False
 
     def __repr__(self):
+        """Return the representation of this entry."""
         return (
             f"DashboardEntry({self.path} "
             f"address={self.address} "
@@ -679,10 +751,12 @@ class DashboardEntry:
 
     @property
     def filename(self):
+        """Return the filename of this entry."""
         return os.path.basename(self.path)
 
     @property
     def storage(self) -> StorageJSON | None:
+        """Return the StorageJSON object for this entry."""
         if not self._loaded_storage:
             self._storage = StorageJSON.load(ext_storage_path(self.filename))
             self._loaded_storage = True
@@ -690,48 +764,56 @@ class DashboardEntry:
 
     @property
     def address(self):
+        """Return the address of this entry."""
         if self.storage is None:
             return None
         return self.storage.address
 
     @property
     def no_mdns(self):
+        """Return the no_mdns of this entry."""
         if self.storage is None:
             return None
         return self.storage.no_mdns
 
     @property
     def web_port(self):
+        """Return the web port of this entry."""
         if self.storage is None:
             return None
         return self.storage.web_port
 
     @property
     def name(self):
+        """Return the name of this entry."""
         if self.storage is None:
             return self.filename.replace(".yml", "").replace(".yaml", "")
         return self.storage.name
 
     @property
     def friendly_name(self):
+        """Return the friendly name of this entry."""
         if self.storage is None:
             return self.name
         return self.storage.friendly_name
 
     @property
     def comment(self):
+        """Return the comment of this entry."""
         if self.storage is None:
             return None
         return self.storage.comment
 
     @property
     def target_platform(self):
+        """Return the target platform of this entry."""
         if self.storage is None:
             return None
         return self.storage.target_platform
 
     @property
     def update_available(self):
+        """Return if an update is available for this entry."""
         if self.storage is None:
             return True
         return self.update_old != self.update_new

From ccffbfd3ae8be5f535701e427de58a935af58f1d Mon Sep 17 00:00:00 2001
From: Angel Nunez Mencias <git@angelnu.com>
Date: Tue, 7 Nov 2023 10:15:13 +0100
Subject: [PATCH 21/41] support spi for sn74hc595 (#5491)

Co-authored-by: Jimmy Hedman <jimmy.hedman@gmail.com>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/sn74hc595/__init__.py   | 65 +++++++++++++++-----
 esphome/components/sn74hc595/sn74hc595.cpp | 70 +++++++++++++++-------
 esphome/components/sn74hc595/sn74hc595.h   | 54 +++++++++++++----
 tests/test1.yaml                           |  8 ++-
 4 files changed, 149 insertions(+), 48 deletions(-)

diff --git a/esphome/components/sn74hc595/__init__.py b/esphome/components/sn74hc595/__init__.py
index e98da72304..e7ba45175c 100644
--- a/esphome/components/sn74hc595/__init__.py
+++ b/esphome/components/sn74hc595/__init__.py
@@ -1,8 +1,10 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome import pins
+from esphome.components import spi
 from esphome.const import (
     CONF_ID,
+    CONF_SPI_ID,
     CONF_MODE,
     CONF_NUMBER,
     CONF_INVERTED,
@@ -10,13 +12,20 @@ from esphome.const import (
     CONF_CLOCK_PIN,
     CONF_OUTPUT,
 )
+from esphome.core import EsphomeError
 
-DEPENDENCIES = []
 MULTI_CONF = True
 
 sn74hc595_ns = cg.esphome_ns.namespace("sn74hc595")
 
 SN74HC595Component = sn74hc595_ns.class_("SN74HC595Component", cg.Component)
+SN74HC595GPIOComponent = sn74hc595_ns.class_(
+    "SN74HC595GPIOComponent", SN74HC595Component
+)
+SN74HC595SPIComponent = sn74hc595_ns.class_(
+    "SN74HC595SPIComponent", SN74HC595Component, spi.SPIDevice
+)
+
 SN74HC595GPIOPin = sn74hc595_ns.class_(
     "SN74HC595GPIOPin", cg.GPIOPin, cg.Parented.template(SN74HC595Component)
 )
@@ -25,25 +34,51 @@ CONF_SN74HC595 = "sn74hc595"
 CONF_LATCH_PIN = "latch_pin"
 CONF_OE_PIN = "oe_pin"
 CONF_SR_COUNT = "sr_count"
-CONFIG_SCHEMA = cv.Schema(
-    {
-        cv.Required(CONF_ID): cv.declare_id(SN74HC595Component),
-        cv.Required(CONF_DATA_PIN): pins.gpio_output_pin_schema,
-        cv.Required(CONF_CLOCK_PIN): pins.gpio_output_pin_schema,
-        cv.Required(CONF_LATCH_PIN): pins.gpio_output_pin_schema,
-        cv.Optional(CONF_OE_PIN): pins.gpio_output_pin_schema,
-        cv.Optional(CONF_SR_COUNT, default=1): cv.int_range(min=1, max=256),
-    }
-).extend(cv.COMPONENT_SCHEMA)
+
+
+CONFIG_SCHEMA = cv.Any(
+    cv.Schema(
+        {
+            cv.Required(CONF_ID): cv.declare_id(SN74HC595GPIOComponent),
+            cv.Required(CONF_DATA_PIN): pins.gpio_output_pin_schema,
+            cv.Required(CONF_CLOCK_PIN): pins.gpio_output_pin_schema,
+            cv.Required(CONF_LATCH_PIN): pins.gpio_output_pin_schema,
+            cv.Optional(CONF_OE_PIN): pins.gpio_output_pin_schema,
+            cv.Optional(CONF_SR_COUNT, default=1): cv.int_range(min=1, max=256),
+        }
+    ).extend(cv.COMPONENT_SCHEMA),
+    cv.Schema(
+        {
+            cv.Required(CONF_ID): cv.declare_id(SN74HC595SPIComponent),
+            cv.Required(CONF_LATCH_PIN): pins.gpio_output_pin_schema,
+            cv.Optional(CONF_OE_PIN): pins.gpio_output_pin_schema,
+            cv.Optional(CONF_SR_COUNT, default=1): cv.int_range(min=1, max=256),
+        }
+    )
+    .extend(cv.COMPONENT_SCHEMA)
+    .extend(spi.spi_device_schema(cs_pin_required=False))
+    .extend(
+        {
+            cv.Required(CONF_SPI_ID): cv.use_id(spi.SPIComponent),
+        }
+    ),
+    msg='Either "data_pin" and "clock_pin" must be set or "spi_id" must be set.',
+)
 
 
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
-    data_pin = await cg.gpio_pin_expression(config[CONF_DATA_PIN])
-    cg.add(var.set_data_pin(data_pin))
-    clock_pin = await cg.gpio_pin_expression(config[CONF_CLOCK_PIN])
-    cg.add(var.set_clock_pin(clock_pin))
+    if CONF_DATA_PIN in config:
+        data_pin = await cg.gpio_pin_expression(config[CONF_DATA_PIN])
+        cg.add(var.set_data_pin(data_pin))
+        clock_pin = await cg.gpio_pin_expression(config[CONF_CLOCK_PIN])
+        cg.add(var.set_clock_pin(clock_pin))
+    elif CONF_SPI_ID in config:
+        await spi.register_spi_device(var, config)
+    else:
+        raise EsphomeError("Not supported")
+
     latch_pin = await cg.gpio_pin_expression(config[CONF_LATCH_PIN])
     cg.add(var.set_latch_pin(latch_pin))
     if CONF_OE_PIN in config:
diff --git a/esphome/components/sn74hc595/sn74hc595.cpp b/esphome/components/sn74hc595/sn74hc595.cpp
index 1895b1d5a6..8a37c3bece 100644
--- a/esphome/components/sn74hc595/sn74hc595.cpp
+++ b/esphome/components/sn74hc595/sn74hc595.cpp
@@ -6,26 +6,39 @@ namespace sn74hc595 {
 
 static const char *const TAG = "sn74hc595";
 
-void SN74HC595Component::setup() {
+void SN74HC595Component::pre_setup_() {
   ESP_LOGCONFIG(TAG, "Setting up SN74HC595...");
 
   if (this->have_oe_pin_) {  // disable output
     this->oe_pin_->setup();
     this->oe_pin_->digital_write(true);
   }
-
-  // initialize output pins
-  this->clock_pin_->setup();
-  this->data_pin_->setup();
+}
+void SN74HC595Component::post_setup_() {
   this->latch_pin_->setup();
-  this->clock_pin_->digital_write(false);
-  this->data_pin_->digital_write(false);
   this->latch_pin_->digital_write(false);
 
   // send state to shift register
-  this->write_gpio_();
+  this->write_gpio();
 }
 
+void SN74HC595GPIOComponent::setup() {
+  this->pre_setup_();
+  this->clock_pin_->setup();
+  this->data_pin_->setup();
+  this->clock_pin_->digital_write(false);
+  this->data_pin_->digital_write(false);
+  this->post_setup_();
+}
+
+#ifdef USE_SPI
+void SN74HC595SPIComponent::setup() {
+  this->pre_setup_();
+  this->spi_setup();
+  this->post_setup_();
+}
+#endif
+
 void SN74HC595Component::dump_config() { ESP_LOGCONFIG(TAG, "SN74HC595:"); }
 
 void SN74HC595Component::digital_write_(uint16_t pin, bool value) {
@@ -34,17 +47,38 @@ void SN74HC595Component::digital_write_(uint16_t pin, bool value) {
              (this->sr_count_ * 8) - 1);
     return;
   }
-  this->output_bits_[pin] = value;
-  this->write_gpio_();
+  if (value) {
+    this->output_bytes_[pin / 8] |= (1 << (pin % 8));
+  } else {
+    this->output_bytes_[pin / 8] &= ~(1 << (pin % 8));
+  }
+  this->write_gpio();
 }
 
-void SN74HC595Component::write_gpio_() {
-  for (auto bit = this->output_bits_.rbegin(); bit != this->output_bits_.rend(); bit++) {
-    this->data_pin_->digital_write(*bit);
-    this->clock_pin_->digital_write(true);
-    this->clock_pin_->digital_write(false);
+void SN74HC595GPIOComponent::write_gpio() {
+  for (auto byte = this->output_bytes_.rbegin(); byte != this->output_bytes_.rend(); byte++) {
+    for (int8_t i = 7; i >= 0; i--) {
+      bool bit = (*byte >> i) & 1;
+      this->data_pin_->digital_write(bit);
+      this->clock_pin_->digital_write(true);
+      this->clock_pin_->digital_write(false);
+    }
   }
+  SN74HC595Component::write_gpio();
+}
 
+#ifdef USE_SPI
+void SN74HC595SPIComponent::write_gpio() {
+  for (auto byte = this->output_bytes_.rbegin(); byte != this->output_bytes_.rend(); byte++) {
+    this->enable();
+    this->transfer_byte(*byte);
+    this->disable();
+  }
+  SN74HC595Component::write_gpio();
+}
+#endif
+
+void SN74HC595Component::write_gpio() {
   // pulse latch to activate new values
   this->latch_pin_->digital_write(true);
   this->latch_pin_->digital_write(false);
@@ -60,11 +94,7 @@ float SN74HC595Component::get_setup_priority() const { return setup_priority::IO
 void SN74HC595GPIOPin::digital_write(bool value) {
   this->parent_->digital_write_(this->pin_, value != this->inverted_);
 }
-std::string SN74HC595GPIOPin::dump_summary() const {
-  char buffer[32];
-  snprintf(buffer, sizeof(buffer), "%u via SN74HC595", pin_);
-  return buffer;
-}
+std::string SN74HC595GPIOPin::dump_summary() const { return str_snprintf("%u via SN74HC595", 18, pin_); }
 
 }  // namespace sn74hc595
 }  // namespace esphome
diff --git a/esphome/components/sn74hc595/sn74hc595.h b/esphome/components/sn74hc595/sn74hc595.h
index 64bf06d881..cb9d7bf140 100644
--- a/esphome/components/sn74hc595/sn74hc595.h
+++ b/esphome/components/sn74hc595/sn74hc595.h
@@ -1,9 +1,14 @@
 #pragma once
 
 #include "esphome/core/component.h"
+#include "esphome/core/defines.h"
 #include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 
+#ifdef USE_SPI
+#include "esphome/components/spi/spi.h"
+#endif
+
 #include <vector>
 
 namespace esphome {
@@ -13,34 +18,33 @@ class SN74HC595Component : public Component {
  public:
   SN74HC595Component() = default;
 
-  void setup() override;
+  void setup() override = 0;
   float get_setup_priority() const override;
   void dump_config() override;
 
-  void set_data_pin(GPIOPin *pin) { data_pin_ = pin; }
-  void set_clock_pin(GPIOPin *pin) { clock_pin_ = pin; }
-  void set_latch_pin(GPIOPin *pin) { latch_pin_ = pin; }
+  void set_latch_pin(GPIOPin *pin) { this->latch_pin_ = pin; }
   void set_oe_pin(GPIOPin *pin) {
-    oe_pin_ = pin;
-    have_oe_pin_ = true;
+    this->oe_pin_ = pin;
+    this->have_oe_pin_ = true;
   }
   void set_sr_count(uint8_t count) {
-    sr_count_ = count;
-    this->output_bits_.resize(count * 8);
+    this->sr_count_ = count;
+    this->output_bytes_.resize(count);
   }
 
  protected:
   friend class SN74HC595GPIOPin;
   void digital_write_(uint16_t pin, bool value);
-  void write_gpio_();
+  virtual void write_gpio();
+
+  void pre_setup_();
+  void post_setup_();
 
-  GPIOPin *data_pin_;
-  GPIOPin *clock_pin_;
   GPIOPin *latch_pin_;
   GPIOPin *oe_pin_;
   uint8_t sr_count_;
   bool have_oe_pin_{false};
-  std::vector<bool> output_bits_;
+  std::vector<uint8_t> output_bytes_;
 };
 
 /// Helper class to expose a SC74HC595 pin as an internal output GPIO pin.
@@ -60,5 +64,31 @@ class SN74HC595GPIOPin : public GPIOPin, public Parented<SN74HC595Component> {
   bool inverted_;
 };
 
+class SN74HC595GPIOComponent : public SN74HC595Component {
+ public:
+  void setup() override;
+  void set_data_pin(GPIOPin *pin) { data_pin_ = pin; }
+  void set_clock_pin(GPIOPin *pin) { clock_pin_ = pin; }
+
+ protected:
+  void write_gpio() override;
+
+  GPIOPin *data_pin_;
+  GPIOPin *clock_pin_;
+};
+
+#ifdef USE_SPI
+class SN74HC595SPIComponent : public SN74HC595Component,
+                              public spi::SPIDevice<spi::BIT_ORDER_MSB_FIRST, spi::CLOCK_POLARITY_LOW,
+                                                    spi::CLOCK_PHASE_LEADING, spi::DATA_RATE_4MHZ> {
+ public:
+  void setup() override;
+
+ protected:
+  void write_gpio() override;
+};
+
+#endif
+
 }  // namespace sn74hc595
 }  // namespace esphome
diff --git a/tests/test1.yaml b/tests/test1.yaml
index 6d1136eb1b..32e92440c1 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -189,6 +189,7 @@ i2c:
   id: i2c_bus
 
 spi:
+  id: spi_bus
   clk_pin: GPIO21
   mosi_pin: GPIO22
   miso_pin: GPIO23
@@ -2737,7 +2738,7 @@ switch:
   - platform: gpio
     name: "SN74HC595 Pin #0"
     pin:
-      sn74hc595: sn74hc595_hub
+      sn74hc595: sn74hc595_hub_2
       # Use pin number 0
       number: 0
       inverted: false
@@ -3385,6 +3386,11 @@ sn74hc595:
     latch_pin: GPIO22
     oe_pin: GPIO32
     sr_count: 2
+  - id: sn74hc595_hub_2
+    latch_pin: GPIO22
+    oe_pin: GPIO32
+    sr_count: 2
+    spi_id: spi_bus
 
 rtttl:
   output: gpio_19

From 9bd4b229e3cdc5d599c4b2612b744c85494fcfad Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Tue, 7 Nov 2023 12:00:36 -0600
Subject: [PATCH 22/41] Handle on_disconnect when Wi-Fi is disabled (#5691)

---
 esphome/components/wifi/wifi_component.cpp | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp
index 95d430de4f..cd87bb48a7 100644
--- a/esphome/components/wifi/wifi_component.cpp
+++ b/esphome/components/wifi/wifi_component.cpp
@@ -109,6 +109,15 @@ void WiFiComponent::loop() {
   const uint32_t now = millis();
 
   if (this->has_sta()) {
+    if (this->is_connected() != this->handled_connected_state_) {
+      if (this->handled_connected_state_) {
+        this->disconnect_trigger_->trigger();
+      } else {
+        this->connect_trigger_->trigger();
+      }
+      this->handled_connected_state_ = this->is_connected();
+    }
+
     switch (this->state_) {
       case WIFI_COMPONENT_STATE_COOLDOWN: {
         this->status_set_warning();
@@ -138,19 +147,9 @@ void WiFiComponent::loop() {
           ESP_LOGW(TAG, "WiFi Connection lost... Reconnecting...");
           this->state_ = WIFI_COMPONENT_STATE_STA_CONNECTING;
           this->retry_connect();
-
-          if (this->handled_connected_state_) {
-            this->disconnect_trigger_->trigger();
-            this->handled_connected_state_ = false;
-          }
         } else {
           this->status_clear_warning();
           this->last_connected_ = now;
-
-          if (!this->handled_connected_state_) {
-            this->connect_trigger_->trigger();
-            this->handled_connected_state_ = true;
-          }
         }
         break;
       }

From 0101ae768ca187dac788f230218fd95737dd7d3d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:44:33 -0600
Subject: [PATCH 23/41] Bump aioesphomeapi from 18.2.1 to 18.2.4 (#5692)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4e9a1c3b0e..69c73fa952 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.2.1
+aioesphomeapi==18.2.4
 zeroconf==0.120.0
 
 # esp-idf requires this, but doesn't bundle it by default

From fde7a04ee73654aaaa1dd9326e684f48a56a29ef Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 8 Nov 2023 13:13:56 +1300
Subject: [PATCH 24/41] Bump version to 2023.12.0-dev

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index c2fa9951ff..2487d7b64c 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.0-dev"
+__version__ = "2023.12.0-dev"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From d81bec860b924de2d4724a0639e90f1d1bd1f0c1 Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Wed, 8 Nov 2023 02:50:45 +0100
Subject: [PATCH 25/41] Nextion support to `esp-idf` (#5667)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/nextion/display.py         |   6 +-
 esphome/components/nextion/nextion.cpp        |  10 +-
 esphome/components/nextion/nextion.h          |  55 +++-
 .../components/nextion/nextion_commands.cpp   |   9 +-
 ..._upload.cpp => nextion_upload_arduino.cpp} |  34 ++-
 .../components/nextion/nextion_upload_idf.cpp | 268 ++++++++++++++++++
 6 files changed, 346 insertions(+), 36 deletions(-)
 rename esphome/components/nextion/{nextion_upload.cpp => nextion_upload_arduino.cpp} (94%)
 create mode 100644 esphome/components/nextion/nextion_upload_idf.cpp

diff --git a/esphome/components/nextion/display.py b/esphome/components/nextion/display.py
index afb64ceeea..0831b12f8a 100644
--- a/esphome/components/nextion/display.py
+++ b/esphome/components/nextion/display.py
@@ -36,7 +36,7 @@ CONFIG_SCHEMA = (
     display.BASIC_DISPLAY_SCHEMA.extend(
         {
             cv.GenerateID(): cv.declare_id(Nextion),
-            cv.Optional(CONF_TFT_URL): cv.All(cv.string, cv.only_with_arduino),
+            cv.Optional(CONF_TFT_URL): cv.url,
             cv.Optional(CONF_BRIGHTNESS, default=1.0): cv.percentage,
             cv.Optional(CONF_ON_SETUP): automation.validate_automation(
                 {
@@ -85,10 +85,10 @@ async def to_code(config):
     if CONF_TFT_URL in config:
         cg.add_define("USE_NEXTION_TFT_UPLOAD")
         cg.add(var.set_tft_url(config[CONF_TFT_URL]))
-        if CORE.is_esp32:
+        if CORE.is_esp32 and CORE.using_arduino:
             cg.add_library("WiFiClientSecure", None)
             cg.add_library("HTTPClient", None)
-        if CORE.is_esp8266:
+        elif CORE.is_esp8266 and CORE.using_arduino:
             cg.add_library("ESP8266HTTPClient", None)
 
     if CONF_TOUCH_SLEEP_TIMEOUT in config:
diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp
index 76bdb283f6..0134595050 100644
--- a/esphome/components/nextion/nextion.cpp
+++ b/esphome/components/nextion/nextion.cpp
@@ -128,7 +128,7 @@ void Nextion::dump_config() {
   ESP_LOGCONFIG(TAG, "  Wake On Touch:    %s", this->auto_wake_on_touch_ ? "True" : "False");
 
   if (this->touch_sleep_timeout_ != 0) {
-    ESP_LOGCONFIG(TAG, "  Touch Timeout:       %d", this->touch_sleep_timeout_);
+    ESP_LOGCONFIG(TAG, "  Touch Timeout:       %" PRIu32, this->touch_sleep_timeout_);
   }
 
   if (this->wake_up_page_ != -1) {
@@ -868,6 +868,12 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool
   start = millis();
 
   while ((timeout == 0 && this->available()) || millis() - start <= timeout) {
+    if (!this->available()) {
+      App.feed_wdt();
+      delay(1);
+      continue;
+    }
+
     this->read_byte(&c);
     if (c == 0xFF) {
       nr_of_ff_bytes++;
@@ -886,7 +892,7 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool
       }
     }
     App.feed_wdt();
-    delay(1);
+    delay(2);
 
     if (exit_flag || ff_flag) {
       break;
diff --git a/esphome/components/nextion/nextion.h b/esphome/components/nextion/nextion.h
index 7b5641b711..f3a13d2170 100644
--- a/esphome/components/nextion/nextion.h
+++ b/esphome/components/nextion/nextion.h
@@ -12,14 +12,18 @@
 #include "esphome/components/display/display_color_utils.h"
 
 #ifdef USE_NEXTION_TFT_UPLOAD
+#ifdef ARDUINO
 #ifdef USE_ESP32
 #include <HTTPClient.h>
-#endif
+#endif  // USE_ESP32
 #ifdef USE_ESP8266
 #include <ESP8266HTTPClient.h>
 #include <WiFiClientSecure.h>
-#endif
-#endif
+#endif  // USE_ESP8266
+#elif defined(USE_ESP_IDF)
+#include <esp_http_client.h>
+#endif  // ARDUINO vs ESP-IDF
+#endif  // USE_NEXTION_TFT_UPLOAD
 
 namespace esphome {
 namespace nextion {
@@ -685,16 +689,18 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
 
 #ifdef USE_NEXTION_TFT_UPLOAD
   /**
-   * Set the tft file URL. https seems problamtic with arduino..
+   * Set the tft file URL. https seems problematic with arduino..
    */
   void set_tft_url(const std::string &tft_url) { this->tft_url_ = tft_url; }
 
 #endif
 
   /**
-   * Upload the tft file and softreset the Nextion
+   * Upload the tft file and soft reset Nextion
+   * @return bool True: Transfer completed successfuly, False: Transfer failed.
    */
-  void upload_tft();
+  bool upload_tft();
+
   void dump_config() override;
 
   /**
@@ -817,16 +823,16 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
   BearSSL::WiFiClientSecure *wifi_client_secure_{nullptr};
   WiFiClient *get_wifi_client_();
 #endif
-
+  int content_length_ = 0;
+  int tft_size_ = 0;
+#ifdef ARDUINO
   /**
    * will request chunk_size chunks from the web server
    * and send each to the nextion
-   * @param int contentLength Total size of the file
-   * @param uint32_t chunk_size
-   * @return true if success, false for failure.
+   * @param HTTPClient http HTTP client handler.
+   * @param int range_start Position of next byte to transfer.
+   * @return position of last byte transferred, -1 for failure.
    */
-  int content_length_ = 0;
-  int tft_size_ = 0;
   int upload_by_chunks_(HTTPClient *http, int range_start);
 
   bool upload_with_range_(uint32_t range_start, uint32_t range_end);
@@ -839,7 +845,30 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
    * @return true if success, false for failure.
    */
   bool upload_from_buffer_(const uint8_t *file_buf, size_t buf_size);
-  void upload_end_();
+  /**
+   * Ends the upload process, restart Nextion and, if successful,
+   * restarts ESP
+   * @param bool url successful True: Transfer completed successfuly, False: Transfer failed.
+   * @return bool True: Transfer completed successfuly, False: Transfer failed.
+   */
+  bool upload_end_(bool successful);
+#elif defined(USE_ESP_IDF)
+  /**
+   * will request 4096 bytes chunks from the web server
+   * and send each to Nextion
+   * @param std::string url Full url for download.
+   * @param int range_start Position of next byte to transfer.
+   * @return position of last byte transferred, -1 for failure.
+   */
+  int upload_range(const std::string &url, int range_start);
+  /**
+   * Ends the upload process, restart Nextion and, if successful,
+   * restarts ESP
+   * @param bool url successful True: Transfer completed successfuly, False: Transfer failed.
+   * @return bool True: Transfer completed successfuly, False: Transfer failed.
+   */
+  bool upload_end(bool successful);
+#endif  // ARDUINO vs ESP-IDF
 
 #endif  // USE_NEXTION_TFT_UPLOAD
 
diff --git a/esphome/components/nextion/nextion_commands.cpp b/esphome/components/nextion/nextion_commands.cpp
index a3157c731a..35530d1a7f 100644
--- a/esphome/components/nextion/nextion_commands.cpp
+++ b/esphome/components/nextion/nextion_commands.cpp
@@ -55,7 +55,7 @@ void Nextion::set_protocol_reparse_mode(bool active_mode) {
 
 // Set Colors
 void Nextion::set_component_background_color(const char *component, uint32_t color) {
-  this->add_no_result_to_queue_with_printf_("set_component_background_color", "%s.bco=%d", component, color);
+  this->add_no_result_to_queue_with_printf_("set_component_background_color", "%s.bco=%" PRIu32, component, color);
 }
 
 void Nextion::set_component_background_color(const char *component, const char *color) {
@@ -68,7 +68,8 @@ void Nextion::set_component_background_color(const char *component, Color color)
 }
 
 void Nextion::set_component_pressed_background_color(const char *component, uint32_t color) {
-  this->add_no_result_to_queue_with_printf_("set_component_pressed_background_color", "%s.bco2=%d", component, color);
+  this->add_no_result_to_queue_with_printf_("set_component_pressed_background_color", "%s.bco2=%" PRIu32, component,
+                                            color);
 }
 
 void Nextion::set_component_pressed_background_color(const char *component, const char *color) {
@@ -89,7 +90,7 @@ void Nextion::set_component_picc(const char *component, uint8_t pic_id) {
 }
 
 void Nextion::set_component_font_color(const char *component, uint32_t color) {
-  this->add_no_result_to_queue_with_printf_("set_component_font_color", "%s.pco=%d", component, color);
+  this->add_no_result_to_queue_with_printf_("set_component_font_color", "%s.pco=%" PRIu32, component, color);
 }
 
 void Nextion::set_component_font_color(const char *component, const char *color) {
@@ -102,7 +103,7 @@ void Nextion::set_component_font_color(const char *component, Color color) {
 }
 
 void Nextion::set_component_pressed_font_color(const char *component, uint32_t color) {
-  this->add_no_result_to_queue_with_printf_("set_component_pressed_font_color", "%s.pco2=%d", component, color);
+  this->add_no_result_to_queue_with_printf_("set_component_pressed_font_color", "%s.pco2=%" PRIu32, component, color);
 }
 
 void Nextion::set_component_pressed_font_color(const char *component, const char *color) {
diff --git a/esphome/components/nextion/nextion_upload.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp
similarity index 94%
rename from esphome/components/nextion/nextion_upload.cpp
rename to esphome/components/nextion/nextion_upload_arduino.cpp
index 9e6884398c..d1f9f44c2b 100644
--- a/esphome/components/nextion/nextion_upload.cpp
+++ b/esphome/components/nextion/nextion_upload_arduino.cpp
@@ -1,5 +1,6 @@
 #include "nextion.h"
 
+#ifdef ARDUINO
 #ifdef USE_NEXTION_TFT_UPLOAD
 
 #include "esphome/core/application.h"
@@ -128,15 +129,15 @@ int Nextion::upload_by_chunks_(HTTPClient *http, int range_start) {
   return range_end + 1;
 }
 
-void Nextion::upload_tft() {
+bool Nextion::upload_tft() {
   if (this->is_updating_) {
     ESP_LOGD(TAG, "Currently updating");
-    return;
+    return false;
   }
 
   if (!network::is_connected()) {
     ESP_LOGD(TAG, "network is not connected");
-    return;
+    return false;
   }
 
   this->is_updating_ = true;
@@ -164,7 +165,7 @@ void Nextion::upload_tft() {
     ESP_LOGD(TAG, "connection failed");
     ExternalRAMAllocator<uint8_t> allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
     allocator.deallocate(this->transfer_buffer_, this->transfer_buffer_size_);
-    return;
+    return false;
   } else {
     ESP_LOGD(TAG, "Connected");
   }
@@ -192,7 +193,7 @@ void Nextion::upload_tft() {
   }
 
   if ((code != 200 && code != 206) || tries > 5) {
-    this->upload_end_();
+    return this->upload_end_(false);
   }
 
   String content_range_string = http.header("Content-Range");
@@ -203,7 +204,7 @@ void Nextion::upload_tft() {
 
   if (this->content_length_ < 4096) {
     ESP_LOGE(TAG, "Failed to get file size");
-    this->upload_end_();
+    return this->upload_end_(false);
   }
 
   ESP_LOGD(TAG, "Updating Nextion %s...", this->device_model_.c_str());
@@ -246,7 +247,7 @@ void Nextion::upload_tft() {
     ESP_LOGD(TAG, "preparation for tft update done");
   } else {
     ESP_LOGD(TAG, "preparation for tft update failed %d \"%s\"", response[0], response.c_str());
-    this->upload_end_();
+    return this->upload_end_(false);
   }
 
   // Nextion wants 4096 bytes at a time. Make chunk_size a multiple of 4096
@@ -280,7 +281,7 @@ void Nextion::upload_tft() {
       this->transfer_buffer_ = allocator.allocate(chunk_size);
 
       if (!this->transfer_buffer_)
-        this->upload_end_();
+        return this->upload_end_(false);
     }
 
     this->transfer_buffer_size_ = chunk_size;
@@ -295,7 +296,7 @@ void Nextion::upload_tft() {
     result = this->upload_by_chunks_(&http, result);
     if (result < 0) {
       ESP_LOGD(TAG, "Error updating Nextion!");
-      this->upload_end_();
+      return this->upload_end_(false);
     }
     App.feed_wdt();
     // NOLINTNEXTLINE(readability-static-accessed-through-instance)
@@ -303,15 +304,19 @@ void Nextion::upload_tft() {
   }
   ESP_LOGD(TAG, "Successfully updated Nextion!");
 
-  this->upload_end_();
+  return this->upload_end_(true);
 }
 
-void Nextion::upload_end_() {
+bool Nextion::upload_end_(bool successful) {
+  this->is_updating_ = false;
   ESP_LOGD(TAG, "Restarting Nextion");
   this->soft_reset();
-  delay(1500);  // NOLINT
-  ESP_LOGD(TAG, "Restarting esphome");
-  ESP.restart();  // NOLINT(readability-static-accessed-through-instance)
+  if (successful) {
+    delay(1500);  // NOLINT
+    ESP_LOGD(TAG, "Restarting esphome");
+    ESP.restart();  // NOLINT(readability-static-accessed-through-instance)
+  }
+  return successful;
 }
 
 #ifdef USE_ESP8266
@@ -337,3 +342,4 @@ WiFiClient *Nextion::get_wifi_client_() {
 }  // namespace esphome
 
 #endif  // USE_NEXTION_TFT_UPLOAD
+#endif  // ARDUINO
diff --git a/esphome/components/nextion/nextion_upload_idf.cpp b/esphome/components/nextion/nextion_upload_idf.cpp
new file mode 100644
index 0000000000..58f5659ade
--- /dev/null
+++ b/esphome/components/nextion/nextion_upload_idf.cpp
@@ -0,0 +1,268 @@
+#include "nextion.h"
+
+#ifdef USE_ESP_IDF
+#ifdef USE_NEXTION_TFT_UPLOAD
+
+#include "esphome/core/application.h"
+#include "esphome/core/defines.h"
+#include "esphome/core/util.h"
+#include "esphome/core/log.h"
+#include "esphome/components/network/util.h"
+
+#include <esp_heap_caps.h>
+#include <esp_http_client.h>
+
+namespace esphome {
+namespace nextion {
+static const char *const TAG = "nextion_upload";
+
+// Followed guide
+// https://unofficialnextion.com/t/nextion-upload-protocol-v1-2-the-fast-one/1044/2
+
+int Nextion::upload_range(const std::string &url, int range_start) {
+  ESP_LOGVV(TAG, "url: %s", url.c_str());
+  uint range_size = this->tft_size_ - range_start;
+  ESP_LOGVV(TAG, "tft_size_: %i", this->tft_size_);
+  ESP_LOGV(TAG, "Available heap: %u", esp_get_free_heap_size());
+  int range_end = (range_start == 0) ? std::min(this->tft_size_, 16383) : this->tft_size_;
+  if (range_size <= 0 or range_end <= range_start) {
+    ESP_LOGE(TAG, "Invalid range");
+    ESP_LOGD(TAG, "Range start: %i", range_start);
+    ESP_LOGD(TAG, "Range end: %i", range_end);
+    ESP_LOGD(TAG, "Range size: %i", range_size);
+    return -1;
+  }
+
+  esp_http_client_config_t config = {
+      .url = url.c_str(),
+      .cert_pem = nullptr,
+  };
+  esp_http_client_handle_t client = esp_http_client_init(&config);
+
+  char range_header[64];
+  sprintf(range_header, "bytes=%d-%d", range_start, range_end);
+  ESP_LOGV(TAG, "Requesting range: %s", range_header);
+  esp_http_client_set_header(client, "Range", range_header);
+  ESP_LOGVV(TAG, "Available heap: %u", esp_get_free_heap_size());
+
+  ESP_LOGV(TAG, "Opening http connetion");
+  esp_err_t err;
+  if ((err = esp_http_client_open(client, 0)) != ESP_OK) {
+    ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
+    esp_http_client_cleanup(client);
+    return -1;
+  }
+
+  ESP_LOGV(TAG, "Fetch content length");
+  int content_length = esp_http_client_fetch_headers(client);
+  ESP_LOGV(TAG, "content_length = %d", content_length);
+  if (content_length <= 0) {
+    ESP_LOGE(TAG, "Failed to get content length: %d", content_length);
+    esp_http_client_cleanup(client);
+    return -1;
+  }
+
+  int total_read_len = 0, read_len;
+
+  ESP_LOGV(TAG, "Allocate buffer");
+  uint8_t *buffer = new uint8_t[4096];
+  std::string recv_string;
+  if (buffer == nullptr) {
+    ESP_LOGE(TAG, "Failed to allocate memory for buffer");
+    ESP_LOGV(TAG, "Available heap: %u", esp_get_free_heap_size());
+  } else {
+    ESP_LOGV(TAG, "Memory for buffer allocated successfully");
+
+    while (true) {
+      App.feed_wdt();
+      ESP_LOGVV(TAG, "Available heap: %u", esp_get_free_heap_size());
+      int read_len = esp_http_client_read(client, reinterpret_cast<char *>(buffer), 4096);
+      ESP_LOGVV(TAG, "Read %d bytes from HTTP client, writing to UART", read_len);
+      if (read_len > 0) {
+        this->write_array(buffer, read_len);
+        ESP_LOGVV(TAG, "Write to UART successful");
+        this->recv_ret_string_(recv_string, 5000, true);
+        this->content_length_ -= read_len;
+        ESP_LOGD(TAG, "Uploaded %0.2f %%, remaining %d bytes",
+                 100.0 * (this->tft_size_ - this->content_length_) / this->tft_size_, this->content_length_);
+        if (recv_string[0] != 0x05) {  // 0x05 == "ok"
+          ESP_LOGD(
+              TAG, "recv_string [%s]",
+              format_hex_pretty(reinterpret_cast<const uint8_t *>(recv_string.data()), recv_string.size()).c_str());
+        }
+        // handle partial upload request
+        if (recv_string[0] == 0x08 && recv_string.size() == 5) {
+          uint32_t result = 0;
+          for (int j = 0; j < 4; ++j) {
+            result += static_cast<uint8_t>(recv_string[j + 1]) << (8 * j);
+          }
+          if (result > 0) {
+            ESP_LOGI(TAG, "Nextion reported new range %" PRIu32, result);
+            this->content_length_ = this->tft_size_ - result;
+            // Deallocate the buffer when done
+            delete[] buffer;
+            ESP_LOGVV(TAG, "Memory for buffer deallocated");
+            esp_http_client_cleanup(client);
+            esp_http_client_close(client);
+            return result;
+          }
+        }
+        recv_string.clear();
+      } else if (read_len == 0) {
+        ESP_LOGV(TAG, "End of HTTP response reached");
+        break;  // Exit the loop if there is no more data to read
+      } else {
+        ESP_LOGE(TAG, "Failed to read from HTTP client, error code: %d", read_len);
+        break;  // Exit the loop on error
+      }
+    }
+
+    // Deallocate the buffer when done
+    delete[] buffer;
+    ESP_LOGVV(TAG, "Memory for buffer deallocated");
+  }
+  esp_http_client_cleanup(client);
+  esp_http_client_close(client);
+  return range_end + 1;
+}
+
+bool Nextion::upload_tft() {
+  ESP_LOGD(TAG, "Nextion TFT upload requested");
+  ESP_LOGD(TAG, "url: %s", this->tft_url_.c_str());
+
+  if (this->is_updating_) {
+    ESP_LOGW(TAG, "Currently updating");
+    return false;
+  }
+
+  if (!network::is_connected()) {
+    ESP_LOGE(TAG, "Network is not connected");
+    return false;
+  }
+
+  this->is_updating_ = true;
+
+  // Define the configuration for the HTTP client
+  ESP_LOGV(TAG, "Establishing connection to HTTP server");
+  ESP_LOGVV(TAG, "Available heap: %u", esp_get_free_heap_size());
+  esp_http_client_config_t config = {
+      .url = this->tft_url_.c_str(),
+      .cert_pem = nullptr,
+      .method = HTTP_METHOD_HEAD,
+      .timeout_ms = 15000,
+  };
+
+  // Initialize the HTTP client with the configuration
+  ESP_LOGV(TAG, "Initializing HTTP client");
+  ESP_LOGV(TAG, "Available heap: %u", esp_get_free_heap_size());
+  esp_http_client_handle_t http = esp_http_client_init(&config);
+  if (!http) {
+    ESP_LOGE(TAG, "Failed to initialize HTTP client.");
+    return this->upload_end(false);
+  }
+
+  // Perform the HTTP request
+  ESP_LOGV(TAG, "Check if the client could connect");
+  ESP_LOGV(TAG, "Available heap: %u", esp_get_free_heap_size());
+  esp_err_t err = esp_http_client_perform(http);
+  if (err != ESP_OK) {
+    ESP_LOGE(TAG, "HTTP request failed: %s", esp_err_to_name(err));
+    esp_http_client_cleanup(http);
+    return this->upload_end(false);
+  }
+
+  // Check the HTTP Status Code
+  int status_code = esp_http_client_get_status_code(http);
+  ESP_LOGV(TAG, "HTTP Status Code: %d", status_code);
+  size_t tft_file_size = esp_http_client_get_content_length(http);
+  ESP_LOGD(TAG, "TFT file size: %zu", tft_file_size);
+
+  if (tft_file_size < 4096) {
+    ESP_LOGE(TAG, "File size check failed. Size: %zu", tft_file_size);
+    esp_http_client_cleanup(http);
+    return this->upload_end(false);
+  } else {
+    ESP_LOGV(TAG, "File size check passed. Proceeding...");
+  }
+  this->content_length_ = tft_file_size;
+  this->tft_size_ = tft_file_size;
+
+  ESP_LOGD(TAG, "Updating Nextion");
+  // The Nextion will ignore the update command if it is sleeping
+
+  this->send_command_("sleep=0");
+  this->set_backlight_brightness(1.0);
+  vTaskDelay(pdMS_TO_TICKS(250));  // NOLINT
+
+  App.feed_wdt();
+  char command[128];
+  // Tells the Nextion the content length of the tft file and baud rate it will be sent at
+  // Once the Nextion accepts the command it will wait until the file is successfully uploaded
+  // If it fails for any reason a power cycle of the display will be needed
+  sprintf(command, "whmi-wris %d,%" PRIu32 ",1", this->content_length_, this->parent_->get_baud_rate());
+
+  // Clear serial receive buffer
+  uint8_t d;
+  while (this->available()) {
+    this->read_byte(&d);
+  };
+
+  this->send_command_(command);
+
+  std::string response;
+  ESP_LOGV(TAG, "Waiting for upgrade response");
+  this->recv_ret_string_(response, 2048, true);  // This can take some time to return
+
+  // The Nextion display will, if it's ready to accept data, send a 0x05 byte.
+  ESP_LOGD(TAG, "Upgrade response is [%s]",
+           format_hex_pretty(reinterpret_cast<const uint8_t *>(response.data()), response.size()).c_str());
+
+  if (response.find(0x05) != std::string::npos) {
+    ESP_LOGV(TAG, "Preparation for tft update done");
+  } else {
+    ESP_LOGE(TAG, "Preparation for tft update failed %d \"%s\"", response[0], response.c_str());
+    esp_http_client_cleanup(http);
+    return this->upload_end(false);
+  }
+
+  ESP_LOGD(TAG, "Updating tft from \"%s\" with a file size of %d, Heap Size %" PRIu32, this->tft_url_.c_str(),
+           content_length_, esp_get_free_heap_size());
+
+  ESP_LOGV(TAG, "Starting transfer by chunks loop");
+  int result = 0;
+  while (content_length_ > 0) {
+    result = upload_range(this->tft_url_.c_str(), result);
+    if (result < 0) {
+      ESP_LOGE(TAG, "Error updating Nextion!");
+      esp_http_client_cleanup(http);
+      return this->upload_end(false);
+    }
+    App.feed_wdt();
+    ESP_LOGV(TAG, "Heap Size %" PRIu32 ", Bytes left %d", esp_get_free_heap_size(), content_length_);
+  }
+
+  ESP_LOGD(TAG, "Successfully updated Nextion!");
+
+  ESP_LOGD(TAG, "Close HTTP connection");
+  esp_http_client_close(http);
+  esp_http_client_cleanup(http);
+  return upload_end(true);
+}
+
+bool Nextion::upload_end(bool successful) {
+  this->is_updating_ = false;
+  ESP_LOGD(TAG, "Restarting Nextion");
+  this->soft_reset();
+  vTaskDelay(pdMS_TO_TICKS(1500));  // NOLINT
+  if (successful) {
+    ESP_LOGD(TAG, "Restarting esphome");
+    esp_restart();  // NOLINT(readability-static-accessed-through-instance)
+  }
+  return successful;
+}
+
+}  // namespace nextion
+}  // namespace esphome
+
+#endif  // USE_NEXTION_TFT_UPLOAD
+#endif  // USE_ESP_IDF

From 972598a698fcbc6114e110760450eb23fb6e1dea Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 8 Nov 2023 21:34:44 +1300
Subject: [PATCH 26/41] Handle nanoseconds in config (#5695)

---
 esphome/config_validation.py  | 21 +++++++++++++++++++-
 esphome/core/__init__.py      | 37 +++++++++++++++++++++++++++--------
 esphome/cpp_generator.py      |  3 +++
 tests/unit_tests/test_core.py |  8 +++++---
 4 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/esphome/config_validation.py b/esphome/config_validation.py
index d2b381215e..eb347d0a4d 100644
--- a/esphome/config_validation.py
+++ b/esphome/config_validation.py
@@ -66,6 +66,7 @@ from esphome.core import (
     TimePeriod,
     TimePeriodMicroseconds,
     TimePeriodMilliseconds,
+    TimePeriodNanoseconds,
     TimePeriodSeconds,
     TimePeriodMinutes,
 )
@@ -718,6 +719,8 @@ def time_period_str_unit(value):
         raise Invalid("Expected string for time period with unit.")
 
     unit_to_kwarg = {
+        "ns": "nanoseconds",
+        "nanoseconds": "nanoseconds",
         "us": "microseconds",
         "microseconds": "microseconds",
         "ms": "milliseconds",
@@ -739,7 +742,10 @@ def time_period_str_unit(value):
         raise Invalid(f"Expected time period with unit, got {value}")
     kwarg = unit_to_kwarg[one_of(*unit_to_kwarg)(match.group(2))]
 
-    return TimePeriod(**{kwarg: float(match.group(1))})
+    try:
+        return TimePeriod(**{kwarg: float(match.group(1))})
+    except ValueError as e:
+        raise Invalid(e) from e
 
 
 def time_period_in_milliseconds_(value):
@@ -749,10 +755,18 @@ def time_period_in_milliseconds_(value):
 
 
 def time_period_in_microseconds_(value):
+    if value.nanoseconds is not None and value.nanoseconds != 0:
+        raise Invalid("Maximum precision is microseconds")
     return TimePeriodMicroseconds(**value.as_dict())
 
 
+def time_period_in_nanoseconds_(value):
+    return TimePeriodNanoseconds(**value.as_dict())
+
+
 def time_period_in_seconds_(value):
+    if value.nanoseconds is not None and value.nanoseconds != 0:
+        raise Invalid("Maximum precision is seconds")
     if value.microseconds is not None and value.microseconds != 0:
         raise Invalid("Maximum precision is seconds")
     if value.milliseconds is not None and value.milliseconds != 0:
@@ -761,6 +775,8 @@ def time_period_in_seconds_(value):
 
 
 def time_period_in_minutes_(value):
+    if value.nanoseconds is not None and value.nanoseconds != 0:
+        raise Invalid("Maximum precision is minutes")
     if value.microseconds is not None and value.microseconds != 0:
         raise Invalid("Maximum precision is minutes")
     if value.milliseconds is not None and value.milliseconds != 0:
@@ -787,6 +803,9 @@ time_period_microseconds = All(time_period, time_period_in_microseconds_)
 positive_time_period_microseconds = All(
     positive_time_period, time_period_in_microseconds_
 )
+positive_time_period_nanoseconds = All(
+    positive_time_period, time_period_in_nanoseconds_
+)
 positive_not_null_time_period = All(
     time_period, Range(min=TimePeriod(), min_included=False)
 )
diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py
index 0b597e0c9e..60bd17b481 100644
--- a/esphome/core/__init__.py
+++ b/esphome/core/__init__.py
@@ -87,6 +87,7 @@ def is_approximately_integer(value):
 class TimePeriod:
     def __init__(
         self,
+        nanoseconds=None,
         microseconds=None,
         milliseconds=None,
         seconds=None,
@@ -136,13 +137,23 @@ class TimePeriod:
 
         if microseconds is not None:
             if not is_approximately_integer(microseconds):
-                raise ValueError("Maximum precision is microseconds")
+                frac_microseconds, microseconds = math.modf(microseconds)
+                nanoseconds = (nanoseconds or 0) + frac_microseconds * 1000
             self.microseconds = int(round(microseconds))
         else:
             self.microseconds = None
 
+        if nanoseconds is not None:
+            if not is_approximately_integer(nanoseconds):
+                raise ValueError("Maximum precision is nanoseconds")
+            self.nanoseconds = int(round(nanoseconds))
+        else:
+            self.nanoseconds = None
+
     def as_dict(self):
         out = OrderedDict()
+        if self.nanoseconds is not None:
+            out["nanoseconds"] = self.nanoseconds
         if self.microseconds is not None:
             out["microseconds"] = self.microseconds
         if self.milliseconds is not None:
@@ -158,6 +169,8 @@ class TimePeriod:
         return out
 
     def __str__(self):
+        if self.nanoseconds is not None:
+            return f"{self.total_nanoseconds}ns"
         if self.microseconds is not None:
             return f"{self.total_microseconds}us"
         if self.milliseconds is not None:
@@ -173,7 +186,11 @@ class TimePeriod:
         return "0s"
 
     def __repr__(self):
-        return f"TimePeriod<{self.total_microseconds}>"
+        return f"TimePeriod<{self.total_nanoseconds}ns>"
+
+    @property
+    def total_nanoseconds(self):
+        return self.total_microseconds * 1000 + (self.nanoseconds or 0)
 
     @property
     def total_microseconds(self):
@@ -201,35 +218,39 @@ class TimePeriod:
 
     def __eq__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds == other.total_microseconds
+            return self.total_nanoseconds == other.total_nanoseconds
         return NotImplemented
 
     def __ne__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds != other.total_microseconds
+            return self.total_nanoseconds != other.total_nanoseconds
         return NotImplemented
 
     def __lt__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds < other.total_microseconds
+            return self.total_nanoseconds < other.total_nanoseconds
         return NotImplemented
 
     def __gt__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds > other.total_microseconds
+            return self.total_nanoseconds > other.total_nanoseconds
         return NotImplemented
 
     def __le__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds <= other.total_microseconds
+            return self.total_nanoseconds <= other.total_nanoseconds
         return NotImplemented
 
     def __ge__(self, other):
         if isinstance(other, TimePeriod):
-            return self.total_microseconds >= other.total_microseconds
+            return self.total_nanoseconds >= other.total_nanoseconds
         return NotImplemented
 
 
+class TimePeriodNanoseconds(TimePeriod):
+    pass
+
+
 class TimePeriodMicroseconds(TimePeriod):
     pass
 
diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py
index 2841be1546..909a786917 100644
--- a/esphome/cpp_generator.py
+++ b/esphome/cpp_generator.py
@@ -17,6 +17,7 @@ from esphome.core import (
     TimePeriodMicroseconds,
     TimePeriodMilliseconds,
     TimePeriodMinutes,
+    TimePeriodNanoseconds,
     TimePeriodSeconds,
 )
 from esphome.helpers import cpp_string_escape, indent_all_but_first_and_last
@@ -351,6 +352,8 @@ def safe_exp(obj: SafeExpType) -> Expression:
         return IntLiteral(obj)
     if isinstance(obj, float):
         return FloatLiteral(obj)
+    if isinstance(obj, TimePeriodNanoseconds):
+        return IntLiteral(int(obj.total_nanoseconds))
     if isinstance(obj, TimePeriodMicroseconds):
         return IntLiteral(int(obj.total_microseconds))
     if isinstance(obj, TimePeriodMilliseconds):
diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py
index 9a15bf0b9c..efa9ff5677 100644
--- a/tests/unit_tests/test_core.py
+++ b/tests/unit_tests/test_core.py
@@ -116,14 +116,16 @@ class TestTimePeriod:
 
         assert actual == expected
 
-    def test_init__microseconds_with_fraction(self):
-        with pytest.raises(ValueError, match="Maximum precision is microseconds"):
-            core.TimePeriod(microseconds=1.1)
+    def test_init__nanoseconds_with_fraction(self):
+        with pytest.raises(ValueError, match="Maximum precision is nanoseconds"):
+            core.TimePeriod(nanoseconds=1.1)
 
     @pytest.mark.parametrize(
         "kwargs, expected",
         (
             ({}, "0s"),
+            ({"nanoseconds": 1}, "1ns"),
+            ({"nanoseconds": 1.0001}, "1ns"),
             ({"microseconds": 1}, "1us"),
             ({"microseconds": 1.0001}, "1us"),
             ({"milliseconds": 2}, "2ms"),

From 511348974ed2cce572b929ad7ea7a2c907abf0f9 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 8 Nov 2023 22:01:26 +1300
Subject: [PATCH 27/41] Fix esp32_rmt_led_strip custom timing units (#5696)

---
 esphome/components/esp32_rmt_led_strip/light.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/esphome/components/esp32_rmt_led_strip/light.py b/esphome/components/esp32_rmt_led_strip/light.py
index 43629bec51..122ee132a7 100644
--- a/esphome/components/esp32_rmt_led_strip/light.py
+++ b/esphome/components/esp32_rmt_led_strip/light.py
@@ -93,19 +93,19 @@ CONFIG_SCHEMA = cv.All(
             cv.Inclusive(
                 CONF_BIT0_HIGH,
                 "custom",
-            ): cv.positive_time_period_microseconds,
+            ): cv.positive_time_period_nanoseconds,
             cv.Inclusive(
                 CONF_BIT0_LOW,
                 "custom",
-            ): cv.positive_time_period_microseconds,
+            ): cv.positive_time_period_nanoseconds,
             cv.Inclusive(
                 CONF_BIT1_HIGH,
                 "custom",
-            ): cv.positive_time_period_microseconds,
+            ): cv.positive_time_period_nanoseconds,
             cv.Inclusive(
                 CONF_BIT1_LOW,
                 "custom",
-            ): cv.positive_time_period_microseconds,
+            ): cv.positive_time_period_nanoseconds,
         }
     ),
     cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH),

From cf22c554306ecec631c7fb91c7195d23a21f4d7b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 8 Nov 2023 15:04:01 -0600
Subject: [PATCH 28/41] Fix static assets cache logic (#5700)

---
 esphome/dashboard/dashboard.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py
index 119094de28..5967c95aba 100644
--- a/esphome/dashboard/dashboard.py
+++ b/esphome/dashboard/dashboard.py
@@ -4,6 +4,7 @@ import base64
 import binascii
 import codecs
 import collections
+import datetime
 import functools
 import gzip
 import hashlib
@@ -1406,15 +1407,17 @@ def make_app(debug=get_bool_env(ENV_DEV)):
         )
 
     class StaticFileHandler(tornado.web.StaticFileHandler):
-        def set_extra_headers(self, path):
-            if "favicon.ico" in path:
-                self.set_header("Cache-Control", "max-age=84600, public")
-            else:
-                if debug:
-                    self.set_header(
-                        "Cache-Control",
-                        "no-store, no-cache, must-revalidate, max-age=0",
-                    )
+        def get_cache_time(
+            self, path: str, modified: datetime.datetime | None, mime_type: str
+        ) -> int:
+            """Override to customize cache control behavior."""
+            if debug:
+                return 0
+            # Assets that are hashed have ?hash= in the URL, all javascript
+            # filenames hashed so we can cache them for a long time
+            if "hash" in self.request.arguments or "/javascript" in mime_type:
+                return self.CACHE_MAX_AGE
+            return super().get_cache_time(path, modified, mime_type)
 
     app_settings = {
         "debug": debug,

From d394b957d13d978a0eee4ff4a0c2bfe275fdb4f9 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 8 Nov 2023 16:50:08 -0600
Subject: [PATCH 29/41] Use piwheels for armv7 docker image builds (#5703)

---
 docker/Dockerfile |  7 ++++---
 docker/build.py   | 18 ++++++++++++++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 72aa9d9a9c..262827dea4 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -14,6 +14,7 @@ FROM base-${BASEIMGTYPE} AS base
 
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG PIP_EXTRA_INDEX_URL
 
 # Note that --break-system-packages is used below because
 # https://peps.python.org/pep-0668/ added a safety check that prevents
@@ -46,7 +47,7 @@ RUN \
           libssl-dev=3.0.11-1~deb12u2 \
           libffi-dev=3.4.4-1 \
           cargo=0.66.0+ds1-1 \
-          pkg-config=1.8.1-1; \
+          pkg-config=1.8.1-1 \
           gcc-arm-linux-gnueabihf=4:12.2.0-3; \
     fi; \
     rm -rf \
@@ -58,8 +59,8 @@ ENV \
   # Fix click python3 lang warning https://click.palletsprojects.com/en/7.x/python3/
   LANG=C.UTF-8 LC_ALL=C.UTF-8 \
   # Store globally installed pio libs in /piolibs
-  PLATFORMIO_GLOBALLIB_DIR=/piolibs
-
+  PLATFORMIO_GLOBALLIB_DIR=/piolibs \
+  PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 
 # Support legacy binaries on Debian multiarch system. There is no "correct" way
 # to do this, other than using properly built toolchains...
diff --git a/docker/build.py b/docker/build.py
index 47461ddf97..ae0f5088d8 100755
--- a/docker/build.py
+++ b/docker/build.py
@@ -143,15 +143,25 @@ def main():
         imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push]
         imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push]
 
+        build_args = [
+            "--build-arg",
+            f"BASEIMGTYPE={params.baseimgtype}",
+            "--build-arg",
+            f"BUILD_VERSION={args.tag}",
+        ]
+
+        if args.arch == ARCH_ARMV7:
+            build_args += [
+                "--build-arg",
+                "PIP_EXTRA_INDEX_URL=https://www.piwheels.org/simple",
+            ]
+
         # 3. build
         cmd = [
             "docker",
             "buildx",
             "build",
-            "--build-arg",
-            f"BASEIMGTYPE={params.baseimgtype}",
-            "--build-arg",
-            f"BUILD_VERSION={args.tag}",
+            *build_args,
             "--cache-from",
             f"type=registry,ref={cache_img}",
             "--file",

From ce020b1f9f49d719c63e2be399236bf3cfee18b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rodrigo=20Mart=C3=ADn?= <contact@rodrigomartin.dev>
Date: Thu, 9 Nov 2023 00:35:37 +0100
Subject: [PATCH 30/41] fix: Fix broken bluetooth_proxy and ble_clients after
 BLE enable/disable (#5704)

---
 .../esp32_ble_client/ble_client_base.cpp      | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp
index 40eff49266..cc6d3d7d4d 100644
--- a/esphome/components/esp32_ble_client/ble_client_base.cpp
+++ b/esphome/components/esp32_ble_client/ble_client_base.cpp
@@ -20,16 +20,21 @@ static const esp_bt_uuid_t NOTIFY_DESC_UUID = {
 void BLEClientBase::setup() {
   static uint8_t connection_index = 0;
   this->connection_index_ = connection_index++;
-
-  auto ret = esp_ble_gattc_app_register(this->app_id);
-  if (ret) {
-    ESP_LOGE(TAG, "gattc app register failed. app_id=%d code=%d", this->app_id, ret);
-    this->mark_failed();
-  }
-  this->set_state(espbt::ClientState::IDLE);
 }
 
 void BLEClientBase::loop() {
+  if (!esp32_ble::global_ble->is_active()) {
+    this->set_state(espbt::ClientState::INIT);
+    return;
+  }
+  if (this->state_ == espbt::ClientState::INIT) {
+    auto ret = esp_ble_gattc_app_register(this->app_id);
+    if (ret) {
+      ESP_LOGE(TAG, "gattc app register failed. app_id=%d code=%d", this->app_id, ret);
+      this->mark_failed();
+    }
+    this->set_state(espbt::ClientState::IDLE);
+  }
   // READY_TO_CONNECT means we have discovered the device
   // and the scanner has been stopped by the tracker.
   if (this->state_ == espbt::ClientState::READY_TO_CONNECT) {

From 3e3266fa744dc234883d7f9aa8837e27f74bef3f Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 8 Nov 2023 20:52:08 -0600
Subject: [PATCH 31/41] Bump aioesphomeapi to 18.2.7 (#5706)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 69c73fa952..33c5e6988e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.2.4
+aioesphomeapi==18.2.7
 zeroconf==0.120.0
 
 # esp-idf requires this, but doesn't bundle it by default

From 28513a0502439b4fca97728b21bcd633c6622360 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 9 Nov 2023 02:04:39 -0600
Subject: [PATCH 32/41] Update Dockerfile to use piwheels for armv7 (#5709)

---
 docker/Dockerfile | 41 +++++++++++++++++++++++++++++++----------
 docker/build.py   | 18 ++++--------------
 2 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 262827dea4..7ca633a982 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -5,6 +5,7 @@
 # One of "docker", "hassio"
 ARG BASEIMGTYPE=docker
 
+
 # https://github.com/hassio-addons/addon-debian-base/releases
 FROM ghcr.io/hassio-addons/debian-base:7.2.0 AS base-hassio
 # https://hub.docker.com/_/debian?tab=tags&page=1&name=bookworm
@@ -12,9 +13,10 @@ FROM debian:12.2-slim AS base-docker
 
 FROM base-${BASEIMGTYPE} AS base
 
+
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG PIP_EXTRA_INDEX_URL
+
 
 # Note that --break-system-packages is used below because
 # https://peps.python.org/pep-0668/ added a safety check that prevents
@@ -59,8 +61,7 @@ ENV \
   # Fix click python3 lang warning https://click.palletsprojects.com/en/7.x/python3/
   LANG=C.UTF-8 LC_ALL=C.UTF-8 \
   # Store globally installed pio libs in /piolibs
-  PLATFORMIO_GLOBALLIB_DIR=/piolibs \
-  PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
+  PLATFORMIO_GLOBALLIB_DIR=/piolibs
 
 # Support legacy binaries on Debian multiarch system. There is no "correct" way
 # to do this, other than using properly built toolchains...
@@ -72,8 +73,12 @@ RUN \
 
 RUN \
     # Ubuntu python3-pip is missing wheel
-    pip3 install --break-system-packages --no-cache-dir \
-        platformio==6.1.11 \
+    if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
+        export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple"; \
+    fi; \
+    pip3 install \
+    --break-system-packages --no-cache-dir \
+    platformio==6.1.11 \
     # Change some platformio settings
     && platformio settings set enable_telemetry No \
     && platformio settings set check_platformio_interval 1000000 \
@@ -84,8 +89,12 @@ RUN \
 # tmpfs is for https://github.com/rust-lang/cargo/issues/8719
 
 COPY requirements.txt requirements_optional.txt script/platformio_install_deps.py platformio.ini /
-RUN --mount=type=tmpfs,target=/root/.cargo CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse CARGO_HOME=/root/.cargo \
-    pip3 install --break-system-packages --no-cache-dir -r /requirements.txt -r /requirements_optional.txt \
+RUN --mount=type=tmpfs,target=/root/.cargo if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
+        export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple"; \
+    fi; \
+    CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse CARGO_HOME=/root/.cargo \
+    pip3 install \
+    --break-system-packages --no-cache-dir -r /requirements.txt -r /requirements_optional.txt \
     && /platformio_install_deps.py /platformio.ini --libraries
 
 
@@ -94,7 +103,11 @@ FROM base AS docker
 
 # Copy esphome and install
 COPY . /esphome
-RUN pip3 install --break-system-packages --no-cache-dir --no-use-pep517 -e /esphome
+RUN if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
+        export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple"; \
+  fi; \
+  pip3 install \
+  --break-system-packages --no-cache-dir --no-use-pep517 -e /esphome
 
 # Settings for dashboard
 ENV USERNAME="" PASSWORD=""
@@ -140,7 +153,11 @@ COPY docker/ha-addon-rootfs/ /
 
 # Copy esphome and install
 COPY . /esphome
-RUN pip3 install --break-system-packages --no-cache-dir --no-use-pep517 -e /esphome
+RUN if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
+        export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple"; \
+  fi; \
+  pip3 install \
+  --break-system-packages --no-cache-dir --no-use-pep517 -e /esphome
 
 # Labels
 LABEL \
@@ -176,7 +193,11 @@ RUN \
         /var/lib/apt/lists/*
 
 COPY requirements_test.txt /
-RUN pip3 install --break-system-packages --no-cache-dir -r /requirements_test.txt
+RUN if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
+        export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple"; \
+  fi; \
+  pip3 install \
+  --break-system-packages --no-cache-dir -r /requirements_test.txt
 
 VOLUME ["/esphome"]
 WORKDIR /esphome
diff --git a/docker/build.py b/docker/build.py
index ae0f5088d8..47461ddf97 100755
--- a/docker/build.py
+++ b/docker/build.py
@@ -143,25 +143,15 @@ def main():
         imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push]
         imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push]
 
-        build_args = [
-            "--build-arg",
-            f"BASEIMGTYPE={params.baseimgtype}",
-            "--build-arg",
-            f"BUILD_VERSION={args.tag}",
-        ]
-
-        if args.arch == ARCH_ARMV7:
-            build_args += [
-                "--build-arg",
-                "PIP_EXTRA_INDEX_URL=https://www.piwheels.org/simple",
-            ]
-
         # 3. build
         cmd = [
             "docker",
             "buildx",
             "build",
-            *build_args,
+            "--build-arg",
+            f"BASEIMGTYPE={params.baseimgtype}",
+            "--build-arg",
+            f"BUILD_VERSION={args.tag}",
             "--cache-from",
             f"type=registry,ref={cache_img}",
             "--file",

From bc7519f645db184a0315bba63d644c6bc604fe36 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Nov 2023 12:26:05 -0600
Subject: [PATCH 33/41] Bump zeroconf from 0.120.0 to 0.122.3 (#5715)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 33c5e6988e..f974967a00 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,7 @@ esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
 aioesphomeapi==18.2.7
-zeroconf==0.120.0
+zeroconf==0.122.3
 
 # esp-idf requires this, but doesn't bundle it by default
 # https://github.com/espressif/esp-idf/blob/220590d599e134d7a5e7f1e683cc4550349ffbf8/requirements.txt#L24

From 01d28ce3fcfe06a16906d4ef4b026ef38d9f347f Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 10 Nov 2023 11:40:07 +1300
Subject: [PATCH 34/41] Add resistance_sampler interface for config validation
 (#5718)

---
 CODEOWNERS                                            |  1 +
 esphome/components/ntc/sensor.py                      |  6 ++++--
 esphome/components/resistance/resistance_sensor.h     |  5 +++--
 esphome/components/resistance/sensor.py               | 11 +++++++++--
 esphome/components/resistance_sampler/__init__.py     |  6 ++++++
 .../resistance_sampler/resistance_sampler.h           | 10 ++++++++++
 6 files changed, 33 insertions(+), 6 deletions(-)
 create mode 100644 esphome/components/resistance_sampler/__init__.py
 create mode 100644 esphome/components/resistance_sampler/resistance_sampler.h

diff --git a/CODEOWNERS b/CODEOWNERS
index 2dcef6c514..dd1586d039 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -246,6 +246,7 @@ esphome/components/radon_eye_rd200/* @jeffeb3
 esphome/components/rc522/* @glmnet
 esphome/components/rc522_i2c/* @glmnet
 esphome/components/rc522_spi/* @glmnet
+esphome/components/resistance_sampler/* @jesserockz
 esphome/components/restart/* @esphome/core
 esphome/components/rf_bridge/* @jesserockz
 esphome/components/rgbct/* @jesserockz
diff --git a/esphome/components/ntc/sensor.py b/esphome/components/ntc/sensor.py
index ba8d3df9d8..06fc55fc43 100644
--- a/esphome/components/ntc/sensor.py
+++ b/esphome/components/ntc/sensor.py
@@ -2,7 +2,7 @@ from math import log
 
 import esphome.config_validation as cv
 import esphome.codegen as cg
-from esphome.components import sensor
+from esphome.components import sensor, resistance_sampler
 from esphome.const import (
     CONF_CALIBRATION,
     CONF_REFERENCE_RESISTANCE,
@@ -15,6 +15,8 @@ from esphome.const import (
     UNIT_CELSIUS,
 )
 
+AUTO_LOAD = ["resistance_sampler"]
+
 ntc_ns = cg.esphome_ns.namespace("ntc")
 NTC = ntc_ns.class_("NTC", cg.Component, sensor.Sensor)
 
@@ -124,7 +126,7 @@ CONFIG_SCHEMA = (
     )
     .extend(
         {
-            cv.Required(CONF_SENSOR): cv.use_id(sensor.Sensor),
+            cv.Required(CONF_SENSOR): cv.use_id(resistance_sampler.ResistanceSampler),
             cv.Required(CONF_CALIBRATION): process_calibration,
         }
     )
diff --git a/esphome/components/resistance/resistance_sensor.h b/esphome/components/resistance/resistance_sensor.h
index b57f90b59c..8fa1f8b570 100644
--- a/esphome/components/resistance/resistance_sensor.h
+++ b/esphome/components/resistance/resistance_sensor.h
@@ -1,7 +1,8 @@
 #pragma once
 
-#include "esphome/core/component.h"
+#include "esphome/components/resistance_sampler/resistance_sampler.h"
 #include "esphome/components/sensor/sensor.h"
+#include "esphome/core/component.h"
 
 namespace esphome {
 namespace resistance {
@@ -11,7 +12,7 @@ enum ResistanceConfiguration {
   DOWNSTREAM,
 };
 
-class ResistanceSensor : public Component, public sensor::Sensor {
+class ResistanceSensor : public Component, public sensor::Sensor, resistance_sampler::ResistanceSampler {
  public:
   void set_sensor(Sensor *sensor) { sensor_ = sensor; }
   void set_configuration(ResistanceConfiguration configuration) { configuration_ = configuration; }
diff --git a/esphome/components/resistance/sensor.py b/esphome/components/resistance/sensor.py
index 55e7ddfc81..a84b439497 100644
--- a/esphome/components/resistance/sensor.py
+++ b/esphome/components/resistance/sensor.py
@@ -1,6 +1,6 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
-from esphome.components import sensor
+from esphome.components import sensor, resistance_sampler
 from esphome.const import (
     CONF_SENSOR,
     STATE_CLASS_MEASUREMENT,
@@ -8,8 +8,15 @@ from esphome.const import (
     ICON_FLASH,
 )
 
+AUTO_LOAD = ["resistance_sampler"]
+
 resistance_ns = cg.esphome_ns.namespace("resistance")
-ResistanceSensor = resistance_ns.class_("ResistanceSensor", cg.Component, sensor.Sensor)
+ResistanceSensor = resistance_ns.class_(
+    "ResistanceSensor",
+    cg.Component,
+    sensor.Sensor,
+    resistance_sampler.ResistanceSampler,
+)
 
 CONF_REFERENCE_VOLTAGE = "reference_voltage"
 CONF_CONFIGURATION = "configuration"
diff --git a/esphome/components/resistance_sampler/__init__.py b/esphome/components/resistance_sampler/__init__.py
new file mode 100644
index 0000000000..d2032848aa
--- /dev/null
+++ b/esphome/components/resistance_sampler/__init__.py
@@ -0,0 +1,6 @@
+import esphome.codegen as cg
+
+resistance_sampler_ns = cg.esphome_ns.namespace("resistance_sampler")
+ResistanceSampler = resistance_sampler_ns.class_("ResistanceSampler")
+
+CODEOWNERS = ["@jesserockz"]
diff --git a/esphome/components/resistance_sampler/resistance_sampler.h b/esphome/components/resistance_sampler/resistance_sampler.h
new file mode 100644
index 0000000000..9e300bebcc
--- /dev/null
+++ b/esphome/components/resistance_sampler/resistance_sampler.h
@@ -0,0 +1,10 @@
+#pragma once
+
+namespace esphome {
+namespace resistance_sampler {
+
+/// Abstract interface to mark components that provide resistance values.
+class ResistanceSampler {};
+
+}  // namespace resistance_sampler
+}  // namespace esphome

From 98ec798bfc006938b3eafc975b9c039a3c9d3cbf Mon Sep 17 00:00:00 2001
From: Samuel Sieb <samuel-github@sieb.net>
Date: Thu, 9 Nov 2023 17:53:35 -0800
Subject: [PATCH 35/41] fix pin range for xl9535 (#5722)

Co-authored-by: Samuel Sieb <samuel@sieb.net>
---
 esphome/components/xl9535/__init__.py | 8 +++++++-
 tests/test4.yaml                      | 9 +++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/esphome/components/xl9535/__init__.py b/esphome/components/xl9535/__init__.py
index 7fcac50ba7..e6f8b28b46 100644
--- a/esphome/components/xl9535/__init__.py
+++ b/esphome/components/xl9535/__init__.py
@@ -43,11 +43,17 @@ def validate_mode(mode):
     return mode
 
 
+def validate_pin(pin):
+    if pin in (8, 9):
+        raise cv.Invalid(f"pin {pin} doesn't exist")
+    return pin
+
+
 XL9535_PIN_SCHEMA = cv.All(
     {
         cv.GenerateID(): cv.declare_id(XL9535GPIOPin),
         cv.Required(CONF_XL9535): cv.use_id(XL9535Component),
-        cv.Required(CONF_NUMBER): cv.int_range(min=0, max=15),
+        cv.Required(CONF_NUMBER): cv.All(cv.int_range(min=0, max=17), validate_pin),
         cv.Optional(CONF_MODE, default={}): cv.All(
             {
                 cv.Optional(CONF_INPUT, default=False): cv.boolean,
diff --git a/tests/test4.yaml b/tests/test4.yaml
index a5e5b05e8b..c27dbb65ac 100644
--- a/tests/test4.yaml
+++ b/tests/test4.yaml
@@ -425,6 +425,15 @@ binary_sensor:
         input: true
       inverted: false
 
+  - platform: gpio
+    name: XL9535 Pin 17
+    pin:
+      xl9535: xl9535_hub
+      number: 17
+      mode:
+        input: true
+      inverted: false
+
 climate:
   - platform: tuya
     id: tuya_climate

From 0f19450ab40fe95adccc1c99d876be979e59ff86 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Nov 2023 21:18:22 -0600
Subject: [PATCH 36/41] Bump black from 23.10.1 to 23.11.0 (#5702)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: J. Nick Koston <nick@koston.org>
---
 .pre-commit-config.yaml | 2 +-
 requirements_test.txt   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ad8562640c..dc22265f1f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.10.1
+    rev: 23.11.0
     hooks:
       - id: black
         args:
diff --git a/requirements_test.txt b/requirements_test.txt
index fade3cda3e..d2ce98cc8c 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -1,6 +1,6 @@
 pylint==2.17.6
 flake8==6.1.0  # also change in .pre-commit-config.yaml when updating
-black==23.10.1  # also change in .pre-commit-config.yaml when updating
+black==23.11.0  # also change in .pre-commit-config.yaml when updating
 pyupgrade==3.15.0  # also change in .pre-commit-config.yaml when updating
 pre-commit
 

From 3b891bc146f371b6e334f0463e80669a39eb3489 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 10 Nov 2023 03:17:40 -0600
Subject: [PATCH 37/41] Speed up YAML by using YAML C loader when available
 (#5721)

---
 esphome/yaml_util.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py
index 3d3fa8c5b4..a954415d12 100644
--- a/esphome/yaml_util.py
+++ b/esphome/yaml_util.py
@@ -23,6 +23,14 @@ from esphome.core import (
 from esphome.helpers import add_class_to_obj
 from esphome.util import OrderedDict, filter_yaml_files
 
+try:
+    from yaml import CSafeLoader as FastestAvailableSafeLoader
+except ImportError:
+    from yaml import (  # type: ignore[assignment]
+        SafeLoader as FastestAvailableSafeLoader,
+    )
+
+
 _LOGGER = logging.getLogger(__name__)
 
 # Mostly copied from Home Assistant because that code works fine and
@@ -89,7 +97,7 @@ def _add_data_ref(fn):
     return wrapped
 
 
-class ESPHomeLoader(yaml.SafeLoader):
+class ESPHomeLoader(FastestAvailableSafeLoader):
     """Loader class that keeps track of line numbers."""
 
     @_add_data_ref

From 3363c8f434f09147877531b97c15143ed3b78bb8 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 10 Nov 2023 03:55:21 -0600
Subject: [PATCH 38/41] Fix zeroconf name resolution refactoring error (#5725)

---
 esphome/zeroconf.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py
index d20111ce20..f4cb7f080b 100644
--- a/esphome/zeroconf.py
+++ b/esphome/zeroconf.py
@@ -147,12 +147,13 @@ class DashboardImportDiscovery:
 
 
 class EsphomeZeroconf(Zeroconf):
-    def resolve_host(self, host: str, timeout=3.0):
+    def resolve_host(self, host: str, timeout: float = 3.0) -> str | None:
         """Resolve a host name to an IP address."""
         name = host.partition(".")[0]
-        info = HostResolver(f"{name}.{ESPHOME_SERVICE_TYPE}", ESPHOME_SERVICE_TYPE)
-        if (info.load_from_cache(self) or info.request(self, timeout * 1000)) and (
-            addresses := info.ip_addresses_by_version(IPVersion.V4Only)
-        ):
+        info = HostResolver(ESPHOME_SERVICE_TYPE, f"{name}.{ESPHOME_SERVICE_TYPE}")
+        if (
+            info.load_from_cache(self)
+            or (timeout and info.request(self, timeout * 1000))
+        ) and (addresses := info.ip_addresses_by_version(IPVersion.V4Only)):
             return str(addresses[0])
         return None

From 6a5cea171ea83156f06adc0dbd6d6696a196539a Mon Sep 17 00:00:00 2001
From: Mike La Spina <mike.laspina@shaw.ca>
Date: Fri, 10 Nov 2023 18:37:39 -0600
Subject: [PATCH 39/41] Missed ifdefs (#5727)

---
 esphome/components/ld2420/ld2420.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp
index 6130617457..bda1764cfc 100644
--- a/esphome/components/ld2420/ld2420.cpp
+++ b/esphome/components/ld2420/ld2420.cpp
@@ -68,6 +68,7 @@ void LD2420Component::dump_config() {
   ESP_LOGCONFIG(TAG, "LD2420:");
   ESP_LOGCONFIG(TAG, "  Firmware Version : %7s", this->ld2420_firmware_ver_);
   ESP_LOGCONFIG(TAG, "LD2420 Number:");
+#ifdef USE_NUMBER
   LOG_NUMBER(TAG, "  Gate Timeout:", this->gate_timeout_number_);
   LOG_NUMBER(TAG, "  Gate Max Distance:", this->max_gate_distance_number_);
   LOG_NUMBER(TAG, "  Gate Min Distance:", this->min_gate_distance_number_);
@@ -76,10 +77,13 @@ void LD2420Component::dump_config() {
     LOG_NUMBER(TAG, "  Gate Move Threshold:", this->gate_move_threshold_numbers_[gate]);
     LOG_NUMBER(TAG, "  Gate Still Threshold::", this->gate_still_threshold_numbers_[gate]);
   }
+#endif
+#ifdef USE_BUTTON
   LOG_BUTTON(TAG, "  Apply Config:", this->apply_config_button_);
   LOG_BUTTON(TAG, "  Revert Edits:", this->revert_config_button_);
   LOG_BUTTON(TAG, "  Factory Reset:", this->factory_reset_button_);
   LOG_BUTTON(TAG, "  Restart Module:", this->restart_module_button_);
+#endif
   ESP_LOGCONFIG(TAG, "LD2420 Select:");
   LOG_SELECT(TAG, "  Operating Mode", this->operating_selector_);
   if (this->get_firmware_int_(ld2420_firmware_ver_) < CALIBRATE_VERSION_MIN) {
@@ -183,9 +187,11 @@ void LD2420Component::factory_reset_action() {
     return;
   }
   this->set_min_max_distances_timeout(FACTORY_MAX_GATE, FACTORY_MIN_GATE, FACTORY_TIMEOUT);
+#ifdef USE_NUMBER
   this->gate_timeout_number_->state = FACTORY_TIMEOUT;
   this->min_gate_distance_number_->state = FACTORY_MIN_GATE;
   this->max_gate_distance_number_->state = FACTORY_MAX_GATE;
+#endif
   for (uint8_t gate = 0; gate < LD2420_TOTAL_GATES; gate++) {
     this->new_config.move_thresh[gate] = FACTORY_MOVE_THRESH[gate];
     this->new_config.still_thresh[gate] = FACTORY_STILL_THRESH[gate];

From 51930a02430f440e3dcf6b9fdddae975a67b6e4d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 11 Nov 2023 21:59:36 +0000
Subject: [PATCH 40/41] Bump aioesphomeapi from 18.2.7 to 18.4.0 (#5735)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f974967a00..51799bc685 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.2.7
+aioesphomeapi==18.4.0
 zeroconf==0.122.3
 
 # esp-idf requires this, but doesn't bundle it by default

From 53f3385c496f22dce6a2bcfbcad990cc217b0efa Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 11 Nov 2023 19:36:56 -0600
Subject: [PATCH 41/41] Migrate to using aioesphomeapi for the log runner to
 fix multiple issues (#5733)

---
 esphome/components/api/client.py | 76 +++++++++++++++-----------------
 1 file changed, 35 insertions(+), 41 deletions(-)

diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py
index 819055ccf4..2c43eca70c 100644
--- a/esphome/components/api/client.py
+++ b/esphome/components/api/client.py
@@ -1,71 +1,65 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 from datetime import datetime
-from typing import Optional
+from typing import Any
 
-from aioesphomeapi import APIClient, ReconnectLogic, APIConnectionError, LogLevel
-import zeroconf
+from aioesphomeapi import APIClient
+from aioesphomeapi.api_pb2 import SubscribeLogsResponse
+from aioesphomeapi.log_runner import async_run
+from zeroconf.asyncio import AsyncZeroconf
+
+from esphome.const import CONF_KEY, CONF_PASSWORD, CONF_PORT, __version__
+from esphome.core import CORE
 
-from esphome.const import CONF_KEY, CONF_PORT, CONF_PASSWORD, __version__
-from esphome.util import safe_print
 from . import CONF_ENCRYPTION
 
 _LOGGER = logging.getLogger(__name__)
 
 
 async def async_run_logs(config, address):
+    """Run the logs command in the event loop."""
     conf = config["api"]
     port: int = int(conf[CONF_PORT])
     password: str = conf[CONF_PASSWORD]
-    noise_psk: Optional[str] = None
+    noise_psk: str | None = None
     if CONF_ENCRYPTION in conf:
         noise_psk = conf[CONF_ENCRYPTION][CONF_KEY]
     _LOGGER.info("Starting log output from %s using esphome API", address)
+    aiozc = AsyncZeroconf()
+
     cli = APIClient(
         address,
         port,
         password,
         client_info=f"ESPHome Logs {__version__}",
         noise_psk=noise_psk,
+        zeroconf_instance=aiozc.zeroconf,
     )
-    first_connect = True
+    dashboard = CORE.dashboard
 
-    def on_log(msg):
-        time_ = datetime.now().time().strftime("[%H:%M:%S]")
-        text = msg.message.decode("utf8", "backslashreplace")
-        safe_print(time_ + text)
-
-    async def on_connect():
-        nonlocal first_connect
-        try:
-            await cli.subscribe_logs(
-                on_log,
-                log_level=LogLevel.LOG_LEVEL_VERY_VERBOSE,
-                dump_config=first_connect,
-            )
-            first_connect = False
-        except APIConnectionError:
-            cli.disconnect()
-
-    async def on_disconnect(expected_disconnect: bool) -> None:
-        _LOGGER.warning("Disconnected from API")
-
-    zc = zeroconf.Zeroconf()
-    reconnect = ReconnectLogic(
-        client=cli,
-        on_connect=on_connect,
-        on_disconnect=on_disconnect,
-        zeroconf_instance=zc,
-    )
-    await reconnect.start()
+    def on_log(msg: SubscribeLogsResponse) -> None:
+        """Handle a new log message."""
+        time_ = datetime.now()
+        message: bytes = msg.message
+        text = message.decode("utf8", "backslashreplace")
+        if dashboard:
+            text = text.replace("\033", "\\033")
+        print(f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}]{text}")
 
+    stop = await async_run(cli, on_log, aio_zeroconf_instance=aiozc)
     try:
         while True:
             await asyncio.sleep(60)
+    finally:
+        await aiozc.async_close()
+        await stop()
+
+
+def run_logs(config: dict[str, Any], address: str) -> None:
+    """Run the logs command."""
+    try:
+        asyncio.run(async_run_logs(config, address))
     except KeyboardInterrupt:
-        await reconnect.stop()
-        zc.close()
-
-
-def run_logs(config, address):
-    asyncio.run(async_run_logs(config, address))
+        pass