From 8e4b9c3c1e38f8aead90f19aa726143e86aad1a6 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Mon, 27 Nov 2023 13:45:26 +1300
Subject: [PATCH 01/75] Voice Assistant improvements (#5827)

---
 .../i2s_audio/speaker/i2s_audio_speaker.cpp   |   2 +
 .../i2s_audio/speaker/i2s_audio_speaker.h     |   2 +
 esphome/components/speaker/speaker.h          |   2 +
 .../voice_assistant/voice_assistant.cpp       | 108 +++++++++++-------
 .../voice_assistant/voice_assistant.h         |   3 +
 5 files changed, 78 insertions(+), 39 deletions(-)

diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index ed13e6b458..e729cdf954 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -220,6 +220,8 @@ size_t I2SAudioSpeaker::play(const uint8_t *data, size_t length) {
   return index;
 }
 
+bool I2SAudioSpeaker::has_buffered_data() const { return uxQueueMessagesWaiting(this->buffer_queue_) > 0; }
+
 }  // namespace i2s_audio
 }  // namespace esphome
 
diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
index b075722e1b..20c36a69d3 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
@@ -56,6 +56,8 @@ class I2SAudioSpeaker : public Component, public speaker::Speaker, public I2SAud
 
   size_t play(const uint8_t *data, size_t length) override;
 
+  bool has_buffered_data() const override;
+
  protected:
   void start_();
   // void stop_();
diff --git a/esphome/components/speaker/speaker.h b/esphome/components/speaker/speaker.h
index 3f520e3c5e..b494873160 100644
--- a/esphome/components/speaker/speaker.h
+++ b/esphome/components/speaker/speaker.h
@@ -18,6 +18,8 @@ class Speaker {
   virtual void start() = 0;
   virtual void stop() = 0;
 
+  virtual bool has_buffered_data() const = 0;
+
   bool is_running() const { return this->state_ == STATE_RUNNING; }
 
  protected:
diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index 9b13a71039..c0e706305d 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -273,28 +273,27 @@ void VoiceAssistant::loop() {
       bool playing = false;
 #ifdef USE_SPEAKER
       if (this->speaker_ != nullptr) {
+        ssize_t received_len = 0;
         if (this->speaker_buffer_index_ + RECEIVE_SIZE < SPEAKER_BUFFER_SIZE) {
-          auto len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE);
-          if (len > 0) {
-            this->speaker_buffer_index_ += len;
-            this->speaker_buffer_size_ += len;
+          received_len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE);
+          if (received_len > 0) {
+            this->speaker_buffer_index_ += received_len;
+            this->speaker_buffer_size_ += received_len;
+            this->speaker_bytes_received_ += received_len;
           }
         } else {
-          ESP_LOGW(TAG, "Receive buffer full");
-        }
-        if (this->speaker_buffer_size_ > 0) {
-          size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
-          if (written > 0) {
-            memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written);
-            this->speaker_buffer_size_ -= written;
-            this->speaker_buffer_index_ -= written;
-            this->set_timeout("speaker-timeout", 2000, [this]() { this->speaker_->stop(); });
-          } else {
-            ESP_LOGW(TAG, "Speaker buffer full");
-          }
+          ESP_LOGD(TAG, "Receive buffer full");
         }
+        // Build a small buffer of audio before sending to the speaker
+        if (this->speaker_bytes_received_ > RECEIVE_SIZE * 4)
+          this->write_speaker_();
         if (this->wait_for_stream_end_) {
           this->cancel_timeout("playing");
+          if (this->stream_ended_ && received_len < 0) {
+            ESP_LOGD(TAG, "End of audio stream received");
+            this->cancel_timeout("speaker-timeout");
+            this->set_state_(State::RESPONSE_FINISHED, State::RESPONSE_FINISHED);
+          }
           break;  // We dont want to timeout here as the STREAM_END event will take care of that.
         }
         playing = this->speaker_->is_running();
@@ -316,14 +315,26 @@ void VoiceAssistant::loop() {
     case State::RESPONSE_FINISHED: {
 #ifdef USE_SPEAKER
       if (this->speaker_ != nullptr) {
+        if (this->speaker_buffer_size_ > 0) {
+          this->write_speaker_();
+          break;
+        }
+        if (this->speaker_->has_buffered_data() || this->speaker_->is_running()) {
+          break;
+        }
+        ESP_LOGD(TAG, "Speaker has finished outputting all audio");
         this->speaker_->stop();
         this->cancel_timeout("speaker-timeout");
         this->cancel_timeout("playing");
         this->speaker_buffer_size_ = 0;
         this->speaker_buffer_index_ = 0;
+        this->speaker_bytes_received_ = 0;
         memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE);
+        this->wait_for_stream_end_ = false;
+        this->stream_ended_ = false;
+
+        this->tts_stream_end_trigger_->trigger();
       }
-      this->wait_for_stream_end_ = false;
 #endif
       this->set_state_(State::IDLE, State::IDLE);
       break;
@@ -333,6 +344,20 @@ void VoiceAssistant::loop() {
   }
 }
 
+void VoiceAssistant::write_speaker_() {
+  if (this->speaker_buffer_size_ > 0) {
+    size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
+    if (written > 0) {
+      memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written);
+      this->speaker_buffer_size_ -= written;
+      this->speaker_buffer_index_ -= written;
+      this->set_timeout("speaker-timeout", 5000, [this]() { this->speaker_->stop(); });
+    } else {
+      ESP_LOGD(TAG, "Speaker buffer full, trying again next loop");
+    }
+  }
+}
+
 void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) {
   if (!subscribe) {
     if (this->api_client_ == nullptr || client != this->api_client_) {
@@ -503,21 +528,20 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
   switch (msg.event_type) {
     case api::enums::VOICE_ASSISTANT_RUN_START:
       ESP_LOGD(TAG, "Assist Pipeline running");
-      this->start_trigger_->trigger();
+      this->defer([this]() { this->start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_WAKE_WORD_START:
       break;
     case api::enums::VOICE_ASSISTANT_WAKE_WORD_END: {
       ESP_LOGD(TAG, "Wake word detected");
-      this->wake_word_detected_trigger_->trigger();
+      this->defer([this]() { this->wake_word_detected_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_START:
       ESP_LOGD(TAG, "STT started");
-      this->listening_trigger_->trigger();
+      this->defer([this]() { this->listening_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_STT_END: {
-      this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE);
       std::string text;
       for (auto arg : msg.data) {
         if (arg.name == "text") {
@@ -529,12 +553,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Speech recognised as: \"%s\"", text.c_str());
-      this->stt_end_trigger_->trigger(text);
+      this->defer([this, text]() { this->stt_end_trigger_->trigger(text); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_INTENT_START:
       ESP_LOGD(TAG, "Intent started");
-      this->intent_start_trigger_->trigger();
+      this->defer([this]() { this->intent_start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_INTENT_END: {
       for (auto arg : msg.data) {
@@ -542,7 +566,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
           this->conversation_id_ = std::move(arg.value);
         }
       }
-      this->intent_end_trigger_->trigger();
+      this->defer([this]() { this->intent_end_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_START: {
@@ -557,10 +581,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Response: \"%s\"", text.c_str());
-      this->tts_start_trigger_->trigger(text);
+      this->defer([this, text]() {
+        this->tts_start_trigger_->trigger(text);
 #ifdef USE_SPEAKER
-      this->speaker_->start();
+        this->speaker_->start();
 #endif
+      });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_END: {
@@ -575,14 +601,16 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Response URL: \"%s\"", url.c_str());
+      this->defer([this, url]() {
 #ifdef USE_MEDIA_PLAYER
-      if (this->media_player_ != nullptr) {
-        this->media_player_->make_call().set_media_url(url).perform();
-      }
+        if (this->media_player_ != nullptr) {
+          this->media_player_->make_call().set_media_url(url).perform();
+        }
 #endif
+        this->tts_end_trigger_->trigger(url);
+      });
       State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE;
       this->set_state_(new_state, new_state);
-      this->tts_end_trigger_->trigger(url);
       break;
     }
     case api::enums::VOICE_ASSISTANT_RUN_END: {
@@ -599,7 +627,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
           this->set_state_(State::IDLE, State::IDLE);
         }
       }
-      this->end_trigger_->trigger();
+      this->defer([this]() { this->end_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_ERROR: {
@@ -617,8 +645,10 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       } else if (code == "wake-provider-missing" || code == "wake-engine-missing") {
         // Wake word is not set up or not ready on Home Assistant so stop and do not retry until user starts again.
-        this->request_stop();
-        this->error_trigger_->trigger(code, message);
+        this->defer([this, code, message]() {
+          this->request_stop();
+          this->error_trigger_->trigger(code, message);
+        });
         return;
       }
       ESP_LOGE(TAG, "Error: %s - %s", code.c_str(), message.c_str());
@@ -626,32 +656,32 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         this->signal_stop_();
         this->set_state_(State::STOP_MICROPHONE, State::IDLE);
       }
-      this->error_trigger_->trigger(code, message);
+      this->defer([this, code, message]() { this->error_trigger_->trigger(code, message); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_START: {
 #ifdef USE_SPEAKER
       this->wait_for_stream_end_ = true;
       ESP_LOGD(TAG, "TTS stream start");
-      this->tts_stream_start_trigger_->trigger();
+      this->defer([this] { this->tts_stream_start_trigger_->trigger(); });
 #endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_END: {
-      this->set_state_(State::RESPONSE_FINISHED, State::IDLE);
 #ifdef USE_SPEAKER
+      this->stream_ended_ = true;
       ESP_LOGD(TAG, "TTS stream end");
-      this->tts_stream_end_trigger_->trigger();
 #endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_VAD_START:
       ESP_LOGD(TAG, "Starting STT by VAD");
-      this->stt_vad_start_trigger_->trigger();
+      this->defer([this]() { this->stt_vad_start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_STT_VAD_END:
       ESP_LOGD(TAG, "STT by VAD end");
-      this->stt_vad_end_trigger_->trigger();
+      this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE);
+      this->defer([this]() { this->stt_vad_end_trigger_->trigger(); });
       break;
     default:
       ESP_LOGD(TAG, "Unhandled event type: %d", msg.event_type);
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index f6dcd1c563..66bf4c3c57 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -156,11 +156,14 @@ class VoiceAssistant : public Component {
 
   microphone::Microphone *mic_{nullptr};
 #ifdef USE_SPEAKER
+  void write_speaker_();
   speaker::Speaker *speaker_{nullptr};
   uint8_t *speaker_buffer_;
   size_t speaker_buffer_index_{0};
   size_t speaker_buffer_size_{0};
+  size_t speaker_bytes_received_{0};
   bool wait_for_stream_end_{false};
+  bool stream_ended_{false};
 #endif
 #ifdef USE_MEDIA_PLAYER
   media_player::MediaPlayer *media_player_{nullptr};

From ff97639f791963bb9e5f35318be960a100f024b0 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:43:41 +1300
Subject: [PATCH 02/75] Fix missing include in remote_base (#5843)

---
 esphome/components/remote_base/remote_base.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp
index 40c699e8ea..095f95053f 100644
--- a/esphome/components/remote_base/remote_base.cpp
+++ b/esphome/components/remote_base/remote_base.cpp
@@ -1,6 +1,8 @@
 #include "remote_base.h"
 #include "esphome/core/log.h"
 
+#include <cinttypes>
+
 namespace esphome {
 namespace remote_base {
 

From 687f5ca6330c7b96e92823bd28f9d68731404152 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:57:40 +1300
Subject: [PATCH 03/75] Add 'voice_assistant.connected' condition (#5845)

---
 esphome/components/voice_assistant/__init__.py       | 12 ++++++++++++
 esphome/components/voice_assistant/voice_assistant.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py
index d05f39072c..59aef901f2 100644
--- a/esphome/components/voice_assistant/__init__.py
+++ b/esphome/components/voice_assistant/__init__.py
@@ -57,6 +57,9 @@ StopAction = voice_assistant_ns.class_(
 IsRunningCondition = voice_assistant_ns.class_(
     "IsRunningCondition", automation.Condition, cg.Parented.template(VoiceAssistant)
 )
+ConnectedCondition = voice_assistant_ns.class_(
+    "ConnectedCondition", automation.Condition, cg.Parented.template(VoiceAssistant)
+)
 
 
 def tts_stream_validate(config):
@@ -298,3 +301,12 @@ async def voice_assistant_is_running_to_code(config, condition_id, template_arg,
     var = cg.new_Pvariable(condition_id, template_arg)
     await cg.register_parented(var, config[CONF_ID])
     return var
+
+
+@register_condition(
+    "voice_assistant.connected", ConnectedCondition, VOICE_ASSISTANT_ACTION_SCHEMA
+)
+async def voice_assistant_connected_to_code(config, condition_id, template_arg, args):
+    var = cg.new_Pvariable(condition_id, template_arg)
+    await cg.register_parented(var, config[CONF_ID])
+    return var
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index 66bf4c3c57..f9325dff54 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -222,6 +222,11 @@ template<typename... Ts> class IsRunningCondition : public Condition<Ts...>, pub
   bool check(Ts... x) override { return this->parent_->is_running() || this->parent_->is_continuous(); }
 };
 
+template<typename... Ts> class ConnectedCondition : public Condition<Ts...>, public Parented<VoiceAssistant> {
+ public:
+  bool check(Ts... x) override { return this->parent_->get_api_connection() != nullptr; }
+};
+
 extern VoiceAssistant *global_voice_assistant;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 }  // namespace voice_assistant

From 28a3cddde374a07c880622cd13095ce7dfbbec4d Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 11:14:26 +1300
Subject: [PATCH 04/75] Bump version to 2023.11.5

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 5596cbcd36..2e693d0ea0 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.4"
+__version__ = "2023.11.5"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 175f00f41bc8305cfa65bef7142ba7cdd4ee8140 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 12:28:24 +1300
Subject: [PATCH 05/75] Fix write_speaker without speaker in config (#5847)

---
 esphome/components/voice_assistant/voice_assistant.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index c0e706305d..29fc664342 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -344,6 +344,7 @@ void VoiceAssistant::loop() {
   }
 }
 
+#ifdef USE_SPEAKER
 void VoiceAssistant::write_speaker_() {
   if (this->speaker_buffer_size_ > 0) {
     size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
@@ -357,6 +358,7 @@ void VoiceAssistant::write_speaker_() {
     }
   }
 }
+#endif
 
 void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) {
   if (!subscribe) {

From ed9fd173a9f791d0d77ef28e68117fa04c6f8e23 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 12:31:55 +1300
Subject: [PATCH 06/75] Bump version to 2023.11.6

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 2e693d0ea0..6f8d83170e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.5"
+__version__ = "2023.11.6"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 8cc44766e6acc236bf2c7e70e24a4c5531a9ffba Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 08:30:45 +0900
Subject: [PATCH 07/75] Bump version to 2023.12.0b1

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 101a690db5..b8495a3a7c 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0-dev"
+__version__ = "2023.12.0b1"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 3c3ac920380dd07c38fa11860da21e944f2a1f01 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 13:24:16 +0900
Subject: [PATCH 08/75] Allow use of CDC/JTAG loggers on esp32 variants with
 Arduino (#4658)

* Allow use of CDC/JTAG loggers on esp32 variants with Arduino

* Only on s2/s3

* Separate C3 from S2/S3

* C code builds & runs correctly, still needs work though

* Works on S2

* It works!

* Remove unnecessary header

---------

Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
---
 esphome/components/logger/__init__.py |  2 --
 esphome/components/logger/logger.cpp  | 32 ++++++++++++++++++++++++---
 esphome/components/logger/logger.h    |  2 --
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py
index e431997276..11a7f996f0 100644
--- a/esphome/components/logger/__init__.py
+++ b/esphome/components/logger/__init__.py
@@ -124,8 +124,6 @@ is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
 
 def uart_selection(value):
     if CORE.is_esp32:
-        if value.upper() in ESP_IDF_UARTS and not CORE.using_esp_idf:
-            raise cv.Invalid(f"Only esp-idf framework supports {value}.")
         variant = get_esp32_variant()
         if variant in UART_SELECTION_ESP32:
             return cv.one_of(*UART_SELECTION_ESP32[variant], upper=True)(value)
diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp
index 2d2524b5f4..e0ca0806cb 100644
--- a/esphome/components/logger/logger.cpp
+++ b/esphome/components/logger/logger.cpp
@@ -236,8 +236,13 @@ void Logger::pre_setup() {
         this->hw_serial_ = &Serial1;
         Serial1.begin(this->baud_rate_);
 #else
+#if ARDUINO_USB_CDC_ON_BOOT
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);
+#else
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif
 #endif
 #ifdef USE_ESP8266
         if (this->uart_ == UART_SELECTION_UART0_SWAP) {
@@ -265,12 +270,35 @@ void Logger::pre_setup() {
         Serial2.begin(this->baud_rate_);
         break;
 #endif
+#if defined(USE_ESP32) && \
+    (defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3))
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+      case UART_SELECTION_USB_CDC:
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+#if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32S3)
+      case UART_SELECTION_USB_SERIAL_JTAG:
+#endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32S3
+#ifdef USE_ESP32_VARIANT_ESP32C3
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif  // USE_ESP32_VARIANT_ESP32C3
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if ARDUINO_USB_CDC_ON_BOOT
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#else
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif  // ARDUINO_USB_CDC_ON_BOOT
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+        break;
+#endif  // USE_ESP32 && (USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3)
 #ifdef USE_RP2040
       case UART_SELECTION_USB_CDC:
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);
         break;
-#endif
+#endif  // USE_RP2040
     }
 #endif  // USE_ARDUINO
 #ifdef USE_ESP_IDF
@@ -393,14 +421,12 @@ const char *const UART_SELECTIONS[] = {
     "UART2",
 #endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARINT_ESP32C6 && !USE_ESP32_VARIANT_ESP32S2 &&
         // !USE_ESP32_VARIANT_ESP32S3 && !USE_ESP32_VARIANT_ESP32H2
-#if defined(USE_ESP_IDF)
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
     "USB_CDC",
 #endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
 #if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32S3)
     "USB_SERIAL_JTAG",
 #endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32S3
-#endif  // USE_ESP_IDF
 };
 #endif  // USE_ESP32
 #ifdef USE_ESP8266
diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h
index 3816b1dd14..68efc056df 100644
--- a/esphome/components/logger/logger.h
+++ b/esphome/components/logger/logger.h
@@ -45,7 +45,6 @@ enum UARTSelection {
   UART_SELECTION_UART2,
 #endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARIANT_ESP32C6 && !USE_ESP32_VARIANT_ESP32S2 &&
         // !USE_ESP32_VARIANT_ESP32S3 && !USE_ESP32_VARIANT_ESP32H2
-#ifdef USE_ESP_IDF
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
   UART_SELECTION_USB_CDC,
 #endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
@@ -54,7 +53,6 @@ enum UARTSelection {
   UART_SELECTION_USB_SERIAL_JTAG,
 #endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C6 || USE_ESP32_VARIANT_ESP32S3 ||
         // USE_ESP32_VARIANT_ESP32H2
-#endif  // USE_ESP_IDF
 #endif  // USE_ESP32
 #ifdef USE_ESP8266
   UART_SELECTION_UART0_SWAP,

From 3e475c21ff9664c95b347375fe43b5647afdad03 Mon Sep 17 00:00:00 2001
From: Fabian <Fabian-Schmidt@users.noreply.github.com>
Date: Thu, 14 Dec 2023 05:47:31 +0100
Subject: [PATCH 09/75] [Logger] ESP32 S3 serial logger (#4853)

* Add support for ESP32 S3 logger.

* fix default

* Remove cpp & h changes to combine with PR #4658

* Not enough attention to details.

* Add build flag

* Validation fix

* Fix validation for real this time

---------

Co-authored-by: Your Name <you@example.com>
Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
---
 esphome/components/logger/__init__.py | 10 ++++-
 esphome/config_validation.py          | 53 +++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py
index 11a7f996f0..be302bd489 100644
--- a/esphome/components/logger/__init__.py
+++ b/esphome/components/logger/__init__.py
@@ -97,7 +97,7 @@ UART_SELECTION_LIBRETINY = {
     COMPONENT_RTL87XX: [DEFAULT, UART0, UART1, UART2],
 }
 
-ESP_IDF_UARTS = [USB_CDC, USB_SERIAL_JTAG]
+ESP_ARDUINO_UNSUPPORTED_USB_UARTS = [USB_SERIAL_JTAG]
 
 UART_SELECTION_RP2040 = [USB_CDC, UART0, UART1]
 
@@ -124,6 +124,8 @@ is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
 
 def uart_selection(value):
     if CORE.is_esp32:
+        if CORE.using_arduino and value.upper() in ESP_ARDUINO_UNSUPPORTED_USB_UARTS:
+            raise cv.Invalid(f"Arduino framework does not support {value}.")
         variant = get_esp32_variant()
         if variant in UART_SELECTION_ESP32:
             return cv.one_of(*UART_SELECTION_ESP32[variant], upper=True)(value)
@@ -169,6 +171,8 @@ CONFIG_SCHEMA = cv.All(
                 CONF_HARDWARE_UART,
                 esp8266=UART0,
                 esp32=UART0,
+                esp32_s2=USB_CDC,
+                esp32_s3=USB_CDC,
                 rp2040=USB_CDC,
                 bk72xx=DEFAULT,
                 rtl87xx=DEFAULT,
@@ -256,6 +260,10 @@ async def to_code(config):
     if config.get(CONF_ESP8266_STORE_LOG_STRINGS_IN_FLASH):
         cg.add_build_flag("-DUSE_STORE_LOG_STR_IN_FLASH")
 
+    if CORE.using_arduino:
+        if config[CONF_HARDWARE_UART] == USB_CDC:
+            cg.add_build_flag("-DARDUINO_USB_CDC_ON_BOOT=1")
+
     if CORE.using_esp_idf:
         if config[CONF_HARDWARE_UART] == USB_CDC:
             add_idf_sdkconfig_option("CONFIG_ESP_CONSOLE_USB_CDC", True)
diff --git a/esphome/config_validation.py b/esphome/config_validation.py
index ad2ee11512..fdbe7d6320 100644
--- a/esphome/config_validation.py
+++ b/esphome/config_validation.py
@@ -1528,6 +1528,12 @@ class SplitDefault(Optional):
         esp32=vol.UNDEFINED,
         esp32_arduino=vol.UNDEFINED,
         esp32_idf=vol.UNDEFINED,
+        esp32_s2=vol.UNDEFINED,
+        esp32_s2_arduino=vol.UNDEFINED,
+        esp32_s2_idf=vol.UNDEFINED,
+        esp32_s3=vol.UNDEFINED,
+        esp32_s3_arduino=vol.UNDEFINED,
+        esp32_s3_idf=vol.UNDEFINED,
         rp2040=vol.UNDEFINED,
         bk72xx=vol.UNDEFINED,
         rtl87xx=vol.UNDEFINED,
@@ -1541,6 +1547,26 @@ class SplitDefault(Optional):
         self._esp32_idf_default = vol.default_factory(
             esp32_idf if esp32 is vol.UNDEFINED else esp32
         )
+        self._esp32_s2_arduino_default = vol.default_factory(
+            (esp32_s2_arduino if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s2 is vol.UNDEFINED
+            else esp32_s2
+        )
+        self._esp32_s2_idf_default = vol.default_factory(
+            (esp32_s2_idf if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s2 is vol.UNDEFINED
+            else esp32_s2
+        )
+        self._esp32_s3_arduino_default = vol.default_factory(
+            (esp32_s3_arduino if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s3 is vol.UNDEFINED
+            else esp32_s3
+        )
+        self._esp32_s3_idf_default = vol.default_factory(
+            (esp32_s3_idf if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s3 is vol.UNDEFINED
+            else esp32_s3
+        )
         self._rp2040_default = vol.default_factory(rp2040)
         self._bk72xx_default = vol.default_factory(bk72xx)
         self._rtl87xx_default = vol.default_factory(rtl87xx)
@@ -1550,10 +1576,29 @@ class SplitDefault(Optional):
     def default(self):
         if CORE.is_esp8266:
             return self._esp8266_default
-        if CORE.is_esp32 and CORE.using_arduino:
-            return self._esp32_arduino_default
-        if CORE.is_esp32 and CORE.using_esp_idf:
-            return self._esp32_idf_default
+        if CORE.is_esp32:
+            from esphome.components.esp32 import get_esp32_variant
+            from esphome.components.esp32.const import (
+                VARIANT_ESP32S2,
+                VARIANT_ESP32S3,
+            )
+
+            variant = get_esp32_variant()
+            if variant == VARIANT_ESP32S2:
+                if CORE.using_arduino:
+                    return self._esp32_s2_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_s2_idf_default
+            elif variant == VARIANT_ESP32S3:
+                if CORE.using_arduino:
+                    return self._esp32_s3_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_s3_idf_default
+            else:
+                if CORE.using_arduino:
+                    return self._esp32_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_idf_default
         if CORE.is_rp2040:
             return self._rp2040_default
         if CORE.is_bk72xx:

From f28cf9348e486b3aab5c2b3e6a04f372c4ad4364 Mon Sep 17 00:00:00 2001
From: jochenvg <jochen@vanguyse.net>
Date: Thu, 14 Dec 2023 05:01:01 +0000
Subject: [PATCH 10/75] Support toggle action for template cover (#5917)

---
 esphome/components/template/cover/__init__.py        |  7 +++++++
 esphome/components/template/cover/template_cover.cpp | 10 ++++++++++
 esphome/components/template/cover/template_cover.h   |  4 ++++
 3 files changed, 21 insertions(+)

diff --git a/esphome/components/template/cover/__init__.py b/esphome/components/template/cover/__init__.py
index 8844ddd6ab..43d0be99b4 100644
--- a/esphome/components/template/cover/__init__.py
+++ b/esphome/components/template/cover/__init__.py
@@ -31,6 +31,7 @@ RESTORE_MODES = {
 }
 
 CONF_HAS_POSITION = "has_position"
+CONF_TOGGLE_ACTION = "toggle_action"
 
 CONFIG_SCHEMA = cover.COVER_SCHEMA.extend(
     {
@@ -44,6 +45,7 @@ CONFIG_SCHEMA = cover.COVER_SCHEMA.extend(
         cv.Optional(CONF_STOP_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_TILT_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_TILT_LAMBDA): cv.returning_lambda,
+        cv.Optional(CONF_TOGGLE_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_POSITION_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum(
             RESTORE_MODES, upper=True
@@ -74,6 +76,11 @@ async def to_code(config):
             var.get_stop_trigger(), [], config[CONF_STOP_ACTION]
         )
         cg.add(var.set_has_stop(True))
+    if CONF_TOGGLE_ACTION in config:
+        await automation.build_automation(
+            var.get_toggle_trigger(), [], config[CONF_TOGGLE_ACTION]
+        )
+        cg.add(var.set_has_toggle(True))
     if CONF_TILT_ACTION in config:
         await automation.build_automation(
             var.get_tilt_trigger(), [(float, "tilt")], config[CONF_TILT_ACTION]
diff --git a/esphome/components/template/cover/template_cover.cpp b/esphome/components/template/cover/template_cover.cpp
index b16e439943..2d6c3087ae 100644
--- a/esphome/components/template/cover/template_cover.cpp
+++ b/esphome/components/template/cover/template_cover.cpp
@@ -12,6 +12,7 @@ TemplateCover::TemplateCover()
     : open_trigger_(new Trigger<>()),
       close_trigger_(new Trigger<>),
       stop_trigger_(new Trigger<>()),
+      toggle_trigger_(new Trigger<>()),
       position_trigger_(new Trigger<float>()),
       tilt_trigger_(new Trigger<float>()) {}
 void TemplateCover::setup() {
@@ -68,6 +69,7 @@ float TemplateCover::get_setup_priority() const { return setup_priority::HARDWAR
 Trigger<> *TemplateCover::get_open_trigger() const { return this->open_trigger_; }
 Trigger<> *TemplateCover::get_close_trigger() const { return this->close_trigger_; }
 Trigger<> *TemplateCover::get_stop_trigger() const { return this->stop_trigger_; }
+Trigger<> *TemplateCover::get_toggle_trigger() const { return this->toggle_trigger_; }
 void TemplateCover::dump_config() { LOG_COVER("", "Template Cover", this); }
 void TemplateCover::control(const CoverCall &call) {
   if (call.get_stop()) {
@@ -76,6 +78,12 @@ void TemplateCover::control(const CoverCall &call) {
     this->prev_command_trigger_ = this->stop_trigger_;
     this->publish_state();
   }
+  if (call.get_toggle().has_value()) {
+    this->stop_prev_trigger_();
+    this->toggle_trigger_->trigger();
+    this->prev_command_trigger_ = this->toggle_trigger_;
+    this->publish_state();
+  }
   if (call.get_position().has_value()) {
     auto pos = *call.get_position();
     this->stop_prev_trigger_();
@@ -110,6 +118,7 @@ CoverTraits TemplateCover::get_traits() {
   auto traits = CoverTraits();
   traits.set_is_assumed_state(this->assumed_state_);
   traits.set_supports_stop(this->has_stop_);
+  traits.set_supports_toggle(this->has_toggle_);
   traits.set_supports_position(this->has_position_);
   traits.set_supports_tilt(this->has_tilt_);
   return traits;
@@ -118,6 +127,7 @@ Trigger<float> *TemplateCover::get_position_trigger() const { return this->posit
 Trigger<float> *TemplateCover::get_tilt_trigger() const { return this->tilt_trigger_; }
 void TemplateCover::set_tilt_lambda(std::function<optional<float>()> &&tilt_f) { this->tilt_f_ = tilt_f; }
 void TemplateCover::set_has_stop(bool has_stop) { this->has_stop_ = has_stop; }
+void TemplateCover::set_has_toggle(bool has_toggle) { this->has_toggle_ = has_toggle; }
 void TemplateCover::set_has_position(bool has_position) { this->has_position_ = has_position; }
 void TemplateCover::set_has_tilt(bool has_tilt) { this->has_tilt_ = has_tilt; }
 void TemplateCover::stop_prev_trigger_() {
diff --git a/esphome/components/template/cover/template_cover.h b/esphome/components/template/cover/template_cover.h
index 4ff5caf1db..958c94b0a6 100644
--- a/esphome/components/template/cover/template_cover.h
+++ b/esphome/components/template/cover/template_cover.h
@@ -21,6 +21,7 @@ class TemplateCover : public cover::Cover, public Component {
   Trigger<> *get_open_trigger() const;
   Trigger<> *get_close_trigger() const;
   Trigger<> *get_stop_trigger() const;
+  Trigger<> *get_toggle_trigger() const;
   Trigger<float> *get_position_trigger() const;
   Trigger<float> *get_tilt_trigger() const;
   void set_optimistic(bool optimistic);
@@ -29,6 +30,7 @@ class TemplateCover : public cover::Cover, public Component {
   void set_has_stop(bool has_stop);
   void set_has_position(bool has_position);
   void set_has_tilt(bool has_tilt);
+  void set_has_toggle(bool has_toggle);
   void set_restore_mode(TemplateCoverRestoreMode restore_mode) { restore_mode_ = restore_mode; }
 
   void setup() override;
@@ -50,7 +52,9 @@ class TemplateCover : public cover::Cover, public Component {
   Trigger<> *open_trigger_;
   Trigger<> *close_trigger_;
   bool has_stop_{false};
+  bool has_toggle_{false};
   Trigger<> *stop_trigger_;
+  Trigger<> *toggle_trigger_;
   Trigger<> *prev_command_trigger_{nullptr};
   Trigger<float> *position_trigger_;
   bool has_position_{false};

From 6b5eb7e656440ea03eaa6784172f662b1194bc3b Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 17:33:04 +0900
Subject: [PATCH 11/75] Fix SplitDefault with variants (#5928)

---
 esphome/config_validation.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/esphome/config_validation.py b/esphome/config_validation.py
index fdbe7d6320..7b94608509 100644
--- a/esphome/config_validation.py
+++ b/esphome/config_validation.py
@@ -1518,6 +1518,13 @@ class GenerateID(Optional):
         super().__init__(key, default=lambda: None)
 
 
+def _get_priority_default(*args):
+    for arg in args:
+        if arg is not vol.UNDEFINED:
+            return arg
+    return vol.UNDEFINED
+
+
 class SplitDefault(Optional):
     """Mark this key to have a split default for ESP8266/ESP32."""
 
@@ -1542,30 +1549,22 @@ class SplitDefault(Optional):
         super().__init__(key)
         self._esp8266_default = vol.default_factory(esp8266)
         self._esp32_arduino_default = vol.default_factory(
-            esp32_arduino if esp32 is vol.UNDEFINED else esp32
+            _get_priority_default(esp32, esp32_arduino)
         )
         self._esp32_idf_default = vol.default_factory(
-            esp32_idf if esp32 is vol.UNDEFINED else esp32
+            _get_priority_default(esp32, esp32_idf)
         )
         self._esp32_s2_arduino_default = vol.default_factory(
-            (esp32_s2_arduino if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s2 is vol.UNDEFINED
-            else esp32_s2
+            _get_priority_default(esp32_s2, esp32, esp32_s2_arduino, esp32_arduino)
         )
         self._esp32_s2_idf_default = vol.default_factory(
-            (esp32_s2_idf if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s2 is vol.UNDEFINED
-            else esp32_s2
+            _get_priority_default(esp32_s2, esp32, esp32_s2_idf, esp32_idf)
         )
         self._esp32_s3_arduino_default = vol.default_factory(
-            (esp32_s3_arduino if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s3 is vol.UNDEFINED
-            else esp32_s3
+            _get_priority_default(esp32_s3, esp32, esp32_s3_arduino, esp32_arduino)
         )
         self._esp32_s3_idf_default = vol.default_factory(
-            (esp32_s3_idf if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s3 is vol.UNDEFINED
-            else esp32_s3
+            _get_priority_default(esp32_s3, esp32, esp32_s3_idf, esp32_idf)
         )
         self._rp2040_default = vol.default_factory(rp2040)
         self._bk72xx_default = vol.default_factory(bk72xx)

From e030c0fc45c1105cc21e2ddb12db9c0c90766a2b Mon Sep 17 00:00:00 2001
From: mrtoy-me <118446898+mrtoy-me@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:39:05 +1000
Subject: [PATCH 12/75] Update ENS160 TVOC device_class and AQI units to match
 required by HA (#5939)

---
 esphome/components/ens160/sensor.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/esphome/components/ens160/sensor.py b/esphome/components/ens160/sensor.py
index 55f0ff7b6f..393b63bae1 100644
--- a/esphome/components/ens160/sensor.py
+++ b/esphome/components/ens160/sensor.py
@@ -9,7 +9,7 @@ from esphome.const import (
     CONF_TVOC,
     DEVICE_CLASS_AQI,
     DEVICE_CLASS_CARBON_DIOXIDE,
-    DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+    DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS_PARTS,
     ICON_CHEMICAL_WEAPON,
     ICON_MOLECULE_CO2,
     ICON_RADIATOR,
@@ -45,11 +45,10 @@ CONFIG_SCHEMA = (
                 unit_of_measurement=UNIT_PARTS_PER_BILLION,
                 icon=ICON_RADIATOR,
                 accuracy_decimals=0,
-                device_class=DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+                device_class=DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS_PARTS,
                 state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Required(CONF_AQI): sensor.sensor_schema(
-                unit_of_measurement=UNIT_INDEX,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_AQI,

From 514db8b26e8614902b0e8c849802d22709df018b Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:02:47 +0900
Subject: [PATCH 13/75] Bump version to 2023.12.0b2

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index b8495a3a7c..6c2a520331 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b1"
+__version__ = "2023.12.0b2"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 70dac541139a7514dd06281152f233fc2efd3382 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 17 Dec 2023 16:27:08 +0000
Subject: [PATCH 14/75] Bump zeroconf from 0.128.4 to 0.130.0 (#5950)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f330ecbf3e..20a5514e71 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,7 @@ esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
 aioesphomeapi==21.0.0
-zeroconf==0.128.4
+zeroconf==0.130.0
 python-magic==0.4.27
 
 # esp-idf requires this, but doesn't bundle it by default

From 917e0f93edd561b13fd25da0b5ee4beb58e10e1d Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Mon, 18 Dec 2023 00:19:30 +0100
Subject: [PATCH 15/75] UARTComponent inline doc (#5930)

---
 esphome/components/uart/uart_component.h | 88 +++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 3 deletions(-)

diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h
index 34bda42bb5..e03784fdd8 100644
--- a/esphome/components/uart/uart_component.h
+++ b/esphome/components/uart/uart_component.h
@@ -31,40 +31,122 @@ const LogString *parity_to_str(UARTParityOptions parity);
 
 class UARTComponent {
  public:
+  // Writes an array of bytes to the UART bus.
+  // @param data A vector of bytes to be written.
   void write_array(const std::vector<uint8_t> &data) { this->write_array(&data[0], data.size()); }
+
+  // Writes a single byte to the UART bus.
+  // @param data The byte to be written.
   void write_byte(uint8_t data) { this->write_array(&data, 1); };
+
+  // Writes a null-terminated string to the UART bus.
+  // @param str Pointer to the null-terminated string.
   void write_str(const char *str) {
     const auto *data = reinterpret_cast<const uint8_t *>(str);
     this->write_array(data, strlen(str));
   };
 
+  // Pure virtual method to write an array of bytes to the UART bus.
+  // @param data Pointer to the array of bytes.
+  // @param len Length of the array.
   virtual void write_array(const uint8_t *data, size_t len) = 0;
 
+  // Reads a single byte from the UART bus.
+  // @param data Pointer to the byte where the read data will be stored.
+  // @return True if a byte was successfully read, false otherwise.
   bool read_byte(uint8_t *data) { return this->read_array(data, 1); };
+
+  // Pure virtual method to peek the next byte in the UART buffer without removing it.
+  // @param data Pointer to the byte where the peeked data will be stored.
+  // @return True if a byte is available to peek, false otherwise.
   virtual bool peek_byte(uint8_t *data) = 0;
+
+  // Pure virtual method to read an array of bytes from the UART bus.
+  // @param data Pointer to the array where the read data will be stored.
+  // @param len Number of bytes to read.
+  // @return True if the specified number of bytes were successfully read, false otherwise.
   virtual bool read_array(uint8_t *data, size_t len) = 0;
 
-  /// Return available number of bytes.
+  // Pure virtual method to return the number of bytes available for reading.
+  // @return Number of available bytes.
   virtual int available() = 0;
-  /// Block until all bytes have been written to the UART bus.
+
+  // Pure virtual method to block until all bytes have been written to the UART bus.
   virtual void flush() = 0;
 
+  // Sets the TX (transmit) pin for the UART bus.
+  // @param tx_pin Pointer to the internal GPIO pin used for transmission.
   void set_tx_pin(InternalGPIOPin *tx_pin) { this->tx_pin_ = tx_pin; }
+
+  // Sets the RX (receive) pin for the UART bus.
+  // @param rx_pin Pointer to the internal GPIO pin used for reception.
   void set_rx_pin(InternalGPIOPin *rx_pin) { this->rx_pin_ = rx_pin; }
+
+  // Sets the size of the RX buffer.
+  // @param rx_buffer_size Size of the RX buffer in bytes.
   void set_rx_buffer_size(size_t rx_buffer_size) { this->rx_buffer_size_ = rx_buffer_size; }
+
+  // Gets the size of the RX buffer.
+  // @return Size of the RX buffer in bytes.
   size_t get_rx_buffer_size() { return this->rx_buffer_size_; }
 
+  // Sets the number of stop bits used in UART communication.
+  // @param stop_bits Number of stop bits.
   void set_stop_bits(uint8_t stop_bits) { this->stop_bits_ = stop_bits; }
+
+  // Gets the number of stop bits used in UART communication.
+  // @return Number of stop bits.
   uint8_t get_stop_bits() const { return this->stop_bits_; }
+
+  // Set the number of data bits used in UART communication.
+  // @param data_bits Number of data bits.
   void set_data_bits(uint8_t data_bits) { this->data_bits_ = data_bits; }
+
+  // Get the number of data bits used in UART communication.
+  // @return Number of data bits.
   uint8_t get_data_bits() const { return this->data_bits_; }
+
+  // Set the parity used in UART communication.
+  // @param parity Parity option.
   void set_parity(UARTParityOptions parity) { this->parity_ = parity; }
+
+  // Get the parity used in UART communication.
+  // @return Parity option.
   UARTParityOptions get_parity() const { return this->parity_; }
+
+  // Set the baud rate for UART communication.
+  // @param baud_rate Baud rate in bits per second.
   void set_baud_rate(uint32_t baud_rate) { baud_rate_ = baud_rate; }
+
+  // Get the baud rate for UART communication.
+  // @return Baud rate in bits per second.
   uint32_t get_baud_rate() const { return baud_rate_; }
+
 #ifdef USE_ESP32
-  virtual void load_settings() = 0;
+  /**
+   * Load the UART settings.
+   * @param dump_config If true (default), output the new settings to logs; otherwise, change settings quietly.
+   *
+   * Example:
+   * ```cpp
+   * id(uart1).load_settings(false);
+   * ```
+   *
+   * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
+   */
   virtual void load_settings(bool dump_config) = 0;
+
+  /**
+   * Load the UART settings.
+   *
+   * Example:
+   * ```cpp
+   * id(uart1).load_settings();
+   * ```
+   *
+   * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
+   */
+  virtual void load_settings() = 0;
 #endif  // USE_ESP32
 
 #ifdef USE_UART_DEBUGGER

From 2309f15ce06b0fe633420b8061293427e62f0051 Mon Sep 17 00:00:00 2001
From: Grant Le Roux <cram42@users.noreply.github.com>
Date: Mon, 18 Dec 2023 07:28:48 +0800
Subject: [PATCH 16/75] Fix - Tuya Fan - Allow integer speed datapoint (#5948)

Co-authored-by: Cram42 <5396871+cram42@users.noreply.github.com>
---
 esphome/components/tuya/fan/tuya_fan.cpp | 23 +++++++++++++++++------
 esphome/components/tuya/fan/tuya_fan.h   |  1 +
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/esphome/components/tuya/fan/tuya_fan.cpp b/esphome/components/tuya/fan/tuya_fan.cpp
index 1b03ea50fa..481c931f2e 100644
--- a/esphome/components/tuya/fan/tuya_fan.cpp
+++ b/esphome/components/tuya/fan/tuya_fan.cpp
@@ -9,13 +9,20 @@ static const char *const TAG = "tuya.fan";
 void TuyaFan::setup() {
   if (this->speed_id_.has_value()) {
     this->parent_->register_listener(*this->speed_id_, [this](const TuyaDatapoint &datapoint) {
-      ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_enum);
-      if (datapoint.value_enum >= this->speed_count_) {
-        ESP_LOGE(TAG, "Speed has invalid value %d", datapoint.value_enum);
-      } else {
-        this->speed = datapoint.value_enum + 1;
+      if (datapoint.type == TuyaDatapointType::ENUM) {
+        ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_enum);
+        if (datapoint.value_enum >= this->speed_count_) {
+          ESP_LOGE(TAG, "Speed has invalid value %d", datapoint.value_enum);
+        } else {
+          this->speed = datapoint.value_enum + 1;
+          this->publish_state();
+        }
+      } else if (datapoint.type == TuyaDatapointType::INTEGER) {
+        ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_int);
+        this->speed = datapoint.value_int;
         this->publish_state();
       }
+      this->speed_type_ = datapoint.type;
     });
   }
   if (this->switch_id_.has_value()) {
@@ -80,7 +87,11 @@ void TuyaFan::control(const fan::FanCall &call) {
     this->parent_->set_enum_datapoint_value(*this->direction_id_, enable);
   }
   if (this->speed_id_.has_value() && call.get_speed().has_value()) {
-    this->parent_->set_enum_datapoint_value(*this->speed_id_, *call.get_speed() - 1);
+    if (this->speed_type_ == TuyaDatapointType::ENUM) {
+      this->parent_->set_enum_datapoint_value(*this->speed_id_, *call.get_speed() - 1);
+    } else if (this->speed_type_ == TuyaDatapointType::INTEGER) {
+      this->parent_->set_integer_datapoint_value(*this->speed_id_, *call.get_speed());
+    }
   }
 }
 
diff --git a/esphome/components/tuya/fan/tuya_fan.h b/esphome/components/tuya/fan/tuya_fan.h
index 4aba1e1c07..77b2cc6383 100644
--- a/esphome/components/tuya/fan/tuya_fan.h
+++ b/esphome/components/tuya/fan/tuya_fan.h
@@ -28,6 +28,7 @@ class TuyaFan : public Component, public fan::Fan {
   optional<uint8_t> oscillation_id_{};
   optional<uint8_t> direction_id_{};
   int speed_count_{};
+  TuyaDatapointType speed_type_{};
 };
 
 }  // namespace tuya

From 168e70413030060eedb3d723939ea7191767a8a0 Mon Sep 17 00:00:00 2001
From: Alex Hermann <gaaf@gmx.net>
Date: Mon, 18 Dec 2023 00:58:13 +0100
Subject: [PATCH 17/75] i2s_audio: Set player_task's prio to 1 (#5945)

---
 esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index e729cdf954..95e63035fe 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -29,7 +29,7 @@ void I2SAudioSpeaker::start_() {
   }
   this->state_ = speaker::STATE_RUNNING;
 
-  xTaskCreate(I2SAudioSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_);
+  xTaskCreate(I2SAudioSpeaker::player_task, "speaker_task", 8192, (void *) this, 1, &this->player_task_handle_);
 }
 
 void I2SAudioSpeaker::player_task(void *params) {

From e8ce780482c914f45df13bff31448352d5ee5e76 Mon Sep 17 00:00:00 2001
From: Alex Hermann <gaaf@gmx.net>
Date: Mon, 18 Dec 2023 01:00:42 +0100
Subject: [PATCH 18/75] esp32_camera: Set framebuffer task prio to 1 (#5943)

---
 esphome/components/esp32_camera/esp32_camera.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/esp32_camera/esp32_camera.cpp b/esphome/components/esp32_camera/esp32_camera.cpp
index 99cb811fe4..555f6ca5f1 100644
--- a/esphome/components/esp32_camera/esp32_camera.cpp
+++ b/esphome/components/esp32_camera/esp32_camera.cpp
@@ -37,7 +37,7 @@ void ESP32Camera::setup() {
                           "framebuffer_task",  // name
                           1024,                // stack size
                           nullptr,             // task pv params
-                          0,                   // priority
+                          1,                   // priority
                           nullptr,             // handle
                           1                    // core
   );

From d0df73769d6e51f0efffab59011298e9f836e4f6 Mon Sep 17 00:00:00 2001
From: dentra <dentra@users.noreply.github.com>
Date: Mon, 18 Dec 2023 03:01:21 +0300
Subject: [PATCH 19/75] web_server_idf: fix call with hardcoded http code
 (#5942)

---
 esphome/components/web_server_idf/web_server_idf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h
index bc64e5231e..2fbc5cd2e9 100644
--- a/esphome/components/web_server_idf/web_server_idf.h
+++ b/esphome/components/web_server_idf/web_server_idf.h
@@ -117,7 +117,7 @@ class AsyncWebServerRequest {
   // NOLINTNEXTLINE(readability-identifier-naming)
   AsyncWebServerResponse *beginResponse(int code, const char *content_type) {
     auto *res = new AsyncWebServerResponseEmpty(this);  // NOLINT(cppcoreguidelines-owning-memory)
-    this->init_response_(res, 200, content_type);
+    this->init_response_(res, code, content_type);
     return res;
   }
   // NOLINTNEXTLINE(readability-identifier-naming)

From eefa1cd3aba5ff2fcf93328b35117cdbf70594b7 Mon Sep 17 00:00:00 2001
From: Jean  Louis-Guerin <DrCoolzic@hotmail.com>
Date: Mon, 18 Dec 2023 01:03:01 +0100
Subject: [PATCH 20/75] Revert pure virtual functions in UART component from
 #5920 (#5932)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/uart/uart_component.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h
index e03784fdd8..6f27f36bcb 100644
--- a/esphome/components/uart/uart_component.h
+++ b/esphome/components/uart/uart_component.h
@@ -134,7 +134,7 @@ class UARTComponent {
    *
    * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
    */
-  virtual void load_settings(bool dump_config) = 0;
+  virtual void load_settings(bool dump_config){};
 
   /**
    * Load the UART settings.
@@ -146,7 +146,7 @@ class UARTComponent {
    *
    * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
    */
-  virtual void load_settings() = 0;
+  virtual void load_settings(){};
 #endif  // USE_ESP32
 
 #ifdef USE_UART_DEBUGGER

From ab22a3da349d3d60b687cbf70dd5a00ec44b67e2 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Mon, 18 Dec 2023 01:33:12 -0600
Subject: [PATCH 21/75] Use the correct UART/`Serial` when CDC is enabled
 (#5957)

---
 esphome/components/logger/logger.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp
index e0ca0806cb..05b97a5f64 100644
--- a/esphome/components/logger/logger.cpp
+++ b/esphome/components/logger/logger.cpp
@@ -237,8 +237,8 @@ void Logger::pre_setup() {
         Serial1.begin(this->baud_rate_);
 #else
 #if ARDUINO_USB_CDC_ON_BOOT
-        this->hw_serial_ = &Serial;
-        Serial.begin(this->baud_rate_);
+        this->hw_serial_ = &Serial0;
+        Serial0.begin(this->baud_rate_);
 #else
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);

From dbfa77cb4bac5f5fc6120a79320e52a142c01812 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:51:11 +0900
Subject: [PATCH 22/75] Bump version to 2023.12.0b3

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 6c2a520331..4153646aea 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b2"
+__version__ = "2023.12.0b3"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 16798bbfb461bf4b79799b715c2575654755548e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 19 Dec 2023 13:41:26 -1000
Subject: [PATCH 23/75] Bump aioesphomeapi to 21.0.1 (#5969)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index fffe011c5a..115f85de3e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.7.0
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==21.0.0
+aioesphomeapi==21.0.1
 zeroconf==0.131.0
 python-magic==0.4.27
 

From 7807f0d89250a890aaec97a524ad1d68310d07c3 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Mon, 18 Dec 2023 17:11:07 -0600
Subject: [PATCH 24/75] Fix build issue with UART component when building with
 Arduino and CDC (#5964)

---
 esphome/components/uart/uart_component_esp32_arduino.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/esphome/components/uart/uart_component_esp32_arduino.cpp b/esphome/components/uart/uart_component_esp32_arduino.cpp
index 75b67bf5c2..f77783e20e 100644
--- a/esphome/components/uart/uart_component_esp32_arduino.cpp
+++ b/esphome/components/uart/uart_component_esp32_arduino.cpp
@@ -88,7 +88,11 @@ void ESP32ArduinoUARTComponent::setup() {
 #endif
   static uint8_t next_uart_num = 0;
   if (is_default_tx && is_default_rx && next_uart_num == 0) {
+#if ARDUINO_USB_CDC_ON_BOOT
+    this->hw_serial_ = &Serial0;
+#else
     this->hw_serial_ = &Serial;
+#endif
     next_uart_num++;
   } else {
 #ifdef USE_LOGGER

From 820f3282480d282914a6a0682908443ad3b5755a Mon Sep 17 00:00:00 2001
From: Jean  Louis-Guerin <DrCoolzic@hotmail.com>
Date: Tue, 19 Dec 2023 00:14:42 +0100
Subject: [PATCH 25/75] Fix I2CBus::write() bug and add i2c documentation
 (#5947)

---
 esphome/components/i2c/i2c.h     | 153 +++++++++++++++++++++++++++++--
 esphome/components/i2c/i2c_bus.h |  79 ++++++++++++----
 2 files changed, 205 insertions(+), 27 deletions(-)

diff --git a/esphome/components/i2c/i2c.h b/esphome/components/i2c/i2c.h
index eb5d463b65..8d8e139c61 100644
--- a/esphome/components/i2c/i2c.h
+++ b/esphome/components/i2c/i2c.h
@@ -11,43 +11,116 @@ namespace i2c {
 
 #define LOG_I2C_DEVICE(this) ESP_LOGCONFIG(TAG, "  Address: 0x%02X", this->address_);
 
-class I2CDevice;
+class I2CDevice;  // forward declaration
+
+/// @brief This class is used to create I2CRegister objects that act as proxies to read/write internal registers on an
+/// I2C device.
+/// @details
+/// @n typical usage:
+/// @code
+/// constexpr uint8_t ADDR_REGISTER_1 = 0x12;
+/// i2c::I2CRegister reg_1 = this->reg(ADDR_REGISTER_1); // declare
+/// reg_1 |= 0x01; // set bit
+/// reg_1 &= ~0x01; // reset bit
+/// reg_1 = 10; // Set value
+/// uint val = reg_1.get(); // get value
+/// @endcode
+/// @details The I²C protocol specifies how to read/write in sets of 8-bits followed by an Acknowledgement (ACK/NACK)
+/// from the device receiving the data. How the device interprets the bits read/written can vary greatly from
+/// device to device. However most of the devices follow the same protocol for reading/writing 8 bit registers using as
+/// implemented in the I2CRegister: after sending the device address, the controller sends one byte with the internal
+/// register address and then read or write the specified register content.
 class I2CRegister {
  public:
+  /// @brief overloads the = operator. This allows to set the value of an i2c register
+  /// @param value value to be set in the register
+  /// @return pointer to current object
   I2CRegister &operator=(uint8_t value);
+
+  /// @brief overloads the compound &= operator. This allows to reset specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister &operator&=(uint8_t value);
+
+  /// @brief overloads the compound |= operator. This allows to set specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister &operator|=(uint8_t value);
 
+  /// @brief overloads the uint8_t() cast operator to return the I²C register value
+  /// @return pointer to current object
   explicit operator uint8_t() const { return get(); }
 
+  /// @brief returns the register value
+  /// @return the register value
   uint8_t get() const;
 
  protected:
   friend class I2CDevice;
 
+  /// @brief protected constructor that stores the owning object and the register address. Note as only friends can
+  /// create an I2CRegister @see I2CDevice::reg()
+  /// @param parent our parent
+  /// @param a_register address of the i2c register
   I2CRegister(I2CDevice *parent, uint8_t a_register) : parent_(parent), register_(a_register) {}
 
-  I2CDevice *parent_;
-  uint8_t register_;
+  I2CDevice *parent_;  ///< I2CDevice object pointer
+  uint8_t register_;   ///< the internal address of the register
 };
 
+/// @brief This class is used to create I2CRegister16 objects that act as proxies to read/write internal registers
+/// (specified with a 16 bit address) on an I2C device.
+/// @details
+/// @n typical usage:
+/// @code
+/// constexpr uint16_t X16_BIT_ADDR_REGISTER_1 = 0x1234;
+/// i2c::I2CRegister16 reg_1 = this->reg16(X16_BIT_ADDR_REGISTER_1); // declare
+/// reg_1 |= 0x01; // set bit
+/// reg_1 &= ~0x01; // reset bit
+/// reg_1 = 10; // Set value
+/// uint val = reg_1.get(); // get value
+/// @endcode
+/// @details The I²C protocol specification, reads/writes in sets of 8-bits followed by an Acknowledgement (ACK/NACK)
+/// from the device receiving the data. How the device interprets the bits read/written to it can vary greatly from
+/// device to device. This class can be used to access in the device 8 bits registers that uses a 16 bits internal
+/// address. After sending the device address, the controller sends the internal register address (using two consecutive
+/// bytes following the big indian convention) and then read or write the register content.
 class I2CRegister16 {
  public:
+  /// @brief overloads the = operator. This allows to set the value of an I²C register
+  /// @param value value to be set in the register
+  /// @return pointer to current object
   I2CRegister16 &operator=(uint8_t value);
+
+  /// @brief overloads the compound &= operator. This allows to reset specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister16 &operator&=(uint8_t value);
+
+  /// @brief overloads the compound |= operator. This allows to set bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister16 &operator|=(uint8_t value);
 
+  /// @brief overloads the uint8_t() cast operator to return the I²C register value
+  /// @return the register value
   explicit operator uint8_t() const { return get(); }
 
+  /// @brief returns the register value
+  /// @return the register value
   uint8_t get() const;
 
  protected:
   friend class I2CDevice;
 
+  /// @brief protected constructor that store the owning object and the register address. Only friends can create an
+  /// I2CRegister16 @see I2CDevice::reg16()
+  /// @param parent our parent
+  /// @param a_register 16 bits address of the i2c register
   I2CRegister16(I2CDevice *parent, uint16_t a_register) : parent_(parent), register_(a_register) {}
 
-  I2CDevice *parent_;
-  uint16_t register_;
+  I2CDevice *parent_;  ///< I2CDevice object pointer
+  uint16_t register_;  ///< the internal 16 bits address of the register
 };
 
 // like ntohs/htons but without including networking headers.
@@ -55,29 +128,91 @@ class I2CRegister16 {
 inline uint16_t i2ctohs(uint16_t i2cshort) { return convert_big_endian(i2cshort); }
 inline uint16_t htoi2cs(uint16_t hostshort) { return convert_big_endian(hostshort); }
 
+/// @brief This Class provides the methods to read/write bytes from/to an i2c device.
+/// Objects keep a list of devices found on bus as well as a pointer to the I2CBus in use.
 class I2CDevice {
  public:
+  /// @brief we use the C++ default constructor
   I2CDevice() = default;
 
+  /// @brief We store the address of the device on the bus
+  /// @param address of the device
   void set_i2c_address(uint8_t address) { address_ = address; }
+
+  /// @brief we store the pointer to the I2CBus to use
+  /// @param bus pointer to the I2CBus object
   void set_i2c_bus(I2CBus *bus) { bus_ = bus; }
 
+  /// @brief calls the I2CRegister constructor
+  /// @param a_register address of the I²C register
+  /// @return an I2CRegister proxy object
   I2CRegister reg(uint8_t a_register) { return {this, a_register}; }
+
+  /// @brief calls the I2CRegister16 constructor
+  /// @param a_register 16 bits address of the I²C register
+  /// @return an I2CRegister16 proxy object
   I2CRegister16 reg16(uint16_t a_register) { return {this, a_register}; }
 
+  /// @brief reads an array of bytes from the device using an I2CBus
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @return an i2c::ErrorCode
   ErrorCode read(uint8_t *data, size_t len) { return bus_->read(address_, data, len); }
+
+  /// @brief reads an array of bytes from a specific register in the I²C device
+  /// @param a_register an 8 bits internal address of the I²C register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode read_register(uint8_t a_register, uint8_t *data, size_t len, bool stop = true);
+
+  /// @brief reads an array of bytes from a specific register in the I²C device
+  /// @param a_register the 16 bits internal address of the I²C register to read from
+  /// @param data pointer to an array of bytes to store the information
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode read_register16(uint16_t a_register, uint8_t *data, size_t len, bool stop = true);
 
-  ErrorCode write(const uint8_t *data, uint8_t len, bool stop = true) { return bus_->write(address_, data, len, stop); }
+  /// @brief writes an array of bytes to a device using an I2CBus
+  /// @param data pointer to an array that contains the bytes to send
+  /// @param len length of the buffer = number of bytes to write
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
+  ErrorCode write(const uint8_t *data, size_t len, bool stop = true) { return bus_->write(address_, data, len, stop); }
+
+  /// @brief writes an array of bytes to a specific register in the I²C device
+  /// @param a_register the internal address of the register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode write_register(uint8_t a_register, const uint8_t *data, size_t len, bool stop = true);
+
+  /// @brief write an array of bytes to a specific register in the I²C device
+  /// @param a_register the 16 bits internal address of the register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode write_register16(uint16_t a_register, const uint8_t *data, size_t len, bool stop = true);
 
-  // Compat APIs
+  ///
+  /// Compat APIs
+  /// All methods below have been added for compatibility reasons. They do not bring any functionality and therefore on
+  /// new code it is not recommend to use them.
+  ///
 
   bool read_bytes(uint8_t a_register, uint8_t *data, uint8_t len) {
     return read_register(a_register, data, len) == ERROR_OK;
   }
+
   bool read_bytes_raw(uint8_t *data, uint8_t len) { return read(data, len) == ERROR_OK; }
 
   template<size_t N> optional<std::array<uint8_t, N>> read_bytes(uint8_t a_register) {
@@ -131,8 +266,8 @@ class I2CDevice {
   bool write_byte_16(uint8_t a_register, uint16_t data) { return write_bytes_16(a_register, &data, 1); }
 
  protected:
-  uint8_t address_{0x00};
-  I2CBus *bus_{nullptr};
+  uint8_t address_{0x00};  ///< store the address of the device on the bus
+  I2CBus *bus_{nullptr};   ///< pointer to I2CBus instance
 };
 
 }  // namespace i2c
diff --git a/esphome/components/i2c/i2c_bus.h b/esphome/components/i2c/i2c_bus.h
index 2633a7adf6..fbfc88323e 100644
--- a/esphome/components/i2c/i2c_bus.h
+++ b/esphome/components/i2c/i2c_bus.h
@@ -7,50 +7,93 @@
 namespace esphome {
 namespace i2c {
 
+/// @brief Error codes returned by I2CBus and I2CDevice methods
 enum ErrorCode {
-  ERROR_OK = 0,
-  ERROR_INVALID_ARGUMENT = 1,
-  ERROR_NOT_ACKNOWLEDGED = 2,
-  ERROR_TIMEOUT = 3,
-  ERROR_NOT_INITIALIZED = 4,
-  ERROR_TOO_LARGE = 5,
-  ERROR_UNKNOWN = 6,
-  ERROR_CRC = 7,
+  NO_ERROR = 0,                ///< No error found during execution of method
+  ERROR_OK = 0,                ///< No error found during execution of method
+  ERROR_INVALID_ARGUMENT = 1,  ///< method called invalid argument(s)
+  ERROR_NOT_ACKNOWLEDGED = 2,  ///< I2C bus acknowledgment not received
+  ERROR_TIMEOUT = 3,           ///< timeout while waiting to receive bytes
+  ERROR_NOT_INITIALIZED = 4,   ///< call method to a not initialized bus
+  ERROR_TOO_LARGE = 5,         ///< requested a transfer larger than buffers can hold
+  ERROR_UNKNOWN = 6,           ///< miscellaneous I2C error during execution
+  ERROR_CRC = 7,               ///< bytes received with a CRC error
 };
 
+/// @brief the ReadBuffer structure stores a pointer to a read buffer and its length
 struct ReadBuffer {
-  uint8_t *data;
-  size_t len;
-};
-struct WriteBuffer {
-  const uint8_t *data;
-  size_t len;
+  uint8_t *data;  ///< pointer to the read buffer
+  size_t len;     ///< length of the buffer
 };
 
+/// @brief the WriteBuffer structure stores a pointer to a write buffer and its length
+struct WriteBuffer {
+  const uint8_t *data;  ///< pointer to the write buffer
+  size_t len;           ///< length of the buffer
+};
+
+/// @brief This Class provides the methods to read and write bytes from an I2CBus.
+/// @note The I2CBus virtual class follows a *Factory design pattern* that provides all the interfaces methods required
+/// by clients while deferring the actual implementation of these methods to a subclasses. I2C-bus specification and
+/// user manual can be found here https://www.nxp.com/docs/en/user-guide/UM10204.pdf and an interesting I²C Application
+/// note https://www.nxp.com/docs/en/application-note/AN10216.pdf
 class I2CBus {
  public:
+  /// @brief Creates a ReadBuffer and calls the virtual readv() method to read bytes into this buffer
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffer pointer to an array of bytes that will be used to store the data received
+  /// @param len length of the buffer = number of bytes to read
+  /// @return an i2c::ErrorCode
   virtual ErrorCode read(uint8_t address, uint8_t *buffer, size_t len) {
     ReadBuffer buf;
     buf.data = buffer;
     buf.len = len;
     return readv(address, &buf, 1);
   }
-  virtual ErrorCode readv(uint8_t address, ReadBuffer *buffers, size_t cnt) = 0;
+
+  /// @brief This virtual method reads bytes from an I2CBus into an array of ReadBuffer.
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffers pointer to an array of ReadBuffer
+  /// @param count number of ReadBuffer to read
+  /// @return an i2c::ErrorCode
+  /// @details This is a pure virtual method that must be implemented in a subclass.
+  virtual ErrorCode readv(uint8_t address, ReadBuffer *buffers, size_t count) = 0;
+
   virtual ErrorCode write(uint8_t address, const uint8_t *buffer, size_t len) {
     return write(address, buffer, len, true);
   }
+
+  /// @brief Creates a WriteBuffer and calls the writev() method to send the bytes from this buffer
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffer pointer to an array of bytes that contains the data to be sent
+  /// @param len length of the buffer = number of bytes to write
+  /// @param stop true or false: True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   virtual ErrorCode write(uint8_t address, const uint8_t *buffer, size_t len, bool stop) {
     WriteBuffer buf;
     buf.data = buffer;
     buf.len = len;
     return writev(address, &buf, 1, stop);
   }
+
   virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t cnt) {
     return writev(address, buffers, cnt, true);
   }
-  virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t cnt, bool stop) = 0;
+
+  /// @brief This virtual method writes bytes to an I2CBus from an array of WriteBuffer.
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffers pointer to an array of WriteBuffer
+  /// @param count number of WriteBuffer to write
+  /// @param stop true or false: True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
+  /// @details This is a pure virtual method that must be implemented in the subclass.
+  virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t count, bool stop) = 0;
 
  protected:
+  /// @brief Scans the I2C bus for devices. Devices presence is kept in an array of std::pair
+  /// that contains the address and the corresponding bool presence flag.
   void i2c_scan_() {
     for (uint8_t address = 8; address < 120; address++) {
       auto err = writev(address, nullptr, 0);
@@ -61,8 +104,8 @@ class I2CBus {
       }
     }
   }
-  std::vector<std::pair<uint8_t, bool>> scan_results_;
-  bool scan_{false};
+  std::vector<std::pair<uint8_t, bool>> scan_results_;  ///< array containing scan results
+  bool scan_{false};                                    ///< Should we scan ? Can be set in the yaml
 };
 
 }  // namespace i2c

From 977e0184a72880f917dbaa8cfe3d48b2cc862b52 Mon Sep 17 00:00:00 2001
From: mathieu-mp <mathieu-mp@users.noreply.github.com>
Date: Tue, 19 Dec 2023 04:29:00 +0100
Subject: [PATCH 26/75] Add deep sleep between updates for waveshare epaper
 1.54in and 1.54inv2 (#5961)

---
 .../waveshare_epaper/waveshare_epaper.cpp     | 33 +++++++++++++++++++
 .../waveshare_epaper/waveshare_epaper.h       | 25 ++++++++++----
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.cpp b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
index 53bfa57f4f..8fdb9a3ac0 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.cpp
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
@@ -167,6 +167,25 @@ void WaveshareEPaper::on_safe_shutdown() { this->deep_sleep(); }
 // ========================================================
 
 void WaveshareEPaperTypeA::initialize() {
+  // Achieve display intialization
+  this->init_display_();
+  // If a reset pin is configured, eligible displays can be set to deep sleep
+  // between updates, as recommended by the hardware provider
+  if (this->reset_pin_ != nullptr) {
+    switch (this->model_) {
+      // More models can be added here to enable deep sleep if eligible
+      case WAVESHARE_EPAPER_1_54_IN:
+      case WAVESHARE_EPAPER_1_54_IN_V2:
+        this->deep_sleep_between_updates_ = true;
+        ESP_LOGI(TAG, "Set the display to deep sleep");
+        this->deep_sleep();
+        break;
+      default:
+        break;
+    }
+  }
+}
+void WaveshareEPaperTypeA::init_display_() {
   if (this->model_ == TTGO_EPAPER_2_13_IN_B74) {
     this->reset_pin_->digital_write(false);
     delay(10);
@@ -261,6 +280,13 @@ void HOT WaveshareEPaperTypeA::display() {
   bool full_update = this->at_update_ == 0;
   bool prev_full_update = this->at_update_ == 1;
 
+  if (this->deep_sleep_between_updates_) {
+    ESP_LOGI(TAG, "Wake up the display");
+    this->reset_();
+    this->wait_until_idle_();
+    this->init_display_();
+  }
+
   if (!this->wait_until_idle_()) {
     this->status_set_warning();
     return;
@@ -384,6 +410,11 @@ void HOT WaveshareEPaperTypeA::display() {
   this->command(0xFF);
 
   this->status_clear_warning();
+
+  if (this->deep_sleep_between_updates_) {
+    ESP_LOGI(TAG, "Set the display back to deep sleep");
+    this->deep_sleep();
+  }
 }
 int WaveshareEPaperTypeA::get_width_internal() {
   switch (this->model_) {
@@ -445,6 +476,8 @@ void WaveshareEPaperTypeA::set_full_update_every(uint32_t full_update_every) {
 
 uint32_t WaveshareEPaperTypeA::idle_timeout_() {
   switch (this->model_) {
+    case WAVESHARE_EPAPER_1_54_IN:
+    case WAVESHARE_EPAPER_1_54_IN_V2:
     case TTGO_EPAPER_2_13_IN_B1:
       return 2500;
     default:
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.h b/esphome/components/waveshare_epaper/waveshare_epaper.h
index f6ccf90861..42e8a16829 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.h
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.h
@@ -92,13 +92,20 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
   void display() override;
 
   void deep_sleep() override {
-    if (this->model_ == WAVESHARE_EPAPER_2_9_IN_V2 || this->model_ == WAVESHARE_EPAPER_1_54_IN_V2) {
-      // COMMAND DEEP SLEEP MODE
-      this->command(0x10);
-      this->data(0x01);
-    } else {
-      // COMMAND DEEP SLEEP MODE
-      this->command(0x10);
+    switch (this->model_) {
+      // Models with specific deep sleep command and data
+      case WAVESHARE_EPAPER_1_54_IN:
+      case WAVESHARE_EPAPER_1_54_IN_V2:
+      case WAVESHARE_EPAPER_2_9_IN_V2:
+        // COMMAND DEEP SLEEP MODE
+        this->command(0x10);
+        this->data(0x01);
+        break;
+      // Other models default to simple deep sleep command
+      default:
+        // COMMAND DEEP SLEEP
+        this->command(0x10);
+        break;
     }
     this->wait_until_idle_();
   }
@@ -108,6 +115,8 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
  protected:
   void write_lut_(const uint8_t *lut, uint8_t size);
 
+  void init_display_();
+
   int get_width_internal() override;
 
   int get_height_internal() override;
@@ -118,6 +127,8 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
   uint32_t at_update_{0};
   WaveshareEPaperTypeAModel model_;
   uint32_t idle_timeout_() override;
+
+  bool deep_sleep_between_updates_{false};
 };
 
 enum WaveshareEPaperTypeBModel {

From e5414d70f5c9f3b6aab84b14a198611df3f0c34e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Mon, 18 Dec 2023 19:24:48 -1000
Subject: [PATCH 27/75] Speed up writing protobuf strings/bytes (#5828)

---
 esphome/components/api/proto.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h
index fea219ecb9..ccc6c0d52c 100644
--- a/esphome/components/api/proto.h
+++ b/esphome/components/api/proto.h
@@ -160,8 +160,7 @@ class ProtoWriteBuffer {
     this->encode_field_raw(field_id, 2);
     this->encode_varint_raw(len);
     auto *data = reinterpret_cast<const uint8_t *>(string);
-    for (size_t i = 0; i < len; i++)
-      this->write(data[i]);
+    this->buffer_->insert(this->buffer_->end(), data, data + len);
   }
   void encode_string(uint32_t field_id, const std::string &value, bool force = false) {
     this->encode_string(field_id, value.data(), value.size());

From 1b3068a4092e72263dc252b43848ae959f9028cd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 19 Dec 2023 13:41:26 -1000
Subject: [PATCH 28/75] Bump aioesphomeapi to 21.0.1 (#5969)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 20a5514e71..b9bcd5dae0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==21.0.0
+aioesphomeapi==21.0.1
 zeroconf==0.130.0
 python-magic==0.4.27
 

From e2a00f66b8c2150f49538af8282b80d6fd8aae30 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 09:59:58 +0900
Subject: [PATCH 29/75] Bump version to 2023.12.0b4

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 4153646aea..20218ab61d 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b3"
+__version__ = "2023.12.0b4"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From fbf3d03a33289aa8f47a664b11a724621f499142 Mon Sep 17 00:00:00 2001
From: NP v/d Spek <github_mail@lumensoft.nl>
Date: Mon, 18 Dec 2023 20:23:22 +0100
Subject: [PATCH 30/75] rename set_raw_touch_position_ to
 add_raw_touch_position_ (#5962)

---
 esphome/components/ektf2232/touchscreen/ektf2232.cpp          | 2 +-
 esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h    | 2 +-
 esphome/components/ft63x6/ft63x6.cpp                          | 4 ++--
 esphome/components/gt911/touchscreen/gt911_touchscreen.cpp    | 2 +-
 .../lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp     | 2 +-
 esphome/components/touchscreen/touchscreen.cpp                | 2 +-
 esphome/components/touchscreen/touchscreen.h                  | 2 +-
 esphome/components/tt21100/touchscreen/tt21100.cpp            | 2 +-
 esphome/components/xpt2046/touchscreen/xpt2046.cpp            | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/esphome/components/ektf2232/touchscreen/ektf2232.cpp b/esphome/components/ektf2232/touchscreen/ektf2232.cpp
index 1a2c0389af..00e00bc7e6 100644
--- a/esphome/components/ektf2232/touchscreen/ektf2232.cpp
+++ b/esphome/components/ektf2232/touchscreen/ektf2232.cpp
@@ -74,7 +74,7 @@ void EKTF2232Touchscreen::update_touches() {
     uint8_t *d = raw + 1 + (i * 3);
     x_raw = (d[0] & 0xF0) << 4 | d[1];
     y_raw = (d[0] & 0x0F) << 8 | d[2];
-    this->set_raw_touch_position_(i, x_raw, y_raw);
+    this->add_raw_touch_position_(i, x_raw, y_raw);
   }
 }
 
diff --git a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
index 497d6c906c..0b3a2c1b86 100644
--- a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
+++ b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
@@ -94,7 +94,7 @@ class FT5x06Touchscreen : public touchscreen::Touchscreen, public i2c::I2CDevice
 
       esph_log_d(TAG, "Read %X status, id: %d, pos %d/%d", status, id, x, y);
       if (status == 0 || status == 2) {
-        this->set_raw_touch_position_(id, x, y);
+        this->add_raw_touch_position_(id, x, y);
       }
     }
   }
diff --git a/esphome/components/ft63x6/ft63x6.cpp b/esphome/components/ft63x6/ft63x6.cpp
index 9198954253..b674ded22c 100644
--- a/esphome/components/ft63x6/ft63x6.cpp
+++ b/esphome/components/ft63x6/ft63x6.cpp
@@ -53,13 +53,13 @@ void FT63X6Touchscreen::update_touches() {
   uint8_t touch_id = this->read_touch_id_(FT63X6_ADDR_TOUCH1_ID);  // id1 = 0 or 1
   int16_t x = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH1_X);
   int16_t y = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH1_Y);
-  this->set_raw_touch_position_(touch_id, x, y);
+  this->add_raw_touch_position_(touch_id, x, y);
 
   if (touch_count >= 2) {
     touch_id = this->read_touch_id_(FT63X6_ADDR_TOUCH2_ID);  // id2 = 0 or 1(~id1 & 0x01)
     x = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH2_X);
     y = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH2_Y);
-    this->set_raw_touch_position_(touch_id, x, y);
+    this->add_raw_touch_position_(touch_id, x, y);
   }
 }
 
diff --git a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
index adc577f5da..84854d5b0d 100644
--- a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
+++ b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
@@ -92,7 +92,7 @@ void GT911Touchscreen::update_touches() {
     uint16_t id = data[i][0];
     uint16_t x = encode_uint16(data[i][2], data[i][1]);
     uint16_t y = encode_uint16(data[i][4], data[i][3]);
-    this->set_raw_touch_position_(id, x, y);
+    this->add_raw_touch_position_(id, x, y);
   }
   auto keys = data[num_of_touches][0];
   for (size_t i = 0; i != 4; i++) {
diff --git a/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp b/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
index eb61b6f31e..64cc7ad4d1 100644
--- a/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
+++ b/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
@@ -84,7 +84,7 @@ void LilygoT547Touchscreen::update_touches() {
     id = (buffer[i * 5] >> 4) & 0x0F;
     y_raw = (uint16_t) ((buffer[i * 5 + 1] << 4) | ((buffer[i * 5 + 3] >> 4) & 0x0F));
     x_raw = (uint16_t) ((buffer[i * 5 + 2] << 4) | (buffer[i * 5 + 3] & 0x0F));
-    this->set_raw_touch_position_(id, x_raw, y_raw);
+    this->add_raw_touch_position_(id, x_raw, y_raw);
   }
 
   this->status_clear_warning();
diff --git a/esphome/components/touchscreen/touchscreen.cpp b/esphome/components/touchscreen/touchscreen.cpp
index 140f46b6f6..f095c2af8c 100644
--- a/esphome/components/touchscreen/touchscreen.cpp
+++ b/esphome/components/touchscreen/touchscreen.cpp
@@ -51,7 +51,7 @@ void Touchscreen::loop() {
   }
 }
 
-void Touchscreen::set_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw) {
+void Touchscreen::add_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw) {
   TouchPoint tp;
   uint16_t x, y;
   if (this->touches_.count(id) == 0) {
diff --git a/esphome/components/touchscreen/touchscreen.h b/esphome/components/touchscreen/touchscreen.h
index 1fe304d967..74747c589c 100644
--- a/esphome/components/touchscreen/touchscreen.h
+++ b/esphome/components/touchscreen/touchscreen.h
@@ -87,7 +87,7 @@ class Touchscreen : public PollingComponent {
 
   void attach_interrupt_(InternalGPIOPin *irq_pin, esphome::gpio::InterruptType type);
 
-  void set_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw = 0);
+  void add_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw = 0);
 
   void send_touches_();
 
diff --git a/esphome/components/tt21100/touchscreen/tt21100.cpp b/esphome/components/tt21100/touchscreen/tt21100.cpp
index ff688fd0b0..6b5cba74cd 100644
--- a/esphome/components/tt21100/touchscreen/tt21100.cpp
+++ b/esphome/components/tt21100/touchscreen/tt21100.cpp
@@ -109,7 +109,7 @@ void TT21100Touchscreen::update_touches() {
                  i, touch->touch_type, touch->tip, touch->event_id, touch->touch_id, touch->x, touch->y,
                  touch->pressure, touch->major_axis_length, touch->orientation);
 
-        this->set_raw_touch_position_(touch->tip, touch->x, touch->y, touch->pressure);
+        this->add_raw_touch_position_(touch->tip, touch->x, touch->y, touch->pressure);
       }
     }
   }
diff --git a/esphome/components/xpt2046/touchscreen/xpt2046.cpp b/esphome/components/xpt2046/touchscreen/xpt2046.cpp
index 1a9c202af0..a268da06dd 100644
--- a/esphome/components/xpt2046/touchscreen/xpt2046.cpp
+++ b/esphome/components/xpt2046/touchscreen/xpt2046.cpp
@@ -55,7 +55,7 @@ void XPT2046Component::update_touches() {
 
     ESP_LOGV(TAG, "Touchscreen Update [%d, %d], z = %d", x_raw, y_raw, z_raw);
 
-    this->set_raw_touch_position_(0, x_raw, y_raw, z_raw);
+    this->add_raw_touch_position_(0, x_raw, y_raw, z_raw);
   }
 }
 

From a64e96e7adbca4879c5d3b239272f8052e5d91ff Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 13:43:43 +0900
Subject: [PATCH 31/75] Bump version to 2023.12.0b5

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 20218ab61d..0df9c27233 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b4"
+__version__ = "2023.12.0b5"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From d582cfa30a0ec8406a7d55cbe35f342c36e438e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 20 Dec 2023 11:33:05 +0100
Subject: [PATCH 32/75] image: allow the image to by auto-loaded by animation
 (#5139)

---
 esphome/__main__.py                  |  4 ++--
 esphome/components/image/__init__.py |  1 +
 esphome/config.py                    | 20 ++++++++++++++++++--
 esphome/loader.py                    |  4 ++++
 esphome/writer.py                    |  8 ++++----
 tests/test8.yaml                     | 10 ++++++++++
 6 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/esphome/__main__.py b/esphome/__main__.py
index 0796dead43..baa5ecde47 100644
--- a/esphome/__main__.py
+++ b/esphome/__main__.py
@@ -12,7 +12,7 @@ import argcomplete
 
 from esphome import const, writer, yaml_util
 import esphome.codegen as cg
-from esphome.config import iter_components, read_config, strip_default_ids
+from esphome.config import iter_component_configs, read_config, strip_default_ids
 from esphome.const import (
     ALLOWED_NAME_CHARS,
     CONF_BAUD_RATE,
@@ -196,7 +196,7 @@ def write_cpp(config):
 def generate_cpp_contents(config):
     _LOGGER.info("Generating C++ source...")
 
-    for name, component, conf in iter_components(CORE.config):
+    for name, component, conf in iter_component_configs(CORE.config):
         if component.to_code is not None:
             coro = wrap_to_code(name, component)
             CORE.add_job(coro, conf)
diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py
index c11021fc9c..73dc73aa45 100644
--- a/esphome/components/image/__init__.py
+++ b/esphome/components/image/__init__.py
@@ -36,6 +36,7 @@ _LOGGER = logging.getLogger(__name__)
 DOMAIN = "image"
 DEPENDENCIES = ["display"]
 MULTI_CONF = True
+MULTI_CONF_NO_DEFAULT = True
 
 image_ns = cg.esphome_ns.namespace("image")
 
diff --git a/esphome/config.py b/esphome/config.py
index 745883c2ef..e9433d537e 100644
--- a/esphome/config.py
+++ b/esphome/config.py
@@ -39,6 +39,17 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def iter_components(config):
+    for domain, conf in config.items():
+        component = get_component(domain)
+        yield domain, component
+        if component.is_platform_component:
+            for p_config in conf:
+                p_name = f"{domain}.{p_config[CONF_PLATFORM]}"
+                platform = get_platform(domain, p_config[CONF_PLATFORM])
+                yield p_name, platform
+
+
+def iter_component_configs(config):
     for domain, conf in config.items():
         component = get_component(domain)
         if component.multi_conf:
@@ -303,8 +314,10 @@ class LoadValidationStep(ConfigValidationStep):
             # Ignore top-level keys starting with a dot
             return
         result.add_output_path([self.domain], self.domain)
-        result[self.domain] = self.conf
         component = get_component(self.domain)
+        if component.multi_conf_no_default and isinstance(self.conf, core.AutoLoad):
+            self.conf = []
+        result[self.domain] = self.conf
         path = [self.domain]
         if component is None:
             result.add_str_error(f"Component not found: {self.domain}", path)
@@ -424,7 +437,10 @@ class MetadataValidationStep(ConfigValidationStep):
 
     def run(self, result: Config) -> None:
         if self.conf is None:
-            result[self.domain] = self.conf = {}
+            if self.comp.multi_conf and self.comp.multi_conf_no_default:
+                result[self.domain] = self.conf = []
+            else:
+                result[self.domain] = self.conf = {}
 
         success = True
         for dependency in self.comp.dependencies:
diff --git a/esphome/loader.py b/esphome/loader.py
index cd21e5a509..40a38d0a14 100644
--- a/esphome/loader.py
+++ b/esphome/loader.py
@@ -57,6 +57,10 @@ class ComponentManifest:
     def multi_conf(self) -> bool:
         return getattr(self.module, "MULTI_CONF", False)
 
+    @property
+    def multi_conf_no_default(self) -> bool:
+        return getattr(self.module, "MULTI_CONF_NO_DEFAULT", False)
+
     @property
     def to_code(self) -> Optional[Callable[[Any], None]]:
         return getattr(self.module, "to_code", None)
diff --git a/esphome/writer.py b/esphome/writer.py
index 83e95614a6..3ad0e60d31 100644
--- a/esphome/writer.py
+++ b/esphome/writer.py
@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 from typing import Union
 
-from esphome.config import iter_components
+from esphome.config import iter_components, iter_component_configs
 from esphome.const import (
     HEADER_FILE_EXTENSIONS,
     SOURCE_FILE_EXTENSIONS,
@@ -70,14 +70,14 @@ UPLOAD_SPEED_OVERRIDE = {
 
 def get_flags(key):
     flags = set()
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         flags |= getattr(component, key)(conf)
     return flags
 
 
 def get_include_text():
     include_text = '#include "esphome.h"\nusing namespace esphome;\n'
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         if not hasattr(component, "includes"):
             continue
         includes = component.includes
@@ -232,7 +232,7 @@ the custom_components folder or the external_components feature.
 
 def copy_src_tree():
     source_files: list[loader.FileResource] = []
-    for _, component, _ in iter_components(CORE.config):
+    for _, component in iter_components(CORE.config):
         source_files += component.resources
     source_files_map = {
         Path(x.package.replace(".", "/") + "/" + x.resource): x for x in source_files
diff --git a/tests/test8.yaml b/tests/test8.yaml
index 558e86e1f9..fafdb76e12 100644
--- a/tests/test8.yaml
+++ b/tests/test8.yaml
@@ -92,3 +92,13 @@ sensor:
       name: "Loop Time"
     psram:
       name: "PSRAM Free"
+
+# Purposely test that `animation:` does auto-load `image:`
+# Keep the `image:` undefined.
+# image:
+
+animation:
+  - id: rgb565_animation
+    file: pnglogo.png
+    type: RGB565
+    use_transparency: no

From 84174aeb8031db638bfa91b73ed8ec93e63ce518 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 19:42:27 +0900
Subject: [PATCH 33/75] Fix pin reuse error with pin expanders (#5973)

Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>
---
 esphome/pins.py    | 20 +++++++++++---------
 tests/test1.yaml   |  4 ----
 tests/test3.1.yaml |  2 --
 3 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/esphome/pins.py b/esphome/pins.py
index e2fd8e98e2..87f7084d4f 100644
--- a/esphome/pins.py
+++ b/esphome/pins.py
@@ -1,7 +1,7 @@
 import operator
 from functools import reduce
 import esphome.config_validation as cv
-from esphome.core import CORE, ID
+from esphome.core import CORE
 
 from esphome.const import (
     CONF_INPUT,
@@ -25,15 +25,16 @@ class PinRegistry(dict):
     def reset(self):
         self.pins_used = {}
 
-    def get_count(self, key, number):
+    def get_count(self, key, id, number):
         """
         Get the number of places a given pin is used.
-        :param key: The ID of the defining component
+        :param key: The key of the registered pin schema.
+        :param id: The ID of the defining component
         :param number: The pin number
         :return: The number of places the pin is used.
         """
-        pin_key = (key, number)
-        return self.pins_used[pin_key] if pin_key in self.pins_used else 0
+        pin_key = (key, id, number)
+        return len(self.pins_used[pin_key]) if pin_key in self.pins_used else 0
 
     def register(self, name, schema, final_validate=None):
         """
@@ -65,9 +66,10 @@ class PinRegistry(dict):
         result = self[key][1](conf)
         if CONF_NUMBER in result:
             # key maps to the pin schema
-            if isinstance(key, ID):
-                key = key.id
-            pin_key = (key, result[CONF_NUMBER])
+            if key != CORE.target_platform:
+                pin_key = (key, conf[key], result[CONF_NUMBER])
+            else:
+                pin_key = (key, key, result[CONF_NUMBER])
             if pin_key not in self.pins_used:
                 self.pins_used[pin_key] = []
             # client_id identifies the instance of the providing component
@@ -101,7 +103,7 @@ class PinRegistry(dict):
         Run the final validation for all pins, and check for reuse
         :param fconf: The full config
         """
-        for (key, _), pin_list in self.pins_used.items():
+        for (key, _, _), pin_list in self.pins_used.items():
             count = len(pin_list)  # number of places same pin used.
             final_val_fun = self[key][2]  # final validation function
             for pin_path, client_id, pin_config in pin_list:
diff --git a/tests/test1.yaml b/tests/test1.yaml
index f7b433cce2..3f8ae0151f 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -1667,7 +1667,6 @@ binary_sensor:
       mcp23xxx: mcp23s08_hub
       # Use pin number 1
       number: 1
-      allow_other_uses: true
       # One of INPUT or INPUT_PULLUP
       mode: INPUT_PULLUP
       inverted: false
@@ -2149,7 +2148,6 @@ output:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2157,7 +2155,6 @@ output:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2597,7 +2594,6 @@ switch:
       mcp23xxx: mcp23s08_hub
       # Use pin number 0
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 63ef4e8ce0..b5428abbfa 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -401,7 +401,6 @@ switch:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: &interlock [gpio_switch1, gpio_switch2, gpio_switch3]
   - platform: gpio
@@ -409,7 +408,6 @@ switch:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: *interlock
   - platform: gpio

From 23ceddafed185ab38a8326782994b9c7e1f07f3c Mon Sep 17 00:00:00 2001
From: Yorick Smilda <y.smilda@gmail.com>
Date: Wed, 20 Dec 2023 11:52:46 +0100
Subject: [PATCH 34/75] Add ability to lock to set mode (#5924)

---
 esphome/components/hlw8012/hlw8012.cpp |  2 +-
 esphome/components/hlw8012/sensor.py   | 11 ++++++++---
 tests/test1.yaml                       |  2 +-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/esphome/components/hlw8012/hlw8012.cpp b/esphome/components/hlw8012/hlw8012.cpp
index 1a9f47faaf..14e83f60e1 100644
--- a/esphome/components/hlw8012/hlw8012.cpp
+++ b/esphome/components/hlw8012/hlw8012.cpp
@@ -96,7 +96,7 @@ void HLW8012Component::update() {
     this->energy_sensor_->publish_state(energy);
   }
 
-  if (this->change_mode_at_++ == this->change_mode_every_) {
+  if (this->change_mode_every_ != 0 && this->change_mode_at_++ == this->change_mode_every_) {
     this->current_mode_ = !this->current_mode_;
     ESP_LOGV(TAG, "Changing mode to %s mode", this->current_mode_ ? "CURRENT" : "VOLTAGE");
     this->change_mode_at_ = 0;
diff --git a/esphome/components/hlw8012/sensor.py b/esphome/components/hlw8012/sensor.py
index 033cccc3d4..2687edaca2 100644
--- a/esphome/components/hlw8012/sensor.py
+++ b/esphome/components/hlw8012/sensor.py
@@ -79,8 +79,9 @@ CONFIG_SCHEMA = cv.Schema(
         cv.Optional(CONF_CURRENT_RESISTOR, default=0.001): cv.resistance,
         cv.Optional(CONF_VOLTAGE_DIVIDER, default=2351): cv.positive_float,
         cv.Optional(CONF_MODEL, default="HLW8012"): cv.enum(MODELS, upper=True),
-        cv.Optional(CONF_CHANGE_MODE_EVERY, default=8): cv.All(
-            cv.uint32_t, cv.Range(min=1)
+        cv.Optional(CONF_CHANGE_MODE_EVERY, default=8): cv.Any(
+            "never",
+            cv.All(cv.uint32_t, cv.Range(min=1)),
         ),
         cv.Optional(CONF_INITIAL_MODE, default=CONF_VOLTAGE): cv.one_of(
             *INITIAL_MODES, lower=True
@@ -114,6 +115,10 @@ async def to_code(config):
         cg.add(var.set_energy_sensor(sens))
     cg.add(var.set_current_resistor(config[CONF_CURRENT_RESISTOR]))
     cg.add(var.set_voltage_divider(config[CONF_VOLTAGE_DIVIDER]))
-    cg.add(var.set_change_mode_every(config[CONF_CHANGE_MODE_EVERY]))
     cg.add(var.set_initial_mode(INITIAL_MODES[config[CONF_INITIAL_MODE]]))
     cg.add(var.set_sensor_model(config[CONF_MODEL]))
+
+    interval = config[CONF_CHANGE_MODE_EVERY]
+    if interval == "never":
+        interval = 0
+    cg.add(var.set_change_mode_every(interval))
diff --git a/tests/test1.yaml b/tests/test1.yaml
index 3f8ae0151f..aaad10ff0a 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -794,7 +794,7 @@ sensor:
     update_interval: 15s
     current_resistor: 0.001 ohm
     voltage_divider: 2351
-    change_mode_every: 16
+    change_mode_every: "never"
     initial_mode: VOLTAGE
     model: hlw8012
   - platform: total_daily_energy

From ab25e32509798e8564bad1b556f8ad552f13880f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 20 Dec 2023 11:33:05 +0100
Subject: [PATCH 35/75] image: allow the image to by auto-loaded by animation
 (#5139)

---
 esphome/__main__.py                  |  4 ++--
 esphome/components/image/__init__.py |  1 +
 esphome/config.py                    | 20 ++++++++++++++++++--
 esphome/loader.py                    |  4 ++++
 esphome/writer.py                    |  8 ++++----
 tests/test8.yaml                     | 10 ++++++++++
 6 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/esphome/__main__.py b/esphome/__main__.py
index 0796dead43..baa5ecde47 100644
--- a/esphome/__main__.py
+++ b/esphome/__main__.py
@@ -12,7 +12,7 @@ import argcomplete
 
 from esphome import const, writer, yaml_util
 import esphome.codegen as cg
-from esphome.config import iter_components, read_config, strip_default_ids
+from esphome.config import iter_component_configs, read_config, strip_default_ids
 from esphome.const import (
     ALLOWED_NAME_CHARS,
     CONF_BAUD_RATE,
@@ -196,7 +196,7 @@ def write_cpp(config):
 def generate_cpp_contents(config):
     _LOGGER.info("Generating C++ source...")
 
-    for name, component, conf in iter_components(CORE.config):
+    for name, component, conf in iter_component_configs(CORE.config):
         if component.to_code is not None:
             coro = wrap_to_code(name, component)
             CORE.add_job(coro, conf)
diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py
index c11021fc9c..73dc73aa45 100644
--- a/esphome/components/image/__init__.py
+++ b/esphome/components/image/__init__.py
@@ -36,6 +36,7 @@ _LOGGER = logging.getLogger(__name__)
 DOMAIN = "image"
 DEPENDENCIES = ["display"]
 MULTI_CONF = True
+MULTI_CONF_NO_DEFAULT = True
 
 image_ns = cg.esphome_ns.namespace("image")
 
diff --git a/esphome/config.py b/esphome/config.py
index 745883c2ef..e9433d537e 100644
--- a/esphome/config.py
+++ b/esphome/config.py
@@ -39,6 +39,17 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def iter_components(config):
+    for domain, conf in config.items():
+        component = get_component(domain)
+        yield domain, component
+        if component.is_platform_component:
+            for p_config in conf:
+                p_name = f"{domain}.{p_config[CONF_PLATFORM]}"
+                platform = get_platform(domain, p_config[CONF_PLATFORM])
+                yield p_name, platform
+
+
+def iter_component_configs(config):
     for domain, conf in config.items():
         component = get_component(domain)
         if component.multi_conf:
@@ -303,8 +314,10 @@ class LoadValidationStep(ConfigValidationStep):
             # Ignore top-level keys starting with a dot
             return
         result.add_output_path([self.domain], self.domain)
-        result[self.domain] = self.conf
         component = get_component(self.domain)
+        if component.multi_conf_no_default and isinstance(self.conf, core.AutoLoad):
+            self.conf = []
+        result[self.domain] = self.conf
         path = [self.domain]
         if component is None:
             result.add_str_error(f"Component not found: {self.domain}", path)
@@ -424,7 +437,10 @@ class MetadataValidationStep(ConfigValidationStep):
 
     def run(self, result: Config) -> None:
         if self.conf is None:
-            result[self.domain] = self.conf = {}
+            if self.comp.multi_conf and self.comp.multi_conf_no_default:
+                result[self.domain] = self.conf = []
+            else:
+                result[self.domain] = self.conf = {}
 
         success = True
         for dependency in self.comp.dependencies:
diff --git a/esphome/loader.py b/esphome/loader.py
index cd21e5a509..40a38d0a14 100644
--- a/esphome/loader.py
+++ b/esphome/loader.py
@@ -57,6 +57,10 @@ class ComponentManifest:
     def multi_conf(self) -> bool:
         return getattr(self.module, "MULTI_CONF", False)
 
+    @property
+    def multi_conf_no_default(self) -> bool:
+        return getattr(self.module, "MULTI_CONF_NO_DEFAULT", False)
+
     @property
     def to_code(self) -> Optional[Callable[[Any], None]]:
         return getattr(self.module, "to_code", None)
diff --git a/esphome/writer.py b/esphome/writer.py
index 83e95614a6..3ad0e60d31 100644
--- a/esphome/writer.py
+++ b/esphome/writer.py
@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 from typing import Union
 
-from esphome.config import iter_components
+from esphome.config import iter_components, iter_component_configs
 from esphome.const import (
     HEADER_FILE_EXTENSIONS,
     SOURCE_FILE_EXTENSIONS,
@@ -70,14 +70,14 @@ UPLOAD_SPEED_OVERRIDE = {
 
 def get_flags(key):
     flags = set()
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         flags |= getattr(component, key)(conf)
     return flags
 
 
 def get_include_text():
     include_text = '#include "esphome.h"\nusing namespace esphome;\n'
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         if not hasattr(component, "includes"):
             continue
         includes = component.includes
@@ -232,7 +232,7 @@ the custom_components folder or the external_components feature.
 
 def copy_src_tree():
     source_files: list[loader.FileResource] = []
-    for _, component, _ in iter_components(CORE.config):
+    for _, component in iter_components(CORE.config):
         source_files += component.resources
     source_files_map = {
         Path(x.package.replace(".", "/") + "/" + x.resource): x for x in source_files
diff --git a/tests/test8.yaml b/tests/test8.yaml
index 558e86e1f9..fafdb76e12 100644
--- a/tests/test8.yaml
+++ b/tests/test8.yaml
@@ -92,3 +92,13 @@ sensor:
       name: "Loop Time"
     psram:
       name: "PSRAM Free"
+
+# Purposely test that `animation:` does auto-load `image:`
+# Keep the `image:` undefined.
+# image:
+
+animation:
+  - id: rgb565_animation
+    file: pnglogo.png
+    type: RGB565
+    use_transparency: no

From 6a9e85438f95b14be54bdf9a961897bdd78224b9 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 19:42:27 +0900
Subject: [PATCH 36/75] Fix pin reuse error with pin expanders (#5973)

Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>
---
 esphome/pins.py    | 20 +++++++++++---------
 tests/test1.yaml   |  4 ----
 tests/test3.1.yaml |  2 --
 3 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/esphome/pins.py b/esphome/pins.py
index e2fd8e98e2..87f7084d4f 100644
--- a/esphome/pins.py
+++ b/esphome/pins.py
@@ -1,7 +1,7 @@
 import operator
 from functools import reduce
 import esphome.config_validation as cv
-from esphome.core import CORE, ID
+from esphome.core import CORE
 
 from esphome.const import (
     CONF_INPUT,
@@ -25,15 +25,16 @@ class PinRegistry(dict):
     def reset(self):
         self.pins_used = {}
 
-    def get_count(self, key, number):
+    def get_count(self, key, id, number):
         """
         Get the number of places a given pin is used.
-        :param key: The ID of the defining component
+        :param key: The key of the registered pin schema.
+        :param id: The ID of the defining component
         :param number: The pin number
         :return: The number of places the pin is used.
         """
-        pin_key = (key, number)
-        return self.pins_used[pin_key] if pin_key in self.pins_used else 0
+        pin_key = (key, id, number)
+        return len(self.pins_used[pin_key]) if pin_key in self.pins_used else 0
 
     def register(self, name, schema, final_validate=None):
         """
@@ -65,9 +66,10 @@ class PinRegistry(dict):
         result = self[key][1](conf)
         if CONF_NUMBER in result:
             # key maps to the pin schema
-            if isinstance(key, ID):
-                key = key.id
-            pin_key = (key, result[CONF_NUMBER])
+            if key != CORE.target_platform:
+                pin_key = (key, conf[key], result[CONF_NUMBER])
+            else:
+                pin_key = (key, key, result[CONF_NUMBER])
             if pin_key not in self.pins_used:
                 self.pins_used[pin_key] = []
             # client_id identifies the instance of the providing component
@@ -101,7 +103,7 @@ class PinRegistry(dict):
         Run the final validation for all pins, and check for reuse
         :param fconf: The full config
         """
-        for (key, _), pin_list in self.pins_used.items():
+        for (key, _, _), pin_list in self.pins_used.items():
             count = len(pin_list)  # number of places same pin used.
             final_val_fun = self[key][2]  # final validation function
             for pin_path, client_id, pin_config in pin_list:
diff --git a/tests/test1.yaml b/tests/test1.yaml
index f7b433cce2..3f8ae0151f 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -1667,7 +1667,6 @@ binary_sensor:
       mcp23xxx: mcp23s08_hub
       # Use pin number 1
       number: 1
-      allow_other_uses: true
       # One of INPUT or INPUT_PULLUP
       mode: INPUT_PULLUP
       inverted: false
@@ -2149,7 +2148,6 @@ output:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2157,7 +2155,6 @@ output:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2597,7 +2594,6 @@ switch:
       mcp23xxx: mcp23s08_hub
       # Use pin number 0
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 63ef4e8ce0..b5428abbfa 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -401,7 +401,6 @@ switch:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: &interlock [gpio_switch1, gpio_switch2, gpio_switch3]
   - platform: gpio
@@ -409,7 +408,6 @@ switch:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: *interlock
   - platform: gpio

From bec1ad9396806086f61eae699ca6d3d8210682a7 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 20:59:46 +0900
Subject: [PATCH 37/75] Bump version to 2023.12.0b6

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 0df9c27233..475f28c35e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b5"
+__version__ = "2023.12.0b6"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 867595561417deb035fcde921a33db72d1b8c3fa Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 21 Dec 2023 08:09:10 +0900
Subject: [PATCH 38/75] Bump version to 2023.12.0

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 475f28c35e..55be91a4f9 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b6"
+__version__ = "2023.12.0"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 991880d53fa2d21512dfeaf84ee1380a4bd32bbe Mon Sep 17 00:00:00 2001
From: Branden Cash <203336+ammmze@users.noreply.github.com>
Date: Wed, 20 Dec 2023 17:07:40 -0700
Subject: [PATCH 39/75] feat: add AS5600 component/sensor (#5174)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 CODEOWNERS                                    |   2 +
 esphome/components/as5600/__init__.py         | 228 ++++++++++++++++++
 esphome/components/as5600/as5600.cpp          | 138 +++++++++++
 esphome/components/as5600/as5600.h            | 105 ++++++++
 esphome/components/as5600/sensor/__init__.py  | 119 +++++++++
 .../as5600/sensor/as5600_sensor.cpp           |  98 ++++++++
 .../components/as5600/sensor/as5600_sensor.h  |  43 ++++
 tests/test1.yaml                              |  22 ++
 8 files changed, 755 insertions(+)
 create mode 100644 esphome/components/as5600/__init__.py
 create mode 100644 esphome/components/as5600/as5600.cpp
 create mode 100644 esphome/components/as5600/as5600.h
 create mode 100644 esphome/components/as5600/sensor/__init__.py
 create mode 100644 esphome/components/as5600/sensor/as5600_sensor.cpp
 create mode 100644 esphome/components/as5600/sensor/as5600_sensor.h

diff --git a/CODEOWNERS b/CODEOWNERS
index 320a23ffaa..fc14b2088e 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -34,6 +34,8 @@ esphome/components/analog_threshold/* @ianchi
 esphome/components/animation/* @syndlex
 esphome/components/anova/* @buxtronix
 esphome/components/api/* @OttoWinter
+esphome/components/as5600/* @ammmze
+esphome/components/as5600/sensor/* @ammmze
 esphome/components/as7341/* @mrgnr
 esphome/components/async_tcp/* @OttoWinter
 esphome/components/atc_mithermometer/* @ahpohl
diff --git a/esphome/components/as5600/__init__.py b/esphome/components/as5600/__init__.py
new file mode 100644
index 0000000000..1840b22768
--- /dev/null
+++ b/esphome/components/as5600/__init__.py
@@ -0,0 +1,228 @@
+from esphome import pins
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import i2c
+from esphome.const import (
+    CONF_ID,
+    CONF_DIR_PIN,
+    CONF_DIRECTION,
+    CONF_HYSTERESIS,
+    CONF_RANGE,
+)
+
+CODEOWNERS = ["@ammmze"]
+DEPENDENCIES = ["i2c"]
+MULTI_CONF = True
+
+as5600_ns = cg.esphome_ns.namespace("as5600")
+AS5600Component = as5600_ns.class_("AS5600Component", cg.Component, i2c.I2CDevice)
+
+DIRECTION = {
+    "CLOCKWISE": 0,
+    "COUNTERCLOCKWISE": 1,
+}
+
+POWER_MODE = {
+    "NOMINAL": 0,
+    "LOW1": 1,
+    "LOW2": 2,
+    "LOW3": 3,
+}
+
+HYSTERESIS = {
+    "NONE": 0,
+    "LSB1": 1,
+    "LSB2": 2,
+    "LSB3": 3,
+}
+
+SLOW_FILTER = {
+    "16X": 0,
+    "8X": 1,
+    "4X": 2,
+    "2X": 3,
+}
+
+FAST_FILTER = {
+    "NONE": 0,
+    "LSB6": 1,
+    "LSB7": 2,
+    "LSB9": 3,
+    "LSB18": 4,
+    "LSB21": 5,
+    "LSB24": 6,
+    "LSB10": 7,
+}
+
+CONF_ANGLE = "angle"
+CONF_RAW_ANGLE = "raw_angle"
+CONF_RAW_POSITION = "raw_position"
+CONF_WATCHDOG = "watchdog"
+CONF_POWER_MODE = "power_mode"
+CONF_SLOW_FILTER = "slow_filter"
+CONF_FAST_FILTER = "fast_filter"
+CONF_START_POSITION = "start_position"
+CONF_END_POSITION = "end_position"
+
+
+RESOLUTION = 4096
+MAX_POSITION = RESOLUTION - 1
+ANGLE_TO_POSITION = RESOLUTION / 360
+POSITION_TO_ANGLE = 360 / RESOLUTION
+# validate min range of 18deg (per datasheet) ... though i seem to get valid values down to a range of 192steps (16.875deg)
+MIN_RANGE = round(18 * ANGLE_TO_POSITION)
+
+
+def angle(min=-360, max=360):
+    return cv.All(
+        cv.float_with_unit("angle", "(°|deg)"), cv.float_range(min=min, max=max)
+    )
+
+
+def angle_to_position(value, min=-360, max=360):
+    try:
+        value = angle(min=min, max=max)(value)
+        return (RESOLUTION + round(value * ANGLE_TO_POSITION)) % RESOLUTION
+    except cv.Invalid as e:
+        raise cv.Invalid(f"When using angle, {e.error_message}")
+
+
+def percent_to_position(value):
+    value = cv.possibly_negative_percentage(value)
+    return (RESOLUTION + round(value * RESOLUTION)) % RESOLUTION
+
+
+def position(min=-MAX_POSITION, max=MAX_POSITION):
+    """Validate that the config option is a position.
+    Accepts integers, degrees, or percentage (of 360 degrees).
+    """
+
+    def validator(value):
+        if isinstance(value, str) and value.endswith("%"):
+            value = percent_to_position(value)
+
+        if isinstance(value, str) and (value.endswith("°") or value.endswith("deg")):
+            return angle_to_position(
+                value,
+                min=round(min * POSITION_TO_ANGLE),
+                max=round(max * POSITION_TO_ANGLE),
+            )
+
+        return cv.int_range(min=min, max=max)(value)
+
+    return validator
+
+
+def position_range():
+    """Validate that value given is a valid range for the device.
+    A valid range is one of the following:
+    - a value of 0 (meaning full range)
+    - 18 thru 360 degrees
+    - negative 360 thru negative 18 degrees (notes: these are normalized to their positive values, accepting negatives is for convenience)
+    """
+    zero_validator = position(min=0, max=0)
+    negative_validator = cv.Any(
+        position(min=-MAX_POSITION, max=-MIN_RANGE),
+        zero_validator,
+    )
+    positive_validator = cv.Any(
+        position(min=MIN_RANGE, max=MAX_POSITION),
+        zero_validator,
+    )
+
+    def validator(value):
+        is_negative_str = isinstance(value, str) and value.startswith("-")
+        is_negative_num = isinstance(value, (float, int)) and value < 0
+        if is_negative_str or is_negative_num:
+            return negative_validator(value)
+        return positive_validator(value)
+
+    return validator
+
+
+def has_valid_range_config():
+    """Validate that that the config start + end position results in a valid
+    positional range, which must be >= 18degrees
+    """
+    range_validator = position_range()
+
+    def validator(config):
+        # if we don't have an end position, then there is nothing to do
+        if CONF_END_POSITION not in config:
+            return config
+
+        # determine the range by taking the difference from the end and start
+        range = config[CONF_END_POSITION] - config[CONF_START_POSITION]
+
+        # but need to account for start position being greater than end position
+        # where the range rolls back around the 0 position
+        if config[CONF_END_POSITION] < config[CONF_START_POSITION]:
+            range = RESOLUTION + config[CONF_END_POSITION] - config[CONF_START_POSITION]
+
+        try:
+            range_validator(range)
+            return config
+        except cv.Invalid as e:
+            raise cv.Invalid(
+                f"The range between start and end position is invalid. It was was {range} but {e.error_message}"
+            )
+
+    return validator
+
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(AS5600Component),
+            cv.Optional(CONF_DIR_PIN): pins.gpio_input_pin_schema,
+            cv.Optional(CONF_DIRECTION, default="CLOCKWISE"): cv.enum(
+                DIRECTION, upper=True
+            ),
+            cv.Optional(CONF_WATCHDOG, default=False): cv.boolean,
+            cv.Optional(CONF_POWER_MODE, default="NOMINAL"): cv.enum(
+                POWER_MODE, upper=True, space=""
+            ),
+            cv.Optional(CONF_HYSTERESIS, default="NONE"): cv.enum(
+                HYSTERESIS, upper=True, space=""
+            ),
+            cv.Optional(CONF_SLOW_FILTER, default="16X"): cv.enum(
+                SLOW_FILTER, upper=True, space=""
+            ),
+            cv.Optional(CONF_FAST_FILTER, default="NONE"): cv.enum(
+                FAST_FILTER, upper=True, space=""
+            ),
+            cv.Optional(CONF_START_POSITION, default=0): position(),
+            cv.Optional(CONF_END_POSITION): position(),
+            cv.Optional(CONF_RANGE): position_range(),
+        }
+    )
+    .extend(cv.COMPONENT_SCHEMA)
+    .extend(i2c.i2c_device_schema(0x36)),
+    # ensure end_position and range are mutually exclusive
+    cv.has_at_most_one_key(CONF_END_POSITION, CONF_RANGE),
+    has_valid_range_config(),
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+    await i2c.register_i2c_device(var, config)
+
+    cg.add(var.set_direction(config[CONF_DIRECTION]))
+    cg.add(var.set_watchdog(config[CONF_WATCHDOG]))
+    cg.add(var.set_power_mode(config[CONF_POWER_MODE]))
+    cg.add(var.set_hysteresis(config[CONF_HYSTERESIS]))
+    cg.add(var.set_slow_filter(config[CONF_SLOW_FILTER]))
+    cg.add(var.set_fast_filter(config[CONF_FAST_FILTER]))
+    cg.add(var.set_start_position(config[CONF_START_POSITION]))
+
+    if dir_pin_config := config.get(CONF_DIR_PIN):
+        pin = await cg.gpio_pin_expression(dir_pin_config)
+        cg.add(var.set_dir_pin(pin))
+
+    if (end_position_config := config.get(CONF_END_POSITION, None)) is not None:
+        cg.add(var.set_end_position(end_position_config))
+
+    if (range_config := config.get(CONF_RANGE, None)) is not None:
+        cg.add(var.set_range(range_config))
diff --git a/esphome/components/as5600/as5600.cpp b/esphome/components/as5600/as5600.cpp
new file mode 100644
index 0000000000..3fe7eab58d
--- /dev/null
+++ b/esphome/components/as5600/as5600.cpp
@@ -0,0 +1,138 @@
+#include "as5600.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace as5600 {
+
+static const char *const TAG = "as5600";
+
+// Configuration registers
+static const uint8_t REGISTER_ZMCO = 0x00;  // 8 bytes  / R
+static const uint8_t REGISTER_ZPOS = 0x01;  // 16 bytes / RW
+static const uint8_t REGISTER_MPOS = 0x03;  // 16 bytes / RW
+static const uint8_t REGISTER_MANG = 0x05;  // 16 bytes / RW
+static const uint8_t REGISTER_CONF = 0x07;  // 16 bytes / RW
+
+// Output registers
+static const uint8_t REGISTER_ANGLE_RAW = 0x0C;  // 16 bytes / R
+static const uint8_t REGISTER_ANGLE = 0x0E;      // 16 bytes / R
+
+// Status registers
+static const uint8_t REGISTER_STATUS = 0x0B;     // 8 bytes  / R
+static const uint8_t REGISTER_AGC = 0x1A;        // 8 bytes  / R
+static const uint8_t REGISTER_MAGNITUDE = 0x1B;  // 16 bytes / R
+
+void AS5600Component::setup() {
+  ESP_LOGCONFIG(TAG, "Setting up AS5600...");
+
+  if (!this->read_byte(REGISTER_STATUS).has_value()) {
+    this->mark_failed();
+    return;
+  }
+
+  // configuration direction pin, if given
+  // the dir pin on the chip should be low for clockwise
+  // and high for counterclockwise. If the pin is left floating
+  // the reported positions will be erratic.
+  if (this->dir_pin_ != nullptr) {
+    this->dir_pin_->pin_mode(gpio::FLAG_OUTPUT);
+    this->dir_pin_->digital_write(this->direction_ == 1);
+  }
+
+  // build config register
+  // take the value, shift it left, and add mask to it to ensure we
+  // are only changing the bits appropriate for that setting in the
+  // off chance we somehow have bad value in there and it makes for
+  // a nice visual for the bit positions.
+  uint16_t config = 0;
+  // clang-format off
+  config |= (this->watchdog_      << 13) & 0b0010000000000000;
+  config |= (this->fast_filter_   << 10) & 0b0001110000000000;
+  config |= (this->slow_filter_   <<  8) & 0b0000001100000000;
+  config |= (this->pwm_frequency_ <<  6) & 0b0000000011000000;
+  config |= (this->output_mode_   <<  4) & 0b0000000000110000;
+  config |= (this->hysteresis_    <<  2) & 0b0000000000001100;
+  config |= (this->power_mode_    <<  0) & 0b0000000000000011;
+  // clang-format on
+
+  // write config to config register
+  if (!this->write_byte_16(REGISTER_CONF, config)) {
+    this->mark_failed();
+    return;
+  }
+
+  // configure the start position
+  this->write_byte_16(REGISTER_ZPOS, this->start_position_);
+
+  // configure either end position or max angle
+  if (this->end_mode_ == END_MODE_POSITION) {
+    this->write_byte_16(REGISTER_MPOS, this->end_position_);
+  } else {
+    this->write_byte_16(REGISTER_MANG, this->end_position_);
+  }
+
+  // calculate the raw max from end position or start + range
+  this->raw_max_ = this->end_mode_ == END_MODE_POSITION ? this->end_position_ & 4095
+                                                        : (this->start_position_ + this->end_position_) & 4095;
+
+  // calculate allowed range of motion by taking the start from the end
+  // but only if the end is greater than the start. If the start is greater
+  // than the end position, then that means we take the start all the way to
+  // reset point (i.e. 0 deg raw) and then we that with the end position
+  uint16_t range = this->raw_max_ > this->start_position_ ? this->raw_max_ - this->start_position_
+                                                          : (4095 - this->start_position_) + this->raw_max_;
+
+  // range scale is ratio of actual allowed range to the full range
+  this->range_scale_ = range / 4095.0f;
+}
+
+void AS5600Component::dump_config() {
+  ESP_LOGCONFIG(TAG, "AS5600:");
+  LOG_I2C_DEVICE(this);
+
+  if (this->is_failed()) {
+    ESP_LOGE(TAG, "Communication with AS5600 failed!");
+    return;
+  }
+
+  ESP_LOGCONFIG(TAG, "  Watchdog: %d", this->watchdog_);
+  ESP_LOGCONFIG(TAG, "  Fast Filter: %d", this->fast_filter_);
+  ESP_LOGCONFIG(TAG, "  Slow Filter: %d", this->slow_filter_);
+  ESP_LOGCONFIG(TAG, "  Hysteresis: %d", this->hysteresis_);
+  ESP_LOGCONFIG(TAG, "  Start Position: %d", this->start_position_);
+  if (this->end_mode_ == END_MODE_POSITION) {
+    ESP_LOGCONFIG(TAG, "  End Position: %d", this->end_position_);
+  } else {
+    ESP_LOGCONFIG(TAG, "  Range: %d", this->end_position_);
+  }
+}
+
+bool AS5600Component::in_range(uint16_t raw_position) {
+  return this->raw_max_ > this->start_position_
+             ? raw_position >= this->start_position_ && raw_position <= this->raw_max_
+             : raw_position >= this->start_position_ || raw_position <= this->raw_max_;
+}
+
+AS5600MagnetStatus AS5600Component::read_magnet_status() {
+  uint8_t status = this->reg(REGISTER_STATUS).get() >> 3 & 0b000111;
+  return static_cast<AS5600MagnetStatus>(status);
+}
+
+optional<uint16_t> AS5600Component::read_position() {
+  uint16_t pos = 0;
+  if (!this->read_byte_16(REGISTER_ANGLE, &pos)) {
+    return {};
+  }
+  return pos;
+}
+
+optional<uint16_t> AS5600Component::read_raw_position() {
+  uint16_t pos = 0;
+  if (!this->read_byte_16(REGISTER_ANGLE_RAW, &pos)) {
+    return {};
+  }
+  return pos;
+}
+
+}  // namespace as5600
+}  // namespace esphome
diff --git a/esphome/components/as5600/as5600.h b/esphome/components/as5600/as5600.h
new file mode 100644
index 0000000000..fbfd18db40
--- /dev/null
+++ b/esphome/components/as5600/as5600.h
@@ -0,0 +1,105 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+#include "esphome/core/preferences.h"
+#include "esphome/components/i2c/i2c.h"
+#include "esphome/components/sensor/sensor.h"
+
+namespace esphome {
+namespace as5600 {
+
+static const uint16_t POSITION_COUNT = 4096;
+static const float RAW_TO_DEGREES = 360.0 / POSITION_COUNT;
+static const float DEGREES_TO_RAW = POSITION_COUNT / 360.0;
+
+enum EndPositionMode : uint8_t {
+  // In this mode, the end position is calculated by taking the start position
+  // and adding the range/positions. For example, you could say start at 90deg,
+  // and have a range of 180deg and effectively the sensor will report values
+  // from the physical 90deg thru 270deg.
+  END_MODE_RANGE,
+  // In this mode, the end position is explicitly set, and changing the start
+  // position will NOT change the end position.
+  END_MODE_POSITION,
+};
+
+enum OutRangeMode : uint8_t {
+  // In this mode, the AS5600 chip itself actually reports these values, but
+  // effectively it splits the out-of-range values in half, and when positioned
+  // over the half closest to the min/start position, it will report 0 and when
+  // positioned over the half closes to the max/end position, it will report the
+  // max/end value.
+  OUT_RANGE_MODE_MIN_MAX,
+  // In this mode, when the magnet is positioned outside the configured
+  // range, the sensor will report NAN, which translates to "Unknown"
+  // in Home Assistant.
+  OUT_RANGE_MODE_NAN,
+};
+
+enum AS5600MagnetStatus : uint8_t {
+  MAGNET_GONE = 2,    // 0b010 / magnet not detected
+  MAGNET_OK = 4,      // 0b100 / magnet just right
+  MAGNET_STRONG = 5,  // 0b101 / magnet too strong
+  MAGNET_WEAK = 6,    // 0b110 / magnet too weak
+};
+
+class AS5600Component : public Component, public i2c::I2CDevice {
+ public:
+  /// Set up the internal sensor array.
+  void setup() override;
+  void dump_config() override;
+  /// HARDWARE_LATE setup priority
+  float get_setup_priority() const override { return setup_priority::DATA; }
+
+  // configuration setters
+  void set_dir_pin(InternalGPIOPin *pin) { this->dir_pin_ = pin; }
+  void set_direction(uint8_t direction) { this->direction_ = direction; }
+  void set_fast_filter(uint8_t fast_filter) { this->fast_filter_ = fast_filter; }
+  void set_hysteresis(uint8_t hysteresis) { this->hysteresis_ = hysteresis; }
+  void set_power_mode(uint8_t power_mode) { this->power_mode_ = power_mode; }
+  void set_slow_filter(uint8_t slow_filter) { this->slow_filter_ = slow_filter; }
+  void set_watchdog(bool watchdog) { this->watchdog_ = watchdog; }
+  bool get_watchdog() { return this->watchdog_; }
+  void set_start_position(uint16_t start_position) { this->start_position_ = start_position % POSITION_COUNT; }
+  void set_end_position(uint16_t end_position) {
+    this->end_position_ = end_position % POSITION_COUNT;
+    this->end_mode_ = END_MODE_POSITION;
+  }
+  void set_range(uint16_t range) {
+    this->end_position_ = range % POSITION_COUNT;
+    this->end_mode_ = END_MODE_RANGE;
+  }
+
+  // Gets the scale value for the configured range.
+  // For example, if configured to start at 0deg and end at 180deg, the
+  // range is 50% of the native/raw range, so the range scale would be 0.5.
+  // If configured to use the full 360deg, the range scale would be 1.0.
+  float get_range_scale() { return this->range_scale_; }
+
+  // Indicates whether the given *raw* position is within the configured range
+  bool in_range(uint16_t raw_position);
+
+  AS5600MagnetStatus read_magnet_status();
+  optional<uint16_t> read_position();
+  optional<uint16_t> read_raw_position();
+
+ protected:
+  InternalGPIOPin *dir_pin_{nullptr};
+  uint8_t direction_;
+  uint8_t fast_filter_;
+  uint8_t hysteresis_;
+  uint8_t power_mode_;
+  uint8_t slow_filter_;
+  uint8_t pwm_frequency_{0};
+  uint8_t output_mode_{0};
+  bool watchdog_;
+  uint16_t start_position_;
+  uint16_t end_position_{0};
+  uint16_t raw_max_;
+  EndPositionMode end_mode_{END_MODE_RANGE};
+  float range_scale_{1.0};
+};
+
+}  // namespace as5600
+}  // namespace esphome
diff --git a/esphome/components/as5600/sensor/__init__.py b/esphome/components/as5600/sensor/__init__.py
new file mode 100644
index 0000000000..589a66950a
--- /dev/null
+++ b/esphome/components/as5600/sensor/__init__.py
@@ -0,0 +1,119 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import sensor
+from esphome.const import (
+    CONF_ID,
+    STATE_CLASS_MEASUREMENT,
+    ICON_MAGNET,
+    ICON_ROTATE_RIGHT,
+    CONF_GAIN,
+    ENTITY_CATEGORY_DIAGNOSTIC,
+    CONF_MAGNITUDE,
+    CONF_STATUS,
+    CONF_POSITION,
+)
+from .. import as5600_ns, AS5600Component
+
+CODEOWNERS = ["@ammmze"]
+DEPENDENCIES = ["as5600"]
+
+AS5600Sensor = as5600_ns.class_("AS5600Sensor", sensor.Sensor, cg.PollingComponent)
+
+CONF_ANGLE = "angle"
+CONF_RAW_ANGLE = "raw_angle"
+CONF_RAW_POSITION = "raw_position"
+CONF_WATCHDOG = "watchdog"
+CONF_POWER_MODE = "power_mode"
+CONF_SLOW_FILTER = "slow_filter"
+CONF_FAST_FILTER = "fast_filter"
+CONF_PWM_FREQUENCY = "pwm_frequency"
+CONF_BURN_COUNT = "burn_count"
+CONF_START_POSITION = "start_position"
+CONF_END_POSITION = "end_position"
+CONF_OUT_OF_RANGE_MODE = "out_of_range_mode"
+
+OutOfRangeMode = as5600_ns.enum("OutRangeMode")
+OUT_OF_RANGE_MODES = {
+    "MIN_MAX": OutOfRangeMode.OUT_RANGE_MODE_MIN_MAX,
+    "NAN": OutOfRangeMode.OUT_RANGE_MODE_NAN,
+}
+
+
+CONF_AS5600_ID = "as5600_id"
+CONFIG_SCHEMA = (
+    sensor.sensor_schema(
+        AS5600Sensor,
+        accuracy_decimals=0,
+        icon=ICON_ROTATE_RIGHT,
+        state_class=STATE_CLASS_MEASUREMENT,
+    )
+    .extend(
+        {
+            cv.GenerateID(CONF_AS5600_ID): cv.use_id(AS5600Component),
+            cv.Optional(CONF_OUT_OF_RANGE_MODE): cv.enum(
+                OUT_OF_RANGE_MODES, upper=True, space="_"
+            ),
+            cv.Optional(CONF_RAW_POSITION): sensor.sensor_schema(
+                accuracy_decimals=0,
+                icon=ICON_ROTATE_RIGHT,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Optional(CONF_GAIN): sensor.sensor_schema(
+                accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
+                entity_category=ENTITY_CATEGORY_DIAGNOSTIC,
+            ),
+            cv.Optional(CONF_MAGNITUDE): sensor.sensor_schema(
+                accuracy_decimals=0,
+                icon=ICON_MAGNET,
+                state_class=STATE_CLASS_MEASUREMENT,
+                entity_category=ENTITY_CATEGORY_DIAGNOSTIC,
+            ),
+            cv.Optional(CONF_STATUS): sensor.sensor_schema(
+                accuracy_decimals=0,
+                icon=ICON_MAGNET,
+                state_class=STATE_CLASS_MEASUREMENT,
+                entity_category=ENTITY_CATEGORY_DIAGNOSTIC,
+            ),
+        }
+    )
+    .extend(cv.polling_component_schema("60s"))
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_parented(var, config[CONF_AS5600_ID])
+    await cg.register_component(var, config)
+    await sensor.register_sensor(var, config)
+
+    if out_of_range_mode_config := config.get(CONF_OUT_OF_RANGE_MODE):
+        cg.add(var.set_out_of_range_mode(out_of_range_mode_config))
+
+    if angle_config := config.get(CONF_ANGLE):
+        sens = await sensor.new_sensor(angle_config)
+        cg.add(var.set_angle_sensor(sens))
+
+    if raw_angle_config := config.get(CONF_RAW_ANGLE):
+        sens = await sensor.new_sensor(raw_angle_config)
+        cg.add(var.set_raw_angle_sensor(sens))
+
+    if position_config := config.get(CONF_POSITION):
+        sens = await sensor.new_sensor(position_config)
+        cg.add(var.set_position_sensor(sens))
+
+    if raw_position_config := config.get(CONF_RAW_POSITION):
+        sens = await sensor.new_sensor(raw_position_config)
+        cg.add(var.set_raw_position_sensor(sens))
+
+    if gain_config := config.get(CONF_GAIN):
+        sens = await sensor.new_sensor(gain_config)
+        cg.add(var.set_gain_sensor(sens))
+
+    if magnitude_config := config.get(CONF_MAGNITUDE):
+        sens = await sensor.new_sensor(magnitude_config)
+        cg.add(var.set_magnitude_sensor(sens))
+
+    if status_config := config.get(CONF_STATUS):
+        sens = await sensor.new_sensor(status_config)
+        cg.add(var.set_status_sensor(sens))
diff --git a/esphome/components/as5600/sensor/as5600_sensor.cpp b/esphome/components/as5600/sensor/as5600_sensor.cpp
new file mode 100644
index 0000000000..feb8f6cebf
--- /dev/null
+++ b/esphome/components/as5600/sensor/as5600_sensor.cpp
@@ -0,0 +1,98 @@
+#include "as5600_sensor.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace as5600 {
+
+static const char *const TAG = "as5600.sensor";
+
+// Configuration registers
+static const uint8_t REGISTER_ZMCO = 0x00;  // 8 bytes  / R
+static const uint8_t REGISTER_ZPOS = 0x01;  // 16 bytes / RW
+static const uint8_t REGISTER_MPOS = 0x03;  // 16 bytes / RW
+static const uint8_t REGISTER_MANG = 0x05;  // 16 bytes / RW
+static const uint8_t REGISTER_CONF = 0x07;  // 16 bytes / RW
+
+// Output registers
+static const uint8_t REGISTER_ANGLE_RAW = 0x0C;  // 16 bytes / R
+static const uint8_t REGISTER_ANGLE = 0x0E;      // 16 bytes / R
+
+// Status registers
+static const uint8_t REGISTER_STATUS = 0x0B;     // 8 bytes  / R
+static const uint8_t REGISTER_AGC = 0x1A;        // 8 bytes  / R
+static const uint8_t REGISTER_MAGNITUDE = 0x1B;  // 16 bytes / R
+
+float AS5600Sensor::get_setup_priority() const { return setup_priority::DATA; }
+
+void AS5600Sensor::dump_config() {
+  LOG_SENSOR("", "AS5600 Sensor", this);
+  ESP_LOGCONFIG(TAG, "  Out of Range Mode: %u", this->out_of_range_mode_);
+  if (this->angle_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Angle Sensor", this->angle_sensor_);
+  }
+  if (this->raw_angle_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Raw Angle Sensor", this->raw_angle_sensor_);
+  }
+  if (this->position_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Position Sensor", this->position_sensor_);
+  }
+  if (this->raw_position_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Raw Position Sensor", this->raw_position_sensor_);
+  }
+  if (this->gain_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Gain Sensor", this->gain_sensor_);
+  }
+  if (this->magnitude_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Magnitude Sensor", this->magnitude_sensor_);
+  }
+  if (this->status_sensor_ != nullptr) {
+    LOG_SENSOR("  ", "Status Sensor", this->status_sensor_);
+  }
+  LOG_UPDATE_INTERVAL(this);
+}
+
+void AS5600Sensor::update() {
+  if (this->gain_sensor_ != nullptr) {
+    this->gain_sensor_->publish_state(this->parent_->reg(REGISTER_AGC).get());
+  }
+
+  if (this->magnitude_sensor_ != nullptr) {
+    uint16_t value = 0;
+    this->parent_->read_byte_16(REGISTER_MAGNITUDE, &value);
+    this->magnitude_sensor_->publish_state(value);
+  }
+
+  // 2 = magnet not detected
+  // 4 = magnet just right
+  // 5 = magnet too strong
+  // 6 = magnet too weak
+  if (this->status_sensor_ != nullptr) {
+    this->status_sensor_->publish_state(this->parent_->read_magnet_status());
+  }
+
+  auto pos = this->parent_->read_position();
+  if (!pos.has_value()) {
+    this->status_set_warning();
+    return;
+  }
+
+  auto raw = this->parent_->read_raw_position();
+  if (!raw.has_value()) {
+    this->status_set_warning();
+    return;
+  }
+
+  if (this->out_of_range_mode_ == OUT_RANGE_MODE_NAN) {
+    this->publish_state(this->parent_->in_range(raw.value()) ? pos.value() : NAN);
+  } else {
+    this->publish_state(pos.value());
+  }
+
+  if (this->raw_position_sensor_ != nullptr) {
+    this->raw_position_sensor_->publish_state(raw.value());
+  }
+  this->status_clear_warning();
+}
+
+}  // namespace as5600
+}  // namespace esphome
diff --git a/esphome/components/as5600/sensor/as5600_sensor.h b/esphome/components/as5600/sensor/as5600_sensor.h
new file mode 100644
index 0000000000..0af9b01ae5
--- /dev/null
+++ b/esphome/components/as5600/sensor/as5600_sensor.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+#include "esphome/core/preferences.h"
+#include "esphome/components/i2c/i2c.h"
+#include "esphome/components/sensor/sensor.h"
+#include "esphome/components/as5600/as5600.h"
+
+namespace esphome {
+namespace as5600 {
+
+class AS5600Sensor : public PollingComponent, public Parented<AS5600Component>, public sensor::Sensor {
+ public:
+  void update() override;
+  void dump_config() override;
+  float get_setup_priority() const override;
+
+  void set_angle_sensor(sensor::Sensor *angle_sensor) { this->angle_sensor_ = angle_sensor; }
+  void set_raw_angle_sensor(sensor::Sensor *raw_angle_sensor) { this->raw_angle_sensor_ = raw_angle_sensor; }
+  void set_position_sensor(sensor::Sensor *position_sensor) { this->position_sensor_ = position_sensor; }
+  void set_raw_position_sensor(sensor::Sensor *raw_position_sensor) {
+    this->raw_position_sensor_ = raw_position_sensor;
+  }
+  void set_gain_sensor(sensor::Sensor *gain_sensor) { this->gain_sensor_ = gain_sensor; }
+  void set_magnitude_sensor(sensor::Sensor *magnitude_sensor) { this->magnitude_sensor_ = magnitude_sensor; }
+  void set_status_sensor(sensor::Sensor *status_sensor) { this->status_sensor_ = status_sensor; }
+  void set_out_of_range_mode(OutRangeMode oor_mode) { this->out_of_range_mode_ = oor_mode; }
+  OutRangeMode get_out_of_range_mode() { return this->out_of_range_mode_; }
+
+ protected:
+  sensor::Sensor *angle_sensor_{nullptr};
+  sensor::Sensor *raw_angle_sensor_{nullptr};
+  sensor::Sensor *position_sensor_{nullptr};
+  sensor::Sensor *raw_position_sensor_{nullptr};
+  sensor::Sensor *gain_sensor_{nullptr};
+  sensor::Sensor *magnitude_sensor_{nullptr};
+  sensor::Sensor *status_sensor_{nullptr};
+  OutRangeMode out_of_range_mode_{OUT_RANGE_MODE_MIN_MAX};
+};
+
+}  // namespace as5600
+}  // namespace esphome
diff --git a/tests/test1.yaml b/tests/test1.yaml
index aaad10ff0a..2748a09c59 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -322,6 +322,18 @@ ads1115:
   address: 0x48
   i2c_id: i2c_bus
 
+as5600:
+  i2c_id: i2c_bus
+  dir_pin: GPIO27
+  direction: clockwise
+  start_position: 90deg
+  range: 180deg
+  watchdog: true
+  power_mode: low1
+  hysteresis: lsb1
+  slow_filter: 8x
+  fast_filter: lsb6
+
 dallas:
   pin:
     allow_other_uses: true
@@ -555,6 +567,16 @@ sensor:
     state_topic: hi/me
     retain: false
     availability:
+  - platform: as5600
+    name: AS5600 Position
+    raw_position:
+        name: AS5600 Raw Position
+    gain:
+        name: AS5600 Gain
+    magnitude:
+        name: AS5600 Magnitude
+    status:
+        name: AS5600 Status
   - platform: as7341
     update_interval: 15s
     gain: X8

From 3c2383e26119c3c05341786aa335ea7653c0d120 Mon Sep 17 00:00:00 2001
From: mknjc <markus.knetschke@gmail.com>
Date: Thu, 21 Dec 2023 01:08:13 +0100
Subject: [PATCH 40/75] Add default substitutions for package includes (#5752)

---
 esphome/const.py     |  1 +
 esphome/yaml_util.py | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 5de34b86cd..58c6c47e3a 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -185,6 +185,7 @@ CONF_DEFAULT_MODE = "default_mode"
 CONF_DEFAULT_TARGET_TEMPERATURE_HIGH = "default_target_temperature_high"
 CONF_DEFAULT_TARGET_TEMPERATURE_LOW = "default_target_temperature_low"
 CONF_DEFAULT_TRANSITION_LENGTH = "default_transition_length"
+CONF_DEFAULTS = "defaults"
 CONF_DELAY = "delay"
 CONF_DELIMITER = "delimiter"
 CONF_DELTA = "delta"
diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py
index f0f755dd61..aa9fe45ebb 100644
--- a/esphome/yaml_util.py
+++ b/esphome/yaml_util.py
@@ -282,7 +282,7 @@ class ESPHomeLoader(FastestAvailableSafeLoader):
             return file, vars
 
         def substitute_vars(config, vars):
-            from esphome.const import CONF_SUBSTITUTIONS
+            from esphome.const import CONF_SUBSTITUTIONS, CONF_DEFAULTS
             from esphome.components import substitutions
 
             org_subs = None
@@ -294,7 +294,15 @@ class ESPHomeLoader(FastestAvailableSafeLoader):
             elif CONF_SUBSTITUTIONS in result:
                 org_subs = result.pop(CONF_SUBSTITUTIONS)
 
+            defaults = {}
+            if CONF_DEFAULTS in result:
+                defaults = result.pop(CONF_DEFAULTS)
+
             result[CONF_SUBSTITUTIONS] = vars
+            for k, v in defaults.items():
+                if k not in result[CONF_SUBSTITUTIONS]:
+                    result[CONF_SUBSTITUTIONS][k] = v
+
             # Ignore missing vars that refer to the top level substitutions
             substitutions.do_substitution_pass(result, None, ignore_missing=True)
             result.pop(CONF_SUBSTITUTIONS)

From c6a37da9da4061e5a9f663b800d694ca27b478f0 Mon Sep 17 00:00:00 2001
From: Matthew Campbell <calvinmc@gmail.com>
Date: Wed, 20 Dec 2023 16:08:44 -0800
Subject: [PATCH 41/75] Add gradient option to addressable color wipe effect
 (#5689)

---
 esphome/components/light/addressable_light_effect.h | 12 ++++++++++--
 esphome/components/light/effects.py                 |  3 +++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/esphome/components/light/addressable_light_effect.h b/esphome/components/light/addressable_light_effect.h
index c2109b2d23..73083a58b7 100644
--- a/esphome/components/light/addressable_light_effect.h
+++ b/esphome/components/light/addressable_light_effect.h
@@ -100,6 +100,7 @@ struct AddressableColorWipeEffectColor {
   uint8_t r, g, b, w;
   bool random;
   size_t num_leds;
+  bool gradient;
 };
 
 class AddressableColorWipeEffect : public AddressableLightEffect {
@@ -117,8 +118,15 @@ class AddressableColorWipeEffect : public AddressableLightEffect {
       it.shift_left(1);
     else
       it.shift_right(1);
-    const AddressableColorWipeEffectColor color = this->colors_[this->at_color_];
-    const Color esp_color = Color(color.r, color.g, color.b, color.w);
+    const AddressableColorWipeEffectColor &color = this->colors_[this->at_color_];
+    Color esp_color = Color(color.r, color.g, color.b, color.w);
+    if (color.gradient) {
+      size_t next_color_index = (this->at_color_ + 1) % this->colors_.size();
+      const AddressableColorWipeEffectColor &next_color = this->colors_[next_color_index];
+      const Color next_esp_color = Color(next_color.r, next_color.g, next_color.b, next_color.w);
+      uint8_t gradient = 255 * ((float) this->leds_added_ / color.num_leds);
+      esp_color = esp_color.gradient(next_esp_color, gradient);
+    }
     if (this->reverse_)
       it[-1] = esp_color;
     else
diff --git a/esphome/components/light/effects.py b/esphome/components/light/effects.py
index 5212e90938..5093ace949 100644
--- a/esphome/components/light/effects.py
+++ b/esphome/components/light/effects.py
@@ -58,6 +58,7 @@ from .types import (
 
 CONF_ADD_LED_INTERVAL = "add_led_interval"
 CONF_REVERSE = "reverse"
+CONF_GRADIENT = "gradient"
 CONF_MOVE_INTERVAL = "move_interval"
 CONF_SCAN_WIDTH = "scan_width"
 CONF_TWINKLE_PROBABILITY = "twinkle_probability"
@@ -386,6 +387,7 @@ async def addressable_rainbow_effect_to_code(config, effect_id):
                 cv.Optional(CONF_WHITE, default=1.0): cv.percentage,
                 cv.Optional(CONF_RANDOM, default=False): cv.boolean,
                 cv.Required(CONF_NUM_LEDS): cv.All(cv.uint32_t, cv.Range(min=1)),
+                cv.Optional(CONF_GRADIENT, default=False): cv.boolean,
             }
         ),
         cv.Optional(
@@ -409,6 +411,7 @@ async def addressable_color_wipe_effect_to_code(config, effect_id):
                 ("w", int(round(color[CONF_WHITE] * 255))),
                 ("random", color[CONF_RANDOM]),
                 ("num_leds", color[CONF_NUM_LEDS]),
+                ("gradient", color[CONF_GRADIENT]),
             )
         )
     cg.add(var.set_colors(colors))

From b5932940eebb085317e67de24c3f93ade537419f Mon Sep 17 00:00:00 2001
From: Pavlo Dudnytskyi <paveldudn@gmail.com>
Date: Thu, 21 Dec 2023 01:10:46 +0100
Subject: [PATCH 42/75] Added alarm processing for Haier component (hOn
 protocol) (#5965)

---
 esphome/components/haier/climate.py           |  51 ++++++--
 esphome/components/haier/haier_base.cpp       |   3 +-
 esphome/components/haier/haier_base.h         |   3 +-
 esphome/components/haier/hon_climate.cpp      | 117 +++++++++++++++++-
 esphome/components/haier/hon_climate.h        |  27 ++++
 esphome/components/haier/hon_packet.h         |  56 +++++++++
 .../components/haier/smartair2_climate.cpp    |  19 ++-
 tests/test3.yaml                              |  23 +++-
 8 files changed, 282 insertions(+), 17 deletions(-)

diff --git a/esphome/components/haier/climate.py b/esphome/components/haier/climate.py
index 49d42a231f..c6998ce0c5 100644
--- a/esphome/components/haier/climate.py
+++ b/esphome/components/haier/climate.py
@@ -18,6 +18,7 @@ from esphome.const import (
     CONF_SUPPORTED_SWING_MODES,
     CONF_TARGET_TEMPERATURE,
     CONF_TEMPERATURE_STEP,
+    CONF_TRIGGER_ID,
     CONF_VISUAL,
     CONF_WIFI,
     DEVICE_CLASS_TEMPERATURE,
@@ -49,6 +50,8 @@ CONF_CONTROL_METHOD = "control_method"
 CONF_CONTROL_PACKET_SIZE = "control_packet_size"
 CONF_DISPLAY = "display"
 CONF_HORIZONTAL_AIRFLOW = "horizontal_airflow"
+CONF_ON_ALARM_START = "on_alarm_start"
+CONF_ON_ALARM_END = "on_alarm_end"
 CONF_OUTDOOR_TEMPERATURE = "outdoor_temperature"
 CONF_VERTICAL_AIRFLOW = "vertical_airflow"
 CONF_WIFI_SIGNAL = "wifi_signal"
@@ -85,8 +88,8 @@ AIRFLOW_HORIZONTAL_DIRECTION_OPTIONS = {
 }
 
 SUPPORTED_SWING_MODES_OPTIONS = {
-    "OFF": ClimateSwingMode.CLIMATE_SWING_OFF,  # always available
-    "VERTICAL": ClimateSwingMode.CLIMATE_SWING_VERTICAL,  # always available
+    "OFF": ClimateSwingMode.CLIMATE_SWING_OFF,
+    "VERTICAL": ClimateSwingMode.CLIMATE_SWING_VERTICAL,
     "HORIZONTAL": ClimateSwingMode.CLIMATE_SWING_HORIZONTAL,
     "BOTH": ClimateSwingMode.CLIMATE_SWING_BOTH,
 }
@@ -101,13 +104,15 @@ SUPPORTED_CLIMATE_MODES_OPTIONS = {
 }
 
 SUPPORTED_CLIMATE_PRESETS_SMARTAIR2_OPTIONS = {
+    "AWAY": ClimatePreset.CLIMATE_PRESET_AWAY,
     "BOOST": ClimatePreset.CLIMATE_PRESET_BOOST,
     "COMFORT": ClimatePreset.CLIMATE_PRESET_COMFORT,
 }
 
 SUPPORTED_CLIMATE_PRESETS_HON_OPTIONS = {
-    "ECO": ClimatePreset.CLIMATE_PRESET_ECO,
+    "AWAY": ClimatePreset.CLIMATE_PRESET_AWAY,
     "BOOST": ClimatePreset.CLIMATE_PRESET_BOOST,
+    "ECO": ClimatePreset.CLIMATE_PRESET_ECO,
     "SLEEP": ClimatePreset.CLIMATE_PRESET_SLEEP,
 }
 
@@ -118,6 +123,16 @@ SUPPORTED_HON_CONTROL_METHODS = {
     "SET_SINGLE_PARAMETER": HonControlMethod.SET_SINGLE_PARAMETER,
 }
 
+HaierAlarmStartTrigger = haier_ns.class_(
+    "HaierAlarmStartTrigger",
+    automation.Trigger.template(cg.uint8, cg.const_char_ptr),
+)
+
+HaierAlarmEndTrigger = haier_ns.class_(
+    "HaierAlarmEndTrigger",
+    automation.Trigger.template(cg.uint8, cg.const_char_ptr),
+)
+
 
 def validate_visual(config):
     if CONF_VISUAL in config:
@@ -200,9 +215,7 @@ CONFIG_SCHEMA = cv.All(
                     ): cv.boolean,
                     cv.Optional(
                         CONF_SUPPORTED_PRESETS,
-                        default=list(
-                            SUPPORTED_CLIMATE_PRESETS_SMARTAIR2_OPTIONS.keys()
-                        ),
+                        default=list(["BOOST", "COMFORT"]),  # No AWAY by default
                     ): cv.ensure_list(
                         cv.enum(SUPPORTED_CLIMATE_PRESETS_SMARTAIR2_OPTIONS, upper=True)
                     ),
@@ -222,7 +235,7 @@ CONFIG_SCHEMA = cv.All(
                     ): cv.int_range(min=PROTOCOL_CONTROL_PACKET_SIZE, max=50),
                     cv.Optional(
                         CONF_SUPPORTED_PRESETS,
-                        default=list(SUPPORTED_CLIMATE_PRESETS_HON_OPTIONS.keys()),
+                        default=list(["BOOST", "ECO", "SLEEP"]),  # No AWAY by default
                     ): cv.ensure_list(
                         cv.enum(SUPPORTED_CLIMATE_PRESETS_HON_OPTIONS, upper=True)
                     ),
@@ -233,6 +246,20 @@ CONFIG_SCHEMA = cv.All(
                         device_class=DEVICE_CLASS_TEMPERATURE,
                         state_class=STATE_CLASS_MEASUREMENT,
                     ),
+                    cv.Optional(CONF_ON_ALARM_START): automation.validate_automation(
+                        {
+                            cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(
+                                HaierAlarmStartTrigger
+                            ),
+                        }
+                    ),
+                    cv.Optional(CONF_ON_ALARM_END): automation.validate_automation(
+                        {
+                            cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(
+                                HaierAlarmEndTrigger
+                            ),
+                        }
+                    ),
                 }
             ),
         },
@@ -457,5 +484,15 @@ async def to_code(config):
                 config[CONF_CONTROL_PACKET_SIZE] - PROTOCOL_CONTROL_PACKET_SIZE
             )
         )
+    for conf in config.get(CONF_ON_ALARM_START, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(
+            trigger, [(cg.uint8, "code"), (cg.const_char_ptr, "message")], conf
+        )
+    for conf in config.get(CONF_ON_ALARM_END, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(
+            trigger, [(cg.uint8, "code"), (cg.const_char_ptr, "message")], conf
+        )
     # https://github.com/paveldn/HaierProtocol
     cg.add_library("pavlodn/HaierProtocol", "0.9.24")
diff --git a/esphome/components/haier/haier_base.cpp b/esphome/components/haier/haier_base.cpp
index 6943fc7d9c..a3f68bb081 100644
--- a/esphome/components/haier/haier_base.cpp
+++ b/esphome/components/haier/haier_base.cpp
@@ -25,13 +25,14 @@ const char *HaierClimateBase::phase_to_string_(ProtocolPhases phase) {
       "SENDING_INIT_1",
       "SENDING_INIT_2",
       "SENDING_FIRST_STATUS_REQUEST",
-      "SENDING_ALARM_STATUS_REQUEST",
+      "SENDING_FIRST_ALARM_STATUS_REQUEST",
       "IDLE",
       "SENDING_STATUS_REQUEST",
       "SENDING_UPDATE_SIGNAL_REQUEST",
       "SENDING_SIGNAL_LEVEL",
       "SENDING_CONTROL",
       "SENDING_ACTION_COMMAND",
+      "SENDING_ALARM_STATUS_REQUEST",
       "UNKNOWN"  // Should be the last!
   };
   static_assert(
diff --git a/esphome/components/haier/haier_base.h b/esphome/components/haier/haier_base.h
index 75abbc20fb..504c841e5f 100644
--- a/esphome/components/haier/haier_base.h
+++ b/esphome/components/haier/haier_base.h
@@ -64,7 +64,7 @@ class HaierClimateBase : public esphome::Component,
     SENDING_INIT_1 = 0,
     SENDING_INIT_2,
     SENDING_FIRST_STATUS_REQUEST,
-    SENDING_ALARM_STATUS_REQUEST,
+    SENDING_FIRST_ALARM_STATUS_REQUEST,
     // FUNCTIONAL STATE
     IDLE,
     SENDING_STATUS_REQUEST,
@@ -72,6 +72,7 @@ class HaierClimateBase : public esphome::Component,
     SENDING_SIGNAL_LEVEL,
     SENDING_CONTROL,
     SENDING_ACTION_COMMAND,
+    SENDING_ALARM_STATUS_REQUEST,
     NUM_PROTOCOL_PHASES
   };
   const char *phase_to_string_(ProtocolPhases phase);
diff --git a/esphome/components/haier/hon_climate.cpp b/esphome/components/haier/hon_climate.cpp
index 09f90fffa8..e5aa88e2c9 100644
--- a/esphome/components/haier/hon_climate.cpp
+++ b/esphome/components/haier/hon_climate.cpp
@@ -16,6 +16,7 @@ constexpr size_t SIGNAL_LEVEL_UPDATE_INTERVAL_MS = 10000;
 constexpr int PROTOCOL_OUTDOOR_TEMPERATURE_OFFSET = -64;
 constexpr uint8_t CONTROL_MESSAGE_RETRIES = 5;
 constexpr std::chrono::milliseconds CONTROL_MESSAGE_RETRIES_INTERVAL = std::chrono::milliseconds(500);
+constexpr size_t ALARM_STATUS_REQUEST_INTERVAL_MS = 600000;
 
 hon_protocol::VerticalSwingMode get_vertical_swing_mode(AirflowVerticalDirection direction) {
   switch (direction) {
@@ -110,6 +111,14 @@ void HonClimate::start_steri_cleaning() {
   }
 }
 
+void HonClimate::add_alarm_start_callback(std::function<void(uint8_t, const char *)> &&callback) {
+  this->alarm_start_callback_.add(std::move(callback));
+}
+
+void HonClimate::add_alarm_end_callback(std::function<void(uint8_t, const char *)> &&callback) {
+  this->alarm_end_callback_.add(std::move(callback));
+}
+
 haier_protocol::HandlerError HonClimate::get_device_version_answer_handler_(haier_protocol::FrameType request_type,
                                                                             haier_protocol::FrameType message_type,
                                                                             const uint8_t *data, size_t data_size) {
@@ -194,7 +203,7 @@ haier_protocol::HandlerError HonClimate::status_handler_(haier_protocol::FrameTy
       switch (this->protocol_phase_) {
         case ProtocolPhases::SENDING_FIRST_STATUS_REQUEST:
           ESP_LOGI(TAG, "First HVAC status received");
-          this->set_phase(ProtocolPhases::SENDING_ALARM_STATUS_REQUEST);
+          this->set_phase(ProtocolPhases::SENDING_FIRST_ALARM_STATUS_REQUEST);
           break;
         case ProtocolPhases::SENDING_ACTION_COMMAND:
           // Do nothing, phase will be changed in process_phase
@@ -251,12 +260,15 @@ haier_protocol::HandlerError HonClimate::get_alarm_status_answer_handler_(haier_
       this->set_phase(ProtocolPhases::IDLE);
       return haier_protocol::HandlerError::UNSUPPORTED_MESSAGE;
     }
-    if (this->protocol_phase_ != ProtocolPhases::SENDING_ALARM_STATUS_REQUEST) {
+    if ((this->protocol_phase_ != ProtocolPhases::SENDING_FIRST_ALARM_STATUS_REQUEST) &&
+        (this->protocol_phase_ != ProtocolPhases::SENDING_ALARM_STATUS_REQUEST)) {
       // Don't expect this answer now
       this->set_phase(ProtocolPhases::IDLE);
       return haier_protocol::HandlerError::UNEXPECTED_MESSAGE;
     }
-    memcpy(this->active_alarms_, data + 2, 8);
+    if (data_size < sizeof(active_alarms_) + 2)
+      return haier_protocol::HandlerError::WRONG_MESSAGE_STRUCTURE;
+    this->process_alarm_message_(data, data_size, this->protocol_phase_ >= ProtocolPhases::IDLE);
     this->set_phase(ProtocolPhases::IDLE);
     return haier_protocol::HandlerError::HANDLER_OK;
   } else {
@@ -265,6 +277,19 @@ haier_protocol::HandlerError HonClimate::get_alarm_status_answer_handler_(haier_
   }
 }
 
+haier_protocol::HandlerError HonClimate::alarm_status_message_handler_(haier_protocol::FrameType type,
+                                                                       const uint8_t *buffer, size_t size) {
+  haier_protocol::HandlerError result = haier_protocol::HandlerError::HANDLER_OK;
+  if (size < sizeof(this->active_alarms_) + 2) {
+    // Log error but confirm anyway to avoid to many messages
+    result = haier_protocol::HandlerError::WRONG_MESSAGE_STRUCTURE;
+  }
+  this->process_alarm_message_(buffer, size, true);
+  this->haier_protocol_.send_answer(haier_protocol::HaierMessage(haier_protocol::FrameType::CONFIRM));
+  this->last_alarm_request_ = std::chrono::steady_clock::now();
+  return result;
+}
+
 void HonClimate::set_handlers() {
   // Set handlers
   this->haier_protocol_.set_answer_handler(
@@ -291,6 +316,10 @@ void HonClimate::set_handlers() {
       haier_protocol::FrameType::REPORT_NETWORK_STATUS,
       std::bind(&HonClimate::report_network_status_answer_handler_, this, std::placeholders::_1, std::placeholders::_2,
                 std::placeholders::_3, std::placeholders::_4));
+  this->haier_protocol_.set_message_handler(
+      haier_protocol::FrameType::ALARM_STATUS,
+      std::bind(&HonClimate::alarm_status_message_handler_, this, std::placeholders::_1, std::placeholders::_2,
+                std::placeholders::_3));
 }
 
 void HonClimate::dump_config() {
@@ -363,10 +392,12 @@ void HonClimate::process_phase(std::chrono::steady_clock::time_point now) {
       this->set_phase(ProtocolPhases::IDLE);
       break;
 #endif
+    case ProtocolPhases::SENDING_FIRST_ALARM_STATUS_REQUEST:
     case ProtocolPhases::SENDING_ALARM_STATUS_REQUEST:
       if (this->can_send_message() && this->is_message_interval_exceeded_(now)) {
         static const haier_protocol::HaierMessage ALARM_STATUS_REQUEST(haier_protocol::FrameType::GET_ALARM_STATUS);
         this->send_message_(ALARM_STATUS_REQUEST, this->use_crc_);
+        this->last_alarm_request_ = now;
       }
       break;
     case ProtocolPhases::SENDING_CONTROL:
@@ -417,12 +448,16 @@ void HonClimate::process_phase(std::chrono::steady_clock::time_point now) {
       if (this->forced_request_status_ || this->is_status_request_interval_exceeded_(now)) {
         this->set_phase(ProtocolPhases::SENDING_STATUS_REQUEST);
         this->forced_request_status_ = false;
+      } else if (std::chrono::duration_cast<std::chrono::milliseconds>(now - this->last_alarm_request_).count() >
+                 ALARM_STATUS_REQUEST_INTERVAL_MS) {
+        this->set_phase(ProtocolPhases::SENDING_ALARM_STATUS_REQUEST);
       }
 #ifdef USE_WIFI
       else if (this->send_wifi_signal_ &&
                (std::chrono::duration_cast<std::chrono::milliseconds>(now - this->last_signal_request_).count() >
-                SIGNAL_LEVEL_UPDATE_INTERVAL_MS))
+                SIGNAL_LEVEL_UPDATE_INTERVAL_MS)) {
         this->set_phase(ProtocolPhases::SENDING_UPDATE_SIGNAL_REQUEST);
+      }
 #endif
     } break;
     default:
@@ -452,6 +487,7 @@ haier_protocol::HaierMessage HonClimate::get_control_message() {
   uint8_t control_out_buffer[sizeof(hon_protocol::HaierPacketControl)];
   memcpy(control_out_buffer, this->last_status_message_.get(), sizeof(hon_protocol::HaierPacketControl));
   hon_protocol::HaierPacketControl *out_data = (hon_protocol::HaierPacketControl *) control_out_buffer;
+  control_out_buffer[4] = 0;  // This byte should be cleared before setting values
   bool has_hvac_settings = false;
   if (this->current_hvac_settings_.valid) {
     has_hvac_settings = true;
@@ -552,31 +588,41 @@ haier_protocol::HaierMessage HonClimate::get_control_message() {
           out_data->quiet_mode = 0;
           out_data->fast_mode = 0;
           out_data->sleep_mode = 0;
+          out_data->ten_degree = 0;
           break;
         case CLIMATE_PRESET_ECO:
           // Eco is not supported in Fan only mode
           out_data->quiet_mode = (this->mode != CLIMATE_MODE_FAN_ONLY) ? 1 : 0;
           out_data->fast_mode = 0;
           out_data->sleep_mode = 0;
+          out_data->ten_degree = 0;
           break;
         case CLIMATE_PRESET_BOOST:
           out_data->quiet_mode = 0;
           // Boost is not supported in Fan only mode
           out_data->fast_mode = (this->mode != CLIMATE_MODE_FAN_ONLY) ? 1 : 0;
           out_data->sleep_mode = 0;
+          out_data->ten_degree = 0;
           break;
         case CLIMATE_PRESET_AWAY:
           out_data->quiet_mode = 0;
           out_data->fast_mode = 0;
           out_data->sleep_mode = 0;
+          // 10 degrees allowed only in heat mode
+          out_data->ten_degree = (this->mode == CLIMATE_MODE_HEAT) ? 1 : 0;
           break;
         case CLIMATE_PRESET_SLEEP:
           out_data->quiet_mode = 0;
           out_data->fast_mode = 0;
           out_data->sleep_mode = 1;
+          out_data->ten_degree = 0;
           break;
         default:
           ESP_LOGE("Control", "Unsupported preset");
+          out_data->quiet_mode = 0;
+          out_data->fast_mode = 0;
+          out_data->sleep_mode = 0;
+          out_data->ten_degree = 0;
           break;
       }
     }
@@ -595,6 +641,50 @@ haier_protocol::HaierMessage HonClimate::get_control_message() {
                                       control_out_buffer, sizeof(hon_protocol::HaierPacketControl));
 }
 
+void HonClimate::process_alarm_message_(const uint8_t *packet, uint8_t size, bool check_new) {
+  constexpr size_t active_alarms_size = sizeof(this->active_alarms_);
+  if (size >= active_alarms_size + 2) {
+    if (check_new) {
+      size_t alarm_code = 0;
+      for (int i = active_alarms_size - 1; i >= 0; i--) {
+        if (packet[2 + i] != active_alarms_[i]) {
+          uint8_t alarm_bit = 1;
+          for (int b = 0; b < 8; b++) {
+            if ((packet[2 + i] & alarm_bit) != (this->active_alarms_[i] & alarm_bit)) {
+              bool alarm_status = (packet[2 + i] & alarm_bit) != 0;
+              int log_level = alarm_status ? ESPHOME_LOG_LEVEL_WARN : ESPHOME_LOG_LEVEL_INFO;
+              const char *alarm_message = alarm_code < esphome::haier::hon_protocol::HON_ALARM_COUNT
+                                              ? esphome::haier::hon_protocol::HON_ALARM_MESSAGES[alarm_code].c_str()
+                                              : "Unknown";
+              esp_log_printf_(log_level, TAG, __LINE__, "Alarm %s (%d): %s", alarm_status ? "activated" : "deactivated",
+                              alarm_code, alarm_message);
+              if (alarm_status) {
+                this->alarm_start_callback_.call(alarm_code, alarm_message);
+                this->active_alarm_count_ += 1.0f;
+              } else {
+                this->alarm_end_callback_.call(alarm_code, alarm_message);
+                this->active_alarm_count_ -= 1.0f;
+              }
+            }
+            alarm_bit <<= 1;
+            alarm_code++;
+          }
+          active_alarms_[i] = packet[2 + i];
+        } else
+          alarm_code += 8;
+      }
+    } else {
+      float alarm_count = 0.0f;
+      static uint8_t nibble_bits_count[] = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
+      for (size_t i = 0; i < sizeof(this->active_alarms_); i++) {
+        alarm_count += (float) (nibble_bits_count[packet[2 + i] & 0x0F] + nibble_bits_count[packet[2 + i] >> 4]);
+      }
+      this->active_alarm_count_ = alarm_count;
+      memcpy(this->active_alarms_, packet + 2, sizeof(this->active_alarms_));
+    }
+  }
+}
+
 haier_protocol::HandlerError HonClimate::process_status_message_(const uint8_t *packet_buffer, uint8_t size) {
   if (size < hon_protocol::HAIER_STATUS_FRAME_SIZE + this->extra_control_packet_bytes_)
     return haier_protocol::HandlerError::WRONG_MESSAGE_STRUCTURE;
@@ -626,6 +716,8 @@ haier_protocol::HandlerError HonClimate::process_status_message_(const uint8_t *
       this->preset = CLIMATE_PRESET_BOOST;
     } else if (packet.control.sleep_mode != 0) {
       this->preset = CLIMATE_PRESET_SLEEP;
+    } else if (packet.control.ten_degree != 0) {
+      this->preset = CLIMATE_PRESET_AWAY;
     } else {
       this->preset = CLIMATE_PRESET_NONE;
     }
@@ -882,25 +974,35 @@ void HonClimate::fill_control_messages_queue_() {
   // CLimate preset
   {
     uint8_t fast_mode_buf[] = {0x00, 0xFF};
+    uint8_t away_mode_buf[] = {0x00, 0xFF};
     if (!new_power) {
       // If AC is off - no presets allowed
       quiet_mode_buf[1] = 0x00;
       fast_mode_buf[1] = 0x00;
+      away_mode_buf[1] = 0x00;
     } else if (climate_control.preset.has_value()) {
       switch (climate_control.preset.value()) {
         case CLIMATE_PRESET_NONE:
           quiet_mode_buf[1] = 0x00;
           fast_mode_buf[1] = 0x00;
+          away_mode_buf[1] = 0x00;
           break;
         case CLIMATE_PRESET_ECO:
           // Eco is not supported in Fan only mode
           quiet_mode_buf[1] = (this->mode != CLIMATE_MODE_FAN_ONLY) ? 0x01 : 0x00;
           fast_mode_buf[1] = 0x00;
+          away_mode_buf[1] = 0x00;
           break;
         case CLIMATE_PRESET_BOOST:
           quiet_mode_buf[1] = 0x00;
           // Boost is not supported in Fan only mode
           fast_mode_buf[1] = (this->mode != CLIMATE_MODE_FAN_ONLY) ? 0x01 : 0x00;
+          away_mode_buf[1] = 0x00;
+          break;
+        case CLIMATE_PRESET_AWAY:
+          quiet_mode_buf[1] = 0x00;
+          fast_mode_buf[1] = 0x00;
+          away_mode_buf[1] = (this->mode == CLIMATE_MODE_HEAT) ? 0x01 : 0x00;
           break;
         default:
           ESP_LOGE("Control", "Unsupported preset");
@@ -921,6 +1023,13 @@ void HonClimate::fill_control_messages_queue_() {
                                            (uint8_t) hon_protocol::DataParameters::FAST_MODE,
                                        fast_mode_buf, 2));
     }
+    if (away_mode_buf[1] != 0xFF) {
+      this->control_messages_queue_.push(
+          haier_protocol::HaierMessage(haier_protocol::FrameType::CONTROL,
+                                       (uint16_t) hon_protocol::SubcommandsControl::SET_SINGLE_PARAMETER +
+                                           (uint8_t) hon_protocol::DataParameters::TEN_DEGREE,
+                                       away_mode_buf, 2));
+    }
   }
   // Target temperature
   if (climate_control.target_temperature.has_value()) {
diff --git a/esphome/components/haier/hon_climate.h b/esphome/components/haier/hon_climate.h
index 1ba6a8e041..9c05e59b87 100644
--- a/esphome/components/haier/hon_climate.h
+++ b/esphome/components/haier/hon_climate.h
@@ -2,6 +2,7 @@
 
 #include <chrono>
 #include "esphome/components/sensor/sensor.h"
+#include "esphome/core/automation.h"
 #include "haier_base.h"
 
 namespace esphome {
@@ -52,6 +53,9 @@ class HonClimate : public HaierClimateBase {
   void start_steri_cleaning();
   void set_extra_control_packet_bytes_size(size_t size) { this->extra_control_packet_bytes_ = size; };
   void set_control_method(HonControlMethod method) { this->control_method_ = method; };
+  void add_alarm_start_callback(std::function<void(uint8_t, const char *)> &&callback);
+  void add_alarm_end_callback(std::function<void(uint8_t, const char *)> &&callback);
+  float get_active_alarm_count() const { return this->active_alarm_count_; }
 
  protected:
   void set_handlers() override;
@@ -77,8 +81,11 @@ class HonClimate : public HaierClimateBase {
   haier_protocol::HandlerError get_alarm_status_answer_handler_(haier_protocol::FrameType request_type,
                                                                 haier_protocol::FrameType message_type,
                                                                 const uint8_t *data, size_t data_size);
+  haier_protocol::HandlerError alarm_status_message_handler_(haier_protocol::FrameType type, const uint8_t *buffer,
+                                                             size_t size);
   // Helper functions
   haier_protocol::HandlerError process_status_message_(const uint8_t *packet, uint8_t size);
+  void process_alarm_message_(const uint8_t *packet, uint8_t size, bool check_new);
   void fill_control_messages_queue_();
   void clear_control_messages_queue_();
 
@@ -101,6 +108,26 @@ class HonClimate : public HaierClimateBase {
   HonControlMethod control_method_;
   esphome::sensor::Sensor *outdoor_sensor_;
   std::queue<haier_protocol::HaierMessage> control_messages_queue_;
+  CallbackManager<void(uint8_t, const char *)> alarm_start_callback_{};
+  CallbackManager<void(uint8_t, const char *)> alarm_end_callback_{};
+  float active_alarm_count_{NAN};
+  std::chrono::steady_clock::time_point last_alarm_request_;
+};
+
+class HaierAlarmStartTrigger : public Trigger<uint8_t, const char *> {
+ public:
+  explicit HaierAlarmStartTrigger(HonClimate *parent) {
+    parent->add_alarm_start_callback(
+        [this](uint8_t alarm_code, const char *alarm_message) { this->trigger(alarm_code, alarm_message); });
+  }
+};
+
+class HaierAlarmEndTrigger : public Trigger<uint8_t, const char *> {
+ public:
+  explicit HaierAlarmEndTrigger(HonClimate *parent) {
+    parent->add_alarm_end_callback(
+        [this](uint8_t alarm_code, const char *alarm_message) { this->trigger(alarm_code, alarm_message); });
+  }
 };
 
 }  // namespace haier
diff --git a/esphome/components/haier/hon_packet.h b/esphome/components/haier/hon_packet.h
index 7724b43854..be1b0ae51c 100644
--- a/esphome/components/haier/hon_packet.h
+++ b/esphome/components/haier/hon_packet.h
@@ -163,6 +163,62 @@ enum class SubcommandsControl : uint16_t {
                                   // content: all values like in status packet)
 };
 
+const std::string HON_ALARM_MESSAGES[] = {
+    "Outdoor module failure",
+    "Outdoor defrost sensor failure",
+    "Outdoor compressor exhaust sensor failure",
+    "Outdoor EEPROM abnormality",
+    "Indoor coil sensor failure",
+    "Indoor-outdoor communication failure",
+    "Power supply overvoltage protection",
+    "Communication failure between panel and indoor unit",
+    "Outdoor compressor overheat protection",
+    "Outdoor environmental sensor abnormality",
+    "Full water protection",
+    "Indoor EEPROM failure",
+    "Outdoor out air sensor failure",
+    "CBD and module communication failure",
+    "Indoor DC fan failure",
+    "Outdoor DC fan failure",
+    "Door switch failure",
+    "Dust filter needs cleaning reminder",
+    "Water shortage protection",
+    "Humidity sensor failure",
+    "Indoor temperature sensor failure",
+    "Manipulator limit failure",
+    "Indoor PM2.5 sensor failure",
+    "Outdoor PM2.5 sensor failure",
+    "Indoor heating overload/high load alarm",
+    "Outdoor AC current protection",
+    "Outdoor compressor operation abnormality",
+    "Outdoor DC current protection",
+    "Outdoor no-load failure",
+    "CT current abnormality",
+    "Indoor cooling freeze protection",
+    "High and low pressure protection",
+    "Compressor out air temperature is too high",
+    "Outdoor evaporator sensor failure",
+    "Outdoor cooling overload",
+    "Water pump drainage failure",
+    "Three-phase power supply failure",
+    "Four-way valve failure",
+    "External alarm/scraper flow switch failure",
+    "Temperature cutoff protection alarm",
+    "Different mode operation failure",
+    "Electronic expansion valve failure",
+    "Dual heat source sensor Tw failure",
+    "Communication failure with the wired controller",
+    "Indoor unit address duplication failure",
+    "50Hz zero crossing failure",
+    "Outdoor unit failure",
+    "Formaldehyde sensor failure",
+    "VOC sensor failure",
+    "CO2 sensor failure",
+    "Firewall failure",
+};
+
+constexpr size_t HON_ALARM_COUNT = sizeof(HON_ALARM_MESSAGES) / sizeof(HON_ALARM_MESSAGES[0]);
+
 }  // namespace hon_protocol
 }  // namespace haier
 }  // namespace esphome
diff --git a/esphome/components/haier/smartair2_climate.cpp b/esphome/components/haier/smartair2_climate.cpp
index c2326883f7..00590694d5 100644
--- a/esphome/components/haier/smartair2_climate.cpp
+++ b/esphome/components/haier/smartair2_climate.cpp
@@ -95,7 +95,7 @@ haier_protocol::HandlerError Smartair2Climate::messages_timeout_handler_with_cyc
   ESP_LOGI(TAG, "Answer timeout for command %02X, phase %s", (uint8_t) message_type,
            phase_to_string_(this->protocol_phase_));
   ProtocolPhases new_phase = (ProtocolPhases) ((int) this->protocol_phase_ + 1);
-  if (new_phase >= ProtocolPhases::SENDING_ALARM_STATUS_REQUEST)
+  if (new_phase >= ProtocolPhases::SENDING_FIRST_ALARM_STATUS_REQUEST)
     new_phase = ProtocolPhases::SENDING_INIT_1;
   this->set_phase(new_phase);
   return haier_protocol::HandlerError::HANDLER_OK;
@@ -170,9 +170,12 @@ void Smartair2Climate::process_phase(std::chrono::steady_clock::time_point now)
     case ProtocolPhases::SENDING_UPDATE_SIGNAL_REQUEST:
       this->set_phase(ProtocolPhases::SENDING_SIGNAL_LEVEL);
       break;
-    case ProtocolPhases::SENDING_ALARM_STATUS_REQUEST:
+    case ProtocolPhases::SENDING_FIRST_ALARM_STATUS_REQUEST:
       this->set_phase(ProtocolPhases::SENDING_INIT_1);
       break;
+    case ProtocolPhases::SENDING_ALARM_STATUS_REQUEST:
+      this->set_phase(ProtocolPhases::IDLE);
+      break;
     case ProtocolPhases::SENDING_CONTROL:
       if (this->can_send_message() && this->is_control_message_interval_exceeded_(now)) {
         ESP_LOGI(TAG, "Sending control packet");
@@ -343,19 +346,29 @@ haier_protocol::HaierMessage Smartair2Climate::get_control_message() {
     } else if (climate_control.preset.has_value()) {
       switch (climate_control.preset.value()) {
         case CLIMATE_PRESET_NONE:
+          out_data->ten_degree = 0;
           out_data->turbo_mode = 0;
           out_data->quiet_mode = 0;
           break;
         case CLIMATE_PRESET_BOOST:
+          out_data->ten_degree = 0;
           out_data->turbo_mode = 1;
           out_data->quiet_mode = 0;
           break;
         case CLIMATE_PRESET_COMFORT:
+          out_data->ten_degree = 0;
           out_data->turbo_mode = 0;
           out_data->quiet_mode = 1;
           break;
+        case CLIMATE_PRESET_AWAY:
+          // Only allowed in heat mode
+          out_data->ten_degree = (this->mode == CLIMATE_MODE_HEAT) ? 1 : 0;
+          out_data->turbo_mode = 0;
+          out_data->quiet_mode = 0;
+          break;
         default:
           ESP_LOGE("Control", "Unsupported preset");
+          out_data->ten_degree = 0;
           out_data->turbo_mode = 0;
           out_data->quiet_mode = 0;
           break;
@@ -381,6 +394,8 @@ haier_protocol::HandlerError Smartair2Climate::process_status_message_(const uin
       this->preset = CLIMATE_PRESET_BOOST;
     } else if (packet.control.quiet_mode != 0) {
       this->preset = CLIMATE_PRESET_COMFORT;
+    } else if (packet.control.ten_degree != 0) {
+      this->preset = CLIMATE_PRESET_AWAY;
     } else {
       this->preset = CLIMATE_PRESET_NONE;
     }
diff --git a/tests/test3.yaml b/tests/test3.yaml
index ab7f38d07f..2c7a7a81f7 100644
--- a/tests/test3.yaml
+++ b/tests/test3.yaml
@@ -1026,11 +1026,13 @@ climate:
     wifi_signal: true
     beeper: true
     outdoor_temperature:
-      name:  Haier AC outdoor temperature
+      name: Haier AC outdoor temperature
     visual:
       min_temperature: 16 °C
       max_temperature: 30 °C
-      temperature_step: 1 °C
+      temperature_step:
+        target_temperature: 1
+        current_temperature: 0.5
     supported_modes:
     - 'OFF'
     - HEAT_COOL
@@ -1043,6 +1045,23 @@ climate:
     - VERTICAL
     - HORIZONTAL
     - BOTH
+    supported_presets:
+    - AWAY
+    - BOOST
+    - ECO
+    - SLEEP
+    on_alarm_start:
+      then:
+        - logger.log:
+            level: DEBUG
+            format: "Alarm activated. Code: %d. Message: \"%s\""
+            args: [ code, message]
+    on_alarm_end:
+      then:
+        - logger.log:
+            level: DEBUG
+            format: "Alarm deactivated. Code: %d. Message: \"%s\""
+            args: [ code, message]
 
 sprinkler:
   - id: yard_sprinkler_ctrlr

From 937a9c96ce87a5f3cb871d9d147022497696e2e1 Mon Sep 17 00:00:00 2001
From: Chris AtLee <chris@atlee.ca>
Date: Wed, 20 Dec 2023 19:11:32 -0500
Subject: [PATCH 43/75] Allow haier remote protocol to use lambdas (#5898)

---
 esphome/components/remote_base/haier_protocol.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/esphome/components/remote_base/haier_protocol.h b/esphome/components/remote_base/haier_protocol.h
index 6a1c4bea72..7a4ee640e8 100644
--- a/esphome/components/remote_base/haier_protocol.h
+++ b/esphome/components/remote_base/haier_protocol.h
@@ -26,12 +26,11 @@ DECLARE_REMOTE_PROTOCOL(Haier)
 
 template<typename... Ts> class HaierAction : public RemoteTransmitterActionBase<Ts...> {
  public:
-  TEMPLATABLE_VALUE(std::vector<uint8_t>, data)
+  TEMPLATABLE_VALUE(std::vector<uint8_t>, code)
 
-  void set_code(const std::vector<uint8_t> &code) { data_ = code; }
   void encode(RemoteTransmitData *dst, Ts... x) override {
     HaierData data{};
-    data.data = this->data_.value(x...);
+    data.data = this->code_.value(x...);
     HaierProtocol().encode(dst, data);
   }
 };

From 2a1d16f17b90544e70dedffb33ba4f7097115e51 Mon Sep 17 00:00:00 2001
From: William Heimbigner <william.heimbigner@gmail.com>
Date: Wed, 20 Dec 2023 21:55:34 -0600
Subject: [PATCH 44/75] PMSx003 add relevant device and state classes to
 default config (#5633)

---
 esphome/components/pmsx003/sensor.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/esphome/components/pmsx003/sensor.py b/esphome/components/pmsx003/sensor.py
index eefcb529f2..08ccd6096e 100644
--- a/esphome/components/pmsx003/sensor.py
+++ b/esphome/components/pmsx003/sensor.py
@@ -92,66 +92,78 @@ CONFIG_SCHEMA = (
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_PM1,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_2_5_STD): sensor.sensor_schema(
                 unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_PM25,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_10_0_STD): sensor.sensor_schema(
                 unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_PM10,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_1_0): sensor.sensor_schema(
                 unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                device_class=DEVICE_CLASS_PM1,
                 state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_2_5): sensor.sensor_schema(
                 unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                device_class=DEVICE_CLASS_PM25,
                 state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_10_0): sensor.sensor_schema(
                 unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                device_class=DEVICE_CLASS_PM10,
                 state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_0_3UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_0_5UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_1_0UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_2_5UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_5_0UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_PM_10_0UM): sensor.sensor_schema(
                 unit_of_measurement=UNIT_COUNT_DECILITRE,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
+                state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Optional(CONF_TEMPERATURE): sensor.sensor_schema(
                 unit_of_measurement=UNIT_CELSIUS,

From 784dff7574ee39554759eb7681e8486bcc4219e6 Mon Sep 17 00:00:00 2001
From: Fabian Pflug <gumulka@users.noreply.github.com>
Date: Thu, 21 Dec 2023 05:30:10 +0100
Subject: [PATCH 45/75] Add waveshare 2.7in V2 model (#5903)

---
 .../components/waveshare_epaper/display.py    |  4 ++
 .../waveshare_epaper/waveshare_epaper.cpp     | 53 +++++++++++++++++++
 .../waveshare_epaper/waveshare_epaper.h       | 16 ++++++
 3 files changed, 73 insertions(+)

diff --git a/esphome/components/waveshare_epaper/display.py b/esphome/components/waveshare_epaper/display.py
index 519b07fca2..432c7b6119 100644
--- a/esphome/components/waveshare_epaper/display.py
+++ b/esphome/components/waveshare_epaper/display.py
@@ -26,6 +26,9 @@ WaveshareEPaperTypeA = waveshare_epaper_ns.class_(
 WaveshareEPaper2P7In = waveshare_epaper_ns.class_(
     "WaveshareEPaper2P7In", WaveshareEPaper
 )
+WaveshareEPaper2P7InV2 = waveshare_epaper_ns.class_(
+    "WaveshareEPaper2P7InV2", WaveshareEPaper
+)
 WaveshareEPaper2P9InB = waveshare_epaper_ns.class_(
     "WaveshareEPaper2P9InB", WaveshareEPaper
 )
@@ -83,6 +86,7 @@ MODELS = {
     "2.90inv2": ("a", WaveshareEPaperTypeAModel.WAVESHARE_EPAPER_2_9_IN_V2),
     "gdey029t94": ("c", GDEY029T94),
     "2.70in": ("b", WaveshareEPaper2P7In),
+    "2.70inv2": ("b", WaveshareEPaper2P7InV2),
     "2.90in-b": ("b", WaveshareEPaper2P9InB),
     "4.20in": ("b", WaveshareEPaper4P2In),
     "4.20in-bv2": ("b", WaveshareEPaper4P2InBV2),
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.cpp b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
index 8fdb9a3ac0..4cd5e3366c 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.cpp
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
@@ -634,6 +634,59 @@ void WaveshareEPaper2P7In::dump_config() {
   LOG_UPDATE_INTERVAL(this);
 }
 
+void WaveshareEPaper2P7InV2::initialize() {
+  this->reset_();
+  this->wait_until_idle_();
+
+  this->command(0x12);  // SWRESET
+  this->wait_until_idle_();
+
+  // SET WINDOWS
+  // XRAM_START_AND_END_POSITION
+  this->command(0x44);
+  this->data(0x00);
+  this->data(((get_width_internal() - 1) >> 3) & 0xFF);
+  // YRAM_START_AND_END_POSITION
+  this->command(0x45);
+  this->data(0x00);
+  this->data(0x00);
+  this->data((get_height_internal() - 1) & 0xFF);
+  this->data(((get_height_internal() - 1) >> 8) & 0xFF);
+
+  // SET CURSOR
+  // XRAM_ADDRESS
+  this->command(0x4E);
+  this->data(0x00);
+  // YRAM_ADDRESS
+  this->command(0x4F);
+  this->data(0x00);
+  this->data(0x00);
+
+  this->command(0x11);  // data entry mode
+  this->data(0x03);
+}
+void HOT WaveshareEPaper2P7InV2::display() {
+  this->command(0x24);
+  this->start_data_();
+  this->write_array(this->buffer_, this->get_buffer_length_());
+  this->end_data_();
+
+  // COMMAND DISPLAY REFRESH
+  this->command(0x22);
+  this->data(0xF7);
+  this->command(0x20);
+}
+int WaveshareEPaper2P7InV2::get_width_internal() { return 176; }
+int WaveshareEPaper2P7InV2::get_height_internal() { return 264; }
+void WaveshareEPaper2P7InV2::dump_config() {
+  LOG_DISPLAY("", "Waveshare E-Paper", this);
+  ESP_LOGCONFIG(TAG, "  Model: 2.7in V2");
+  LOG_PIN("  Reset Pin: ", this->reset_pin_);
+  LOG_PIN("  DC Pin: ", this->dc_pin_);
+  LOG_PIN("  Busy Pin: ", this->busy_pin_);
+  LOG_UPDATE_INTERVAL(this);
+}
+
 // ========================================================
 //               2.90in Type B (LUT from OTP)
 // Datasheet:
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.h b/esphome/components/waveshare_epaper/waveshare_epaper.h
index 42e8a16829..bc5dbea11d 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.h
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.h
@@ -160,6 +160,22 @@ class WaveshareEPaper2P7In : public WaveshareEPaper {
   int get_height_internal() override;
 };
 
+class WaveshareEPaper2P7InV2 : public WaveshareEPaper {
+ public:
+  void initialize() override;
+
+  void display() override;
+
+  void dump_config() override;
+
+  void deep_sleep() override { ; }
+
+ protected:
+  int get_width_internal() override;
+
+  int get_height_internal() override;
+};
+
 class GDEY029T94 : public WaveshareEPaper {
  public:
   void initialize() override;

From c305f61020ecd390600763d930fcb47328842f01 Mon Sep 17 00:00:00 2001
From: Fabian Pflug <gumulka@users.noreply.github.com>
Date: Thu, 21 Dec 2023 05:36:43 +0100
Subject: [PATCH 46/75] Add support for waveshare 2.9in B V3 version (#5902)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 .../components/waveshare_epaper/display.py    |  4 ++
 .../waveshare_epaper/waveshare_epaper.cpp     | 69 +++++++++++++++++++
 .../waveshare_epaper/waveshare_epaper.h       | 20 ++++++
 3 files changed, 93 insertions(+)

diff --git a/esphome/components/waveshare_epaper/display.py b/esphome/components/waveshare_epaper/display.py
index 432c7b6119..1dd4b7fc54 100644
--- a/esphome/components/waveshare_epaper/display.py
+++ b/esphome/components/waveshare_epaper/display.py
@@ -32,6 +32,9 @@ WaveshareEPaper2P7InV2 = waveshare_epaper_ns.class_(
 WaveshareEPaper2P9InB = waveshare_epaper_ns.class_(
     "WaveshareEPaper2P9InB", WaveshareEPaper
 )
+WaveshareEPaper2P9InBV3 = waveshare_epaper_ns.class_(
+    "WaveshareEPaper2P9InBV3", WaveshareEPaper
+)
 GDEY029T94 = waveshare_epaper_ns.class_("GDEY029T94", WaveshareEPaper)
 WaveshareEPaper4P2In = waveshare_epaper_ns.class_(
     "WaveshareEPaper4P2In", WaveshareEPaper
@@ -88,6 +91,7 @@ MODELS = {
     "2.70in": ("b", WaveshareEPaper2P7In),
     "2.70inv2": ("b", WaveshareEPaper2P7InV2),
     "2.90in-b": ("b", WaveshareEPaper2P9InB),
+    "2.90in-bv3": ("b", WaveshareEPaper2P9InBV3),
     "4.20in": ("b", WaveshareEPaper4P2In),
     "4.20in-bv2": ("b", WaveshareEPaper4P2InBV2),
     "5.83in": ("b", WaveshareEPaper5P8In),
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.cpp b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
index 4cd5e3366c..0e9b129988 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.cpp
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
@@ -766,6 +766,75 @@ void WaveshareEPaper2P9InB::dump_config() {
   LOG_UPDATE_INTERVAL(this);
 }
 
+// ========================================================
+//               2.90in Type B (LUT from OTP)
+// Datasheet:
+//  - https://files.waveshare.com/upload/a/af/2.9inch-e-paper-b-v3-specification.pdf
+// ========================================================
+
+void WaveshareEPaper2P9InBV3::initialize() {
+  // from https://github.com/waveshareteam/e-Paper/blob/master/Arduino/epd2in9b_V3/epd2in9b_V3.cpp
+  this->reset_();
+
+  // COMMAND POWER ON
+  this->command(0x04);
+  this->wait_until_idle_();
+
+  // COMMAND PANEL SETTING
+  this->command(0x00);
+  this->data(0x0F);
+  this->data(0x89);
+
+  // COMMAND RESOLUTION SETTING
+  this->command(0x61);
+  this->data(0x80);
+  this->data(0x01);
+  this->data(0x28);
+
+  // COMMAND VCOM AND DATA INTERVAL SETTING
+  this->command(0x50);
+  this->data(0x77);
+}
+void HOT WaveshareEPaper2P9InBV3::display() {
+  // COMMAND DATA START TRANSMISSION 1 (B/W data)
+  this->command(0x10);
+  delay(2);
+  this->start_data_();
+  this->write_array(this->buffer_, this->get_buffer_length_());
+  this->end_data_();
+  this->command(0x92);
+  delay(2);
+
+  // COMMAND DATA START TRANSMISSION 2 (RED data)
+  this->command(0x13);
+  delay(2);
+  this->start_data_();
+  for (size_t i = 0; i < this->get_buffer_length_(); i++)
+    this->write_byte(0xFF);
+  this->end_data_();
+  this->command(0x92);
+  delay(2);
+
+  // COMMAND DISPLAY REFRESH
+  this->command(0x12);
+  delay(2);
+  this->wait_until_idle_();
+
+  // COMMAND POWER OFF
+  // NOTE: power off < deep sleep
+  this->command(0x02);
+}
+int WaveshareEPaper2P9InBV3::get_width_internal() { return 128; }
+int WaveshareEPaper2P9InBV3::get_height_internal() { return 296; }
+void WaveshareEPaper2P9InBV3::dump_config() {
+  LOG_DISPLAY("", "Waveshare E-Paper", this);
+  ESP_LOGCONFIG(TAG, "  Model: 2.9in (B) V3");
+  LOG_PIN("  Reset Pin: ", this->reset_pin_);
+  LOG_PIN("  DC Pin: ", this->dc_pin_);
+  LOG_PIN("  Busy Pin: ", this->busy_pin_);
+  LOG_UPDATE_INTERVAL(this);
+}
+
 // ========================================================
 //     Good Display 2.9in black/white/grey
 // Datasheet:
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.h b/esphome/components/waveshare_epaper/waveshare_epaper.h
index bc5dbea11d..ee9443e8be 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.h
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.h
@@ -256,6 +256,26 @@ class WaveshareEPaper2P9InB : public WaveshareEPaper {
   int get_height_internal() override;
 };
 
+class WaveshareEPaper2P9InBV3 : public WaveshareEPaper {
+ public:
+  void initialize() override;
+
+  void display() override;
+
+  void dump_config() override;
+
+  void deep_sleep() override {
+    // COMMAND DEEP SLEEP
+    this->command(0x07);
+    this->data(0xA5);  // check byte
+  }
+
+ protected:
+  int get_width_internal() override;
+
+  int get_height_internal() override;
+};
+
 class WaveshareEPaper4P2In : public WaveshareEPaper {
  public:
   void initialize() override;

From c92715e403b766061669bd19658f0e1dafb37640 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 21 Dec 2023 13:37:02 +0900
Subject: [PATCH 47/75] Fix pin reuse in test1 (#5978)

---
 tests/test1.yaml | 64 +++++++++++++++++++++++-------------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/tests/test1.yaml b/tests/test1.yaml
index 2748a09c59..eb72f90b21 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -324,7 +324,9 @@ ads1115:
 
 as5600:
   i2c_id: i2c_bus
-  dir_pin: GPIO27
+  dir_pin:
+    number: 27
+    allow_other_uses: true
   direction: clockwise
   start_position: 90deg
   range: 180deg
@@ -570,13 +572,13 @@ sensor:
   - platform: as5600
     name: AS5600 Position
     raw_position:
-        name: AS5600 Raw Position
+      name: AS5600 Raw Position
     gain:
-        name: AS5600 Gain
+      name: AS5600 Gain
     magnitude:
-        name: AS5600 Magnitude
+      name: AS5600 Magnitude
     status:
-        name: AS5600 Status
+      name: AS5600 Status
   - platform: as7341
     update_interval: 15s
     gain: X8
@@ -2037,21 +2039,21 @@ my9231:
 
 sm2235:
   data_pin:
-      allow_other_uses: true
-      number: GPIO4
+    allow_other_uses: true
+    number: GPIO4
   clock_pin:
-      allow_other_uses: true
-      number: GPIO5
+    allow_other_uses: true
+    number: GPIO5
   max_power_color_channels: 9
   max_power_white_channels: 9
 
 sm2335:
   data_pin:
-      allow_other_uses: true
-      number: GPIO4
+    allow_other_uses: true
+    number: GPIO4
   clock_pin:
-      allow_other_uses: true
-      number: GPIO5
+    allow_other_uses: true
+    number: GPIO5
   max_power_color_channels: 9
   max_power_white_channels: 9
 
@@ -3040,17 +3042,13 @@ display:
     id: my_lcd_gpio
     dimensions: 18x4
     data_pins:
-      -
-        allow_other_uses: true
+      - allow_other_uses: true
         number: GPIO19
-      -
-        allow_other_uses: true
+      - allow_other_uses: true
         number: GPIO21
-      -
-        allow_other_uses: true
+      - allow_other_uses: true
         number: GPIO22
-      -
-        allow_other_uses: true
+      - allow_other_uses: true
         number: GPIO23
     enable_pin:
       allow_other_uses: true
@@ -4201,25 +4199,25 @@ graphical_display_menu:
       lambda: 'ESP_LOGI("graphical_display_menu", "root leave");'
   items:
     - type: back
-      text: 'Back'
+      text: "Back"
     - type: label
     - type: menu
-      text: 'Submenu 1'
+      text: "Submenu 1"
       items:
         - type: back
-          text: 'Back'
+          text: "Back"
         - type: menu
-          text: 'Submenu 21'
+          text: "Submenu 21"
           items:
             - type: back
-              text: 'Back'
+              text: "Back"
             - type: command
-              text: 'Show Main'
+              text: "Show Main"
               on_value:
                 then:
                   - display_menu.show_main: test_graphical_display_menu
     - type: select
-      text: 'Enum Item'
+      text: "Enum Item"
       immediate_edit: true
       select: test_select
       on_enter:
@@ -4232,7 +4230,7 @@ graphical_display_menu:
         then:
           lambda: 'ESP_LOGI("graphical_display_menu", "select value: %s, %s", it->get_text().c_str(), it->get_value_text().c_str());'
     - type: number
-      text: 'Number'
+      text: "Number"
       number: test_number
       on_enter:
         then:
@@ -4244,15 +4242,15 @@ graphical_display_menu:
         then:
           lambda: 'ESP_LOGI("graphical_display_menu", "number value: %s, %s", it->get_text().c_str(), it->get_value_text().c_str());'
     - type: command
-      text: 'Hide'
+      text: "Hide"
       on_value:
         then:
           - display_menu.hide: test_graphical_display_menu
     - type: switch
-      text: 'Switch'
+      text: "Switch"
       switch: my_switch
-      on_text: 'Bright'
-      off_text: 'Dark'
+      on_text: "Bright"
+      off_text: "Dark"
       immediate_edit: false
       on_value:
         then:

From a784f1e69127daafc6457f0be897bf51bb1cb5e6 Mon Sep 17 00:00:00 2001
From: mrtoy-me <118446898+mrtoy-me@users.noreply.github.com>
Date: Thu, 21 Dec 2023 14:38:11 +1000
Subject: [PATCH 48/75] Add Waveshare 1.47in 172x320 to ST7789v component
 (#5884)

---
 esphome/components/st7789v/display.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/esphome/components/st7789v/display.py b/esphome/components/st7789v/display.py
index 41970afd26..5b2b5a126c 100644
--- a/esphome/components/st7789v/display.py
+++ b/esphome/components/st7789v/display.py
@@ -97,6 +97,19 @@ MODELS = {
             CONF_BACKLIGHT_PIN: "GPIO15",
         }
     ),
+    "WAVESHARE_1.47IN_172X320": model_spec(
+        presets={
+            CONF_HEIGHT: 320,
+            CONF_WIDTH: 172,
+            CONF_OFFSET_HEIGHT: 34,
+            CONF_OFFSET_WIDTH: 0,
+            CONF_ROTATION: 90,
+            CONF_CS_PIN: "GPIO21",
+            CONF_DC_PIN: "GPIO22",
+            CONF_RESET_PIN: "GPIO23",
+            CONF_BACKLIGHT_PIN: "GPIO4",
+        }
+    ),
     "CUSTOM": model_spec(),
 }
 

From 04b354799251808f85a3fe12c2a2579977de5770 Mon Sep 17 00:00:00 2001
From: Ruben van Dijk <15885455+RubenNL@users.noreply.github.com>
Date: Thu, 21 Dec 2023 05:39:55 +0100
Subject: [PATCH 49/75] (fingerprint_grow) Added on_finger_scan_invalid
 automation. (#5885)

---
 esphome/components/fingerprint_grow/__init__.py  | 16 ++++++++++++++++
 .../fingerprint_grow/fingerprint_grow.cpp        |  4 ++++
 .../fingerprint_grow/fingerprint_grow.h          | 11 +++++++++++
 esphome/const.py                                 |  1 +
 tests/test3.yaml                                 |  3 +++
 5 files changed, 35 insertions(+)

diff --git a/esphome/components/fingerprint_grow/__init__.py b/esphome/components/fingerprint_grow/__init__.py
index ecbbc3d477..5249107f17 100644
--- a/esphome/components/fingerprint_grow/__init__.py
+++ b/esphome/components/fingerprint_grow/__init__.py
@@ -15,6 +15,7 @@ from esphome.const import (
     CONF_ON_ENROLLMENT_SCAN,
     CONF_ON_FINGER_SCAN_MATCHED,
     CONF_ON_FINGER_SCAN_UNMATCHED,
+    CONF_ON_FINGER_SCAN_INVALID,
     CONF_PASSWORD,
     CONF_SENSING_PIN,
     CONF_SPEED,
@@ -42,6 +43,10 @@ FingerScanUnmatchedTrigger = fingerprint_grow_ns.class_(
     "FingerScanUnmatchedTrigger", automation.Trigger.template()
 )
 
+FingerScanInvalidTrigger = fingerprint_grow_ns.class_(
+    "FingerScanInvalidTrigger", automation.Trigger.template()
+)
+
 EnrollmentScanTrigger = fingerprint_grow_ns.class_(
     "EnrollmentScanTrigger", automation.Trigger.template(cg.uint8, cg.uint16)
 )
@@ -108,6 +113,13 @@ CONFIG_SCHEMA = (
                     ),
                 }
             ),
+            cv.Optional(CONF_ON_FINGER_SCAN_INVALID): automation.validate_automation(
+                {
+                    cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(
+                        FingerScanInvalidTrigger
+                    ),
+                }
+            ),
             cv.Optional(CONF_ON_ENROLLMENT_SCAN): automation.validate_automation(
                 {
                     cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(
@@ -162,6 +174,10 @@ async def to_code(config):
         trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
         await automation.build_automation(trigger, [], conf)
 
+    for conf in config.get(CONF_ON_FINGER_SCAN_INVALID, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(trigger, [], conf)
+
     for conf in config.get(CONF_ON_ENROLLMENT_SCAN, []):
         trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
         await automation.build_automation(
diff --git a/esphome/components/fingerprint_grow/fingerprint_grow.cpp b/esphome/components/fingerprint_grow/fingerprint_grow.cpp
index 8729e0f4bf..2486e02964 100644
--- a/esphome/components/fingerprint_grow/fingerprint_grow.cpp
+++ b/esphome/components/fingerprint_grow/fingerprint_grow.cpp
@@ -134,12 +134,14 @@ uint8_t FingerprintGrowComponent::scan_image_(uint8_t buffer) {
     case NO_FINGER:
       if (this->sensing_pin_ != nullptr) {
         ESP_LOGD(TAG, "No finger");
+        this->finger_scan_invalid_callback_.call();
       } else {
         ESP_LOGV(TAG, "No finger");
       }
       return this->data_[0];
     case IMAGE_FAIL:
       ESP_LOGE(TAG, "Imaging error");
+      this->finger_scan_invalid_callback_.call();
     default:
       return this->data_[0];
   }
@@ -152,10 +154,12 @@ uint8_t FingerprintGrowComponent::scan_image_(uint8_t buffer) {
       break;
     case IMAGE_MESS:
       ESP_LOGE(TAG, "Image too messy");
+      this->finger_scan_invalid_callback_.call();
       break;
     case FEATURE_FAIL:
     case INVALID_IMAGE:
       ESP_LOGE(TAG, "Could not find fingerprint features");
+      this->finger_scan_invalid_callback_.call();
       break;
   }
   return this->data_[0];
diff --git a/esphome/components/fingerprint_grow/fingerprint_grow.h b/esphome/components/fingerprint_grow/fingerprint_grow.h
index f414146e64..9aad94fc2a 100644
--- a/esphome/components/fingerprint_grow/fingerprint_grow.h
+++ b/esphome/components/fingerprint_grow/fingerprint_grow.h
@@ -124,6 +124,9 @@ class FingerprintGrowComponent : public PollingComponent, public uart::UARTDevic
   void add_on_finger_scan_unmatched_callback(std::function<void()> callback) {
     this->finger_scan_unmatched_callback_.add(std::move(callback));
   }
+  void add_on_finger_scan_invalid_callback(std::function<void()> callback) {
+    this->finger_scan_invalid_callback_.add(std::move(callback));
+  }
   void add_on_enrollment_scan_callback(std::function<void(uint8_t, uint16_t)> callback) {
     this->enrollment_scan_callback_.add(std::move(callback));
   }
@@ -172,6 +175,7 @@ class FingerprintGrowComponent : public PollingComponent, public uart::UARTDevic
   sensor::Sensor *last_finger_id_sensor_{nullptr};
   sensor::Sensor *last_confidence_sensor_{nullptr};
   binary_sensor::BinarySensor *enrolling_binary_sensor_{nullptr};
+  CallbackManager<void()> finger_scan_invalid_callback_;
   CallbackManager<void(uint16_t, uint16_t)> finger_scan_matched_callback_;
   CallbackManager<void()> finger_scan_unmatched_callback_;
   CallbackManager<void(uint8_t, uint16_t)> enrollment_scan_callback_;
@@ -194,6 +198,13 @@ class FingerScanUnmatchedTrigger : public Trigger<> {
   }
 };
 
+class FingerScanInvalidTrigger : public Trigger<> {
+ public:
+  explicit FingerScanInvalidTrigger(FingerprintGrowComponent *parent) {
+    parent->add_on_finger_scan_invalid_callback([this]() { this->trigger(); });
+  }
+};
+
 class EnrollmentScanTrigger : public Trigger<uint8_t, uint16_t> {
  public:
   explicit EnrollmentScanTrigger(FingerprintGrowComponent *parent) {
diff --git a/esphome/const.py b/esphome/const.py
index 58c6c47e3a..4a4a59f659 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -500,6 +500,7 @@ CONF_ON_DOUBLE_CLICK = "on_double_click"
 CONF_ON_ENROLLMENT_DONE = "on_enrollment_done"
 CONF_ON_ENROLLMENT_FAILED = "on_enrollment_failed"
 CONF_ON_ENROLLMENT_SCAN = "on_enrollment_scan"
+CONF_ON_FINGER_SCAN_INVALID = "on_finger_scan_invalid"
 CONF_ON_FINGER_SCAN_MATCHED = "on_finger_scan_matched"
 CONF_ON_FINGER_SCAN_UNMATCHED = "on_finger_scan_unmatched"
 CONF_ON_JSON_MESSAGE = "on_json_message"
diff --git a/tests/test3.yaml b/tests/test3.yaml
index 2c7a7a81f7..e39e711ab3 100644
--- a/tests/test3.yaml
+++ b/tests/test3.yaml
@@ -1257,6 +1257,9 @@ fingerprint_grow:
     number: 4
   password: 0x12FE37DC
   new_password: 0xA65B9840
+  on_finger_scan_invalid:
+    - homeassistant.event:
+        event: esphome.${device_name}_fingerprint_grow_finger_scan_invalid
   on_finger_scan_matched:
     - homeassistant.event:
         event: esphome.${device_name}_fingerprint_grow_finger_scan_matched

From 223e6e8f137ae442cce15125e80889ec76c85b65 Mon Sep 17 00:00:00 2001
From: Steve Rodgers <hwstar@users.noreply.github.com>
Date: Wed, 20 Dec 2023 21:10:47 -0800
Subject: [PATCH 50/75] Alarm panel: Add changes to support enhanced features
 (#5671)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 CODEOWNERS                                    |   4 +-
 .../alarm_control_panel/__init__.py           |  53 +++++++-
 .../alarm_control_panel.cpp                   |   8 ++
 .../alarm_control_panel/alarm_control_panel.h |  16 +++
 .../alarm_control_panel/automation.h          |  14 +++
 .../template/alarm_control_panel/__init__.py  |  25 +++-
 .../template_alarm_control_panel.cpp          | 116 ++++++++++++++----
 .../template_alarm_control_panel.h            |  32 ++++-
 8 files changed, 237 insertions(+), 31 deletions(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index fc14b2088e..b8c870f20f 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -25,7 +25,7 @@ esphome/components/airthings_ble/* @jeromelaban
 esphome/components/airthings_wave_base/* @jeromelaban @kpfleming @ncareau
 esphome/components/airthings_wave_mini/* @ncareau
 esphome/components/airthings_wave_plus/* @jeromelaban
-esphome/components/alarm_control_panel/* @grahambrown11
+esphome/components/alarm_control_panel/* @grahambrown11 @hwstar
 esphome/components/alpha3/* @jan-hofmeier
 esphome/components/am43/* @buxtronix
 esphome/components/am43/cover/* @buxtronix
@@ -327,7 +327,7 @@ esphome/components/tca9548a/* @andreashergert1984
 esphome/components/tcl112/* @glmnet
 esphome/components/tee501/* @Stock-M
 esphome/components/teleinfo/* @0hax
-esphome/components/template/alarm_control_panel/* @grahambrown11
+esphome/components/template/alarm_control_panel/* @grahambrown11 @hwstar
 esphome/components/text/* @mauritskorse
 esphome/components/thermostat/* @kbx81
 esphome/components/time/* @OttoWinter
diff --git a/esphome/components/alarm_control_panel/__init__.py b/esphome/components/alarm_control_panel/__init__.py
index d9cafb4f30..35d239c267 100644
--- a/esphome/components/alarm_control_panel/__init__.py
+++ b/esphome/components/alarm_control_panel/__init__.py
@@ -11,7 +11,7 @@ from esphome.const import (
 )
 from esphome.cpp_helpers import setup_entity
 
-CODEOWNERS = ["@grahambrown11"]
+CODEOWNERS = ["@grahambrown11", "@hwstar"]
 IS_PLATFORM_COMPONENT = True
 
 CONF_ON_TRIGGERED = "on_triggered"
@@ -22,6 +22,8 @@ CONF_ON_ARMED_HOME = "on_armed_home"
 CONF_ON_ARMED_NIGHT = "on_armed_night"
 CONF_ON_ARMED_AWAY = "on_armed_away"
 CONF_ON_DISARMED = "on_disarmed"
+CONF_ON_CHIME = "on_chime"
+CONF_ON_READY = "on_ready"
 
 alarm_control_panel_ns = cg.esphome_ns.namespace("alarm_control_panel")
 AlarmControlPanel = alarm_control_panel_ns.class_("AlarmControlPanel", cg.EntityBase)
@@ -53,12 +55,22 @@ ArmedAwayTrigger = alarm_control_panel_ns.class_(
 DisarmedTrigger = alarm_control_panel_ns.class_(
     "DisarmedTrigger", automation.Trigger.template()
 )
+ChimeTrigger = alarm_control_panel_ns.class_(
+    "ChimeTrigger", automation.Trigger.template()
+)
+ReadyTrigger = alarm_control_panel_ns.class_(
+    "ReadyTrigger", automation.Trigger.template()
+)
+
 ArmAwayAction = alarm_control_panel_ns.class_("ArmAwayAction", automation.Action)
 ArmHomeAction = alarm_control_panel_ns.class_("ArmHomeAction", automation.Action)
 ArmNightAction = alarm_control_panel_ns.class_("ArmNightAction", automation.Action)
 DisarmAction = alarm_control_panel_ns.class_("DisarmAction", automation.Action)
 PendingAction = alarm_control_panel_ns.class_("PendingAction", automation.Action)
 TriggeredAction = alarm_control_panel_ns.class_("TriggeredAction", automation.Action)
+ChimeAction = alarm_control_panel_ns.class_("ChimeAction", automation.Action)
+ReadyAction = alarm_control_panel_ns.class_("ReadyAction", automation.Action)
+
 AlarmControlPanelCondition = alarm_control_panel_ns.class_(
     "AlarmControlPanelCondition", automation.Condition
 )
@@ -111,6 +123,16 @@ ALARM_CONTROL_PANEL_SCHEMA = cv.ENTITY_BASE_SCHEMA.extend(
                 cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(ClearedTrigger),
             }
         ),
+        cv.Optional(CONF_ON_CHIME): automation.validate_automation(
+            {
+                cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(ChimeTrigger),
+            }
+        ),
+        cv.Optional(CONF_ON_READY): automation.validate_automation(
+            {
+                cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(ReadyTrigger),
+            }
+        ),
     }
 )
 
@@ -157,6 +179,12 @@ async def setup_alarm_control_panel_core_(var, config):
     for conf in config.get(CONF_ON_CLEARED, []):
         trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
         await automation.build_automation(trigger, [], conf)
+    for conf in config.get(CONF_ON_CHIME, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(trigger, [], conf)
+    for conf in config.get(CONF_ON_READY, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(trigger, [], conf)
 
 
 async def register_alarm_control_panel(var, config):
@@ -232,6 +260,29 @@ async def alarm_action_trigger_to_code(config, action_id, template_arg, args):
     return var
 
 
+@automation.register_action(
+    "alarm_control_panel.chime", ChimeAction, ALARM_CONTROL_PANEL_ACTION_SCHEMA
+)
+async def alarm_action_chime_to_code(config, action_id, template_arg, args):
+    paren = await cg.get_variable(config[CONF_ID])
+    var = cg.new_Pvariable(action_id, template_arg, paren)
+    return var
+
+
+@automation.register_action(
+    "alarm_control_panel.ready", ReadyAction, ALARM_CONTROL_PANEL_ACTION_SCHEMA
+)
+@automation.register_condition(
+    "alarm_control_panel.ready",
+    AlarmControlPanelCondition,
+    ALARM_CONTROL_PANEL_CONDITION_SCHEMA,
+)
+async def alarm_action_ready_to_code(config, action_id, template_arg, args):
+    paren = await cg.get_variable(config[CONF_ID])
+    var = cg.new_Pvariable(action_id, template_arg, paren)
+    return var
+
+
 @automation.register_condition(
     "alarm_control_panel.is_armed",
     AlarmControlPanelCondition,
diff --git a/esphome/components/alarm_control_panel/alarm_control_panel.cpp b/esphome/components/alarm_control_panel/alarm_control_panel.cpp
index 9dc083c004..9f1485ee90 100644
--- a/esphome/components/alarm_control_panel/alarm_control_panel.cpp
+++ b/esphome/components/alarm_control_panel/alarm_control_panel.cpp
@@ -96,6 +96,14 @@ void AlarmControlPanel::add_on_cleared_callback(std::function<void()> &&callback
   this->cleared_callback_.add(std::move(callback));
 }
 
+void AlarmControlPanel::add_on_chime_callback(std::function<void()> &&callback) {
+  this->chime_callback_.add(std::move(callback));
+}
+
+void AlarmControlPanel::add_on_ready_callback(std::function<void()> &&callback) {
+  this->ready_callback_.add(std::move(callback));
+}
+
 void AlarmControlPanel::arm_away(optional<std::string> code) {
   auto call = this->make_call();
   call.arm_away();
diff --git a/esphome/components/alarm_control_panel/alarm_control_panel.h b/esphome/components/alarm_control_panel/alarm_control_panel.h
index dc0b92df76..85c2b2148e 100644
--- a/esphome/components/alarm_control_panel/alarm_control_panel.h
+++ b/esphome/components/alarm_control_panel/alarm_control_panel.h
@@ -89,6 +89,18 @@ class AlarmControlPanel : public EntityBase {
    */
   void add_on_cleared_callback(std::function<void()> &&callback);
 
+  /** Add a callback for when a chime zone goes from closed to open
+   *
+   * @param callback The callback function
+   */
+  void add_on_chime_callback(std::function<void()> &&callback);
+
+  /** Add a callback for when a ready state changes
+   *
+   * @param callback The callback function
+   */
+  void add_on_ready_callback(std::function<void()> &&callback);
+
   /** A numeric representation of the supported features as per HomeAssistant
    *
    */
@@ -178,6 +190,10 @@ class AlarmControlPanel : public EntityBase {
   CallbackManager<void()> disarmed_callback_{};
   // clear callback
   CallbackManager<void()> cleared_callback_{};
+  // chime callback
+  CallbackManager<void()> chime_callback_{};
+  // ready callback
+  CallbackManager<void()> ready_callback_{};
 };
 
 }  // namespace alarm_control_panel
diff --git a/esphome/components/alarm_control_panel/automation.h b/esphome/components/alarm_control_panel/automation.h
index 8538020c53..2177fb710f 100644
--- a/esphome/components/alarm_control_panel/automation.h
+++ b/esphome/components/alarm_control_panel/automation.h
@@ -69,6 +69,20 @@ class ClearedTrigger : public Trigger<> {
   }
 };
 
+class ChimeTrigger : public Trigger<> {
+ public:
+  explicit ChimeTrigger(AlarmControlPanel *alarm_control_panel) {
+    alarm_control_panel->add_on_chime_callback([this]() { this->trigger(); });
+  }
+};
+
+class ReadyTrigger : public Trigger<> {
+ public:
+  explicit ReadyTrigger(AlarmControlPanel *alarm_control_panel) {
+    alarm_control_panel->add_on_ready_callback([this]() { this->trigger(); });
+  }
+};
+
 template<typename... Ts> class ArmAwayAction : public Action<Ts...> {
  public:
   explicit ArmAwayAction(AlarmControlPanel *alarm_control_panel) : alarm_control_panel_(alarm_control_panel) {}
diff --git a/esphome/components/template/alarm_control_panel/__init__.py b/esphome/components/template/alarm_control_panel/__init__.py
index 27b7e92b4f..3555f2fafd 100644
--- a/esphome/components/template/alarm_control_panel/__init__.py
+++ b/esphome/components/template/alarm_control_panel/__init__.py
@@ -12,11 +12,13 @@ from esphome.const import (
 )
 from .. import template_ns
 
-CODEOWNERS = ["@grahambrown11"]
+CODEOWNERS = ["@grahambrown11", "@hwstar"]
 
 CONF_CODES = "codes"
 CONF_BYPASS_ARMED_HOME = "bypass_armed_home"
 CONF_BYPASS_ARMED_NIGHT = "bypass_armed_night"
+CONF_CHIME = "chime"
+CONF_TRIGGER_MODE = "trigger_mode"
 CONF_REQUIRES_CODE_TO_ARM = "requires_code_to_arm"
 CONF_ARMING_HOME_TIME = "arming_home_time"
 CONF_ARMING_NIGHT_TIME = "arming_night_time"
@@ -24,16 +26,20 @@ CONF_ARMING_AWAY_TIME = "arming_away_time"
 CONF_PENDING_TIME = "pending_time"
 CONF_TRIGGER_TIME = "trigger_time"
 
+
 FLAG_NORMAL = "normal"
 FLAG_BYPASS_ARMED_HOME = "bypass_armed_home"
 FLAG_BYPASS_ARMED_NIGHT = "bypass_armed_night"
+FLAG_CHIME = "chime"
 
 BinarySensorFlags = {
     FLAG_NORMAL: 1 << 0,
     FLAG_BYPASS_ARMED_HOME: 1 << 1,
     FLAG_BYPASS_ARMED_NIGHT: 1 << 2,
+    FLAG_CHIME: 1 << 3,
 }
 
+
 TemplateAlarmControlPanel = template_ns.class_(
     "TemplateAlarmControlPanel", alarm_control_panel.AlarmControlPanel, cg.Component
 )
@@ -46,6 +52,14 @@ RESTORE_MODES = {
     "RESTORE_DEFAULT_DISARMED": TemplateAlarmControlPanelRestoreMode.ALARM_CONTROL_PANEL_RESTORE_DEFAULT_DISARMED,
 }
 
+AlarmSensorType = template_ns.enum("AlarmSensorType")
+
+ALARM_SENSOR_TYPES = {
+    "DELAYED": AlarmSensorType.ALARM_SENSOR_TYPE_DELAYED,
+    "INSTANT": AlarmSensorType.ALARM_SENSOR_TYPE_INSTANT,
+    "DELAYED_FOLLOWER": AlarmSensorType.ALARM_SENSOR_TYPE_DELAYED_FOLLOWER,
+}
+
 
 def validate_config(config):
     if config.get(CONF_REQUIRES_CODE_TO_ARM, False) and not config.get(CONF_CODES, []):
@@ -60,6 +74,10 @@ TEMPLATE_ALARM_CONTROL_PANEL_BINARY_SENSOR_SCHEMA = cv.maybe_simple_value(
         cv.Required(CONF_INPUT): cv.use_id(binary_sensor.BinarySensor),
         cv.Optional(CONF_BYPASS_ARMED_HOME, default=False): cv.boolean,
         cv.Optional(CONF_BYPASS_ARMED_NIGHT, default=False): cv.boolean,
+        cv.Optional(CONF_CHIME, default=False): cv.boolean,
+        cv.Optional(CONF_TRIGGER_MODE, default="DELAYED"): cv.enum(
+            ALARM_SENSOR_TYPES, upper=True, space="_"
+        ),
     },
     key=CONF_INPUT,
 )
@@ -123,6 +141,7 @@ async def to_code(config):
 
     for sensor in config.get(CONF_BINARY_SENSORS, []):
         bs = await cg.get_variable(sensor[CONF_INPUT])
+
         flags = BinarySensorFlags[FLAG_NORMAL]
         if sensor[CONF_BYPASS_ARMED_HOME]:
             flags |= BinarySensorFlags[FLAG_BYPASS_ARMED_HOME]
@@ -130,7 +149,9 @@ async def to_code(config):
         if sensor[CONF_BYPASS_ARMED_NIGHT]:
             flags |= BinarySensorFlags[FLAG_BYPASS_ARMED_NIGHT]
             supports_arm_night = True
-        cg.add(var.add_sensor(bs, flags))
+        if sensor[CONF_CHIME]:
+            flags |= BinarySensorFlags[FLAG_CHIME]
+        cg.add(var.add_sensor(bs, flags, sensor[CONF_TRIGGER_MODE]))
 
     cg.add(var.set_supports_arm_home(supports_arm_home))
     cg.add(var.set_supports_arm_night(supports_arm_night))
diff --git a/esphome/components/template/alarm_control_panel/template_alarm_control_panel.cpp b/esphome/components/template/alarm_control_panel/template_alarm_control_panel.cpp
index b39b587811..99843417fa 100644
--- a/esphome/components/template/alarm_control_panel/template_alarm_control_panel.cpp
+++ b/esphome/components/template/alarm_control_panel/template_alarm_control_panel.cpp
@@ -1,3 +1,4 @@
+
 #include "template_alarm_control_panel.h"
 #include <utility>
 #include "esphome/components/alarm_control_panel/alarm_control_panel.h"
@@ -15,8 +16,14 @@ static const char *const TAG = "template.alarm_control_panel";
 TemplateAlarmControlPanel::TemplateAlarmControlPanel(){};
 
 #ifdef USE_BINARY_SENSOR
-void TemplateAlarmControlPanel::add_sensor(binary_sensor::BinarySensor *sensor, uint16_t flags) {
-  this->sensor_map_[sensor] = flags;
+void TemplateAlarmControlPanel::add_sensor(binary_sensor::BinarySensor *sensor, uint16_t flags, AlarmSensorType type) {
+  // Save the flags and type. Assign a store index for the per sensor data type.
+  SensorDataStore sd;
+  sd.last_chime_state = false;
+  this->sensor_map_[sensor].flags = flags;
+  this->sensor_map_[sensor].type = type;
+  this->sensor_data_.push_back(sd);
+  this->sensor_map_[sensor].store_index = this->next_store_index_++;
 };
 #endif
 
@@ -35,13 +42,27 @@ void TemplateAlarmControlPanel::dump_config() {
   ESP_LOGCONFIG(TAG, "  Trigger Time: %" PRIu32 "s", (this->trigger_time_ / 1000));
   ESP_LOGCONFIG(TAG, "  Supported Features: %" PRIu32, this->get_supported_features());
 #ifdef USE_BINARY_SENSOR
-  for (auto sensor_pair : this->sensor_map_) {
-    ESP_LOGCONFIG(TAG, "  Binary Sesnsor:");
-    ESP_LOGCONFIG(TAG, "    Name: %s", sensor_pair.first->get_name().c_str());
+  for (auto sensor_info : this->sensor_map_) {
+    ESP_LOGCONFIG(TAG, "  Binary Sensor:");
+    ESP_LOGCONFIG(TAG, "    Name: %s", sensor_info.first->get_name().c_str());
     ESP_LOGCONFIG(TAG, "    Armed home bypass: %s",
-                  TRUEFALSE(sensor_pair.second & BINARY_SENSOR_MODE_BYPASS_ARMED_HOME));
+                  TRUEFALSE(sensor_info.second.flags & BINARY_SENSOR_MODE_BYPASS_ARMED_HOME));
     ESP_LOGCONFIG(TAG, "    Armed night bypass: %s",
-                  TRUEFALSE(sensor_pair.second & BINARY_SENSOR_MODE_BYPASS_ARMED_NIGHT));
+                  TRUEFALSE(sensor_info.second.flags & BINARY_SENSOR_MODE_BYPASS_ARMED_NIGHT));
+    ESP_LOGCONFIG(TAG, "    Chime mode: %s", TRUEFALSE(sensor_info.second.flags & BINARY_SENSOR_MODE_CHIME));
+    const char *sensor_type;
+    switch (sensor_info.second.type) {
+      case ALARM_SENSOR_TYPE_INSTANT:
+        sensor_type = "instant";
+        break;
+      case ALARM_SENSOR_TYPE_DELAYED_FOLLOWER:
+        sensor_type = "delayed_follower";
+        break;
+      case ALARM_SENSOR_TYPE_DELAYED:
+      default:
+        sensor_type = "delayed";
+    }
+    ESP_LOGCONFIG(TAG, "    Sensor type: %s", sensor_type);
   }
 #endif
 }
@@ -92,31 +113,80 @@ void TemplateAlarmControlPanel::loop() {
       (millis() - this->last_update_) > this->trigger_time_) {
     future_state = this->desired_state_;
   }
-  bool trigger = false;
+
+  bool delayed_sensor_not_ready = false;
+  bool instant_sensor_not_ready = false;
+
 #ifdef USE_BINARY_SENSOR
-  if (this->is_state_armed(future_state)) {
-    // TODO might be better to register change for each sensor in setup...
-    for (auto sensor_pair : this->sensor_map_) {
-      if (sensor_pair.first->state) {
-        if (this->current_state_ == ACP_STATE_ARMED_HOME &&
-            (sensor_pair.second & BINARY_SENSOR_MODE_BYPASS_ARMED_HOME)) {
-          continue;
+  // Test all of the sensors in the list regardless of the alarm panel state
+  for (auto sensor_info : this->sensor_map_) {
+    // Check for chime zones
+    if ((sensor_info.second.flags & BINARY_SENSOR_MODE_CHIME)) {
+      // Look for the transition from closed to open
+      if ((!this->sensor_data_[sensor_info.second.store_index].last_chime_state) && (sensor_info.first->state)) {
+        // Must be disarmed to chime
+        if (this->current_state_ == ACP_STATE_DISARMED) {
+          this->chime_callback_.call();
         }
-        if (this->current_state_ == ACP_STATE_ARMED_NIGHT &&
-            (sensor_pair.second & BINARY_SENSOR_MODE_BYPASS_ARMED_NIGHT)) {
-          continue;
+      }
+      // Record the sensor state change
+      this->sensor_data_[sensor_info.second.store_index].last_chime_state = sensor_info.first->state;
+    }
+    // Check for triggered sensors
+    if (sensor_info.first->state) {  // Sensor triggered?
+      // Skip if bypass armed home
+      if (this->current_state_ == ACP_STATE_ARMED_HOME &&
+          (sensor_info.second.flags & BINARY_SENSOR_MODE_BYPASS_ARMED_HOME)) {
+        continue;
+      }
+      // Skip if bypass armed night
+      if (this->current_state_ == ACP_STATE_ARMED_NIGHT &&
+          (sensor_info.second.flags & BINARY_SENSOR_MODE_BYPASS_ARMED_NIGHT)) {
+        continue;
+      }
+
+      // If sensor type is of type instant
+      if (sensor_info.second.type == ALARM_SENSOR_TYPE_INSTANT) {
+        instant_sensor_not_ready = true;
+        break;
+      }
+      // If sensor type is of type interior follower
+      if (sensor_info.second.type == ALARM_SENSOR_TYPE_DELAYED_FOLLOWER) {
+        // Look to see if we are in the pending state
+        if (this->current_state_ == ACP_STATE_PENDING) {
+          delayed_sensor_not_ready = true;
+        } else {
+          instant_sensor_not_ready = true;
         }
-        trigger = true;
+      }
+      // If sensor type is of type delayed
+      if (sensor_info.second.type == ALARM_SENSOR_TYPE_DELAYED) {
+        delayed_sensor_not_ready = true;
         break;
       }
     }
   }
+  // Update all sensors not ready flag
+  this->sensors_ready_ = ((!instant_sensor_not_ready) && (!delayed_sensor_not_ready));
+
+  // Call the ready state change callback if there was a change
+  if (this->sensors_ready_ != this->sensors_ready_last_) {
+    this->ready_callback_.call();
+    this->sensors_ready_last_ = this->sensors_ready_;
+  }
+
 #endif
-  if (trigger) {
-    if (this->pending_time_ > 0 && this->current_state_ != ACP_STATE_TRIGGERED) {
-      this->publish_state(ACP_STATE_PENDING);
-    } else {
+  if (this->is_state_armed(future_state) && (!this->sensors_ready_)) {
+    // Instant sensors
+    if (instant_sensor_not_ready) {
       this->publish_state(ACP_STATE_TRIGGERED);
+    } else if (delayed_sensor_not_ready) {
+      // Delayed sensors
+      if ((this->pending_time_ > 0) && (this->current_state_ != ACP_STATE_TRIGGERED)) {
+        this->publish_state(ACP_STATE_PENDING);
+      } else {
+        this->publish_state(ACP_STATE_TRIGGERED);
+      }
     }
   } else if (future_state != this->current_state_) {
     this->publish_state(future_state);
diff --git a/esphome/components/template/alarm_control_panel/template_alarm_control_panel.h b/esphome/components/template/alarm_control_panel/template_alarm_control_panel.h
index 9582ed157c..9ae69a0422 100644
--- a/esphome/components/template/alarm_control_panel/template_alarm_control_panel.h
+++ b/esphome/components/template/alarm_control_panel/template_alarm_control_panel.h
@@ -21,7 +21,15 @@ enum BinarySensorFlags : uint16_t {
   BINARY_SENSOR_MODE_NORMAL = 1 << 0,
   BINARY_SENSOR_MODE_BYPASS_ARMED_HOME = 1 << 1,
   BINARY_SENSOR_MODE_BYPASS_ARMED_NIGHT = 1 << 2,
+  BINARY_SENSOR_MODE_CHIME = 1 << 3,
 };
+
+enum AlarmSensorType : uint16_t {
+  ALARM_SENSOR_TYPE_DELAYED = 0,
+  ALARM_SENSOR_TYPE_INSTANT,
+  ALARM_SENSOR_TYPE_DELAYED_FOLLOWER
+};
+
 #endif
 
 enum TemplateAlarmControlPanelRestoreMode {
@@ -29,6 +37,16 @@ enum TemplateAlarmControlPanelRestoreMode {
   ALARM_CONTROL_PANEL_RESTORE_DEFAULT_DISARMED,
 };
 
+struct SensorDataStore {
+  bool last_chime_state;
+};
+
+struct SensorInfo {
+  uint16_t flags;
+  AlarmSensorType type;
+  uint8_t store_index;
+};
+
 class TemplateAlarmControlPanel : public alarm_control_panel::AlarmControlPanel, public Component {
  public:
   TemplateAlarmControlPanel();
@@ -38,6 +56,7 @@ class TemplateAlarmControlPanel : public alarm_control_panel::AlarmControlPanel,
   uint32_t get_supported_features() const override;
   bool get_requires_code() const override;
   bool get_requires_code_to_arm() const override { return this->requires_code_to_arm_; }
+  bool get_all_sensors_ready() { return this->sensors_ready_; };
   void set_restore_mode(TemplateAlarmControlPanelRestoreMode restore_mode) { this->restore_mode_ = restore_mode; }
 
 #ifdef USE_BINARY_SENSOR
@@ -46,7 +65,8 @@ class TemplateAlarmControlPanel : public alarm_control_panel::AlarmControlPanel,
    * @param sensor The BinarySensor instance.
    * @param ignore_when_home if this should be ignored when armed_home mode
    */
-  void add_sensor(binary_sensor::BinarySensor *sensor, uint16_t flags = 0);
+  void add_sensor(binary_sensor::BinarySensor *sensor, uint16_t flags = 0,
+                  AlarmSensorType type = ALARM_SENSOR_TYPE_DELAYED);
 #endif
 
   /** add a code
@@ -98,8 +118,9 @@ class TemplateAlarmControlPanel : public alarm_control_panel::AlarmControlPanel,
  protected:
   void control(const alarm_control_panel::AlarmControlPanelCall &call) override;
 #ifdef USE_BINARY_SENSOR
-  // the map of binary sensors that the alarm_panel monitors with their modes
-  std::map<binary_sensor::BinarySensor *, uint16_t> sensor_map_;
+  // This maps a binary sensor to its type and attribute bits
+  std::map<binary_sensor::BinarySensor *, SensorInfo> sensor_map_;
+
 #endif
   TemplateAlarmControlPanelRestoreMode restore_mode_{};
 
@@ -115,10 +136,15 @@ class TemplateAlarmControlPanel : public alarm_control_panel::AlarmControlPanel,
   uint32_t trigger_time_;
   // a list of codes
   std::vector<std::string> codes_;
+  // Per sensor data store
+  std::vector<SensorDataStore> sensor_data_;
   // requires a code to arm
   bool requires_code_to_arm_ = false;
   bool supports_arm_home_ = false;
   bool supports_arm_night_ = false;
+  bool sensors_ready_ = false;
+  bool sensors_ready_last_ = false;
+  uint8_t next_store_index_ = 0;
   // check if the code is valid
   bool is_code_valid_(optional<std::string> code);
 

From f096f107e238236e75cb7f6b68b9f90433bb2e59 Mon Sep 17 00:00:00 2001
From: sbrudenell <steven.brudenell@gmail.com>
Date: Wed, 20 Dec 2023 21:13:54 -0900
Subject: [PATCH 51/75] support default pins for adafruit esp32 feather v2
 (#5482)

---
 esphome/components/esp32/boards.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/esphome/components/esp32/boards.py b/esphome/components/esp32/boards.py
index e6c23c4d96..d361ca9917 100644
--- a/esphome/components/esp32/boards.py
+++ b/esphome/components/esp32/boards.py
@@ -42,6 +42,34 @@ ESP32_BASE_PINS = {
 }
 
 ESP32_BOARD_PINS = {
+    "adafruit_feather_esp32_v2": {
+        "A0": 26,
+        "A1": 25,
+        "A2": 34,
+        "A3": 39,
+        "A4": 36,
+        "A5": 4,
+        "SCK": 5,
+        "MOSI": 19,
+        "MISO": 21,
+        "RX": 7,
+        "TX": 8,
+        "D37": 37,
+        "LED": 13,
+        "LED_BUILTIN": 13,
+        "D12": 12,
+        "D27": 27,
+        "D33": 33,
+        "D15": 15,
+        "D32": 32,
+        "D14": 14,
+        "SCL": 20,
+        "SDA": 22,
+        "BUTTON": 38,
+        "NEOPIXEL": 0,
+        "PIN_NEOPIXEL": 0,
+        "NEOPIXEL_POWER": 2,
+    },
     "adafruit_feather_esp32s2_tft": {
         "BUTTON": 0,
         "A0": 18,

From d73ad39aedebf0bb4941aca26e4fce789f637a6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Wojciech=20Bana=C5=9B?= <fizista@gmail.com>
Date: Thu, 21 Dec 2023 08:03:57 +0100
Subject: [PATCH 52/75] Bug: Unwanted change resistance in x9c component
 (#5483)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/x9c/x9c.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/esphome/components/x9c/x9c.cpp b/esphome/components/x9c/x9c.cpp
index ff7777e71f..1b283a68e5 100644
--- a/esphome/components/x9c/x9c.cpp
+++ b/esphome/components/x9c/x9c.cpp
@@ -7,6 +7,10 @@ namespace x9c {
 static const char *const TAG = "x9c.output";
 
 void X9cOutput::trim_value(int change_amount) {
+  if (change_amount == 0) {
+    return;
+  }
+
   if (change_amount > 0) {  // Set change direction
     this->ud_pin_->digital_write(true);
   } else {

From 222bb9b495ceeac459c65bff08fd5187d0b4a797 Mon Sep 17 00:00:00 2001
From: marshn <marshn@spa-technology.co.uk>
Date: Thu, 21 Dec 2023 07:17:01 +0000
Subject: [PATCH 53/75] Improvements to RF receiver for Drayton Digistat
 heating controller (#5504)

---
 .../remote_base/drayton_protocol.cpp          | 142 ++++++++++--------
 1 file changed, 80 insertions(+), 62 deletions(-)

diff --git a/esphome/components/remote_base/drayton_protocol.cpp b/esphome/components/remote_base/drayton_protocol.cpp
index fb5f37b470..6c617f56c8 100644
--- a/esphome/components/remote_base/drayton_protocol.cpp
+++ b/esphome/components/remote_base/drayton_protocol.cpp
@@ -13,7 +13,8 @@ static const uint8_t NBITS_SYNC = 4;
 static const uint8_t NBITS_ADDRESS = 16;
 static const uint8_t NBITS_CHANNEL = 5;
 static const uint8_t NBITS_COMMAND = 7;
-static const uint8_t NBITS = NBITS_ADDRESS + NBITS_CHANNEL + NBITS_COMMAND;
+static const uint8_t NDATABITS = NBITS_ADDRESS + NBITS_CHANNEL + NBITS_COMMAND;
+static const uint8_t MIN_RX_SRC = (NDATABITS * 2 + NBITS_SYNC / 2);
 
 static const uint8_t CMD_ON = 0x41;
 static const uint8_t CMD_OFF = 0x02;
@@ -116,7 +117,7 @@ void DraytonProtocol::encode(RemoteTransmitData *dst, const DraytonData &data) {
 
   ESP_LOGV(TAG, "Send Drayton: out_data %08" PRIx32, out_data);
 
-  for (uint32_t mask = 1UL << (NBITS - 1); mask != 0; mask >>= 1) {
+  for (uint32_t mask = 1UL << (NDATABITS - 1); mask != 0; mask >>= 1) {
     if (out_data & mask) {
       dst->mark(BIT_TIME_US);
       dst->space(BIT_TIME_US);
@@ -134,79 +135,96 @@ optional<DraytonData> DraytonProtocol::decode(RemoteReceiveData src) {
       .command = 0,
   };
 
-  if (src.size() < 45) {
-    return {};
-  }
+  while (src.size() - src.get_index() > MIN_RX_SRC) {
+    ESP_LOGVV(TAG,
+              "Decode Drayton: %" PRId32 ", %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
+              " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
+              " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 "",
+              src.size() - src.get_index(), src.peek(0), src.peek(1), src.peek(2), src.peek(3), src.peek(4),
+              src.peek(5), src.peek(6), src.peek(7), src.peek(8), src.peek(9), src.peek(10), src.peek(11), src.peek(12),
+              src.peek(13), src.peek(14), src.peek(15), src.peek(16), src.peek(17), src.peek(18), src.peek(19));
 
-  ESP_LOGVV(TAG,
-            "Decode Drayton: %" PRId32 ", %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
-            " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32
-            " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32 "",
-            src.size(), src.peek(0), src.peek(1), src.peek(2), src.peek(3), src.peek(4), src.peek(5), src.peek(6),
-            src.peek(7), src.peek(8), src.peek(9), src.peek(10), src.peek(11), src.peek(12), src.peek(13), src.peek(14),
-            src.peek(15), src.peek(16), src.peek(17), src.peek(18), src.peek(19));
+    // If first preamble item is a space, skip it
+    if (src.peek_space_at_least(1)) {
+      src.advance(1);
+    }
 
-  // If first preamble item is a space, skip it
-  if (src.peek_space_at_least(1)) {
-    src.advance(1);
-  }
+    // Look for sync pulse, after. If sucessful index points to space of sync symbol
+    while (src.size() - src.get_index() >= NDATABITS) {
+      ESP_LOGVV(TAG, "Decode Drayton: sync search %d, %" PRId32 " %" PRId32, src.size() - src.get_index(), src.peek(),
+                src.peek(1));
+      if (src.peek_mark(2 * BIT_TIME_US) &&
+          (src.peek_space(2 * BIT_TIME_US, 1) || src.peek_space(3 * BIT_TIME_US, 1))) {
+        src.advance(1);
+        ESP_LOGVV(TAG, "Decode Drayton: Found SYNC, - %d", src.get_index());
+        break;
+      } else {
+        src.advance(2);
+      }
+    }
 
-  // Look for sync pulse, after. If sucessful index points to space of sync symbol
-  for (uint16_t preamble = 0; preamble <= NBITS_PREAMBLE * 2; preamble += 2) {
-    ESP_LOGVV(TAG, "Decode Drayton: preamble %d  %" PRId32 " %" PRId32, preamble, src.peek(preamble),
-              src.peek(preamble + 1));
-    if (src.peek_mark(2 * BIT_TIME_US, preamble) &&
-        (src.peek_space(2 * BIT_TIME_US, preamble + 1) || src.peek_space(3 * BIT_TIME_US, preamble + 1))) {
-      src.advance(preamble + 1);
+    // No point continuing if not enough samples remaining to complete a packet
+    if (src.size() - src.get_index() < NDATABITS) {
+      ESP_LOGV(TAG, "Decode Drayton: Fail 1, - %" PRIu32, src.get_index());
       break;
     }
-  }
 
-  // Read data. Index points to space of sync symbol
-  // Extract first bit
-  // Checks next bit to leave index pointing correctly
-  uint32_t out_data = 0;
-  uint8_t bit = NBITS_ADDRESS + NBITS_COMMAND + NBITS_CHANNEL - 1;
-  if (src.expect_space(3 * BIT_TIME_US) && (src.expect_mark(BIT_TIME_US) || src.peek_mark(2 * BIT_TIME_US))) {
-    out_data |= 0 << bit;
-  } else if (src.expect_space(2 * BIT_TIME_US) && src.expect_mark(BIT_TIME_US) &&
-             (src.expect_space(BIT_TIME_US) || src.peek_space(2 * BIT_TIME_US))) {
-    out_data |= 1 << bit;
-  } else {
-    ESP_LOGV(TAG, "Decode Drayton: Fail 1, - %" PRIu32, src.get_index());
-    return {};
-  }
-
-  // Before/after each bit is read the index points to the transition at the start of the bit period or,
-  // if there is no transition at the start of the bit period, then the transition in the middle of
-  // the previous bit period.
-  while (--bit >= 1) {
-    ESP_LOGVV(TAG, "Decode Drayton: Data, %2d %08" PRIx32, bit, out_data);
-    if ((src.expect_space(BIT_TIME_US) || src.expect_space(2 * BIT_TIME_US)) &&
-        (src.expect_mark(BIT_TIME_US) || src.peek_mark(2 * BIT_TIME_US))) {
+    // Read data. Index points to space of sync symbol
+    // Extract first bit
+    // Checks next bit to leave index pointing correctly
+    uint32_t out_data = 0;
+    uint8_t bit = NDATABITS - 1;
+    ESP_LOGVV(TAG, "Decode Drayton: first bit %d  %" PRId32 ", %" PRId32, src.peek(0), src.peek(1), src.peek(2));
+    if (src.expect_space(3 * BIT_TIME_US) && (src.expect_mark(BIT_TIME_US) || src.peek_mark(2 * BIT_TIME_US))) {
       out_data |= 0 << bit;
-    } else if ((src.expect_mark(BIT_TIME_US) || src.expect_mark(2 * BIT_TIME_US)) &&
+    } else if (src.expect_space(2 * BIT_TIME_US) && src.expect_mark(BIT_TIME_US) &&
                (src.expect_space(BIT_TIME_US) || src.peek_space(2 * BIT_TIME_US))) {
       out_data |= 1 << bit;
     } else {
-      ESP_LOGVV(TAG, "Decode Drayton: Fail 2, %2d %08" PRIx32, bit, out_data);
-      return {};
+      ESP_LOGV(TAG, "Decode Drayton: Fail 2, - %d %d %d", src.peek(-1), src.peek(0), src.peek(1));
+      continue;
     }
-  }
-  if (src.expect_space(BIT_TIME_US) || src.expect_space(2 * BIT_TIME_US)) {
-    out_data |= 0;
-  } else if (src.expect_mark(BIT_TIME_US) || src.expect_mark(2 * BIT_TIME_US)) {
-    out_data |= 1;
-  }
-  ESP_LOGV(TAG, "Decode Drayton: Data, %2d %08" PRIx32, bit, out_data);
 
-  out.channel = (uint8_t) (out_data & 0x1F);
-  out_data >>= NBITS_CHANNEL;
-  out.command = (uint8_t) (out_data & 0x7F);
-  out_data >>= NBITS_COMMAND;
-  out.address = (uint16_t) (out_data & 0xFFFF);
+    // Before/after each bit is read the index points to the transition at the start of the bit period or,
+    // if there is no transition at the start of the bit period, then the transition in the middle of
+    // the previous bit period.
+    while (--bit >= 1) {
+      ESP_LOGVV(TAG, "Decode Drayton: Data, %2d %08" PRIx32, bit, out_data);
+      if ((src.expect_space(BIT_TIME_US) || src.expect_space(2 * BIT_TIME_US)) &&
+          (src.expect_mark(BIT_TIME_US) || src.peek_mark(2 * BIT_TIME_US))) {
+        out_data |= 0 << bit;
+      } else if ((src.expect_mark(BIT_TIME_US) || src.expect_mark(2 * BIT_TIME_US)) &&
+                 (src.expect_space(BIT_TIME_US) || src.peek_space(2 * BIT_TIME_US))) {
+        out_data |= 1 << bit;
+      } else {
+        break;
+      }
+    }
 
-  return out;
+    if (bit > 0) {
+      ESP_LOGVV(TAG, "Decode Drayton: Fail 3, %d %" PRId32 " %" PRId32, src.peek(-1), src.peek(0), src.peek(1));
+      continue;
+    }
+
+    if (src.expect_space(BIT_TIME_US) || src.expect_space(2 * BIT_TIME_US)) {
+      out_data |= 0;
+    } else if (src.expect_mark(BIT_TIME_US) || src.expect_mark(2 * BIT_TIME_US)) {
+      out_data |= 1;
+    } else {
+      continue;
+    }
+
+    ESP_LOGV(TAG, "Decode Drayton: Data, %2d %08x", bit, out_data);
+
+    out.channel = (uint8_t) (out_data & 0x1F);
+    out_data >>= NBITS_CHANNEL;
+    out.command = (uint8_t) (out_data & 0x7F);
+    out_data >>= NBITS_COMMAND;
+    out.address = (uint16_t) (out_data & 0xFFFF);
+
+    return out;
+  }
+  return {};
 }
 void DraytonProtocol::dump(const DraytonData &data) {
   ESP_LOGI(TAG, "Received Drayton: address=0x%04X (0x%04x), channel=0x%03x command=0x%03X", data.address,

From 74281b93c4491f80bfd02db9e56696a86081083c Mon Sep 17 00:00:00 2001
From: kahrendt <kahrendt@gmail.com>
Date: Thu, 21 Dec 2023 02:19:15 -0500
Subject: [PATCH 54/75] Reduce memory usage with StringRef in MQTT Components
 (#5719)

---
 esphome/components/mqtt/mqtt_binary_sensor.cpp |  2 +-
 esphome/components/mqtt/mqtt_component.cpp     | 12 ++++++------
 esphome/components/mqtt/mqtt_component.h       | 13 ++++++++-----
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/esphome/components/mqtt/mqtt_binary_sensor.cpp b/esphome/components/mqtt/mqtt_binary_sensor.cpp
index 79e6989a8f..6d12e88391 100644
--- a/esphome/components/mqtt/mqtt_binary_sensor.cpp
+++ b/esphome/components/mqtt/mqtt_binary_sensor.cpp
@@ -25,7 +25,7 @@ void MQTTBinarySensorComponent::dump_config() {
 MQTTBinarySensorComponent::MQTTBinarySensorComponent(binary_sensor::BinarySensor *binary_sensor)
     : binary_sensor_(binary_sensor) {
   if (this->binary_sensor_->is_status_binary_sensor()) {
-    this->set_custom_state_topic(mqtt::global_mqtt_client->get_availability().topic);
+    this->set_custom_state_topic(mqtt::global_mqtt_client->get_availability().topic.c_str());
   }
 }
 
diff --git a/esphome/components/mqtt/mqtt_component.cpp b/esphome/components/mqtt/mqtt_component.cpp
index af4d6f13a5..8855b4d8e3 100644
--- a/esphome/components/mqtt/mqtt_component.cpp
+++ b/esphome/components/mqtt/mqtt_component.cpp
@@ -34,13 +34,13 @@ std::string MQTTComponent::get_default_topic_for_(const std::string &suffix) con
 
 std::string MQTTComponent::get_state_topic_() const {
   if (this->has_custom_state_topic_)
-    return this->custom_state_topic_;
+    return this->custom_state_topic_.str();
   return this->get_default_topic_for_("state");
 }
 
 std::string MQTTComponent::get_command_topic_() const {
   if (this->has_custom_command_topic_)
-    return this->custom_command_topic_;
+    return this->custom_command_topic_.str();
   return this->get_default_topic_for_("command");
 }
 
@@ -180,12 +180,12 @@ MQTTComponent::MQTTComponent() = default;
 
 float MQTTComponent::get_setup_priority() const { return setup_priority::AFTER_CONNECTION; }
 void MQTTComponent::disable_discovery() { this->discovery_enabled_ = false; }
-void MQTTComponent::set_custom_state_topic(const std::string &custom_state_topic) {
-  this->custom_state_topic_ = custom_state_topic;
+void MQTTComponent::set_custom_state_topic(const char *custom_state_topic) {
+  this->custom_state_topic_ = StringRef(custom_state_topic);
   this->has_custom_state_topic_ = true;
 }
-void MQTTComponent::set_custom_command_topic(const std::string &custom_command_topic) {
-  this->custom_command_topic_ = custom_command_topic;
+void MQTTComponent::set_custom_command_topic(const char *custom_command_topic) {
+  this->custom_command_topic_ = StringRef(custom_command_topic);
   this->has_custom_command_topic_ = true;
 }
 void MQTTComponent::set_command_retain(bool command_retain) { this->command_retain_ = command_retain; }
diff --git a/esphome/components/mqtt/mqtt_component.h b/esphome/components/mqtt/mqtt_component.h
index aacfe8891f..9fc8c795d3 100644
--- a/esphome/components/mqtt/mqtt_component.h
+++ b/esphome/components/mqtt/mqtt_component.h
@@ -8,6 +8,7 @@
 
 #include "esphome/core/component.h"
 #include "esphome/core/entity_base.h"
+#include "esphome/core/string_ref.h"
 #include "mqtt_client.h"
 
 namespace esphome {
@@ -88,9 +89,9 @@ class MQTTComponent : public Component {
   virtual std::string component_type() const = 0;
 
   /// Set a custom state topic. Set to "" for default behavior.
-  void set_custom_state_topic(const std::string &custom_state_topic);
+  void set_custom_state_topic(const char *custom_state_topic);
   /// Set a custom command topic. Set to "" for default behavior.
-  void set_custom_command_topic(const std::string &custom_command_topic);
+  void set_custom_command_topic(const char *custom_command_topic);
   /// Set whether command message should be retained.
   void set_command_retain(bool command_retain);
 
@@ -188,15 +189,17 @@ class MQTTComponent : public Component {
   /// Generate the Home Assistant MQTT discovery object id by automatically transforming the friendly name.
   std::string get_default_object_id_() const;
 
-  std::string custom_state_topic_{};
-  std::string custom_command_topic_{};
+  StringRef custom_state_topic_{};
+  StringRef custom_command_topic_{};
+
+  std::unique_ptr<Availability> availability_;
+
   bool has_custom_state_topic_{false};
   bool has_custom_command_topic_{false};
 
   bool command_retain_{false};
   bool retain_{true};
   bool discovery_enabled_{true};
-  std::unique_ptr<Availability> availability_;
   bool resend_state_{false};
 };
 

From 5e2df0b6a27821bc978a8beb6e8d8c5e7e7c91b9 Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Thu, 21 Dec 2023 09:34:33 +0100
Subject: [PATCH 55/75] Nextion allow underscore on names (#5979)

---
 esphome/components/nextion/base_component.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/nextion/base_component.py b/esphome/components/nextion/base_component.py
index 5bd6643cb8..784da35371 100644
--- a/esphome/components/nextion/base_component.py
+++ b/esphome/components/nextion/base_component.py
@@ -33,14 +33,14 @@ CONF_EXIT_REPARSE_ON_START = "exit_reparse_on_start"
 
 
 def NextionName(value):
-    valid_chars = f"{ascii_letters + digits}."
+    valid_chars = f"{ascii_letters + digits + '_'}."
     if not isinstance(value, str) or len(value) > 29:
         raise cv.Invalid("Must be a string less than 29 characters")
 
     for char in value:
         if char not in valid_chars:
             raise cv.Invalid(
-                f"Must only consist of upper/lowercase characters, numbers and the period '.'. The character '{char}' cannot be used."
+                f"Must only consist of upper/lowercase characters, numbers, the underscore '_', and the period '.'. The character '{char}' cannot be used."
             )
 
     return value

From 442820deafd1b17d0da003ad4577f80c98d6c513 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 22 Dec 2023 03:28:25 +1300
Subject: [PATCH 56/75] Fix replaced - in allowed characters during object_id
 sanitizing (#5983)

---
 esphome/helpers.py               | 2 +-
 tests/unit_tests/test_helpers.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/esphome/helpers.py b/esphome/helpers.py
index 00416b591f..254c950b5d 100644
--- a/esphome/helpers.py
+++ b/esphome/helpers.py
@@ -357,7 +357,7 @@ def snake_case(value):
     return value.replace(" ", "_").lower()
 
 
-_DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9_]")
+_DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9-_]")
 
 
 def sanitize(value):
diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py
index 79d39901f0..fc6bdbcdec 100644
--- a/tests/unit_tests/test_helpers.py
+++ b/tests/unit_tests/test_helpers.py
@@ -261,6 +261,7 @@ def test_snake_case(text, expected):
         ('!"§$%&/()=?foo_bar', "___________foo_bar"),
         ('foo_!"§$%&/()=?bar', "foo____________bar"),
         ('foo_bar!"§$%&/()=?', "foo_bar___________"),
+        ('foo-bar!"§$%&/()=?', "foo-bar___________"),
     ),
 )
 def test_sanitize(text, expected):

From 0a779a9299edf1b8af2813a245e6dfb51814e394 Mon Sep 17 00:00:00 2001
From: CVan <cvandesande@opendmz.com>
Date: Thu, 21 Dec 2023 17:55:10 -0500
Subject: [PATCH 57/75] Update libtiff6 (#5985)

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index ee7c70bb0f..7c162fc316 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -50,7 +50,7 @@ RUN \
           libssl-dev=3.0.11-1~deb12u2 \
           libffi-dev=3.4.4-1 \
           libopenjp2-7=2.5.0-2 \
-          libtiff6=4.5.0-6 \
+          libtiff6=4.5.0-6+deb12u1 \
           cargo=0.66.0+ds1-1 \
           pkg-config=1.8.1-1 \
           gcc-arm-linux-gnueabihf=4:12.2.0-3; \

From 31448a4fcd3f12e0d486e8197eddf7e022a8ac25 Mon Sep 17 00:00:00 2001
From: davidmonro <davidm@davidmonro.net>
Date: Fri, 22 Dec 2023 09:57:12 +1100
Subject: [PATCH 58/75] Override GPIOs 12 and 13 on the airm2m (LuatOS) board
 (#5982)

Co-authored-by: David Monro <david.monro@anu.edu.au>
---
 esphome/components/esp32/boards.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/esphome/components/esp32/boards.py b/esphome/components/esp32/boards.py
index d361ca9917..cd85f3da97 100644
--- a/esphome/components/esp32/boards.py
+++ b/esphome/components/esp32/boards.py
@@ -161,6 +161,10 @@ ESP32_BOARD_PINS = {
         "BUTTON": 0,
         "SWITCH": 0,
     },
+    "airm2m_core_esp32c3": {
+        "LED1_BUILTIN": 12,
+        "LED2_BUILTIN": 13,
+    },
     "alksesp32": {
         "A0": 32,
         "A1": 33,

From 513a02ce11f8879e869afcba8bd118febf823cce Mon Sep 17 00:00:00 2001
From: marshn <marshn@spa-technology.co.uk>
Date: Fri, 22 Dec 2023 00:30:23 +0000
Subject: [PATCH 59/75] Add Keeloq RF protocol (#5511)

---
 esphome/components/remote_base/__init__.py    |  56 ++++++
 .../remote_base/keeloq_protocol.cpp           | 188 ++++++++++++++++++
 .../components/remote_base/keeloq_protocol.h  |  53 +++++
 3 files changed, 297 insertions(+)
 create mode 100644 esphome/components/remote_base/keeloq_protocol.cpp
 create mode 100644 esphome/components/remote_base/keeloq_protocol.h

diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py
index 3accd5038c..1d8c6e0967 100644
--- a/esphome/components/remote_base/__init__.py
+++ b/esphome/components/remote_base/__init__.py
@@ -633,6 +633,62 @@ async def magiquest_action(var, config, args):
     cg.add(var.set_magnitude(template_))
 
 
+# Microchip HCS301 KeeLoq OOK
+(
+    KeeloqData,
+    KeeloqBinarySensor,
+    KeeloqTrigger,
+    KeeloqAction,
+    KeeloqDumper,
+) = declare_protocol("Keeloq")
+KEELOQ_SCHEMA = cv.Schema(
+    {
+        cv.Required(CONF_ADDRESS): cv.All(cv.hex_int, cv.Range(min=0, max=0xFFFFFFF)),
+        cv.Required(CONF_CODE): cv.All(cv.hex_int, cv.Range(min=0, max=0xFFFFFFFF)),
+        cv.Optional(CONF_COMMAND, default=0x10): cv.All(
+            cv.hex_int,
+            cv.Range(min=0, max=0x10),
+        ),
+        cv.Optional(CONF_LEVEL, default=False): cv.boolean,
+    }
+)
+
+
+@register_binary_sensor("keeloq", KeeloqBinarySensor, KEELOQ_SCHEMA)
+def Keeloq_binary_sensor(var, config):
+    cg.add(
+        var.set_data(
+            cg.StructInitializer(
+                KeeloqData,
+                ("address", config[CONF_ADDRESS]),
+                ("command", config[CONF_COMMAND]),
+            )
+        )
+    )
+
+
+@register_trigger("keeloq", KeeloqTrigger, KeeloqData)
+def keeloq_trigger(var, config):
+    pass
+
+
+@register_dumper("keeloq", KeeloqDumper)
+def keeloq_dumper(var, config):
+    pass
+
+
+@register_action("keeloq", KeeloqAction, KEELOQ_SCHEMA)
+async def keeloq_action(var, config, args):
+    template_ = await cg.templatable(config[CONF_ADDRESS], args, cg.uint32)
+    cg.add(var.set_address(template_))
+    template_ = await cg.templatable(config[CONF_CODE], args, cg.uint32)
+    cg.add(var.set_encrypted(template_))
+    template_ = await cg.templatable(config[CONF_COMMAND], args, cg.uint8)
+    cg.add(var.set_command(template_))
+    template_ = await cg.templatable(config[CONF_LEVEL], args, bool)
+    cg.add(var.set_vlow(template_))
+
+
 # NEC
 NECData, NECBinarySensor, NECTrigger, NECAction, NECDumper = declare_protocol("NEC")
 NEC_SCHEMA = cv.Schema(
diff --git a/esphome/components/remote_base/keeloq_protocol.cpp b/esphome/components/remote_base/keeloq_protocol.cpp
new file mode 100644
index 0000000000..77a2f9be6c
--- /dev/null
+++ b/esphome/components/remote_base/keeloq_protocol.cpp
@@ -0,0 +1,188 @@
+#include "keeloq_protocol.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace remote_base {
+
+static const char *const TAG = "remote.keeloq";
+
+static const uint32_t BIT_TIME_US = 380;
+static const uint8_t NBITS_PREAMBLE = 12;
+static const uint8_t NBITS_REPEAT = 1;
+static const uint8_t NBITS_VLOW = 1;
+static const uint8_t NBITS_SERIAL = 28;
+static const uint8_t NBITS_BUTTONS = 4;
+static const uint8_t NBITS_DISC = 12;
+static const uint8_t NBITS_SYNC_CNT = 16;
+
+static const uint8_t NBITS_FIXED_DATA = NBITS_REPEAT + NBITS_VLOW + NBITS_BUTTONS + NBITS_SERIAL;
+static const uint8_t NBITS_ENCRYPTED_DATA = NBITS_BUTTONS + NBITS_DISC + NBITS_SYNC_CNT;
+static const uint8_t NBITS_DATA = NBITS_FIXED_DATA + NBITS_ENCRYPTED_DATA;
+
+/*
+KeeLoq Protocol
+
+Coded using information from datasheet for Microchip HCS301 KeeLow Code Hopping Encoder
+
+Encoder - Hopping code is generated at random.
+
+Decoder - Hopping code is ignored and not checked when received. Serial number of
+transmitter and nutton command is decoded.
+
+*/
+
+void KeeloqProtocol::encode(RemoteTransmitData *dst, const KeeloqData &data) {
+  uint32_t out_data = 0x0;
+
+  ESP_LOGD(TAG, "Send Keeloq: address=%07x command=%03x encrypted=%08x", data.address, data.command, data.encrypted);
+  ESP_LOGV(TAG, "Send Keeloq: data bits (%d + %d)", NBITS_ENCRYPTED_DATA, NBITS_FIXED_DATA);
+
+  // Preamble = '01' x 12
+  for (uint8_t cnt = NBITS_PREAMBLE; cnt; cnt--) {
+    dst->space(BIT_TIME_US);
+    dst->mark(BIT_TIME_US);
+  }
+
+  // Header = 10 bit space
+  dst->space(10 * BIT_TIME_US);
+
+  // Encrypted field
+  out_data = data.encrypted;
+
+  ESP_LOGV(TAG, "Send Keeloq: Encrypted data %04x", out_data);
+
+  for (uint32_t mask = 1, cnt = 0; cnt < NBITS_ENCRYPTED_DATA; cnt++, mask <<= 1) {
+    if (out_data & mask) {
+      dst->mark(1 * BIT_TIME_US);
+      dst->space(2 * BIT_TIME_US);
+    } else {
+      dst->mark(2 * BIT_TIME_US);
+      dst->space(1 * BIT_TIME_US);
+    }
+  }
+
+  // first 32 bits of fixed portion
+  out_data = (data.command & 0x0f);
+  out_data <<= NBITS_SERIAL;
+  out_data |= data.address;
+  ESP_LOGV(TAG, "Send Keeloq: Fixed data %04x", out_data);
+
+  for (uint32_t mask = 1, cnt = 0; cnt < (NBITS_FIXED_DATA - 2); cnt++, mask <<= 1) {
+    if (out_data & mask) {
+      dst->mark(1 * BIT_TIME_US);
+      dst->space(2 * BIT_TIME_US);
+    } else {
+      dst->mark(2 * BIT_TIME_US);
+      dst->space(1 * BIT_TIME_US);
+    }
+  }
+
+  // low battery flag
+  if (data.vlow) {
+    dst->mark(1 * BIT_TIME_US);
+    dst->space(2 * BIT_TIME_US);
+  } else {
+    dst->mark(2 * BIT_TIME_US);
+    dst->space(1 * BIT_TIME_US);
+  }
+
+  // repeat flag - always sent as a '1'
+  dst->mark(1 * BIT_TIME_US);
+  dst->space(2 * BIT_TIME_US);
+
+  // Guard time  at end of packet
+  dst->space(39 * BIT_TIME_US);
+}
+
+optional<KeeloqData> KeeloqProtocol::decode(RemoteReceiveData src) {
+  KeeloqData out{
+      .encrypted = 0,
+      .address = 0,
+      .command = 0,
+      .repeat = false,
+      .vlow = false,
+
+  };
+
+  if (src.size() != (NBITS_PREAMBLE + NBITS_DATA) * 2) {
+    return {};
+  }
+
+  ESP_LOGVV(TAG, "%2d: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", src.size(), src.peek(0),
+            src.peek(1), src.peek(2), src.peek(3), src.peek(4), src.peek(5), src.peek(6), src.peek(7), src.peek(8),
+            src.peek(9), src.peek(10), src.peek(11), src.peek(12), src.peek(13), src.peek(14), src.peek(15),
+            src.peek(16), src.peek(17), src.peek(18), src.peek(19));
+
+  // Check preamble bits
+  int8_t bit = NBITS_PREAMBLE - 1;
+  while (--bit >= 0) {
+    if (!src.expect_mark(BIT_TIME_US) || !src.expect_space(BIT_TIME_US)) {
+      ESP_LOGV(TAG, "Decode KeeLoq: Fail 1, %d %d", bit + 1, src.peek());
+      return {};
+    }
+  }
+  if (!src.expect_mark(BIT_TIME_US) || !src.expect_space(10 * BIT_TIME_US)) {
+    ESP_LOGV(TAG, "Decode KeeLoq: Fail 1, %d %d", bit + 1, src.peek());
+    return {};
+  }
+
+  // Read encrypted bits
+  uint32_t out_data = 0;
+  for (bit = 0; bit < NBITS_ENCRYPTED_DATA; bit++) {
+    if (src.expect_mark(2 * BIT_TIME_US) && src.expect_space(BIT_TIME_US)) {
+      out_data |= 0 << bit;
+    } else if (src.expect_mark(BIT_TIME_US) && src.expect_space(2 * BIT_TIME_US)) {
+      out_data |= 1 << bit;
+    } else {
+      ESP_LOGV(TAG, "Decode KeeLoq: Fail 2, %d %d", src.get_index(), src.peek());
+      return {};
+    }
+  }
+  ESP_LOGVV(TAG, "Decode KeeLoq: Data, %d %08x", bit, out_data);
+  out.encrypted = out_data;
+
+  // Read Serial Number and Button Status
+  out_data = 0;
+  for (bit = 0; bit < NBITS_SERIAL + NBITS_BUTTONS; bit++) {
+    if (src.expect_mark(2 * BIT_TIME_US) && src.expect_space(BIT_TIME_US)) {
+      out_data |= 0 << bit;
+    } else if (src.expect_mark(BIT_TIME_US) && src.expect_space(2 * BIT_TIME_US)) {
+      out_data |= 1 << bit;
+    } else {
+      ESP_LOGV(TAG, "Decode KeeLoq: Fail 3, %d %d", src.get_index(), src.peek());
+      return {};
+    }
+  }
+  ESP_LOGVV(TAG, "Decode KeeLoq: Data, %2d %08x", bit, out_data);
+  out.command = (out_data >> 28) & 0xf;
+  out.address = out_data & 0xfffffff;
+
+  // Read Vlow bit
+  if (src.expect_mark(2 * BIT_TIME_US) && src.expect_space(BIT_TIME_US)) {
+    out.vlow = false;
+  } else if (src.expect_mark(BIT_TIME_US) && src.expect_space(2 * BIT_TIME_US)) {
+    out.vlow = true;
+  } else {
+    ESP_LOGV(TAG, "Decode KeeLoq: Fail 4, %08x", src.peek());
+    return {};
+  }
+
+  // Read Repeat bit
+  if (src.expect_mark(2 * BIT_TIME_US) && src.peek_space_at_least(BIT_TIME_US)) {
+    out.repeat = false;
+  } else if (src.expect_mark(BIT_TIME_US) && src.peek_space_at_least(2 * BIT_TIME_US)) {
+    out.repeat = true;
+  } else {
+    ESP_LOGV(TAG, "Decode KeeLoq: Fail 5, %08x", src.peek());
+    return {};
+  }
+
+  return out;
+}
+
+void KeeloqProtocol::dump(const KeeloqData &data) {
+  ESP_LOGD(TAG, "Received Keeloq: address=0x%08X, command=0x%02x", data.address, data.command);
+}
+
+}  // namespace remote_base
+}  // namespace esphome
diff --git a/esphome/components/remote_base/keeloq_protocol.h b/esphome/components/remote_base/keeloq_protocol.h
new file mode 100644
index 0000000000..47125c151b
--- /dev/null
+++ b/esphome/components/remote_base/keeloq_protocol.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "remote_base.h"
+
+namespace esphome {
+namespace remote_base {
+
+struct KeeloqData {
+  uint32_t encrypted;  // 32 bit encrypted field
+  uint32_t address;    // 28 bit serial number
+  uint8_t command;     // Button Status S2-S1-S0-S3
+  bool repeat;         // Repeated command bit
+  bool vlow;           // Battery status bit
+
+  bool operator==(const KeeloqData &rhs) const {
+    // Treat 0x10 as a special, wildcard button press
+    // This allows us to match on just the address if wanted.
+    if (address != rhs.address) {
+      return false;
+    }
+    return (rhs.command == 0x10 || command == rhs.command);
+  }
+};
+
+class KeeloqProtocol : public RemoteProtocol<KeeloqData> {
+ public:
+  void encode(RemoteTransmitData *dst, const KeeloqData &data) override;
+  optional<KeeloqData> decode(RemoteReceiveData src) override;
+  void dump(const KeeloqData &data) override;
+};
+
+DECLARE_REMOTE_PROTOCOL(Keeloq)
+
+template<typename... Ts> class KeeloqAction : public RemoteTransmitterActionBase<Ts...> {
+ public:
+  TEMPLATABLE_VALUE(uint32_t, address)
+  TEMPLATABLE_VALUE(uint32_t, encrypted)
+  TEMPLATABLE_VALUE(uint8_t, command)
+  TEMPLATABLE_VALUE(bool, vlow)
+
+  void encode(RemoteTransmitData *dst, Ts... x) override {
+    KeeloqData data{};
+    data.address = this->address_.value(x...);
+    data.encrypted = this->encrypted_.value(x...);
+    data.command = this->command_.value(x...);
+    data.vlow = this->vlow_.value(x...);
+    KeeloqProtocol().encode(dst, data);
+  }
+};
+
+}  // namespace remote_base
+}  // namespace esphome

From 059e4cee58def2e433eb5fe02451981b45a6731f Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Thu, 21 Dec 2023 19:42:12 -0600
Subject: [PATCH 60/75] Add workaround for crash in Arduino 2.0.9 when CDC is
 configured (#5987)

---
 esphome/components/logger/logger.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp
index 05b97a5f64..e0f7e77d2c 100644
--- a/esphome/components/logger/logger.cpp
+++ b/esphome/components/logger/logger.cpp
@@ -285,6 +285,7 @@ void Logger::pre_setup() {
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
 #if ARDUINO_USB_CDC_ON_BOOT
         this->hw_serial_ = &Serial;
+        Serial.setTxTimeoutMs(0);  // workaround for 2.0.9 crash when there's no data connection
         Serial.begin(this->baud_rate_);
 #else
         this->hw_serial_ = &Serial;

From 70fdc3c3f8a1250fb5476d26d24f0ad21985b491 Mon Sep 17 00:00:00 2001
From: Jessica Hamilton <jessica@jesikat.nz>
Date: Fri, 22 Dec 2023 14:58:30 +1300
Subject: [PATCH 61/75] web_server.py: return empty content when file doesn't
 exist (#5980)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/dashboard/web_server.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py
index 4552aebf7b..c4b84d3fe3 100644
--- a/esphome/dashboard/web_server.py
+++ b/esphome/dashboard/web_server.py
@@ -792,13 +792,22 @@ class EditRequestHandler(BaseHandler):
         """Get the content of a file."""
         loop = asyncio.get_running_loop()
         filename = settings.rel_path(configuration)
-        content = await loop.run_in_executor(None, self._read_file, filename)
-        self.write(content)
+        content = await loop.run_in_executor(
+            None, self._read_file, filename, configuration
+        )
+        if content is not None:
+            self.write(content)
 
-    def _read_file(self, filename: str) -> bytes:
+    def _read_file(self, filename: str, configuration: str) -> bytes | None:
         """Read a file and return the content as bytes."""
-        with open(file=filename, encoding="utf-8") as f:
-            return f.read()
+        try:
+            with open(file=filename, encoding="utf-8") as f:
+                return f.read()
+        except FileNotFoundError:
+            if configuration in const.SECRETS_FILES:
+                return ""
+            self.set_status(404)
+            return None
 
     def _write_file(self, filename: str, content: bytes) -> None:
         """Write a file with the given content."""

From 3de5b26d7726128de941dbfa8c32ea168e497dff Mon Sep 17 00:00:00 2001
From: Scott K Logan <logans@cottsay.net>
Date: Thu, 21 Dec 2023 20:33:29 -0600
Subject: [PATCH 62/75] Add a Binary Sensor Filter for state settling (#5900)

---
 esphome/components/binary_sensor/__init__.py | 14 ++++++++++++++
 esphome/components/binary_sensor/filter.cpp  | 17 +++++++++++++++++
 esphome/components/binary_sensor/filter.h    | 13 +++++++++++++
 tests/test1.yaml                             |  2 ++
 4 files changed, 46 insertions(+)

diff --git a/esphome/components/binary_sensor/__init__.py b/esphome/components/binary_sensor/__init__.py
index babe46e082..2f788d7103 100644
--- a/esphome/components/binary_sensor/__init__.py
+++ b/esphome/components/binary_sensor/__init__.py
@@ -141,6 +141,7 @@ DelayedOffFilter = binary_sensor_ns.class_("DelayedOffFilter", Filter, cg.Compon
 InvertFilter = binary_sensor_ns.class_("InvertFilter", Filter)
 AutorepeatFilter = binary_sensor_ns.class_("AutorepeatFilter", Filter, cg.Component)
 LambdaFilter = binary_sensor_ns.class_("LambdaFilter", Filter)
+SettleFilter = binary_sensor_ns.class_("SettleFilter", Filter, cg.Component)
 
 FILTER_REGISTRY = Registry()
 validate_filters = cv.validate_registry("filter", FILTER_REGISTRY)
@@ -259,6 +260,19 @@ async def lambda_filter_to_code(config, filter_id):
     return cg.new_Pvariable(filter_id, lambda_)
 
 
+@register_filter(
+    "settle",
+    SettleFilter,
+    cv.templatable(cv.positive_time_period_milliseconds),
+)
+async def settle_filter_to_code(config, filter_id):
+    var = cg.new_Pvariable(filter_id)
+    await cg.register_component(var, {})
+    template_ = await cg.templatable(config, [], cg.uint32)
+    cg.add(var.set_delay(template_))
+    return var
+
+
 MULTI_CLICK_TIMING_SCHEMA = cv.Schema(
     {
         cv.Optional(CONF_STATE): cv.boolean,
diff --git a/esphome/components/binary_sensor/filter.cpp b/esphome/components/binary_sensor/filter.cpp
index 46957383c3..8f94b108ac 100644
--- a/esphome/components/binary_sensor/filter.cpp
+++ b/esphome/components/binary_sensor/filter.cpp
@@ -111,6 +111,23 @@ LambdaFilter::LambdaFilter(std::function<optional<bool>(bool)> f) : f_(std::move
 
 optional<bool> LambdaFilter::new_value(bool value, bool is_initial) { return this->f_(value); }
 
+optional<bool> SettleFilter::new_value(bool value, bool is_initial) {
+  if (!this->steady_) {
+    this->set_timeout("SETTLE", this->delay_.value(), [this, value, is_initial]() {
+      this->steady_ = true;
+      this->output(value, is_initial);
+    });
+    return {};
+  } else {
+    this->steady_ = false;
+    this->output(value, is_initial);
+    this->set_timeout("SETTLE", this->delay_.value(), [this]() { this->steady_ = true; });
+    return value;
+  }
+}
+
+float SettleFilter::get_setup_priority() const { return setup_priority::HARDWARE; }
+
 }  // namespace binary_sensor
 
 }  // namespace esphome
diff --git a/esphome/components/binary_sensor/filter.h b/esphome/components/binary_sensor/filter.h
index 9514cb3fe2..f7342db2fb 100644
--- a/esphome/components/binary_sensor/filter.h
+++ b/esphome/components/binary_sensor/filter.h
@@ -108,6 +108,19 @@ class LambdaFilter : public Filter {
   std::function<optional<bool>(bool)> f_;
 };
 
+class SettleFilter : public Filter, public Component {
+ public:
+  optional<bool> new_value(bool value, bool is_initial) override;
+
+  float get_setup_priority() const override;
+
+  template<typename T> void set_delay(T delay) { this->delay_ = delay; }
+
+ protected:
+  TemplatableValue<uint32_t> delay_{};
+  bool steady_{true};
+};
+
 }  // namespace binary_sensor
 
 }  // namespace esphome
diff --git a/tests/test1.yaml b/tests/test1.yaml
index eb72f90b21..720d4e8e82 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -1732,6 +1732,8 @@ binary_sensor:
       - delayed_on_off: !lambda "return 10;"
       - delayed_on: !lambda "return 1000;"
       - delayed_off: !lambda "return 0;"
+      - settle: 40ms
+      - settle: !lambda "return 10;"
     on_press:
       then:
         - lambda: >-

From 4fb7e945f86c143e69bb95e77213d45b2e278950 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 21 Dec 2023 16:59:24 -1000
Subject: [PATCH 63/75] Fix unexpected disconnects when outgoing buffer is full
 during keepalive (#5988)

---
 esphome/components/api/api_connection.cpp | 24 +++++++++++++++++++----
 esphome/components/api/api_connection.h   |  3 +++
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp
index d5ab00a822..4ebfb7582e 100644
--- a/esphome/components/api/api_connection.cpp
+++ b/esphome/components/api/api_connection.cpp
@@ -118,7 +118,9 @@ void APIConnection::loop() {
   this->list_entities_iterator_.advance();
   this->initial_state_iterator_.advance();
 
-  const uint32_t keepalive = 60000;
+  static uint32_t keepalive = 60000;
+  static uint8_t max_ping_retries = 60;
+  static uint16_t ping_retry_interval = 1000;
   const uint32_t now = millis();
   if (this->sent_ping_) {
     // Disconnect if not responded within 2.5*keepalive
@@ -126,10 +128,24 @@ void APIConnection::loop() {
       on_fatal_error();
       ESP_LOGW(TAG, "%s didn't respond to ping request in time. Disconnecting...", this->client_combined_info_.c_str());
     }
-  } else if (now - this->last_traffic_ > keepalive) {
+  } else if (now - this->last_traffic_ > keepalive && now > this->next_ping_retry_) {
     ESP_LOGVV(TAG, "Sending keepalive PING...");
-    this->sent_ping_ = true;
-    this->send_ping_request(PingRequest());
+    this->sent_ping_ = this->send_ping_request(PingRequest());
+    if (!this->sent_ping_) {
+      this->next_ping_retry_ = now + ping_retry_interval;
+      this->ping_retries_++;
+      if (this->ping_retries_ >= max_ping_retries) {
+        on_fatal_error();
+        ESP_LOGE(TAG, "%s: Sending keepalive failed %d time(s). Disconnecting...", this->client_combined_info_.c_str(),
+                 this->ping_retries_);
+      } else if (this->ping_retries_ >= 10) {
+        ESP_LOGW(TAG, "%s: Sending keepalive failed %d time(s), will retry in %d ms",
+                 this->client_combined_info_.c_str(), this->ping_retries_, ping_retry_interval);
+      } else {
+        ESP_LOGD(TAG, "%s: Sending keepalive failed %d time(s), will retry in %d ms",
+                 this->client_combined_info_.c_str(), this->ping_retries_, ping_retry_interval);
+      }
+    }
   }
 
 #ifdef USE_ESP32_CAMERA
diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h
index 09b595bb71..9d01468807 100644
--- a/esphome/components/api/api_connection.h
+++ b/esphome/components/api/api_connection.h
@@ -140,6 +140,7 @@ class APIConnection : public APIServerConnection {
   void on_disconnect_response(const DisconnectResponse &value) override;
   void on_ping_response(const PingResponse &value) override {
     // we initiated ping
+    this->ping_retries_ = 0;
     this->sent_ping_ = false;
   }
   void on_home_assistant_state_response(const HomeAssistantStateResponse &msg) override;
@@ -217,6 +218,8 @@ class APIConnection : public APIServerConnection {
   bool state_subscription_{false};
   int log_subscription_{ESPHOME_LOG_LEVEL_NONE};
   uint32_t last_traffic_;
+  uint32_t next_ping_retry_{0};
+  uint8_t ping_retries_{0};
   bool sent_ping_{false};
   bool service_call_subscription_{false};
   bool next_close_ = false;

From bd6fa29f7765c0cc5b6f9ee1705c89f11b965179 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 22 Dec 2023 18:29:10 +1300
Subject: [PATCH 64/75] Regenerate api_pb2 after manual changes were added
 incorrectly in #5732 (#5990)

---
 esphome/components/api/api_pb2.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp
index 8dd34e7ef1..f81bf04e99 100644
--- a/esphome/components/api/api_pb2.cpp
+++ b/esphome/components/api/api_pb2.cpp
@@ -3848,6 +3848,7 @@ void ListEntitiesClimateResponse::dump_to(std::string &out) const {
   sprintf(buffer, "%g", this->visual_max_humidity);
   out.append(buffer);
   out.append("\n");
+  out.append("}");
 }
 #endif
 bool ClimateStateResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
@@ -4015,6 +4016,7 @@ void ClimateStateResponse::dump_to(std::string &out) const {
   sprintf(buffer, "%g", this->target_humidity);
   out.append(buffer);
   out.append("\n");
+  out.append("}");
 }
 #endif
 bool ClimateCommandRequest::decode_varint(uint32_t field_id, ProtoVarInt value) {

From 676ae6b26e1d759f809373d60b047a6f929266d2 Mon Sep 17 00:00:00 2001
From: matzman666 <matzman@matzman.net>
Date: Fri, 22 Dec 2023 07:58:17 +0100
Subject: [PATCH 65/75] Improved sensor readings in htu21d component. (#5839)

---
 esphome/components/htu21d/htu21d.cpp | 67 ++++++++++++++++------------
 1 file changed, 38 insertions(+), 29 deletions(-)

diff --git a/esphome/components/htu21d/htu21d.cpp b/esphome/components/htu21d/htu21d.cpp
index a8133ae32e..d0dbb15a43 100644
--- a/esphome/components/htu21d/htu21d.cpp
+++ b/esphome/components/htu21d/htu21d.cpp
@@ -39,45 +39,54 @@ void HTU21DComponent::dump_config() {
   LOG_SENSOR("  ", "Humidity", this->humidity_);
 }
 void HTU21DComponent::update() {
-  uint16_t raw_temperature;
   if (this->write(&HTU21D_REGISTER_TEMPERATURE, 1) != i2c::ERROR_OK) {
     this->status_set_warning();
     return;
   }
-  delay(50);  // NOLINT
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_temperature), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_temperature = i2c::i2ctohs(raw_temperature);
 
-  float temperature = (float(raw_temperature & 0xFFFC)) * 175.72f / 65536.0f - 46.85f;
+  // According to the datasheet sht21 temperature readings can take up to 85ms
+  this->set_timeout(85, [this]() {
+    uint16_t raw_temperature;
+    if (this->read(reinterpret_cast<uint8_t *>(&raw_temperature), 2) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
+    raw_temperature = i2c::i2ctohs(raw_temperature);
 
-  uint16_t raw_humidity;
-  if (this->write(&HTU21D_REGISTER_HUMIDITY, 1) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  delay(50);  // NOLINT
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_humidity = i2c::i2ctohs(raw_humidity);
+    float temperature = (float(raw_temperature & 0xFFFC)) * 175.72f / 65536.0f - 46.85f;
 
-  float humidity = (float(raw_humidity & 0xFFFC)) * 125.0f / 65536.0f - 6.0f;
+    ESP_LOGD(TAG, "Got Temperature=%.1f°C", temperature);
 
-  int8_t heater_level = this->get_heater_level();
+    if (this->temperature_ != nullptr)
+      this->temperature_->publish_state(temperature);
+    this->status_clear_warning();
 
-  ESP_LOGD(TAG, "Got Temperature=%.1f°C Humidity=%.1f%% Heater Level=%d", temperature, humidity, heater_level);
+    if (this->write(&HTU21D_REGISTER_HUMIDITY, 1) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
 
-  if (this->temperature_ != nullptr)
-    this->temperature_->publish_state(temperature);
-  if (this->humidity_ != nullptr)
-    this->humidity_->publish_state(humidity);
-  if (this->heater_ != nullptr)
-    this->heater_->publish_state(heater_level);
-  this->status_clear_warning();
+    this->set_timeout(50, [this]() {
+      uint16_t raw_humidity;
+      if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
+        this->status_set_warning();
+        return;
+      }
+      raw_humidity = i2c::i2ctohs(raw_humidity);
+
+      float humidity = (float(raw_humidity & 0xFFFC)) * 125.0f / 65536.0f - 6.0f;
+
+      int8_t heater_level = this->get_heater_level();
+
+      ESP_LOGD(TAG, "Got Humidity=%.1f%% Heater Level=%d", humidity, heater_level);
+
+      if (this->humidity_ != nullptr)
+        this->humidity_->publish_state(humidity);
+      if (this->heater_ != nullptr)
+        this->heater_->publish_state(heater_level);
+      this->status_clear_warning();
+    });
+  });
 }
 
 bool HTU21DComponent::is_heater_enabled() {

From d2d005838694774d9a6e4d0f1721a89d7588e79f Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 22 Dec 2023 20:03:47 +1300
Subject: [PATCH 66/75] Lint the script folder files (#5991)

---
 script/api_protobuf/api_protobuf.py | 478 ++++++++++++++--------------
 script/build_language_schema.py     |  29 +-
 script/bump-version.py              |   5 +-
 script/ci-custom.py                 | 233 +++++++-------
 script/helpers.py                   |  23 +-
 script/sync-device_class.py         |   7 +-
 6 files changed, 398 insertions(+), 377 deletions(-)

diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py
index b1292095d8..a2bc3abf64 100755
--- a/script/api_protobuf/api_protobuf.py
+++ b/script/api_protobuf/api_protobuf.py
@@ -17,28 +17,22 @@ then run this script with python3 and the files
 will be generated, they still need to be formatted
 """
 
-import re
 import os
+import re
+import sys
+from abc import ABC, abstractmethod
 from pathlib import Path
-from textwrap import dedent
 from subprocess import call
+from textwrap import dedent
 
 # Generate with
 # protoc --python_out=script/api_protobuf -I esphome/components/api/ api_options.proto
-
 import aioesphomeapi.api_options_pb2 as pb
 import google.protobuf.descriptor_pb2 as descriptor
 
-file_header = "// This file was automatically generated with a tool.\n"
-file_header += "// See scripts/api_protobuf/api_protobuf.py\n"
-
-cwd = Path(__file__).resolve().parent
-root = cwd.parent.parent / "esphome" / "components" / "api"
-prot = root / "api.protoc"
-call(["protoc", "-o", str(prot), "-I", str(root), "api.proto"])
-content = prot.read_bytes()
-
-d = descriptor.FileDescriptorSet.FromString(content)
+FILE_HEADER = """// This file was automatically generated with a tool.
+// See scripts/api_protobuf/api_protobuf.py
+"""
 
 
 def indent_list(text, padding="  "):
@@ -64,7 +58,7 @@ def camel_to_snake(name):
     return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
 
 
-class TypeInfo:
+class TypeInfo(ABC):
     def __init__(self, field):
         self._field = field
 
@@ -186,10 +180,12 @@ class TypeInfo:
     def dump_content(self):
         o = f'out.append("  {self.name}: ");\n'
         o += self.dump(f"this->{self.field_name}") + "\n"
-        o += f'out.append("\\n");\n'
+        o += 'out.append("\\n");\n'
         return o
 
-    dump = None
+    @abstractmethod
+    def dump(self, name: str):
+        pass
 
 
 TYPE_INFO = {}
@@ -212,7 +208,7 @@ class DoubleType(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%g", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -225,7 +221,7 @@ class FloatType(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%g", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -238,7 +234,7 @@ class Int64Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%lld", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -251,7 +247,7 @@ class UInt64Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%llu", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -264,7 +260,7 @@ class Int32Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%" PRId32, {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -277,7 +273,7 @@ class Fixed64Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%llu", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -290,7 +286,7 @@ class Fixed32Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%" PRIu32, {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -372,7 +368,7 @@ class UInt32Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%" PRIu32, {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -406,7 +402,7 @@ class SFixed32Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%" PRId32, {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -419,7 +415,7 @@ class SFixed64Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%lld", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -432,7 +428,7 @@ class SInt32Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%" PRId32, {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -445,7 +441,7 @@ class SInt64Type(TypeInfo):
 
     def dump(self, name):
         o = f'sprintf(buffer, "%lld", {name});\n'
-        o += f"out.append(buffer);"
+        o += "out.append(buffer);"
         return o
 
 
@@ -527,7 +523,7 @@ class RepeatedTypeInfo(TypeInfo):
     def encode_content(self):
         o = f"for (auto {'' if self._ti_is_bool else '&'}it : this->{self.field_name}) {{\n"
         o += f"  buffer.{self._ti.encode_func}({self.number}, it, true);\n"
-        o += f"}}"
+        o += "}"
         return o
 
     @property
@@ -535,10 +531,13 @@ class RepeatedTypeInfo(TypeInfo):
         o = f'for (const auto {"" if self._ti_is_bool else "&"}it : this->{self.field_name}) {{\n'
         o += f'  out.append("  {self.name}: ");\n'
         o += indent(self._ti.dump("it")) + "\n"
-        o += f'  out.append("\\n");\n'
-        o += f"}}\n"
+        o += '  out.append("\\n");\n'
+        o += "}\n"
         return o
 
+    def dump(self, _: str):
+        pass
+
 
 def build_enum_type(desc):
     name = desc.name
@@ -547,17 +546,17 @@ def build_enum_type(desc):
         out += f"  {v.name} = {v.number},\n"
     out += "};\n"
 
-    cpp = f"#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+    cpp = "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
     cpp += f"template<> const char *proto_enum_to_string<enums::{name}>(enums::{name} value) {{\n"
-    cpp += f"  switch (value) {{\n"
+    cpp += "  switch (value) {\n"
     for v in desc.value:
         cpp += f"    case enums::{v.name}:\n"
         cpp += f'      return "{v.name}";\n'
-    cpp += f"    default:\n"
-    cpp += f'      return "UNKNOWN";\n'
-    cpp += f"  }}\n"
-    cpp += f"}}\n"
-    cpp += f"#endif\n"
+    cpp += "    default:\n"
+    cpp += '      return "UNKNOWN";\n'
+    cpp += "  }\n"
+    cpp += "}\n"
+    cpp += "#endif\n"
 
     return out, cpp
 
@@ -652,10 +651,10 @@ def build_message_type(desc):
             o += f" {dump[0]} "
         else:
             o += "\n"
-            o += f"  __attribute__((unused)) char buffer[64];\n"
+            o += "  __attribute__((unused)) char buffer[64];\n"
             o += f'  out.append("{desc.name} {{\\n");\n'
             o += indent("\n".join(dump)) + "\n"
-            o += f'  out.append("}}");\n'
+            o += '  out.append("}");\n'
     else:
         o2 = f'out.append("{desc.name} {{}}");'
         if len(o) + len(o2) + 3 < 120:
@@ -664,9 +663,9 @@ def build_message_type(desc):
             o += "\n"
             o += f"  {o2}\n"
     o += "}\n"
-    cpp += f"#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+    cpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
     cpp += o
-    cpp += f"#endif\n"
+    cpp += "#endif\n"
     prot = "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
     prot += "void dump_to(std::string &out) const override;\n"
     prot += "#endif\n"
@@ -684,71 +683,12 @@ def build_message_type(desc):
     return out, cpp
 
 
-file = d.file[0]
-content = file_header
-content += """\
-#pragma once
-
-#include "proto.h"
-
-namespace esphome {
-namespace api {
-
-"""
-
-cpp = file_header
-cpp += """\
-#include "api_pb2.h"
-#include "esphome/core/log.h"
-
-#include <cinttypes>
-
-namespace esphome {
-namespace api {
-
-"""
-
-content += "namespace enums {\n\n"
-
-for enum in file.enum_type:
-    s, c = build_enum_type(enum)
-    content += s
-    cpp += c
-
-content += "\n}  // namespace enums\n\n"
-
-mt = file.message_type
-
-for m in mt:
-    s, c = build_message_type(m)
-    content += s
-    cpp += c
-
-content += """\
-
-}  // namespace api
-}  // namespace esphome
-"""
-cpp += """\
-
-}  // namespace api
-}  // namespace esphome
-"""
-
-with open(root / "api_pb2.h", "w") as f:
-    f.write(content)
-
-with open(root / "api_pb2.cpp", "w") as f:
-    f.write(cpp)
-
 SOURCE_BOTH = 0
 SOURCE_SERVER = 1
 SOURCE_CLIENT = 2
 
 RECEIVE_CASES = {}
 
-class_name = "APIServerConnectionBase"
-
 ifdefs = {}
 
 
@@ -768,7 +708,6 @@ def build_service_message_type(mt):
 
     ifdef = get_opt(mt, pb.ifdef)
     log = get_opt(mt, pb.log, True)
-    nodelay = get_opt(mt, pb.no_delay, False)
     hout = ""
     cout = ""
 
@@ -781,14 +720,14 @@ def build_service_message_type(mt):
         # Generate send
         func = f"send_{snake}"
         hout += f"bool {func}(const {mt.name} &msg);\n"
-        cout += f"bool {class_name}::{func}(const {mt.name} &msg) {{\n"
+        cout += f"bool APIServerConnectionBase::{func}(const {mt.name} &msg) {{\n"
         if log:
-            cout += f"#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+            cout += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
             cout += f'  ESP_LOGVV(TAG, "{func}: %s", msg.dump().c_str());\n'
-            cout += f"#endif\n"
+            cout += "#endif\n"
         # cout += f'  this->set_nodelay({str(nodelay).lower()});\n'
         cout += f"  return this->send_message_<{mt.name}>(msg, {id_});\n"
-        cout += f"}}\n"
+        cout += "}\n"
     if source in (SOURCE_BOTH, SOURCE_CLIENT):
         # Generate receive
         func = f"on_{snake}"
@@ -797,169 +736,242 @@ def build_service_message_type(mt):
         if ifdef is not None:
             case += f"#ifdef {ifdef}\n"
         case += f"{mt.name} msg;\n"
-        case += f"msg.decode(msg_data, msg_size);\n"
+        case += "msg.decode(msg_data, msg_size);\n"
         if log:
-            case += f"#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+            case += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
             case += f'ESP_LOGVV(TAG, "{func}: %s", msg.dump().c_str());\n'
-            case += f"#endif\n"
+            case += "#endif\n"
         case += f"this->{func}(msg);\n"
         if ifdef is not None:
-            case += f"#endif\n"
+            case += "#endif\n"
         case += "break;"
         RECEIVE_CASES[id_] = case
 
     if ifdef is not None:
-        hout += f"#endif\n"
-        cout += f"#endif\n"
+        hout += "#endif\n"
+        cout += "#endif\n"
 
     return hout, cout
 
 
-hpp = file_header
-hpp += """\
-#pragma once
+def main():
+    cwd = Path(__file__).resolve().parent
+    root = cwd.parent.parent / "esphome" / "components" / "api"
+    prot_file = root / "api.protoc"
+    call(["protoc", "-o", str(prot_file), "-I", str(root), "api.proto"])
+    proto_content = prot_file.read_bytes()
 
-#include "api_pb2.h"
-#include "esphome/core/defines.h"
+    # pylint: disable-next=no-member
+    d = descriptor.FileDescriptorSet.FromString(proto_content)
 
-namespace esphome {
-namespace api {
+    file = d.file[0]
+    content = FILE_HEADER
+    content += """\
+    #pragma once
 
-"""
+    #include "proto.h"
 
-cpp = file_header
-cpp += """\
-#include "api_pb2_service.h"
-#include "esphome/core/log.h"
+    namespace esphome {
+    namespace api {
 
-namespace esphome {
-namespace api {
+    """
 
-static const char *const TAG = "api.service";
+    cpp = FILE_HEADER
+    cpp += """\
+    #include "api_pb2.h"
+    #include "esphome/core/log.h"
 
-"""
+    #include <cinttypes>
 
-hpp += f"class {class_name} : public ProtoService {{\n"
-hpp += " public:\n"
+    namespace esphome {
+    namespace api {
 
-for mt in file.message_type:
-    obj = build_service_message_type(mt)
-    if obj is None:
-        continue
-    hout, cout = obj
-    hpp += indent(hout) + "\n"
-    cpp += cout
+    """
 
-cases = list(RECEIVE_CASES.items())
-cases.sort()
-hpp += " protected:\n"
-hpp += f"  bool read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) override;\n"
-out = f"bool {class_name}::read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) {{\n"
-out += f"  switch (msg_type) {{\n"
-for i, case in cases:
-    c = f"case {i}: {{\n"
-    c += indent(case) + "\n"
-    c += f"}}"
-    out += indent(c, "    ") + "\n"
-out += "    default:\n"
-out += "      return false;\n"
-out += "  }\n"
-out += "  return true;\n"
-out += "}\n"
-cpp += out
-hpp += "};\n"
+    content += "namespace enums {\n\n"
 
-serv = file.service[0]
-class_name = "APIServerConnection"
-hpp += "\n"
-hpp += f"class {class_name} : public {class_name}Base {{\n"
-hpp += " public:\n"
-hpp_protected = ""
-cpp += "\n"
+    for enum in file.enum_type:
+        s, c = build_enum_type(enum)
+        content += s
+        cpp += c
 
-m = serv.method[0]
-for m in serv.method:
-    func = m.name
-    inp = m.input_type[1:]
-    ret = m.output_type[1:]
-    is_void = ret == "void"
-    snake = camel_to_snake(inp)
-    on_func = f"on_{snake}"
-    needs_conn = get_opt(m, pb.needs_setup_connection, True)
-    needs_auth = get_opt(m, pb.needs_authentication, True)
+    content += "\n}  // namespace enums\n\n"
 
-    ifdef = ifdefs.get(inp, None)
+    mt = file.message_type
 
-    if ifdef is not None:
-        hpp += f"#ifdef {ifdef}\n"
-        hpp_protected += f"#ifdef {ifdef}\n"
-        cpp += f"#ifdef {ifdef}\n"
+    for m in mt:
+        s, c = build_message_type(m)
+        content += s
+        cpp += c
 
-    hpp_protected += f"  void {on_func}(const {inp} &msg) override;\n"
-    hpp += f"  virtual {ret} {func}(const {inp} &msg) = 0;\n"
-    cpp += f"void {class_name}::{on_func}(const {inp} &msg) {{\n"
-    body = ""
-    if needs_conn:
-        body += "if (!this->is_connection_setup()) {\n"
-        body += "  this->on_no_setup_connection();\n"
-        body += "  return;\n"
-        body += "}\n"
-    if needs_auth:
-        body += "if (!this->is_authenticated()) {\n"
-        body += "  this->on_unauthenticated_access();\n"
-        body += "  return;\n"
-        body += "}\n"
+    content += """\
 
-    if is_void:
-        body += f"this->{func}(msg);\n"
-    else:
-        body += f"{ret} ret = this->{func}(msg);\n"
-        ret_snake = camel_to_snake(ret)
-        body += f"if (!this->send_{ret_snake}(ret)) {{\n"
-        body += f"  this->on_fatal_error();\n"
-        body += "}\n"
-    cpp += indent(body) + "\n" + "}\n"
+    }  // namespace api
+    }  // namespace esphome
+    """
+    cpp += """\
 
-    if ifdef is not None:
-        hpp += f"#endif\n"
-        hpp_protected += f"#endif\n"
-        cpp += f"#endif\n"
+    }  // namespace api
+    }  // namespace esphome
+    """
 
-hpp += " protected:\n"
-hpp += hpp_protected
-hpp += "};\n"
+    with open(root / "api_pb2.h", "w", encoding="utf-8") as f:
+        f.write(content)
 
-hpp += """\
+    with open(root / "api_pb2.cpp", "w", encoding="utf-8") as f:
+        f.write(cpp)
 
-}  // namespace api
-}  // namespace esphome
-"""
-cpp += """\
+    hpp = FILE_HEADER
+    hpp += """\
+    #pragma once
 
-}  // namespace api
-}  // namespace esphome
-"""
+    #include "api_pb2.h"
+    #include "esphome/core/defines.h"
 
-with open(root / "api_pb2_service.h", "w") as f:
-    f.write(hpp)
+    namespace esphome {
+    namespace api {
 
-with open(root / "api_pb2_service.cpp", "w") as f:
-    f.write(cpp)
+    """
 
-prot.unlink()
+    cpp = FILE_HEADER
+    cpp += """\
+    #include "api_pb2_service.h"
+    #include "esphome/core/log.h"
 
-try:
-    import clang_format
+    namespace esphome {
+    namespace api {
 
-    def exec_clang_format(path):
-        clang_format_path = os.path.join(
-            os.path.dirname(clang_format.__file__), "data", "bin", "clang-format"
-        )
-        call([clang_format_path, "-i", path])
+    static const char *const TAG = "api.service";
 
-    exec_clang_format(root / "api_pb2_service.h")
-    exec_clang_format(root / "api_pb2_service.cpp")
-    exec_clang_format(root / "api_pb2.h")
-    exec_clang_format(root / "api_pb2.cpp")
-except ImportError:
-    pass
+    """
+
+    class_name = "APIServerConnectionBase"
+
+    hpp += f"class {class_name} : public ProtoService {{\n"
+    hpp += " public:\n"
+
+    for mt in file.message_type:
+        obj = build_service_message_type(mt)
+        if obj is None:
+            continue
+        hout, cout = obj
+        hpp += indent(hout) + "\n"
+        cpp += cout
+
+    cases = list(RECEIVE_CASES.items())
+    cases.sort()
+    hpp += " protected:\n"
+    hpp += "  bool read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) override;\n"
+    out = f"bool {class_name}::read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) {{\n"
+    out += "  switch (msg_type) {\n"
+    for i, case in cases:
+        c = f"case {i}: {{\n"
+        c += indent(case) + "\n"
+        c += "}"
+        out += indent(c, "    ") + "\n"
+    out += "    default:\n"
+    out += "      return false;\n"
+    out += "  }\n"
+    out += "  return true;\n"
+    out += "}\n"
+    cpp += out
+    hpp += "};\n"
+
+    serv = file.service[0]
+    class_name = "APIServerConnection"
+    hpp += "\n"
+    hpp += f"class {class_name} : public {class_name}Base {{\n"
+    hpp += " public:\n"
+    hpp_protected = ""
+    cpp += "\n"
+
+    m = serv.method[0]
+    for m in serv.method:
+        func = m.name
+        inp = m.input_type[1:]
+        ret = m.output_type[1:]
+        is_void = ret == "void"
+        snake = camel_to_snake(inp)
+        on_func = f"on_{snake}"
+        needs_conn = get_opt(m, pb.needs_setup_connection, True)
+        needs_auth = get_opt(m, pb.needs_authentication, True)
+
+        ifdef = ifdefs.get(inp, None)
+
+        if ifdef is not None:
+            hpp += f"#ifdef {ifdef}\n"
+            hpp_protected += f"#ifdef {ifdef}\n"
+            cpp += f"#ifdef {ifdef}\n"
+
+        hpp_protected += f"  void {on_func}(const {inp} &msg) override;\n"
+        hpp += f"  virtual {ret} {func}(const {inp} &msg) = 0;\n"
+        cpp += f"void {class_name}::{on_func}(const {inp} &msg) {{\n"
+        body = ""
+        if needs_conn:
+            body += "if (!this->is_connection_setup()) {\n"
+            body += "  this->on_no_setup_connection();\n"
+            body += "  return;\n"
+            body += "}\n"
+        if needs_auth:
+            body += "if (!this->is_authenticated()) {\n"
+            body += "  this->on_unauthenticated_access();\n"
+            body += "  return;\n"
+            body += "}\n"
+
+        if is_void:
+            body += f"this->{func}(msg);\n"
+        else:
+            body += f"{ret} ret = this->{func}(msg);\n"
+            ret_snake = camel_to_snake(ret)
+            body += f"if (!this->send_{ret_snake}(ret)) {{\n"
+            body += "  this->on_fatal_error();\n"
+            body += "}\n"
+        cpp += indent(body) + "\n" + "}\n"
+
+        if ifdef is not None:
+            hpp += "#endif\n"
+            hpp_protected += "#endif\n"
+            cpp += "#endif\n"
+
+    hpp += " protected:\n"
+    hpp += hpp_protected
+    hpp += "};\n"
+
+    hpp += """\
+
+    }  // namespace api
+    }  // namespace esphome
+    """
+    cpp += """\
+
+    }  // namespace api
+    }  // namespace esphome
+    """
+
+    with open(root / "api_pb2_service.h", "w", encoding="utf-8") as f:
+        f.write(hpp)
+
+    with open(root / "api_pb2_service.cpp", "w", encoding="utf-8") as f:
+        f.write(cpp)
+
+    prot_file.unlink()
+
+    try:
+        import clang_format
+
+        def exec_clang_format(path):
+            clang_format_path = os.path.join(
+                os.path.dirname(clang_format.__file__), "data", "bin", "clang-format"
+            )
+            call([clang_format_path, "-i", path])
+
+        exec_clang_format(root / "api_pb2_service.h")
+        exec_clang_format(root / "api_pb2_service.cpp")
+        exec_clang_format(root / "api_pb2.h")
+        exec_clang_format(root / "api_pb2.cpp")
+    except ImportError:
+        pass
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/script/build_language_schema.py b/script/build_language_schema.py
index fb2010fe3e..fc6ccadc5f 100644
--- a/script/build_language_schema.py
+++ b/script/build_language_schema.py
@@ -61,6 +61,7 @@ solve_registry = []
 
 
 def get_component_names():
+    # pylint: disable-next=redefined-outer-name,reimported
     from esphome.loader import CORE_COMPONENTS_PATH
 
     component_names = ["esphome", "sensor", "esp32", "esp8266"]
@@ -82,9 +83,12 @@ def load_components():
         components[domain] = get_component(domain)
 
 
+# pylint: disable=wrong-import-position
 from esphome.const import CONF_TYPE, KEY_CORE
 from esphome.core import CORE
 
+# pylint: enable=wrong-import-position
+
 CORE.data[KEY_CORE] = {}
 load_components()
 
@@ -114,7 +118,7 @@ def write_file(name, obj):
 
 def delete_extra_files(keep_names):
     for d in os.listdir(args.output_path):
-        if d.endswith(".json") and not d[:-5] in keep_names:
+        if d.endswith(".json") and d[:-5] not in keep_names:
             os.remove(os.path.join(args.output_path, d))
             print(f"Deleted {d}")
 
@@ -552,11 +556,11 @@ def shrink():
             s = f"{domain}.{schema_name}"
             if (
                 not s.endswith("." + S_CONFIG_SCHEMA)
-                and s not in referenced_schemas.keys()
+                and s not in referenced_schemas
                 and not is_platform_schema(s)
             ):
                 print(f"Removing {s}")
-                output[domain][S_SCHEMAS].pop(schema_name)
+                domain_schemas[S_SCHEMAS].pop(schema_name)
 
 
 def build_schema():
@@ -564,7 +568,7 @@ def build_schema():
 
     # check esphome was not loaded globally (IDE auto imports)
     if len(ejs.extended_schemas) == 0:
-        raise Exception(
+        raise LookupError(
             "no data collected. Did you globally import an ESPHome component?"
         )
 
@@ -703,7 +707,7 @@ def convert(schema, config_var, path):
             if schema_instance is schema:
                 assert S_CONFIG_VARS not in config_var
                 assert S_EXTENDS not in config_var
-                if not S_TYPE in config_var:
+                if S_TYPE not in config_var:
                     config_var[S_TYPE] = S_SCHEMA
                 # assert config_var[S_TYPE] == S_SCHEMA
 
@@ -765,9 +769,9 @@ def convert(schema, config_var, path):
     elif schema == automation.validate_potentially_and_condition:
         config_var[S_TYPE] = "registry"
         config_var["registry"] = "condition"
-    elif schema == cv.int_ or schema == cv.int_range:
+    elif schema in (cv.int_, cv.int_range):
         config_var[S_TYPE] = "integer"
-    elif schema == cv.string or schema == cv.string_strict or schema == cv.valid_name:
+    elif schema in (cv.string, cv.string_strict, cv.valid_name):
         config_var[S_TYPE] = "string"
 
     elif isinstance(schema, vol.Schema):
@@ -779,6 +783,7 @@ def convert(schema, config_var, path):
         config_var |= pin_validators[repr_schema]
         config_var[S_TYPE] = "pin"
 
+    # pylint: disable-next=too-many-nested-blocks
     elif repr_schema in ejs.hidden_schemas:
         schema_type = ejs.hidden_schemas[repr_schema]
 
@@ -869,7 +874,7 @@ def convert(schema, config_var, path):
                     config_var["use_id_type"] = str(data.base)
                     config_var[S_TYPE] = "use_id"
         else:
-            raise Exception("Unknown extracted schema type")
+            raise TypeError("Unknown extracted schema type")
     elif config_var.get("key") == "GeneratedID":
         if path.startswith("i2c/CONFIG_SCHEMA/") and path.endswith("/id"):
             config_var["id_type"] = {
@@ -884,7 +889,7 @@ def convert(schema, config_var, path):
         elif path == "pins/esp32/val 1/id":
             config_var["id_type"] = "pin"
         else:
-            raise Exception("Cannot determine id_type for " + path)
+            raise TypeError("Cannot determine id_type for " + path)
 
     elif repr_schema in ejs.registry_schemas:
         solve_registry.append((ejs.registry_schemas[repr_schema], config_var))
@@ -948,11 +953,7 @@ def convert_keys(converted, schema, path):
             result["key"] = "GeneratedID"
         elif isinstance(k, cv.Required):
             result["key"] = "Required"
-        elif (
-            isinstance(k, cv.Optional)
-            or isinstance(k, cv.Inclusive)
-            or isinstance(k, cv.Exclusive)
-        ):
+        elif isinstance(k, (cv.Optional, cv.Inclusive, cv.Exclusive)):
             result["key"] = "Optional"
         else:
             converted["key"] = "String"
diff --git a/script/bump-version.py b/script/bump-version.py
index 3e1e473c4b..a55bb65cd6 100755
--- a/script/bump-version.py
+++ b/script/bump-version.py
@@ -2,7 +2,6 @@
 
 import argparse
 import re
-import subprocess
 from dataclasses import dataclass
 import sys
 
@@ -40,12 +39,12 @@ class Version:
 
 
 def sub(path, pattern, repl, expected_count=1):
-    with open(path) as fh:
+    with open(path, encoding="utf-8") as fh:
         content = fh.read()
     content, count = re.subn(pattern, repl, content, flags=re.MULTILINE)
     if expected_count is not None:
         assert count == expected_count, f"Pattern {pattern} replacement failed!"
-    with open(path, "w") as fh:
+    with open(path, "w", encoding="utf-8") as fh:
         fh.write(content)
 
 
diff --git a/script/ci-custom.py b/script/ci-custom.py
index cc9bdcadbb..41ce030d48 100755
--- a/script/ci-custom.py
+++ b/script/ci-custom.py
@@ -1,10 +1,8 @@
 #!/usr/bin/env python3
 
-from helpers import styled, print_error_for_file, git_ls_files, filter_changed
 import argparse
 import codecs
 import collections
-import colorama
 import fnmatch
 import functools
 import os.path
@@ -12,6 +10,9 @@ import re
 import sys
 import time
 
+import colorama
+from helpers import filter_changed, git_ls_files, print_error_for_file, styled
+
 sys.path.append(os.path.dirname(__file__))
 
 
@@ -30,31 +31,6 @@ def find_all(a_str, sub):
             column += len(sub)
 
 
-colorama.init()
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "files", nargs="*", default=[], help="files to be processed (regex on path)"
-)
-parser.add_argument(
-    "-c", "--changed", action="store_true", help="Only run on changed files"
-)
-parser.add_argument(
-    "--print-slowest", action="store_true", help="Print the slowest checks"
-)
-args = parser.parse_args()
-
-EXECUTABLE_BIT = git_ls_files()
-files = list(EXECUTABLE_BIT.keys())
-# Match against re
-file_name_re = re.compile("|".join(args.files))
-files = [p for p in files if file_name_re.search(p)]
-
-if args.changed:
-    files = filter_changed(files)
-
-files.sort()
-
 file_types = (
     ".h",
     ".c",
@@ -86,6 +62,30 @@ ignore_types = (".ico", ".png", ".woff", ".woff2", "")
 LINT_FILE_CHECKS = []
 LINT_CONTENT_CHECKS = []
 LINT_POST_CHECKS = []
+EXECUTABLE_BIT = {}
+
+errors = collections.defaultdict(list)
+
+
+def add_errors(fname, errs):
+    if not isinstance(errs, list):
+        errs = [errs]
+    for err in errs:
+        if err is None:
+            continue
+        try:
+            lineno, col, msg = err
+        except ValueError:
+            lineno = 1
+            col = 1
+            msg = err
+        if not isinstance(msg, str):
+            raise ValueError("Error is not instance of string!")
+        if not isinstance(lineno, int):
+            raise ValueError("Line number is not an int!")
+        if not isinstance(col, int):
+            raise ValueError("Column number is not an int!")
+        errors[fname].append((lineno, col, msg))
 
 
 def run_check(lint_obj, fname, *args):
@@ -155,7 +155,7 @@ def lint_re_check(regex, **kwargs):
     def decorator(func):
         @functools.wraps(func)
         def new_func(fname, content):
-            errors = []
+            errs = []
             for match in prog.finditer(content):
                 if "NOLINT" in match.group(0):
                     continue
@@ -165,8 +165,8 @@ def lint_re_check(regex, **kwargs):
                 err = func(fname, match)
                 if err is None:
                     continue
-                errors.append((lineno, col + 1, err))
-            return errors
+                errs.append((lineno, col + 1, err))
+            return errs
 
         return decor(new_func)
 
@@ -182,13 +182,13 @@ def lint_content_find_check(find, only_first=False, **kwargs):
             find_ = find
             if callable(find):
                 find_ = find(fname, content)
-            errors = []
+            errs = []
             for line, col in find_all(content, find_):
                 err = func(fname)
-                errors.append((line + 1, col + 1, err))
+                errs.append((line + 1, col + 1, err))
                 if only_first:
                     break
-            return errors
+            return errs
 
         return decor(new_func)
 
@@ -235,8 +235,8 @@ def lint_executable_bit(fname):
     ex = EXECUTABLE_BIT[fname]
     if ex != 100644:
         return (
-            "File has invalid executable bit {}. If running from a windows machine please "
-            "see disabling executable bit in git.".format(ex)
+            f"File has invalid executable bit {ex}. If running from a windows machine please "
+            "see disabling executable bit in git."
         )
     return None
 
@@ -285,8 +285,8 @@ def lint_no_defines(fname, match):
     s = highlight(f"static const uint8_t {match.group(1)} = {match.group(2)};")
     return (
         "#define macros for integer constants are not allowed, please use "
-        "{} style instead (replace uint8_t with the appropriate "
-        "datatype). See also Google style guide.".format(s)
+        f"{s} style instead (replace uint8_t with the appropriate "
+        "datatype). See also Google style guide."
     )
 
 
@@ -296,11 +296,11 @@ def lint_no_long_delays(fname, match):
     if duration_ms < 50:
         return None
     return (
-        "{} - long calls to delay() are not allowed in ESPHome because everything executes "
-        "in one thread. Calling delay() will block the main thread and slow down ESPHome.\n"
+        f"{highlight(match.group(0).strip())} - long calls to delay() are not allowed "
+        "in ESPHome because everything executes in one thread. Calling delay() will "
+        "block the main thread and slow down ESPHome.\n"
         "If there's no way to work around the delay() and it doesn't execute often, please add "
         "a '// NOLINT' comment to the line."
-        "".format(highlight(match.group(0).strip()))
     )
 
 
@@ -311,28 +311,28 @@ def lint_const_ordered(fname, content):
     Reason: Otherwise people add it to the end, and then that results in merge conflicts.
     """
     lines = content.splitlines()
-    errors = []
+    errs = []
     for start in ["CONF_", "ICON_", "UNIT_"]:
         matching = [
             (i + 1, line) for i, line in enumerate(lines) if line.startswith(start)
         ]
         ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " ")))
         ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)]
-        for (mi, ml), (oi, ol) in zip(matching, ordered):
-            if ml == ol:
+        for (mi, mline), (_, ol) in zip(matching, ordered):
+            if mline == ol:
                 continue
-            target = next(i for i, l in ordered if l == ml)
-            target_text = next(l for i, l in matching if target == i)
-            errors.append(
+            target = next(i for i, line in ordered if line == mline)
+            target_text = next(line for i, line in matching if target == i)
+            errs.append(
                 (
                     mi,
                     1,
-                    f"Constant {highlight(ml)} is not ordered, please make sure all "
+                    f"Constant {highlight(mline)} is not ordered, please make sure all "
                     f"constants are ordered. See line {mi} (should go to line {target}, "
                     f"{target_text})",
                 )
             )
-    return errors
+    return errs
 
 
 @lint_re_check(r'^\s*CONF_([A-Z_0-9a-z]+)\s+=\s+[\'"](.*?)[\'"]\s*?$', include=["*.py"])
@@ -344,15 +344,14 @@ def lint_conf_matches(fname, match):
     if const_norm == value_norm:
         return None
     return (
-        "Constant {} does not match value {}! Please make sure the constant's name matches its "
-        "value!"
-        "".format(highlight("CONF_" + const), highlight(value))
+        f"Constant {highlight('CONF_' + const)} does not match value {highlight(value)}! "
+        "Please make sure the constant's name matches its value!"
     )
 
 
 CONF_RE = r'^(CONF_[a-zA-Z0-9_]+)\s*=\s*[\'"].*?[\'"]\s*?$'
-with codecs.open("esphome/const.py", "r", encoding="utf-8") as f_handle:
-    constants_content = f_handle.read()
+with codecs.open("esphome/const.py", "r", encoding="utf-8") as const_f_handle:
+    constants_content = const_f_handle.read()
 CONSTANTS = [m.group(1) for m in re.finditer(CONF_RE, constants_content, re.MULTILINE)]
 
 CONSTANTS_USES = collections.defaultdict(list)
@@ -365,8 +364,8 @@ def lint_conf_from_const_py(fname, match):
         CONSTANTS_USES[name].append(fname)
         return None
     return (
-        "Constant {} has already been defined in const.py - please import the constant from "
-        "const.py directly.".format(highlight(name))
+        f"Constant {highlight(name)} has already been defined in const.py - "
+        "please import the constant from const.py directly."
     )
 
 
@@ -473,16 +472,15 @@ def lint_no_byte_datatype(fname, match):
 
 @lint_post_check
 def lint_constants_usage():
-    errors = []
+    errs = []
     for constant, uses in CONSTANTS_USES.items():
         if len(uses) < 4:
             continue
-        errors.append(
-            "Constant {} is defined in {} files. Please move all definitions of the "
-            "constant to const.py (Uses: {})"
-            "".format(highlight(constant), len(uses), ", ".join(uses))
+        errs.append(
+            f"Constant {highlight(constant)} is defined in {len(uses)} files. Please move all definitions of the "
+            f"constant to const.py (Uses: {', '.join(uses)})"
         )
-    return errors
+    return errs
 
 
 def relative_cpp_search_text(fname, content):
@@ -553,7 +551,7 @@ def lint_namespace(fname, content):
     return (
         "Invalid namespace found in C++ file. All integration C++ files should put all "
         "functions in a separate namespace that matches the integration's name. "
-        "Please make sure the file contains {}".format(highlight(search))
+        f"Please make sure the file contains {highlight(search)}"
     )
 
 
@@ -639,66 +637,73 @@ def lint_log_in_header(fname):
     )
 
 
-errors = collections.defaultdict(list)
+def main():
+    colorama.init()
 
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "files", nargs="*", default=[], help="files to be processed (regex on path)"
+    )
+    parser.add_argument(
+        "-c", "--changed", action="store_true", help="Only run on changed files"
+    )
+    parser.add_argument(
+        "--print-slowest", action="store_true", help="Print the slowest checks"
+    )
+    args = parser.parse_args()
 
-def add_errors(fname, errs):
-    if not isinstance(errs, list):
-        errs = [errs]
-    for err in errs:
-        if err is None:
+    global EXECUTABLE_BIT
+    EXECUTABLE_BIT = git_ls_files()
+    files = list(EXECUTABLE_BIT.keys())
+    # Match against re
+    file_name_re = re.compile("|".join(args.files))
+    files = [p for p in files if file_name_re.search(p)]
+
+    if args.changed:
+        files = filter_changed(files)
+
+    files.sort()
+
+    for fname in files:
+        _, ext = os.path.splitext(fname)
+        run_checks(LINT_FILE_CHECKS, fname, fname)
+        if ext in ignore_types:
             continue
         try:
-            lineno, col, msg = err
-        except ValueError:
-            lineno = 1
-            col = 1
-            msg = err
-        if not isinstance(msg, str):
-            raise ValueError("Error is not instance of string!")
-        if not isinstance(lineno, int):
-            raise ValueError("Line number is not an int!")
-        if not isinstance(col, int):
-            raise ValueError("Column number is not an int!")
-        errors[fname].append((lineno, col, msg))
+            with codecs.open(fname, "r", encoding="utf-8") as f_handle:
+                content = f_handle.read()
+        except UnicodeDecodeError:
+            add_errors(
+                fname,
+                "File is not readable as UTF-8. Please set your editor to UTF-8 mode.",
+            )
+            continue
+        run_checks(LINT_CONTENT_CHECKS, fname, fname, content)
 
+    run_checks(LINT_POST_CHECKS, "POST")
 
-for fname in files:
-    _, ext = os.path.splitext(fname)
-    run_checks(LINT_FILE_CHECKS, fname, fname)
-    if ext in ignore_types:
-        continue
-    try:
-        with codecs.open(fname, "r", encoding="utf-8") as f_handle:
-            content = f_handle.read()
-    except UnicodeDecodeError:
-        add_errors(
-            fname,
-            "File is not readable as UTF-8. Please set your editor to UTF-8 mode.",
+    for f, errs in sorted(errors.items()):
+        bold = functools.partial(styled, colorama.Style.BRIGHT)
+        bold_red = functools.partial(styled, (colorama.Style.BRIGHT, colorama.Fore.RED))
+        err_str = (
+            f"{bold(f'{f}:{lineno}:{col}:')} {bold_red('lint:')} {msg}\n"
+            for lineno, col, msg in errs
         )
-        continue
-    run_checks(LINT_CONTENT_CHECKS, fname, fname, content)
+        print_error_for_file(f, "\n".join(err_str))
 
-run_checks(LINT_POST_CHECKS, "POST")
+    if args.print_slowest:
+        lint_times = []
+        for lint in LINT_FILE_CHECKS + LINT_CONTENT_CHECKS + LINT_POST_CHECKS:
+            durations = lint.get("durations", [])
+            lint_times.append((sum(durations), len(durations), lint["func"].__name__))
+        lint_times.sort(key=lambda x: -x[0])
+        for i in range(min(len(lint_times), 10)):
+            dur, invocations, name = lint_times[i]
+            print(f" - '{name}' took {dur:.2f}s total (ran on {invocations} files)")
+        print(f"Total time measured: {sum(x[0] for x in lint_times):.2f}s")
 
-for f, errs in sorted(errors.items()):
-    bold = functools.partial(styled, colorama.Style.BRIGHT)
-    bold_red = functools.partial(styled, (colorama.Style.BRIGHT, colorama.Fore.RED))
-    err_str = (
-        f"{bold(f'{f}:{lineno}:{col}:')} {bold_red('lint:')} {msg}\n"
-        for lineno, col, msg in errs
-    )
-    print_error_for_file(f, "\n".join(err_str))
+    return len(errors)
 
-if args.print_slowest:
-    lint_times = []
-    for lint in LINT_FILE_CHECKS + LINT_CONTENT_CHECKS + LINT_POST_CHECKS:
-        durations = lint.get("durations", [])
-        lint_times.append((sum(durations), len(durations), lint["func"].__name__))
-    lint_times.sort(key=lambda x: -x[0])
-    for i in range(min(len(lint_times), 10)):
-        dur, invocations, name = lint_times[i]
-        print(f" - '{name}' took {dur:.2f}s total (ran on {invocations} files)")
-    print(f"Total time measured: {sum(x[0] for x in lint_times):.2f}s")
 
-sys.exit(len(errors))
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/script/helpers.py b/script/helpers.py
index c042362aeb..b1908e9875 100644
--- a/script/helpers.py
+++ b/script/helpers.py
@@ -1,10 +1,11 @@
-import colorama
+import json
 import os.path
 import re
 import subprocess
-import json
 from pathlib import Path
 
+import colorama
+
 root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", "..")))
 basepath = os.path.join(root_path, "esphome")
 temp_folder = os.path.join(root_path, ".temp")
@@ -44,7 +45,7 @@ def build_all_include():
     content = "\n".join(headers)
     p = Path(temp_header_file)
     p.parent.mkdir(exist_ok=True)
-    p.write_text(content)
+    p.write_text(content, encoding="utf-8")
 
 
 def walk_files(path):
@@ -54,14 +55,14 @@ def walk_files(path):
 
 
 def get_output(*args):
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output, err = proc.communicate()
+    with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
+        output, _ = proc.communicate()
     return output.decode("utf-8")
 
 
 def get_err(*args):
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output, err = proc.communicate()
+    with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
+        _, err = proc.communicate()
     return err.decode("utf-8")
 
 
@@ -78,7 +79,7 @@ def changed_files():
             merge_base = splitlines_no_ends(get_output(*command))[0]
             break
         # pylint: disable=bare-except
-        except:
+        except:  # noqa: E722
             pass
     else:
         raise ValueError("Git not configured")
@@ -103,7 +104,7 @@ def filter_changed(files):
 def filter_grep(files, value):
     matched = []
     for file in files:
-        with open(file) as handle:
+        with open(file, encoding="utf-8") as handle:
             contents = handle.read()
         if value in contents:
             matched.append(file)
@@ -114,8 +115,8 @@ def git_ls_files(patterns=None):
     command = ["git", "ls-files", "-s"]
     if patterns is not None:
         command.extend(patterns)
-    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
-    output, err = proc.communicate()
+    with subprocess.Popen(command, stdout=subprocess.PIPE) as proc:
+        output, _ = proc.communicate()
     lines = [x.split() for x in output.decode("utf-8").splitlines()]
     return {s[3].strip(): int(s[0]) for s in lines}
 
diff --git a/script/sync-device_class.py b/script/sync-device_class.py
index ae6f4be0c8..8f91b97997 100755
--- a/script/sync-device_class.py
+++ b/script/sync-device_class.py
@@ -2,6 +2,7 @@
 
 import re
 
+# pylint: disable=import-error
 from homeassistant.components.binary_sensor import BinarySensorDeviceClass
 from homeassistant.components.button import ButtonDeviceClass
 from homeassistant.components.cover import CoverDeviceClass
@@ -9,6 +10,8 @@ from homeassistant.components.number import NumberDeviceClass
 from homeassistant.components.sensor import SensorDeviceClass
 from homeassistant.components.switch import SwitchDeviceClass
 
+# pylint: enable=import-error
+
 BLOCKLIST = (
     # requires special support on HA side
     "enum",
@@ -25,10 +28,10 @@ DOMAINS = {
 
 
 def sub(path, pattern, repl):
-    with open(path) as handle:
+    with open(path, encoding="utf-8") as handle:
         content = handle.read()
     content = re.sub(pattern, repl, content, flags=re.MULTILINE)
-    with open(path, "w") as handle:
+    with open(path, "w", encoding="utf-8") as handle:
         handle.write(content)
 
 

From 46255ad4dfcd00fff079192d197879f91fd7dd3b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 21 Dec 2023 21:35:31 -1000
Subject: [PATCH 67/75] Fix dashboard logs when api is disabled and using MQTT
 (#5992)

---
 esphome/dashboard/web_server.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py
index c4b84d3fe3..6a80865906 100644
--- a/esphome/dashboard/web_server.py
+++ b/esphome/dashboard/web_server.py
@@ -301,11 +301,16 @@ class EsphomePortCommandWebSocket(EsphomeCommandWebSocket):
         config_file = settings.rel_path(configuration)
         port = json_message["port"]
         if (
-            port == "OTA"
+            port == "OTA"  # pylint: disable=too-many-boolean-expressions
             and (mdns := dashboard.mdns_status)
             and (entry := entries.get(config_file))
+            and entry.loaded_integrations
+            and "api" in entry.loaded_integrations
             and (address := await mdns.async_resolve_host(entry.name))
         ):
+            # Use the IP address if available but only
+            # if the API is loaded and the device is online
+            # since MQTT logging will not work otherwise
             port = address
 
         return [

From 46c4c61b40114f3cce38842c2900e9f1af4c8245 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 22 Dec 2023 21:10:35 +1300
Subject: [PATCH 68/75] Fix broken configs with non-existent components (#5993)

---
 esphome/config.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/esphome/config.py b/esphome/config.py
index e9433d537e..4aca0d6056 100644
--- a/esphome/config.py
+++ b/esphome/config.py
@@ -315,7 +315,11 @@ class LoadValidationStep(ConfigValidationStep):
             return
         result.add_output_path([self.domain], self.domain)
         component = get_component(self.domain)
-        if component.multi_conf_no_default and isinstance(self.conf, core.AutoLoad):
+        if (
+            component is not None
+            and component.multi_conf_no_default
+            and isinstance(self.conf, core.AutoLoad)
+        ):
             self.conf = []
         result[self.domain] = self.conf
         path = [self.domain]

From a97fc4f75810776227f862c78476aca565f2d0c1 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 22 Dec 2023 15:43:17 -1000
Subject: [PATCH 69/75] dashboard: Only ping when polling is active (#6001)

fixes https://github.com/esphome/issues/issues/5257
---
 esphome/dashboard/status/ping.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/esphome/dashboard/status/ping.py b/esphome/dashboard/status/ping.py
index d8281d9de1..989cd1570f 100644
--- a/esphome/dashboard/status/ping.py
+++ b/esphome/dashboard/status/ping.py
@@ -31,6 +31,7 @@ class PingStatus:
         while not dashboard.stop_event.is_set():
             # Only ping if the dashboard is open
             await dashboard.ping_request.wait()
+            dashboard.ping_request.clear()
             current_entries = dashboard.entries.async_all()
             to_ping: list[DashboardEntry] = [
                 entry for entry in current_entries if entry.address is not None

From 8e674990b0c829ee130927f4af06b9f02ec60a63 Mon Sep 17 00:00:00 2001
From: Attila Farago <attila.farago.hu@gmail.com>
Date: Sat, 23 Dec 2023 15:17:00 +0100
Subject: [PATCH 70/75] web_server support for home assistant like styling
 (#5854)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/web_server/web_server.cpp | 38 ++++++++++++--------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp
index 0d72e274cd..54e9e6ebcc 100644
--- a/esphome/components/web_server/web_server.cpp
+++ b/esphome/components/web_server/web_server.cpp
@@ -397,19 +397,21 @@ void WebServer::handle_js_request(AsyncWebServerRequest *request) {
 
 #define set_json_id(root, obj, sensor, start_config) \
   (root)["id"] = sensor; \
-  if (((start_config) == DETAIL_ALL)) \
-    (root)["name"] = (obj)->get_name();
+  if (((start_config) == DETAIL_ALL)) { \
+    (root)["name"] = (obj)->get_name(); \
+    (root)["icon"] = (obj)->get_icon(); \
+    (root)["entity_category"] = (obj)->get_entity_category(); \
+    if ((obj)->is_disabled_by_default()) \
+      (root)["is_disabled_by_default"] = (obj)->is_disabled_by_default(); \
+  }
 
 #define set_json_value(root, obj, sensor, value, start_config) \
-  set_json_id((root), (obj), sensor, start_config)(root)["value"] = value;
-
-#define set_json_state_value(root, obj, sensor, state, value, start_config) \
-  set_json_value(root, obj, sensor, value, start_config)(root)["state"] = state;
+  set_json_id((root), (obj), sensor, start_config); \
+  (root)["value"] = value;
 
 #define set_json_icon_state_value(root, obj, sensor, state, value, start_config) \
-  set_json_value(root, obj, sensor, value, start_config)(root)["state"] = state; \
-  if (((start_config) == DETAIL_ALL)) \
-    (root)["icon"] = (obj)->get_icon();
+  set_json_value(root, obj, sensor, value, start_config); \
+  (root)["state"] = state;
 
 #ifdef USE_SENSOR
 void WebServer::on_sensor_update(sensor::Sensor *obj, float state) {
@@ -436,6 +438,10 @@ std::string WebServer::sensor_json(sensor::Sensor *obj, float value, JsonDetail
         state += " " + obj->get_unit_of_measurement();
     }
     set_json_icon_state_value(root, obj, "sensor-" + obj->get_object_id(), state, value, start_config);
+    if (start_config == DETAIL_ALL) {
+      if (!obj->get_unit_of_measurement().empty())
+        root["uom"] = obj->get_unit_of_measurement();
+    }
   });
 }
 #endif
@@ -529,7 +535,8 @@ void WebServer::on_binary_sensor_update(binary_sensor::BinarySensor *obj, bool s
 }
 std::string WebServer::binary_sensor_json(binary_sensor::BinarySensor *obj, bool value, JsonDetail start_config) {
   return json::build_json([obj, value, start_config](JsonObject root) {
-    set_json_state_value(root, obj, "binary_sensor-" + obj->get_object_id(), value ? "ON" : "OFF", value, start_config);
+    set_json_icon_state_value(root, obj, "binary_sensor-" + obj->get_object_id(), value ? "ON" : "OFF", value,
+                              start_config);
   });
 }
 void WebServer::handle_binary_sensor_request(AsyncWebServerRequest *request, const UrlMatch &match) {
@@ -548,7 +555,8 @@ void WebServer::handle_binary_sensor_request(AsyncWebServerRequest *request, con
 void WebServer::on_fan_update(fan::Fan *obj) { this->events_.send(this->fan_json(obj, DETAIL_STATE).c_str(), "state"); }
 std::string WebServer::fan_json(fan::Fan *obj, JsonDetail start_config) {
   return json::build_json([obj, start_config](JsonObject root) {
-    set_json_state_value(root, obj, "fan-" + obj->get_object_id(), obj->state ? "ON" : "OFF", obj->state, start_config);
+    set_json_icon_state_value(root, obj, "fan-" + obj->get_object_id(), obj->state ? "ON" : "OFF", obj->state,
+                              start_config);
     const auto traits = obj->get_traits();
     if (traits.supports_speed()) {
       root["speed_level"] = obj->speed;
@@ -773,8 +781,8 @@ void WebServer::handle_cover_request(AsyncWebServerRequest *request, const UrlMa
 }
 std::string WebServer::cover_json(cover::Cover *obj, JsonDetail start_config) {
   return json::build_json([obj, start_config](JsonObject root) {
-    set_json_state_value(root, obj, "cover-" + obj->get_object_id(), obj->is_fully_closed() ? "CLOSED" : "OPEN",
-                         obj->position, start_config);
+    set_json_icon_state_value(root, obj, "cover-" + obj->get_object_id(), obj->is_fully_closed() ? "CLOSED" : "OPEN",
+                              obj->position, start_config);
     root["current_operation"] = cover::cover_operation_to_str(obj->current_operation);
 
     if (obj->get_traits().get_supports_tilt())
@@ -824,6 +832,8 @@ std::string WebServer::number_json(number::Number *obj, float value, JsonDetail
       root["max_value"] = obj->traits.get_max_value();
       root["step"] = obj->traits.get_step();
       root["mode"] = (int) obj->traits.get_mode();
+      if (!obj->traits.get_unit_of_measurement().empty())
+        root["uom"] = obj->traits.get_unit_of_measurement();
     }
     if (std::isnan(value)) {
       root["value"] = "\"NaN\"";
@@ -930,7 +940,7 @@ void WebServer::handle_select_request(AsyncWebServerRequest *request, const UrlM
 }
 std::string WebServer::select_json(select::Select *obj, const std::string &value, JsonDetail start_config) {
   return json::build_json([obj, value, start_config](JsonObject root) {
-    set_json_state_value(root, obj, "select-" + obj->get_object_id(), value, value, start_config);
+    set_json_icon_state_value(root, obj, "select-" + obj->get_object_id(), value, value, start_config);
     if (start_config == DETAIL_ALL) {
       JsonArray opt = root.createNestedArray("option");
       for (auto &option : obj->traits.get_options()) {

From 6583026e1473f4e5246f850b813d3999ade49f95 Mon Sep 17 00:00:00 2001
From: NP v/d Spek <github_mail@lumensoft.nl>
Date: Sun, 24 Dec 2023 14:54:53 +0100
Subject: [PATCH 71/75] tt21100: restore init read (#6008)

---
 esphome/components/tt21100/touchscreen/tt21100.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/esphome/components/tt21100/touchscreen/tt21100.cpp b/esphome/components/tt21100/touchscreen/tt21100.cpp
index 6b5cba74cd..ba4b0ee02d 100644
--- a/esphome/components/tt21100/touchscreen/tt21100.cpp
+++ b/esphome/components/tt21100/touchscreen/tt21100.cpp
@@ -64,6 +64,9 @@ void TT21100Touchscreen::setup() {
   // Update display dimensions if they were updated during display setup
   this->x_raw_max_ = this->get_width_();
   this->y_raw_max_ = this->get_height_();
+
+  // Trigger initial read to activate the interrupt
+  this->store_.touched = true;
 }
 
 void TT21100Touchscreen::update_touches() {

From fe15d993f9b79a59531fb0cf46fe02be7514316e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 24 Dec 2023 03:56:31 -1000
Subject: [PATCH 72/75] dashboard: Fix file writes on Windows (#6013)

---
 esphome/dashboard/util/file.py    | 10 +++++++++-
 tests/dashboard/util/test_file.py |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/esphome/dashboard/util/file.py b/esphome/dashboard/util/file.py
index 5f3c5f5f1b..661d5f34cf 100644
--- a/esphome/dashboard/util/file.py
+++ b/esphome/dashboard/util/file.py
@@ -30,6 +30,7 @@ def write_file(
     """
 
     tmp_filename = ""
+    missing_fchmod = False
     try:
         # Modern versions of Python tempfile create this file with mode 0o600
         with tempfile.NamedTemporaryFile(
@@ -38,8 +39,15 @@ def write_file(
             fdesc.write(utf8_data)
             tmp_filename = fdesc.name
             if not private:
-                os.fchmod(fdesc.fileno(), 0o644)
+                try:
+                    os.fchmod(fdesc.fileno(), 0o644)
+                except AttributeError:
+                    # os.fchmod is not available on Windows
+                    missing_fchmod = True
+
         os.replace(tmp_filename, filename)
+        if missing_fchmod:
+            os.chmod(filename, 0o644)
     finally:
         if os.path.exists(tmp_filename):
             try:
diff --git a/tests/dashboard/util/test_file.py b/tests/dashboard/util/test_file.py
index 89e6b97086..270ab565f1 100644
--- a/tests/dashboard/util/test_file.py
+++ b/tests/dashboard/util/test_file.py
@@ -13,7 +13,7 @@ def test_write_utf8_file(tmp_path: Path) -> None:
     assert tmp_path.joinpath("foo.txt").read_text() == "foo"
 
     with pytest.raises(OSError):
-        write_utf8_file(Path("/not-writable"), "bar")
+        write_utf8_file(Path("/dev/not-writable"), "bar")
 
 
 def test_write_file(tmp_path: Path) -> None:

From de6fc6b1dd2ea4edf077eb5e478d69946ea513e3 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 24 Dec 2023 03:57:15 -1000
Subject: [PATCH 73/75] Fix docker builds (#6012)

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7c162fc316..468124e3ed 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -34,7 +34,7 @@ RUN \
         python3-wheel=0.38.4-2 \
         iputils-ping=3:20221126-1 \
         git=1:2.39.2-1.1 \
-        curl=7.88.1-10+deb12u4 \
+        curl=7.88.1-10+deb12u5 \
         openssh-client=1:9.2p1-2+deb12u1 \
         python3-cffi=1.15.1-5 \
         libcairo2=1.16.0-7 \

From 46fc37b6910491cefb3bb76c438db1ee05643c76 Mon Sep 17 00:00:00 2001
From: NP v/d Spek <github_mail@lumensoft.nl>
Date: Sun, 24 Dec 2023 14:58:27 +0100
Subject: [PATCH 74/75] Display: fix class inherence in Python script (#6009)

---
 esphome/components/display/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/display/__init__.py b/esphome/components/display/__init__.py
index 9f4e922a37..91f10c5458 100644
--- a/esphome/components/display/__init__.py
+++ b/esphome/components/display/__init__.py
@@ -18,8 +18,8 @@ from esphome.core import coroutine_with_priority
 IS_PLATFORM_COMPONENT = True
 
 display_ns = cg.esphome_ns.namespace("display")
-Display = display_ns.class_("Display")
-DisplayBuffer = display_ns.class_("DisplayBuffer")
+Display = display_ns.class_("Display", cg.PollingComponent)
+DisplayBuffer = display_ns.class_("DisplayBuffer", Display)
 DisplayPage = display_ns.class_("DisplayPage")
 DisplayPagePtr = DisplayPage.operator("ptr")
 DisplayRef = Display.operator("ref")

From 93ac765425e3e57b5ee14df804c6ce24d5331f5a Mon Sep 17 00:00:00 2001
From: Fabian <Fabian-Schmidt@users.noreply.github.com>
Date: Sun, 24 Dec 2023 20:16:53 +0100
Subject: [PATCH 75/75] [Touchscreen] Add expire of touch record. (#5986)

* Add expire of touch record.

* Implement suggested changes.

* Alternative implementation to detect touch release.

* add `cancel_timeout`.

* Add touch timeout as configurable element.

---------

Co-authored-by: Your Name <you@example.com>
---
 .../lilygo_t5_47/touchscreen/__init__.py      |  2 +-
 esphome/components/touchscreen/__init__.py    | 42 ++++++++++++-------
 .../components/touchscreen/touchscreen.cpp    |  8 ++++
 esphome/components/touchscreen/touchscreen.h  |  2 +
 4 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/esphome/components/lilygo_t5_47/touchscreen/__init__.py b/esphome/components/lilygo_t5_47/touchscreen/__init__.py
index 01b03c807f..17f7262785 100644
--- a/esphome/components/lilygo_t5_47/touchscreen/__init__.py
+++ b/esphome/components/lilygo_t5_47/touchscreen/__init__.py
@@ -18,7 +18,7 @@ LilygoT547Touchscreen = lilygo_t5_47_ns.class_(
 
 CONF_LILYGO_T5_47_TOUCHSCREEN_ID = "lilygo_t5_47_touchscreen_id"
 
-CONFIG_SCHEMA = touchscreen.TOUCHSCREEN_SCHEMA.extend(
+CONFIG_SCHEMA = touchscreen.touchscreen_schema("250ms").extend(
     cv.Schema(
         {
             cv.GenerateID(): cv.declare_id(LilygoT547Touchscreen),
diff --git a/esphome/components/touchscreen/__init__.py b/esphome/components/touchscreen/__init__.py
index bc09c6364d..99aee5c9fb 100644
--- a/esphome/components/touchscreen/__init__.py
+++ b/esphome/components/touchscreen/__init__.py
@@ -24,6 +24,7 @@ CONF_DISPLAY = "display"
 CONF_TOUCHSCREEN_ID = "touchscreen_id"
 CONF_REPORT_INTERVAL = "report_interval"  # not used yet:
 CONF_ON_UPDATE = "on_update"
+CONF_TOUCH_TIMEOUT = "touch_timeout"
 
 CONF_MIRROR_X = "mirror_x"
 CONF_MIRROR_Y = "mirror_y"
@@ -31,21 +32,29 @@ CONF_SWAP_XY = "swap_xy"
 CONF_TRANSFORM = "transform"
 
 
-TOUCHSCREEN_SCHEMA = cv.Schema(
-    {
-        cv.GenerateID(CONF_DISPLAY): cv.use_id(display.Display),
-        cv.Optional(CONF_TRANSFORM): cv.Schema(
-            {
-                cv.Optional(CONF_SWAP_XY, default=False): cv.boolean,
-                cv.Optional(CONF_MIRROR_X, default=False): cv.boolean,
-                cv.Optional(CONF_MIRROR_Y, default=False): cv.boolean,
-            }
-        ),
-        cv.Optional(CONF_ON_TOUCH): automation.validate_automation(single=True),
-        cv.Optional(CONF_ON_UPDATE): automation.validate_automation(single=True),
-        cv.Optional(CONF_ON_RELEASE): automation.validate_automation(single=True),
-    }
-).extend(cv.polling_component_schema("50ms"))
+def touchscreen_schema(default_touch_timeout):
+    return cv.Schema(
+        {
+            cv.GenerateID(CONF_DISPLAY): cv.use_id(display.Display),
+            cv.Optional(CONF_TRANSFORM): cv.Schema(
+                {
+                    cv.Optional(CONF_SWAP_XY, default=False): cv.boolean,
+                    cv.Optional(CONF_MIRROR_X, default=False): cv.boolean,
+                    cv.Optional(CONF_MIRROR_Y, default=False): cv.boolean,
+                }
+            ),
+            cv.Optional(CONF_TOUCH_TIMEOUT, default=default_touch_timeout): cv.All(
+                cv.positive_time_period_milliseconds,
+                cv.Range(max=cv.TimePeriod(milliseconds=65535)),
+            ),
+            cv.Optional(CONF_ON_TOUCH): automation.validate_automation(single=True),
+            cv.Optional(CONF_ON_UPDATE): automation.validate_automation(single=True),
+            cv.Optional(CONF_ON_RELEASE): automation.validate_automation(single=True),
+        }
+    ).extend(cv.polling_component_schema("50ms"))
+
+
+TOUCHSCREEN_SCHEMA = touchscreen_schema(cv.UNDEFINED)
 
 
 async def register_touchscreen(var, config):
@@ -54,6 +63,9 @@ async def register_touchscreen(var, config):
     disp = await cg.get_variable(config[CONF_DISPLAY])
     cg.add(var.set_display(disp))
 
+    if CONF_TOUCH_TIMEOUT in config:
+        cg.add(var.set_touch_timeout(config[CONF_TOUCH_TIMEOUT]))
+
     if CONF_TRANSFORM in config:
         transform = config[CONF_TRANSFORM]
         cg.add(var.set_swap_xy(transform[CONF_SWAP_XY]))
diff --git a/esphome/components/touchscreen/touchscreen.cpp b/esphome/components/touchscreen/touchscreen.cpp
index f095c2af8c..18a4230197 100644
--- a/esphome/components/touchscreen/touchscreen.cpp
+++ b/esphome/components/touchscreen/touchscreen.cpp
@@ -47,6 +47,11 @@ void Touchscreen::loop() {
     } else {
       this->store_.touched = false;
       this->defer([this]() { this->send_touches_(); });
+      if (this->touch_timeout_ > 0) {
+        // Simulate a touch after <this->touch_timeout_> ms. This will reset any existing timeout operation.
+        // This is to detect touch release.
+        this->set_timeout(TAG, this->touch_timeout_, [this]() { this->store_.touched = true; });
+      }
     }
   }
 }
@@ -90,6 +95,9 @@ void Touchscreen::add_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_r
 
 void Touchscreen::send_touches_() {
   if (!this->is_touched_) {
+    if (this->touch_timeout_ > 0) {
+      this->cancel_timeout(TAG);
+    }
     this->release_trigger_.trigger();
     for (auto *listener : this->touch_listeners_)
       listener->release();
diff --git a/esphome/components/touchscreen/touchscreen.h b/esphome/components/touchscreen/touchscreen.h
index 74747c589c..06aff68f07 100644
--- a/esphome/components/touchscreen/touchscreen.h
+++ b/esphome/components/touchscreen/touchscreen.h
@@ -46,6 +46,7 @@ class Touchscreen : public PollingComponent {
   void set_display(display::Display *display) { this->display_ = display; }
   display::Display *get_display() const { return this->display_; }
 
+  void set_touch_timeout(uint16_t val) { this->touch_timeout_ = val; }
   void set_mirror_x(bool invert_x) { this->invert_x_ = invert_x; }
   void set_mirror_y(bool invert_y) { this->invert_y_ = invert_y; }
   void set_swap_xy(bool swap) { this->swap_x_y_ = swap; }
@@ -100,6 +101,7 @@ class Touchscreen : public PollingComponent {
   display::Display *display_{nullptr};
 
   int16_t x_raw_min_{0}, x_raw_max_{0}, y_raw_min_{0}, y_raw_max_{0};
+  uint16_t touch_timeout_{0};
   bool invert_x_{false}, invert_y_{false}, swap_x_y_{false};
 
   Trigger<TouchPoint, const TouchPoints_t &> touch_trigger_;