From 8e4b9c3c1e38f8aead90f19aa726143e86aad1a6 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Mon, 27 Nov 2023 13:45:26 +1300
Subject: [PATCH 01/34] Voice Assistant improvements (#5827)

---
 .../i2s_audio/speaker/i2s_audio_speaker.cpp   |   2 +
 .../i2s_audio/speaker/i2s_audio_speaker.h     |   2 +
 esphome/components/speaker/speaker.h          |   2 +
 .../voice_assistant/voice_assistant.cpp       | 108 +++++++++++-------
 .../voice_assistant/voice_assistant.h         |   3 +
 5 files changed, 78 insertions(+), 39 deletions(-)

diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index ed13e6b458..e729cdf954 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -220,6 +220,8 @@ size_t I2SAudioSpeaker::play(const uint8_t *data, size_t length) {
   return index;
 }
 
+bool I2SAudioSpeaker::has_buffered_data() const { return uxQueueMessagesWaiting(this->buffer_queue_) > 0; }
+
 }  // namespace i2s_audio
 }  // namespace esphome
 
diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
index b075722e1b..20c36a69d3 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
@@ -56,6 +56,8 @@ class I2SAudioSpeaker : public Component, public speaker::Speaker, public I2SAud
 
   size_t play(const uint8_t *data, size_t length) override;
 
+  bool has_buffered_data() const override;
+
  protected:
   void start_();
   // void stop_();
diff --git a/esphome/components/speaker/speaker.h b/esphome/components/speaker/speaker.h
index 3f520e3c5e..b494873160 100644
--- a/esphome/components/speaker/speaker.h
+++ b/esphome/components/speaker/speaker.h
@@ -18,6 +18,8 @@ class Speaker {
   virtual void start() = 0;
   virtual void stop() = 0;
 
+  virtual bool has_buffered_data() const = 0;
+
   bool is_running() const { return this->state_ == STATE_RUNNING; }
 
  protected:
diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index 9b13a71039..c0e706305d 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -273,28 +273,27 @@ void VoiceAssistant::loop() {
       bool playing = false;
 #ifdef USE_SPEAKER
       if (this->speaker_ != nullptr) {
+        ssize_t received_len = 0;
         if (this->speaker_buffer_index_ + RECEIVE_SIZE < SPEAKER_BUFFER_SIZE) {
-          auto len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE);
-          if (len > 0) {
-            this->speaker_buffer_index_ += len;
-            this->speaker_buffer_size_ += len;
+          received_len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE);
+          if (received_len > 0) {
+            this->speaker_buffer_index_ += received_len;
+            this->speaker_buffer_size_ += received_len;
+            this->speaker_bytes_received_ += received_len;
           }
         } else {
-          ESP_LOGW(TAG, "Receive buffer full");
-        }
-        if (this->speaker_buffer_size_ > 0) {
-          size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
-          if (written > 0) {
-            memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written);
-            this->speaker_buffer_size_ -= written;
-            this->speaker_buffer_index_ -= written;
-            this->set_timeout("speaker-timeout", 2000, [this]() { this->speaker_->stop(); });
-          } else {
-            ESP_LOGW(TAG, "Speaker buffer full");
-          }
+          ESP_LOGD(TAG, "Receive buffer full");
         }
+        // Build a small buffer of audio before sending to the speaker
+        if (this->speaker_bytes_received_ > RECEIVE_SIZE * 4)
+          this->write_speaker_();
         if (this->wait_for_stream_end_) {
           this->cancel_timeout("playing");
+          if (this->stream_ended_ && received_len < 0) {
+            ESP_LOGD(TAG, "End of audio stream received");
+            this->cancel_timeout("speaker-timeout");
+            this->set_state_(State::RESPONSE_FINISHED, State::RESPONSE_FINISHED);
+          }
           break;  // We dont want to timeout here as the STREAM_END event will take care of that.
         }
         playing = this->speaker_->is_running();
@@ -316,14 +315,26 @@ void VoiceAssistant::loop() {
     case State::RESPONSE_FINISHED: {
 #ifdef USE_SPEAKER
       if (this->speaker_ != nullptr) {
+        if (this->speaker_buffer_size_ > 0) {
+          this->write_speaker_();
+          break;
+        }
+        if (this->speaker_->has_buffered_data() || this->speaker_->is_running()) {
+          break;
+        }
+        ESP_LOGD(TAG, "Speaker has finished outputting all audio");
         this->speaker_->stop();
         this->cancel_timeout("speaker-timeout");
         this->cancel_timeout("playing");
         this->speaker_buffer_size_ = 0;
         this->speaker_buffer_index_ = 0;
+        this->speaker_bytes_received_ = 0;
         memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE);
+        this->wait_for_stream_end_ = false;
+        this->stream_ended_ = false;
+
+        this->tts_stream_end_trigger_->trigger();
       }
-      this->wait_for_stream_end_ = false;
 #endif
       this->set_state_(State::IDLE, State::IDLE);
       break;
@@ -333,6 +344,20 @@ void VoiceAssistant::loop() {
   }
 }
 
+void VoiceAssistant::write_speaker_() {
+  if (this->speaker_buffer_size_ > 0) {
+    size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
+    if (written > 0) {
+      memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written);
+      this->speaker_buffer_size_ -= written;
+      this->speaker_buffer_index_ -= written;
+      this->set_timeout("speaker-timeout", 5000, [this]() { this->speaker_->stop(); });
+    } else {
+      ESP_LOGD(TAG, "Speaker buffer full, trying again next loop");
+    }
+  }
+}
+
 void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) {
   if (!subscribe) {
     if (this->api_client_ == nullptr || client != this->api_client_) {
@@ -503,21 +528,20 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
   switch (msg.event_type) {
     case api::enums::VOICE_ASSISTANT_RUN_START:
       ESP_LOGD(TAG, "Assist Pipeline running");
-      this->start_trigger_->trigger();
+      this->defer([this]() { this->start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_WAKE_WORD_START:
       break;
     case api::enums::VOICE_ASSISTANT_WAKE_WORD_END: {
       ESP_LOGD(TAG, "Wake word detected");
-      this->wake_word_detected_trigger_->trigger();
+      this->defer([this]() { this->wake_word_detected_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_START:
       ESP_LOGD(TAG, "STT started");
-      this->listening_trigger_->trigger();
+      this->defer([this]() { this->listening_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_STT_END: {
-      this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE);
       std::string text;
       for (auto arg : msg.data) {
         if (arg.name == "text") {
@@ -529,12 +553,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Speech recognised as: \"%s\"", text.c_str());
-      this->stt_end_trigger_->trigger(text);
+      this->defer([this, text]() { this->stt_end_trigger_->trigger(text); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_INTENT_START:
       ESP_LOGD(TAG, "Intent started");
-      this->intent_start_trigger_->trigger();
+      this->defer([this]() { this->intent_start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_INTENT_END: {
       for (auto arg : msg.data) {
@@ -542,7 +566,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
           this->conversation_id_ = std::move(arg.value);
         }
       }
-      this->intent_end_trigger_->trigger();
+      this->defer([this]() { this->intent_end_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_START: {
@@ -557,10 +581,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Response: \"%s\"", text.c_str());
-      this->tts_start_trigger_->trigger(text);
+      this->defer([this, text]() {
+        this->tts_start_trigger_->trigger(text);
 #ifdef USE_SPEAKER
-      this->speaker_->start();
+        this->speaker_->start();
 #endif
+      });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_END: {
@@ -575,14 +601,16 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       }
       ESP_LOGD(TAG, "Response URL: \"%s\"", url.c_str());
+      this->defer([this, url]() {
 #ifdef USE_MEDIA_PLAYER
-      if (this->media_player_ != nullptr) {
-        this->media_player_->make_call().set_media_url(url).perform();
-      }
+        if (this->media_player_ != nullptr) {
+          this->media_player_->make_call().set_media_url(url).perform();
+        }
 #endif
+        this->tts_end_trigger_->trigger(url);
+      });
       State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE;
       this->set_state_(new_state, new_state);
-      this->tts_end_trigger_->trigger(url);
       break;
     }
     case api::enums::VOICE_ASSISTANT_RUN_END: {
@@ -599,7 +627,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
           this->set_state_(State::IDLE, State::IDLE);
         }
       }
-      this->end_trigger_->trigger();
+      this->defer([this]() { this->end_trigger_->trigger(); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_ERROR: {
@@ -617,8 +645,10 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         return;
       } else if (code == "wake-provider-missing" || code == "wake-engine-missing") {
         // Wake word is not set up or not ready on Home Assistant so stop and do not retry until user starts again.
-        this->request_stop();
-        this->error_trigger_->trigger(code, message);
+        this->defer([this, code, message]() {
+          this->request_stop();
+          this->error_trigger_->trigger(code, message);
+        });
         return;
       }
       ESP_LOGE(TAG, "Error: %s - %s", code.c_str(), message.c_str());
@@ -626,32 +656,32 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         this->signal_stop_();
         this->set_state_(State::STOP_MICROPHONE, State::IDLE);
       }
-      this->error_trigger_->trigger(code, message);
+      this->defer([this, code, message]() { this->error_trigger_->trigger(code, message); });
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_START: {
 #ifdef USE_SPEAKER
       this->wait_for_stream_end_ = true;
       ESP_LOGD(TAG, "TTS stream start");
-      this->tts_stream_start_trigger_->trigger();
+      this->defer([this] { this->tts_stream_start_trigger_->trigger(); });
 #endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_END: {
-      this->set_state_(State::RESPONSE_FINISHED, State::IDLE);
 #ifdef USE_SPEAKER
+      this->stream_ended_ = true;
       ESP_LOGD(TAG, "TTS stream end");
-      this->tts_stream_end_trigger_->trigger();
 #endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_VAD_START:
       ESP_LOGD(TAG, "Starting STT by VAD");
-      this->stt_vad_start_trigger_->trigger();
+      this->defer([this]() { this->stt_vad_start_trigger_->trigger(); });
       break;
     case api::enums::VOICE_ASSISTANT_STT_VAD_END:
       ESP_LOGD(TAG, "STT by VAD end");
-      this->stt_vad_end_trigger_->trigger();
+      this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE);
+      this->defer([this]() { this->stt_vad_end_trigger_->trigger(); });
       break;
     default:
       ESP_LOGD(TAG, "Unhandled event type: %d", msg.event_type);
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index f6dcd1c563..66bf4c3c57 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -156,11 +156,14 @@ class VoiceAssistant : public Component {
 
   microphone::Microphone *mic_{nullptr};
 #ifdef USE_SPEAKER
+  void write_speaker_();
   speaker::Speaker *speaker_{nullptr};
   uint8_t *speaker_buffer_;
   size_t speaker_buffer_index_{0};
   size_t speaker_buffer_size_{0};
+  size_t speaker_bytes_received_{0};
   bool wait_for_stream_end_{false};
+  bool stream_ended_{false};
 #endif
 #ifdef USE_MEDIA_PLAYER
   media_player::MediaPlayer *media_player_{nullptr};

From ff97639f791963bb9e5f35318be960a100f024b0 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:43:41 +1300
Subject: [PATCH 02/34] Fix missing include in remote_base (#5843)

---
 esphome/components/remote_base/remote_base.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp
index 40c699e8ea..095f95053f 100644
--- a/esphome/components/remote_base/remote_base.cpp
+++ b/esphome/components/remote_base/remote_base.cpp
@@ -1,6 +1,8 @@
 #include "remote_base.h"
 #include "esphome/core/log.h"
 
+#include <cinttypes>
+
 namespace esphome {
 namespace remote_base {
 

From 687f5ca6330c7b96e92823bd28f9d68731404152 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:57:40 +1300
Subject: [PATCH 03/34] Add 'voice_assistant.connected' condition (#5845)

---
 esphome/components/voice_assistant/__init__.py       | 12 ++++++++++++
 esphome/components/voice_assistant/voice_assistant.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py
index d05f39072c..59aef901f2 100644
--- a/esphome/components/voice_assistant/__init__.py
+++ b/esphome/components/voice_assistant/__init__.py
@@ -57,6 +57,9 @@ StopAction = voice_assistant_ns.class_(
 IsRunningCondition = voice_assistant_ns.class_(
     "IsRunningCondition", automation.Condition, cg.Parented.template(VoiceAssistant)
 )
+ConnectedCondition = voice_assistant_ns.class_(
+    "ConnectedCondition", automation.Condition, cg.Parented.template(VoiceAssistant)
+)
 
 
 def tts_stream_validate(config):
@@ -298,3 +301,12 @@ async def voice_assistant_is_running_to_code(config, condition_id, template_arg,
     var = cg.new_Pvariable(condition_id, template_arg)
     await cg.register_parented(var, config[CONF_ID])
     return var
+
+
+@register_condition(
+    "voice_assistant.connected", ConnectedCondition, VOICE_ASSISTANT_ACTION_SCHEMA
+)
+async def voice_assistant_connected_to_code(config, condition_id, template_arg, args):
+    var = cg.new_Pvariable(condition_id, template_arg)
+    await cg.register_parented(var, config[CONF_ID])
+    return var
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index 66bf4c3c57..f9325dff54 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -222,6 +222,11 @@ template<typename... Ts> class IsRunningCondition : public Condition<Ts...>, pub
   bool check(Ts... x) override { return this->parent_->is_running() || this->parent_->is_continuous(); }
 };
 
+template<typename... Ts> class ConnectedCondition : public Condition<Ts...>, public Parented<VoiceAssistant> {
+ public:
+  bool check(Ts... x) override { return this->parent_->get_api_connection() != nullptr; }
+};
+
 extern VoiceAssistant *global_voice_assistant;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 }  // namespace voice_assistant

From 28a3cddde374a07c880622cd13095ce7dfbbec4d Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 11:14:26 +1300
Subject: [PATCH 04/34] Bump version to 2023.11.5

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 5596cbcd36..2e693d0ea0 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.4"
+__version__ = "2023.11.5"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 175f00f41bc8305cfa65bef7142ba7cdd4ee8140 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 12:28:24 +1300
Subject: [PATCH 05/34] Fix write_speaker without speaker in config (#5847)

---
 esphome/components/voice_assistant/voice_assistant.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index c0e706305d..29fc664342 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -344,6 +344,7 @@ void VoiceAssistant::loop() {
   }
 }
 
+#ifdef USE_SPEAKER
 void VoiceAssistant::write_speaker_() {
   if (this->speaker_buffer_size_ > 0) {
     size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
@@ -357,6 +358,7 @@ void VoiceAssistant::write_speaker_() {
     }
   }
 }
+#endif
 
 void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) {
   if (!subscribe) {

From ed9fd173a9f791d0d77ef28e68117fa04c6f8e23 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 28 Nov 2023 12:31:55 +1300
Subject: [PATCH 06/34] Bump version to 2023.11.6

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 2e693d0ea0..6f8d83170e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.5"
+__version__ = "2023.11.6"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 8cc44766e6acc236bf2c7e70e24a4c5531a9ffba Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 08:30:45 +0900
Subject: [PATCH 07/34] Bump version to 2023.12.0b1

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 101a690db5..b8495a3a7c 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0-dev"
+__version__ = "2023.12.0b1"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 3c3ac920380dd07c38fa11860da21e944f2a1f01 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 13:24:16 +0900
Subject: [PATCH 08/34] Allow use of CDC/JTAG loggers on esp32 variants with
 Arduino (#4658)

* Allow use of CDC/JTAG loggers on esp32 variants with Arduino

* Only on s2/s3

* Separate C3 from S2/S3

* C code builds & runs correctly, still needs work though

* Works on S2

* It works!

* Remove unnecessary header

---------

Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
---
 esphome/components/logger/__init__.py |  2 --
 esphome/components/logger/logger.cpp  | 32 ++++++++++++++++++++++++---
 esphome/components/logger/logger.h    |  2 --
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py
index e431997276..11a7f996f0 100644
--- a/esphome/components/logger/__init__.py
+++ b/esphome/components/logger/__init__.py
@@ -124,8 +124,6 @@ is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
 
 def uart_selection(value):
     if CORE.is_esp32:
-        if value.upper() in ESP_IDF_UARTS and not CORE.using_esp_idf:
-            raise cv.Invalid(f"Only esp-idf framework supports {value}.")
         variant = get_esp32_variant()
         if variant in UART_SELECTION_ESP32:
             return cv.one_of(*UART_SELECTION_ESP32[variant], upper=True)(value)
diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp
index 2d2524b5f4..e0ca0806cb 100644
--- a/esphome/components/logger/logger.cpp
+++ b/esphome/components/logger/logger.cpp
@@ -236,8 +236,13 @@ void Logger::pre_setup() {
         this->hw_serial_ = &Serial1;
         Serial1.begin(this->baud_rate_);
 #else
+#if ARDUINO_USB_CDC_ON_BOOT
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);
+#else
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif
 #endif
 #ifdef USE_ESP8266
         if (this->uart_ == UART_SELECTION_UART0_SWAP) {
@@ -265,12 +270,35 @@ void Logger::pre_setup() {
         Serial2.begin(this->baud_rate_);
         break;
 #endif
+#if defined(USE_ESP32) && \
+    (defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3))
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+      case UART_SELECTION_USB_CDC:
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+#if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32S3)
+      case UART_SELECTION_USB_SERIAL_JTAG:
+#endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32S3
+#ifdef USE_ESP32_VARIANT_ESP32C3
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif  // USE_ESP32_VARIANT_ESP32C3
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if ARDUINO_USB_CDC_ON_BOOT
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#else
+        this->hw_serial_ = &Serial;
+        Serial.begin(this->baud_rate_);
+#endif  // ARDUINO_USB_CDC_ON_BOOT
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+        break;
+#endif  // USE_ESP32 && (USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3)
 #ifdef USE_RP2040
       case UART_SELECTION_USB_CDC:
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);
         break;
-#endif
+#endif  // USE_RP2040
     }
 #endif  // USE_ARDUINO
 #ifdef USE_ESP_IDF
@@ -393,14 +421,12 @@ const char *const UART_SELECTIONS[] = {
     "UART2",
 #endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARINT_ESP32C6 && !USE_ESP32_VARIANT_ESP32S2 &&
         // !USE_ESP32_VARIANT_ESP32S3 && !USE_ESP32_VARIANT_ESP32H2
-#if defined(USE_ESP_IDF)
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
     "USB_CDC",
 #endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
 #if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32S3)
     "USB_SERIAL_JTAG",
 #endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32S3
-#endif  // USE_ESP_IDF
 };
 #endif  // USE_ESP32
 #ifdef USE_ESP8266
diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h
index 3816b1dd14..68efc056df 100644
--- a/esphome/components/logger/logger.h
+++ b/esphome/components/logger/logger.h
@@ -45,7 +45,6 @@ enum UARTSelection {
   UART_SELECTION_UART2,
 #endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARIANT_ESP32C6 && !USE_ESP32_VARIANT_ESP32S2 &&
         // !USE_ESP32_VARIANT_ESP32S3 && !USE_ESP32_VARIANT_ESP32H2
-#ifdef USE_ESP_IDF
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
   UART_SELECTION_USB_CDC,
 #endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
@@ -54,7 +53,6 @@ enum UARTSelection {
   UART_SELECTION_USB_SERIAL_JTAG,
 #endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C6 || USE_ESP32_VARIANT_ESP32S3 ||
         // USE_ESP32_VARIANT_ESP32H2
-#endif  // USE_ESP_IDF
 #endif  // USE_ESP32
 #ifdef USE_ESP8266
   UART_SELECTION_UART0_SWAP,

From 3e475c21ff9664c95b347375fe43b5647afdad03 Mon Sep 17 00:00:00 2001
From: Fabian <Fabian-Schmidt@users.noreply.github.com>
Date: Thu, 14 Dec 2023 05:47:31 +0100
Subject: [PATCH 09/34] [Logger] ESP32 S3 serial logger (#4853)

* Add support for ESP32 S3 logger.

* fix default

* Remove cpp & h changes to combine with PR #4658

* Not enough attention to details.

* Add build flag

* Validation fix

* Fix validation for real this time

---------

Co-authored-by: Your Name <you@example.com>
Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
---
 esphome/components/logger/__init__.py | 10 ++++-
 esphome/config_validation.py          | 53 +++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py
index 11a7f996f0..be302bd489 100644
--- a/esphome/components/logger/__init__.py
+++ b/esphome/components/logger/__init__.py
@@ -97,7 +97,7 @@ UART_SELECTION_LIBRETINY = {
     COMPONENT_RTL87XX: [DEFAULT, UART0, UART1, UART2],
 }
 
-ESP_IDF_UARTS = [USB_CDC, USB_SERIAL_JTAG]
+ESP_ARDUINO_UNSUPPORTED_USB_UARTS = [USB_SERIAL_JTAG]
 
 UART_SELECTION_RP2040 = [USB_CDC, UART0, UART1]
 
@@ -124,6 +124,8 @@ is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
 
 def uart_selection(value):
     if CORE.is_esp32:
+        if CORE.using_arduino and value.upper() in ESP_ARDUINO_UNSUPPORTED_USB_UARTS:
+            raise cv.Invalid(f"Arduino framework does not support {value}.")
         variant = get_esp32_variant()
         if variant in UART_SELECTION_ESP32:
             return cv.one_of(*UART_SELECTION_ESP32[variant], upper=True)(value)
@@ -169,6 +171,8 @@ CONFIG_SCHEMA = cv.All(
                 CONF_HARDWARE_UART,
                 esp8266=UART0,
                 esp32=UART0,
+                esp32_s2=USB_CDC,
+                esp32_s3=USB_CDC,
                 rp2040=USB_CDC,
                 bk72xx=DEFAULT,
                 rtl87xx=DEFAULT,
@@ -256,6 +260,10 @@ async def to_code(config):
     if config.get(CONF_ESP8266_STORE_LOG_STRINGS_IN_FLASH):
         cg.add_build_flag("-DUSE_STORE_LOG_STR_IN_FLASH")
 
+    if CORE.using_arduino:
+        if config[CONF_HARDWARE_UART] == USB_CDC:
+            cg.add_build_flag("-DARDUINO_USB_CDC_ON_BOOT=1")
+
     if CORE.using_esp_idf:
         if config[CONF_HARDWARE_UART] == USB_CDC:
             add_idf_sdkconfig_option("CONFIG_ESP_CONSOLE_USB_CDC", True)
diff --git a/esphome/config_validation.py b/esphome/config_validation.py
index ad2ee11512..fdbe7d6320 100644
--- a/esphome/config_validation.py
+++ b/esphome/config_validation.py
@@ -1528,6 +1528,12 @@ class SplitDefault(Optional):
         esp32=vol.UNDEFINED,
         esp32_arduino=vol.UNDEFINED,
         esp32_idf=vol.UNDEFINED,
+        esp32_s2=vol.UNDEFINED,
+        esp32_s2_arduino=vol.UNDEFINED,
+        esp32_s2_idf=vol.UNDEFINED,
+        esp32_s3=vol.UNDEFINED,
+        esp32_s3_arduino=vol.UNDEFINED,
+        esp32_s3_idf=vol.UNDEFINED,
         rp2040=vol.UNDEFINED,
         bk72xx=vol.UNDEFINED,
         rtl87xx=vol.UNDEFINED,
@@ -1541,6 +1547,26 @@ class SplitDefault(Optional):
         self._esp32_idf_default = vol.default_factory(
             esp32_idf if esp32 is vol.UNDEFINED else esp32
         )
+        self._esp32_s2_arduino_default = vol.default_factory(
+            (esp32_s2_arduino if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s2 is vol.UNDEFINED
+            else esp32_s2
+        )
+        self._esp32_s2_idf_default = vol.default_factory(
+            (esp32_s2_idf if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s2 is vol.UNDEFINED
+            else esp32_s2
+        )
+        self._esp32_s3_arduino_default = vol.default_factory(
+            (esp32_s3_arduino if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s3 is vol.UNDEFINED
+            else esp32_s3
+        )
+        self._esp32_s3_idf_default = vol.default_factory(
+            (esp32_s3_idf if esp32 is vol.UNDEFINED else esp32)
+            if esp32_s3 is vol.UNDEFINED
+            else esp32_s3
+        )
         self._rp2040_default = vol.default_factory(rp2040)
         self._bk72xx_default = vol.default_factory(bk72xx)
         self._rtl87xx_default = vol.default_factory(rtl87xx)
@@ -1550,10 +1576,29 @@ class SplitDefault(Optional):
     def default(self):
         if CORE.is_esp8266:
             return self._esp8266_default
-        if CORE.is_esp32 and CORE.using_arduino:
-            return self._esp32_arduino_default
-        if CORE.is_esp32 and CORE.using_esp_idf:
-            return self._esp32_idf_default
+        if CORE.is_esp32:
+            from esphome.components.esp32 import get_esp32_variant
+            from esphome.components.esp32.const import (
+                VARIANT_ESP32S2,
+                VARIANT_ESP32S3,
+            )
+
+            variant = get_esp32_variant()
+            if variant == VARIANT_ESP32S2:
+                if CORE.using_arduino:
+                    return self._esp32_s2_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_s2_idf_default
+            elif variant == VARIANT_ESP32S3:
+                if CORE.using_arduino:
+                    return self._esp32_s3_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_s3_idf_default
+            else:
+                if CORE.using_arduino:
+                    return self._esp32_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_idf_default
         if CORE.is_rp2040:
             return self._rp2040_default
         if CORE.is_bk72xx:

From f28cf9348e486b3aab5c2b3e6a04f372c4ad4364 Mon Sep 17 00:00:00 2001
From: jochenvg <jochen@vanguyse.net>
Date: Thu, 14 Dec 2023 05:01:01 +0000
Subject: [PATCH 10/34] Support toggle action for template cover (#5917)

---
 esphome/components/template/cover/__init__.py        |  7 +++++++
 esphome/components/template/cover/template_cover.cpp | 10 ++++++++++
 esphome/components/template/cover/template_cover.h   |  4 ++++
 3 files changed, 21 insertions(+)

diff --git a/esphome/components/template/cover/__init__.py b/esphome/components/template/cover/__init__.py
index 8844ddd6ab..43d0be99b4 100644
--- a/esphome/components/template/cover/__init__.py
+++ b/esphome/components/template/cover/__init__.py
@@ -31,6 +31,7 @@ RESTORE_MODES = {
 }
 
 CONF_HAS_POSITION = "has_position"
+CONF_TOGGLE_ACTION = "toggle_action"
 
 CONFIG_SCHEMA = cover.COVER_SCHEMA.extend(
     {
@@ -44,6 +45,7 @@ CONFIG_SCHEMA = cover.COVER_SCHEMA.extend(
         cv.Optional(CONF_STOP_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_TILT_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_TILT_LAMBDA): cv.returning_lambda,
+        cv.Optional(CONF_TOGGLE_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_POSITION_ACTION): automation.validate_automation(single=True),
         cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum(
             RESTORE_MODES, upper=True
@@ -74,6 +76,11 @@ async def to_code(config):
             var.get_stop_trigger(), [], config[CONF_STOP_ACTION]
         )
         cg.add(var.set_has_stop(True))
+    if CONF_TOGGLE_ACTION in config:
+        await automation.build_automation(
+            var.get_toggle_trigger(), [], config[CONF_TOGGLE_ACTION]
+        )
+        cg.add(var.set_has_toggle(True))
     if CONF_TILT_ACTION in config:
         await automation.build_automation(
             var.get_tilt_trigger(), [(float, "tilt")], config[CONF_TILT_ACTION]
diff --git a/esphome/components/template/cover/template_cover.cpp b/esphome/components/template/cover/template_cover.cpp
index b16e439943..2d6c3087ae 100644
--- a/esphome/components/template/cover/template_cover.cpp
+++ b/esphome/components/template/cover/template_cover.cpp
@@ -12,6 +12,7 @@ TemplateCover::TemplateCover()
     : open_trigger_(new Trigger<>()),
       close_trigger_(new Trigger<>),
       stop_trigger_(new Trigger<>()),
+      toggle_trigger_(new Trigger<>()),
       position_trigger_(new Trigger<float>()),
       tilt_trigger_(new Trigger<float>()) {}
 void TemplateCover::setup() {
@@ -68,6 +69,7 @@ float TemplateCover::get_setup_priority() const { return setup_priority::HARDWAR
 Trigger<> *TemplateCover::get_open_trigger() const { return this->open_trigger_; }
 Trigger<> *TemplateCover::get_close_trigger() const { return this->close_trigger_; }
 Trigger<> *TemplateCover::get_stop_trigger() const { return this->stop_trigger_; }
+Trigger<> *TemplateCover::get_toggle_trigger() const { return this->toggle_trigger_; }
 void TemplateCover::dump_config() { LOG_COVER("", "Template Cover", this); }
 void TemplateCover::control(const CoverCall &call) {
   if (call.get_stop()) {
@@ -76,6 +78,12 @@ void TemplateCover::control(const CoverCall &call) {
     this->prev_command_trigger_ = this->stop_trigger_;
     this->publish_state();
   }
+  if (call.get_toggle().has_value()) {
+    this->stop_prev_trigger_();
+    this->toggle_trigger_->trigger();
+    this->prev_command_trigger_ = this->toggle_trigger_;
+    this->publish_state();
+  }
   if (call.get_position().has_value()) {
     auto pos = *call.get_position();
     this->stop_prev_trigger_();
@@ -110,6 +118,7 @@ CoverTraits TemplateCover::get_traits() {
   auto traits = CoverTraits();
   traits.set_is_assumed_state(this->assumed_state_);
   traits.set_supports_stop(this->has_stop_);
+  traits.set_supports_toggle(this->has_toggle_);
   traits.set_supports_position(this->has_position_);
   traits.set_supports_tilt(this->has_tilt_);
   return traits;
@@ -118,6 +127,7 @@ Trigger<float> *TemplateCover::get_position_trigger() const { return this->posit
 Trigger<float> *TemplateCover::get_tilt_trigger() const { return this->tilt_trigger_; }
 void TemplateCover::set_tilt_lambda(std::function<optional<float>()> &&tilt_f) { this->tilt_f_ = tilt_f; }
 void TemplateCover::set_has_stop(bool has_stop) { this->has_stop_ = has_stop; }
+void TemplateCover::set_has_toggle(bool has_toggle) { this->has_toggle_ = has_toggle; }
 void TemplateCover::set_has_position(bool has_position) { this->has_position_ = has_position; }
 void TemplateCover::set_has_tilt(bool has_tilt) { this->has_tilt_ = has_tilt; }
 void TemplateCover::stop_prev_trigger_() {
diff --git a/esphome/components/template/cover/template_cover.h b/esphome/components/template/cover/template_cover.h
index 4ff5caf1db..958c94b0a6 100644
--- a/esphome/components/template/cover/template_cover.h
+++ b/esphome/components/template/cover/template_cover.h
@@ -21,6 +21,7 @@ class TemplateCover : public cover::Cover, public Component {
   Trigger<> *get_open_trigger() const;
   Trigger<> *get_close_trigger() const;
   Trigger<> *get_stop_trigger() const;
+  Trigger<> *get_toggle_trigger() const;
   Trigger<float> *get_position_trigger() const;
   Trigger<float> *get_tilt_trigger() const;
   void set_optimistic(bool optimistic);
@@ -29,6 +30,7 @@ class TemplateCover : public cover::Cover, public Component {
   void set_has_stop(bool has_stop);
   void set_has_position(bool has_position);
   void set_has_tilt(bool has_tilt);
+  void set_has_toggle(bool has_toggle);
   void set_restore_mode(TemplateCoverRestoreMode restore_mode) { restore_mode_ = restore_mode; }
 
   void setup() override;
@@ -50,7 +52,9 @@ class TemplateCover : public cover::Cover, public Component {
   Trigger<> *open_trigger_;
   Trigger<> *close_trigger_;
   bool has_stop_{false};
+  bool has_toggle_{false};
   Trigger<> *stop_trigger_;
+  Trigger<> *toggle_trigger_;
   Trigger<> *prev_command_trigger_{nullptr};
   Trigger<float> *position_trigger_;
   bool has_position_{false};

From 6b5eb7e656440ea03eaa6784172f662b1194bc3b Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 14 Dec 2023 17:33:04 +0900
Subject: [PATCH 11/34] Fix SplitDefault with variants (#5928)

---
 esphome/config_validation.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/esphome/config_validation.py b/esphome/config_validation.py
index fdbe7d6320..7b94608509 100644
--- a/esphome/config_validation.py
+++ b/esphome/config_validation.py
@@ -1518,6 +1518,13 @@ class GenerateID(Optional):
         super().__init__(key, default=lambda: None)
 
 
+def _get_priority_default(*args):
+    for arg in args:
+        if arg is not vol.UNDEFINED:
+            return arg
+    return vol.UNDEFINED
+
+
 class SplitDefault(Optional):
     """Mark this key to have a split default for ESP8266/ESP32."""
 
@@ -1542,30 +1549,22 @@ class SplitDefault(Optional):
         super().__init__(key)
         self._esp8266_default = vol.default_factory(esp8266)
         self._esp32_arduino_default = vol.default_factory(
-            esp32_arduino if esp32 is vol.UNDEFINED else esp32
+            _get_priority_default(esp32, esp32_arduino)
         )
         self._esp32_idf_default = vol.default_factory(
-            esp32_idf if esp32 is vol.UNDEFINED else esp32
+            _get_priority_default(esp32, esp32_idf)
         )
         self._esp32_s2_arduino_default = vol.default_factory(
-            (esp32_s2_arduino if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s2 is vol.UNDEFINED
-            else esp32_s2
+            _get_priority_default(esp32_s2, esp32, esp32_s2_arduino, esp32_arduino)
         )
         self._esp32_s2_idf_default = vol.default_factory(
-            (esp32_s2_idf if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s2 is vol.UNDEFINED
-            else esp32_s2
+            _get_priority_default(esp32_s2, esp32, esp32_s2_idf, esp32_idf)
         )
         self._esp32_s3_arduino_default = vol.default_factory(
-            (esp32_s3_arduino if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s3 is vol.UNDEFINED
-            else esp32_s3
+            _get_priority_default(esp32_s3, esp32, esp32_s3_arduino, esp32_arduino)
         )
         self._esp32_s3_idf_default = vol.default_factory(
-            (esp32_s3_idf if esp32 is vol.UNDEFINED else esp32)
-            if esp32_s3 is vol.UNDEFINED
-            else esp32_s3
+            _get_priority_default(esp32_s3, esp32, esp32_s3_idf, esp32_idf)
         )
         self._rp2040_default = vol.default_factory(rp2040)
         self._bk72xx_default = vol.default_factory(bk72xx)

From e030c0fc45c1105cc21e2ddb12db9c0c90766a2b Mon Sep 17 00:00:00 2001
From: mrtoy-me <118446898+mrtoy-me@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:39:05 +1000
Subject: [PATCH 12/34] Update ENS160 TVOC device_class and AQI units to match
 required by HA (#5939)

---
 esphome/components/ens160/sensor.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/esphome/components/ens160/sensor.py b/esphome/components/ens160/sensor.py
index 55f0ff7b6f..393b63bae1 100644
--- a/esphome/components/ens160/sensor.py
+++ b/esphome/components/ens160/sensor.py
@@ -9,7 +9,7 @@ from esphome.const import (
     CONF_TVOC,
     DEVICE_CLASS_AQI,
     DEVICE_CLASS_CARBON_DIOXIDE,
-    DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+    DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS_PARTS,
     ICON_CHEMICAL_WEAPON,
     ICON_MOLECULE_CO2,
     ICON_RADIATOR,
@@ -45,11 +45,10 @@ CONFIG_SCHEMA = (
                 unit_of_measurement=UNIT_PARTS_PER_BILLION,
                 icon=ICON_RADIATOR,
                 accuracy_decimals=0,
-                device_class=DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+                device_class=DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS_PARTS,
                 state_class=STATE_CLASS_MEASUREMENT,
             ),
             cv.Required(CONF_AQI): sensor.sensor_schema(
-                unit_of_measurement=UNIT_INDEX,
                 icon=ICON_CHEMICAL_WEAPON,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_AQI,

From 514db8b26e8614902b0e8c849802d22709df018b Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:02:47 +0900
Subject: [PATCH 13/34] Bump version to 2023.12.0b2

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index b8495a3a7c..6c2a520331 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b1"
+__version__ = "2023.12.0b2"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 70dac541139a7514dd06281152f233fc2efd3382 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 17 Dec 2023 16:27:08 +0000
Subject: [PATCH 14/34] Bump zeroconf from 0.128.4 to 0.130.0 (#5950)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f330ecbf3e..20a5514e71 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,7 @@ esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
 aioesphomeapi==21.0.0
-zeroconf==0.128.4
+zeroconf==0.130.0
 python-magic==0.4.27
 
 # esp-idf requires this, but doesn't bundle it by default

From 917e0f93edd561b13fd25da0b5ee4beb58e10e1d Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Mon, 18 Dec 2023 00:19:30 +0100
Subject: [PATCH 15/34] UARTComponent inline doc (#5930)

---
 esphome/components/uart/uart_component.h | 88 +++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 3 deletions(-)

diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h
index 34bda42bb5..e03784fdd8 100644
--- a/esphome/components/uart/uart_component.h
+++ b/esphome/components/uart/uart_component.h
@@ -31,40 +31,122 @@ const LogString *parity_to_str(UARTParityOptions parity);
 
 class UARTComponent {
  public:
+  // Writes an array of bytes to the UART bus.
+  // @param data A vector of bytes to be written.
   void write_array(const std::vector<uint8_t> &data) { this->write_array(&data[0], data.size()); }
+
+  // Writes a single byte to the UART bus.
+  // @param data The byte to be written.
   void write_byte(uint8_t data) { this->write_array(&data, 1); };
+
+  // Writes a null-terminated string to the UART bus.
+  // @param str Pointer to the null-terminated string.
   void write_str(const char *str) {
     const auto *data = reinterpret_cast<const uint8_t *>(str);
     this->write_array(data, strlen(str));
   };
 
+  // Pure virtual method to write an array of bytes to the UART bus.
+  // @param data Pointer to the array of bytes.
+  // @param len Length of the array.
   virtual void write_array(const uint8_t *data, size_t len) = 0;
 
+  // Reads a single byte from the UART bus.
+  // @param data Pointer to the byte where the read data will be stored.
+  // @return True if a byte was successfully read, false otherwise.
   bool read_byte(uint8_t *data) { return this->read_array(data, 1); };
+
+  // Pure virtual method to peek the next byte in the UART buffer without removing it.
+  // @param data Pointer to the byte where the peeked data will be stored.
+  // @return True if a byte is available to peek, false otherwise.
   virtual bool peek_byte(uint8_t *data) = 0;
+
+  // Pure virtual method to read an array of bytes from the UART bus.
+  // @param data Pointer to the array where the read data will be stored.
+  // @param len Number of bytes to read.
+  // @return True if the specified number of bytes were successfully read, false otherwise.
   virtual bool read_array(uint8_t *data, size_t len) = 0;
 
-  /// Return available number of bytes.
+  // Pure virtual method to return the number of bytes available for reading.
+  // @return Number of available bytes.
   virtual int available() = 0;
-  /// Block until all bytes have been written to the UART bus.
+
+  // Pure virtual method to block until all bytes have been written to the UART bus.
   virtual void flush() = 0;
 
+  // Sets the TX (transmit) pin for the UART bus.
+  // @param tx_pin Pointer to the internal GPIO pin used for transmission.
   void set_tx_pin(InternalGPIOPin *tx_pin) { this->tx_pin_ = tx_pin; }
+
+  // Sets the RX (receive) pin for the UART bus.
+  // @param rx_pin Pointer to the internal GPIO pin used for reception.
   void set_rx_pin(InternalGPIOPin *rx_pin) { this->rx_pin_ = rx_pin; }
+
+  // Sets the size of the RX buffer.
+  // @param rx_buffer_size Size of the RX buffer in bytes.
   void set_rx_buffer_size(size_t rx_buffer_size) { this->rx_buffer_size_ = rx_buffer_size; }
+
+  // Gets the size of the RX buffer.
+  // @return Size of the RX buffer in bytes.
   size_t get_rx_buffer_size() { return this->rx_buffer_size_; }
 
+  // Sets the number of stop bits used in UART communication.
+  // @param stop_bits Number of stop bits.
   void set_stop_bits(uint8_t stop_bits) { this->stop_bits_ = stop_bits; }
+
+  // Gets the number of stop bits used in UART communication.
+  // @return Number of stop bits.
   uint8_t get_stop_bits() const { return this->stop_bits_; }
+
+  // Set the number of data bits used in UART communication.
+  // @param data_bits Number of data bits.
   void set_data_bits(uint8_t data_bits) { this->data_bits_ = data_bits; }
+
+  // Get the number of data bits used in UART communication.
+  // @return Number of data bits.
   uint8_t get_data_bits() const { return this->data_bits_; }
+
+  // Set the parity used in UART communication.
+  // @param parity Parity option.
   void set_parity(UARTParityOptions parity) { this->parity_ = parity; }
+
+  // Get the parity used in UART communication.
+  // @return Parity option.
   UARTParityOptions get_parity() const { return this->parity_; }
+
+  // Set the baud rate for UART communication.
+  // @param baud_rate Baud rate in bits per second.
   void set_baud_rate(uint32_t baud_rate) { baud_rate_ = baud_rate; }
+
+  // Get the baud rate for UART communication.
+  // @return Baud rate in bits per second.
   uint32_t get_baud_rate() const { return baud_rate_; }
+
 #ifdef USE_ESP32
-  virtual void load_settings() = 0;
+  /**
+   * Load the UART settings.
+   * @param dump_config If true (default), output the new settings to logs; otherwise, change settings quietly.
+   *
+   * Example:
+   * ```cpp
+   * id(uart1).load_settings(false);
+   * ```
+   *
+   * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
+   */
   virtual void load_settings(bool dump_config) = 0;
+
+  /**
+   * Load the UART settings.
+   *
+   * Example:
+   * ```cpp
+   * id(uart1).load_settings();
+   * ```
+   *
+   * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
+   */
+  virtual void load_settings() = 0;
 #endif  // USE_ESP32
 
 #ifdef USE_UART_DEBUGGER

From 2309f15ce06b0fe633420b8061293427e62f0051 Mon Sep 17 00:00:00 2001
From: Grant Le Roux <cram42@users.noreply.github.com>
Date: Mon, 18 Dec 2023 07:28:48 +0800
Subject: [PATCH 16/34] Fix - Tuya Fan - Allow integer speed datapoint (#5948)

Co-authored-by: Cram42 <5396871+cram42@users.noreply.github.com>
---
 esphome/components/tuya/fan/tuya_fan.cpp | 23 +++++++++++++++++------
 esphome/components/tuya/fan/tuya_fan.h   |  1 +
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/esphome/components/tuya/fan/tuya_fan.cpp b/esphome/components/tuya/fan/tuya_fan.cpp
index 1b03ea50fa..481c931f2e 100644
--- a/esphome/components/tuya/fan/tuya_fan.cpp
+++ b/esphome/components/tuya/fan/tuya_fan.cpp
@@ -9,13 +9,20 @@ static const char *const TAG = "tuya.fan";
 void TuyaFan::setup() {
   if (this->speed_id_.has_value()) {
     this->parent_->register_listener(*this->speed_id_, [this](const TuyaDatapoint &datapoint) {
-      ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_enum);
-      if (datapoint.value_enum >= this->speed_count_) {
-        ESP_LOGE(TAG, "Speed has invalid value %d", datapoint.value_enum);
-      } else {
-        this->speed = datapoint.value_enum + 1;
+      if (datapoint.type == TuyaDatapointType::ENUM) {
+        ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_enum);
+        if (datapoint.value_enum >= this->speed_count_) {
+          ESP_LOGE(TAG, "Speed has invalid value %d", datapoint.value_enum);
+        } else {
+          this->speed = datapoint.value_enum + 1;
+          this->publish_state();
+        }
+      } else if (datapoint.type == TuyaDatapointType::INTEGER) {
+        ESP_LOGV(TAG, "MCU reported speed of: %d", datapoint.value_int);
+        this->speed = datapoint.value_int;
         this->publish_state();
       }
+      this->speed_type_ = datapoint.type;
     });
   }
   if (this->switch_id_.has_value()) {
@@ -80,7 +87,11 @@ void TuyaFan::control(const fan::FanCall &call) {
     this->parent_->set_enum_datapoint_value(*this->direction_id_, enable);
   }
   if (this->speed_id_.has_value() && call.get_speed().has_value()) {
-    this->parent_->set_enum_datapoint_value(*this->speed_id_, *call.get_speed() - 1);
+    if (this->speed_type_ == TuyaDatapointType::ENUM) {
+      this->parent_->set_enum_datapoint_value(*this->speed_id_, *call.get_speed() - 1);
+    } else if (this->speed_type_ == TuyaDatapointType::INTEGER) {
+      this->parent_->set_integer_datapoint_value(*this->speed_id_, *call.get_speed());
+    }
   }
 }
 
diff --git a/esphome/components/tuya/fan/tuya_fan.h b/esphome/components/tuya/fan/tuya_fan.h
index 4aba1e1c07..77b2cc6383 100644
--- a/esphome/components/tuya/fan/tuya_fan.h
+++ b/esphome/components/tuya/fan/tuya_fan.h
@@ -28,6 +28,7 @@ class TuyaFan : public Component, public fan::Fan {
   optional<uint8_t> oscillation_id_{};
   optional<uint8_t> direction_id_{};
   int speed_count_{};
+  TuyaDatapointType speed_type_{};
 };
 
 }  // namespace tuya

From 168e70413030060eedb3d723939ea7191767a8a0 Mon Sep 17 00:00:00 2001
From: Alex Hermann <gaaf@gmx.net>
Date: Mon, 18 Dec 2023 00:58:13 +0100
Subject: [PATCH 17/34] i2s_audio: Set player_task's prio to 1 (#5945)

---
 esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index e729cdf954..95e63035fe 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -29,7 +29,7 @@ void I2SAudioSpeaker::start_() {
   }
   this->state_ = speaker::STATE_RUNNING;
 
-  xTaskCreate(I2SAudioSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_);
+  xTaskCreate(I2SAudioSpeaker::player_task, "speaker_task", 8192, (void *) this, 1, &this->player_task_handle_);
 }
 
 void I2SAudioSpeaker::player_task(void *params) {

From e8ce780482c914f45df13bff31448352d5ee5e76 Mon Sep 17 00:00:00 2001
From: Alex Hermann <gaaf@gmx.net>
Date: Mon, 18 Dec 2023 01:00:42 +0100
Subject: [PATCH 18/34] esp32_camera: Set framebuffer task prio to 1 (#5943)

---
 esphome/components/esp32_camera/esp32_camera.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/esp32_camera/esp32_camera.cpp b/esphome/components/esp32_camera/esp32_camera.cpp
index 99cb811fe4..555f6ca5f1 100644
--- a/esphome/components/esp32_camera/esp32_camera.cpp
+++ b/esphome/components/esp32_camera/esp32_camera.cpp
@@ -37,7 +37,7 @@ void ESP32Camera::setup() {
                           "framebuffer_task",  // name
                           1024,                // stack size
                           nullptr,             // task pv params
-                          0,                   // priority
+                          1,                   // priority
                           nullptr,             // handle
                           1                    // core
   );

From d0df73769d6e51f0efffab59011298e9f836e4f6 Mon Sep 17 00:00:00 2001
From: dentra <dentra@users.noreply.github.com>
Date: Mon, 18 Dec 2023 03:01:21 +0300
Subject: [PATCH 19/34] web_server_idf: fix call with hardcoded http code
 (#5942)

---
 esphome/components/web_server_idf/web_server_idf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h
index bc64e5231e..2fbc5cd2e9 100644
--- a/esphome/components/web_server_idf/web_server_idf.h
+++ b/esphome/components/web_server_idf/web_server_idf.h
@@ -117,7 +117,7 @@ class AsyncWebServerRequest {
   // NOLINTNEXTLINE(readability-identifier-naming)
   AsyncWebServerResponse *beginResponse(int code, const char *content_type) {
     auto *res = new AsyncWebServerResponseEmpty(this);  // NOLINT(cppcoreguidelines-owning-memory)
-    this->init_response_(res, 200, content_type);
+    this->init_response_(res, code, content_type);
     return res;
   }
   // NOLINTNEXTLINE(readability-identifier-naming)

From eefa1cd3aba5ff2fcf93328b35117cdbf70594b7 Mon Sep 17 00:00:00 2001
From: Jean  Louis-Guerin <DrCoolzic@hotmail.com>
Date: Mon, 18 Dec 2023 01:03:01 +0100
Subject: [PATCH 20/34] Revert pure virtual functions in UART component from
 #5920 (#5932)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 esphome/components/uart/uart_component.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h
index e03784fdd8..6f27f36bcb 100644
--- a/esphome/components/uart/uart_component.h
+++ b/esphome/components/uart/uart_component.h
@@ -134,7 +134,7 @@ class UARTComponent {
    *
    * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
    */
-  virtual void load_settings(bool dump_config) = 0;
+  virtual void load_settings(bool dump_config){};
 
   /**
    * Load the UART settings.
@@ -146,7 +146,7 @@ class UARTComponent {
    *
    * This will load the current UART interface with the latest settings (baud_rate, parity, etc).
    */
-  virtual void load_settings() = 0;
+  virtual void load_settings(){};
 #endif  // USE_ESP32
 
 #ifdef USE_UART_DEBUGGER

From ab22a3da349d3d60b687cbf70dd5a00ec44b67e2 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Mon, 18 Dec 2023 01:33:12 -0600
Subject: [PATCH 21/34] Use the correct UART/`Serial` when CDC is enabled
 (#5957)

---
 esphome/components/logger/logger.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp
index e0ca0806cb..05b97a5f64 100644
--- a/esphome/components/logger/logger.cpp
+++ b/esphome/components/logger/logger.cpp
@@ -237,8 +237,8 @@ void Logger::pre_setup() {
         Serial1.begin(this->baud_rate_);
 #else
 #if ARDUINO_USB_CDC_ON_BOOT
-        this->hw_serial_ = &Serial;
-        Serial.begin(this->baud_rate_);
+        this->hw_serial_ = &Serial0;
+        Serial0.begin(this->baud_rate_);
 #else
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);

From dbfa77cb4bac5f5fc6120a79320e52a142c01812 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:51:11 +0900
Subject: [PATCH 22/34] Bump version to 2023.12.0b3

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 6c2a520331..4153646aea 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b2"
+__version__ = "2023.12.0b3"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 7807f0d89250a890aaec97a524ad1d68310d07c3 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Mon, 18 Dec 2023 17:11:07 -0600
Subject: [PATCH 23/34] Fix build issue with UART component when building with
 Arduino and CDC (#5964)

---
 esphome/components/uart/uart_component_esp32_arduino.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/esphome/components/uart/uart_component_esp32_arduino.cpp b/esphome/components/uart/uart_component_esp32_arduino.cpp
index 75b67bf5c2..f77783e20e 100644
--- a/esphome/components/uart/uart_component_esp32_arduino.cpp
+++ b/esphome/components/uart/uart_component_esp32_arduino.cpp
@@ -88,7 +88,11 @@ void ESP32ArduinoUARTComponent::setup() {
 #endif
   static uint8_t next_uart_num = 0;
   if (is_default_tx && is_default_rx && next_uart_num == 0) {
+#if ARDUINO_USB_CDC_ON_BOOT
+    this->hw_serial_ = &Serial0;
+#else
     this->hw_serial_ = &Serial;
+#endif
     next_uart_num++;
   } else {
 #ifdef USE_LOGGER

From 820f3282480d282914a6a0682908443ad3b5755a Mon Sep 17 00:00:00 2001
From: Jean  Louis-Guerin <DrCoolzic@hotmail.com>
Date: Tue, 19 Dec 2023 00:14:42 +0100
Subject: [PATCH 24/34] Fix I2CBus::write() bug and add i2c documentation
 (#5947)

---
 esphome/components/i2c/i2c.h     | 153 +++++++++++++++++++++++++++++--
 esphome/components/i2c/i2c_bus.h |  79 ++++++++++++----
 2 files changed, 205 insertions(+), 27 deletions(-)

diff --git a/esphome/components/i2c/i2c.h b/esphome/components/i2c/i2c.h
index eb5d463b65..8d8e139c61 100644
--- a/esphome/components/i2c/i2c.h
+++ b/esphome/components/i2c/i2c.h
@@ -11,43 +11,116 @@ namespace i2c {
 
 #define LOG_I2C_DEVICE(this) ESP_LOGCONFIG(TAG, "  Address: 0x%02X", this->address_);
 
-class I2CDevice;
+class I2CDevice;  // forward declaration
+
+/// @brief This class is used to create I2CRegister objects that act as proxies to read/write internal registers on an
+/// I2C device.
+/// @details
+/// @n typical usage:
+/// @code
+/// constexpr uint8_t ADDR_REGISTER_1 = 0x12;
+/// i2c::I2CRegister reg_1 = this->reg(ADDR_REGISTER_1); // declare
+/// reg_1 |= 0x01; // set bit
+/// reg_1 &= ~0x01; // reset bit
+/// reg_1 = 10; // Set value
+/// uint val = reg_1.get(); // get value
+/// @endcode
+/// @details The I²C protocol specifies how to read/write in sets of 8-bits followed by an Acknowledgement (ACK/NACK)
+/// from the device receiving the data. How the device interprets the bits read/written can vary greatly from
+/// device to device. However most of the devices follow the same protocol for reading/writing 8 bit registers using as
+/// implemented in the I2CRegister: after sending the device address, the controller sends one byte with the internal
+/// register address and then read or write the specified register content.
 class I2CRegister {
  public:
+  /// @brief overloads the = operator. This allows to set the value of an i2c register
+  /// @param value value to be set in the register
+  /// @return pointer to current object
   I2CRegister &operator=(uint8_t value);
+
+  /// @brief overloads the compound &= operator. This allows to reset specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister &operator&=(uint8_t value);
+
+  /// @brief overloads the compound |= operator. This allows to set specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister &operator|=(uint8_t value);
 
+  /// @brief overloads the uint8_t() cast operator to return the I²C register value
+  /// @return pointer to current object
   explicit operator uint8_t() const { return get(); }
 
+  /// @brief returns the register value
+  /// @return the register value
   uint8_t get() const;
 
  protected:
   friend class I2CDevice;
 
+  /// @brief protected constructor that stores the owning object and the register address. Note as only friends can
+  /// create an I2CRegister @see I2CDevice::reg()
+  /// @param parent our parent
+  /// @param a_register address of the i2c register
   I2CRegister(I2CDevice *parent, uint8_t a_register) : parent_(parent), register_(a_register) {}
 
-  I2CDevice *parent_;
-  uint8_t register_;
+  I2CDevice *parent_;  ///< I2CDevice object pointer
+  uint8_t register_;   ///< the internal address of the register
 };
 
+/// @brief This class is used to create I2CRegister16 objects that act as proxies to read/write internal registers
+/// (specified with a 16 bit address) on an I2C device.
+/// @details
+/// @n typical usage:
+/// @code
+/// constexpr uint16_t X16_BIT_ADDR_REGISTER_1 = 0x1234;
+/// i2c::I2CRegister16 reg_1 = this->reg16(X16_BIT_ADDR_REGISTER_1); // declare
+/// reg_1 |= 0x01; // set bit
+/// reg_1 &= ~0x01; // reset bit
+/// reg_1 = 10; // Set value
+/// uint val = reg_1.get(); // get value
+/// @endcode
+/// @details The I²C protocol specification, reads/writes in sets of 8-bits followed by an Acknowledgement (ACK/NACK)
+/// from the device receiving the data. How the device interprets the bits read/written to it can vary greatly from
+/// device to device. This class can be used to access in the device 8 bits registers that uses a 16 bits internal
+/// address. After sending the device address, the controller sends the internal register address (using two consecutive
+/// bytes following the big indian convention) and then read or write the register content.
 class I2CRegister16 {
  public:
+  /// @brief overloads the = operator. This allows to set the value of an I²C register
+  /// @param value value to be set in the register
+  /// @return pointer to current object
   I2CRegister16 &operator=(uint8_t value);
+
+  /// @brief overloads the compound &= operator. This allows to reset specific bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister16 &operator&=(uint8_t value);
+
+  /// @brief overloads the compound |= operator. This allows to set bits of an I²C register
+  /// @param value used for the & operation
+  /// @return pointer to current object
   I2CRegister16 &operator|=(uint8_t value);
 
+  /// @brief overloads the uint8_t() cast operator to return the I²C register value
+  /// @return the register value
   explicit operator uint8_t() const { return get(); }
 
+  /// @brief returns the register value
+  /// @return the register value
   uint8_t get() const;
 
  protected:
   friend class I2CDevice;
 
+  /// @brief protected constructor that store the owning object and the register address. Only friends can create an
+  /// I2CRegister16 @see I2CDevice::reg16()
+  /// @param parent our parent
+  /// @param a_register 16 bits address of the i2c register
   I2CRegister16(I2CDevice *parent, uint16_t a_register) : parent_(parent), register_(a_register) {}
 
-  I2CDevice *parent_;
-  uint16_t register_;
+  I2CDevice *parent_;  ///< I2CDevice object pointer
+  uint16_t register_;  ///< the internal 16 bits address of the register
 };
 
 // like ntohs/htons but without including networking headers.
@@ -55,29 +128,91 @@ class I2CRegister16 {
 inline uint16_t i2ctohs(uint16_t i2cshort) { return convert_big_endian(i2cshort); }
 inline uint16_t htoi2cs(uint16_t hostshort) { return convert_big_endian(hostshort); }
 
+/// @brief This Class provides the methods to read/write bytes from/to an i2c device.
+/// Objects keep a list of devices found on bus as well as a pointer to the I2CBus in use.
 class I2CDevice {
  public:
+  /// @brief we use the C++ default constructor
   I2CDevice() = default;
 
+  /// @brief We store the address of the device on the bus
+  /// @param address of the device
   void set_i2c_address(uint8_t address) { address_ = address; }
+
+  /// @brief we store the pointer to the I2CBus to use
+  /// @param bus pointer to the I2CBus object
   void set_i2c_bus(I2CBus *bus) { bus_ = bus; }
 
+  /// @brief calls the I2CRegister constructor
+  /// @param a_register address of the I²C register
+  /// @return an I2CRegister proxy object
   I2CRegister reg(uint8_t a_register) { return {this, a_register}; }
+
+  /// @brief calls the I2CRegister16 constructor
+  /// @param a_register 16 bits address of the I²C register
+  /// @return an I2CRegister16 proxy object
   I2CRegister16 reg16(uint16_t a_register) { return {this, a_register}; }
 
+  /// @brief reads an array of bytes from the device using an I2CBus
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @return an i2c::ErrorCode
   ErrorCode read(uint8_t *data, size_t len) { return bus_->read(address_, data, len); }
+
+  /// @brief reads an array of bytes from a specific register in the I²C device
+  /// @param a_register an 8 bits internal address of the I²C register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode read_register(uint8_t a_register, uint8_t *data, size_t len, bool stop = true);
+
+  /// @brief reads an array of bytes from a specific register in the I²C device
+  /// @param a_register the 16 bits internal address of the I²C register to read from
+  /// @param data pointer to an array of bytes to store the information
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode read_register16(uint16_t a_register, uint8_t *data, size_t len, bool stop = true);
 
-  ErrorCode write(const uint8_t *data, uint8_t len, bool stop = true) { return bus_->write(address_, data, len, stop); }
+  /// @brief writes an array of bytes to a device using an I2CBus
+  /// @param data pointer to an array that contains the bytes to send
+  /// @param len length of the buffer = number of bytes to write
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
+  ErrorCode write(const uint8_t *data, size_t len, bool stop = true) { return bus_->write(address_, data, len, stop); }
+
+  /// @brief writes an array of bytes to a specific register in the I²C device
+  /// @param a_register the internal address of the register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode write_register(uint8_t a_register, const uint8_t *data, size_t len, bool stop = true);
+
+  /// @brief write an array of bytes to a specific register in the I²C device
+  /// @param a_register the 16 bits internal address of the register to read from
+  /// @param data pointer to an array to store the bytes
+  /// @param len length of the buffer = number of bytes to read
+  /// @param stop (true/false): True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   ErrorCode write_register16(uint16_t a_register, const uint8_t *data, size_t len, bool stop = true);
 
-  // Compat APIs
+  ///
+  /// Compat APIs
+  /// All methods below have been added for compatibility reasons. They do not bring any functionality and therefore on
+  /// new code it is not recommend to use them.
+  ///
 
   bool read_bytes(uint8_t a_register, uint8_t *data, uint8_t len) {
     return read_register(a_register, data, len) == ERROR_OK;
   }
+
   bool read_bytes_raw(uint8_t *data, uint8_t len) { return read(data, len) == ERROR_OK; }
 
   template<size_t N> optional<std::array<uint8_t, N>> read_bytes(uint8_t a_register) {
@@ -131,8 +266,8 @@ class I2CDevice {
   bool write_byte_16(uint8_t a_register, uint16_t data) { return write_bytes_16(a_register, &data, 1); }
 
  protected:
-  uint8_t address_{0x00};
-  I2CBus *bus_{nullptr};
+  uint8_t address_{0x00};  ///< store the address of the device on the bus
+  I2CBus *bus_{nullptr};   ///< pointer to I2CBus instance
 };
 
 }  // namespace i2c
diff --git a/esphome/components/i2c/i2c_bus.h b/esphome/components/i2c/i2c_bus.h
index 2633a7adf6..fbfc88323e 100644
--- a/esphome/components/i2c/i2c_bus.h
+++ b/esphome/components/i2c/i2c_bus.h
@@ -7,50 +7,93 @@
 namespace esphome {
 namespace i2c {
 
+/// @brief Error codes returned by I2CBus and I2CDevice methods
 enum ErrorCode {
-  ERROR_OK = 0,
-  ERROR_INVALID_ARGUMENT = 1,
-  ERROR_NOT_ACKNOWLEDGED = 2,
-  ERROR_TIMEOUT = 3,
-  ERROR_NOT_INITIALIZED = 4,
-  ERROR_TOO_LARGE = 5,
-  ERROR_UNKNOWN = 6,
-  ERROR_CRC = 7,
+  NO_ERROR = 0,                ///< No error found during execution of method
+  ERROR_OK = 0,                ///< No error found during execution of method
+  ERROR_INVALID_ARGUMENT = 1,  ///< method called invalid argument(s)
+  ERROR_NOT_ACKNOWLEDGED = 2,  ///< I2C bus acknowledgment not received
+  ERROR_TIMEOUT = 3,           ///< timeout while waiting to receive bytes
+  ERROR_NOT_INITIALIZED = 4,   ///< call method to a not initialized bus
+  ERROR_TOO_LARGE = 5,         ///< requested a transfer larger than buffers can hold
+  ERROR_UNKNOWN = 6,           ///< miscellaneous I2C error during execution
+  ERROR_CRC = 7,               ///< bytes received with a CRC error
 };
 
+/// @brief the ReadBuffer structure stores a pointer to a read buffer and its length
 struct ReadBuffer {
-  uint8_t *data;
-  size_t len;
-};
-struct WriteBuffer {
-  const uint8_t *data;
-  size_t len;
+  uint8_t *data;  ///< pointer to the read buffer
+  size_t len;     ///< length of the buffer
 };
 
+/// @brief the WriteBuffer structure stores a pointer to a write buffer and its length
+struct WriteBuffer {
+  const uint8_t *data;  ///< pointer to the write buffer
+  size_t len;           ///< length of the buffer
+};
+
+/// @brief This Class provides the methods to read and write bytes from an I2CBus.
+/// @note The I2CBus virtual class follows a *Factory design pattern* that provides all the interfaces methods required
+/// by clients while deferring the actual implementation of these methods to a subclasses. I2C-bus specification and
+/// user manual can be found here https://www.nxp.com/docs/en/user-guide/UM10204.pdf and an interesting I²C Application
+/// note https://www.nxp.com/docs/en/application-note/AN10216.pdf
 class I2CBus {
  public:
+  /// @brief Creates a ReadBuffer and calls the virtual readv() method to read bytes into this buffer
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffer pointer to an array of bytes that will be used to store the data received
+  /// @param len length of the buffer = number of bytes to read
+  /// @return an i2c::ErrorCode
   virtual ErrorCode read(uint8_t address, uint8_t *buffer, size_t len) {
     ReadBuffer buf;
     buf.data = buffer;
     buf.len = len;
     return readv(address, &buf, 1);
   }
-  virtual ErrorCode readv(uint8_t address, ReadBuffer *buffers, size_t cnt) = 0;
+
+  /// @brief This virtual method reads bytes from an I2CBus into an array of ReadBuffer.
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffers pointer to an array of ReadBuffer
+  /// @param count number of ReadBuffer to read
+  /// @return an i2c::ErrorCode
+  /// @details This is a pure virtual method that must be implemented in a subclass.
+  virtual ErrorCode readv(uint8_t address, ReadBuffer *buffers, size_t count) = 0;
+
   virtual ErrorCode write(uint8_t address, const uint8_t *buffer, size_t len) {
     return write(address, buffer, len, true);
   }
+
+  /// @brief Creates a WriteBuffer and calls the writev() method to send the bytes from this buffer
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffer pointer to an array of bytes that contains the data to be sent
+  /// @param len length of the buffer = number of bytes to write
+  /// @param stop true or false: True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
   virtual ErrorCode write(uint8_t address, const uint8_t *buffer, size_t len, bool stop) {
     WriteBuffer buf;
     buf.data = buffer;
     buf.len = len;
     return writev(address, &buf, 1, stop);
   }
+
   virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t cnt) {
     return writev(address, buffers, cnt, true);
   }
-  virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t cnt, bool stop) = 0;
+
+  /// @brief This virtual method writes bytes to an I2CBus from an array of WriteBuffer.
+  /// @param address address of the I²C component on the i2c bus
+  /// @param buffers pointer to an array of WriteBuffer
+  /// @param count number of WriteBuffer to write
+  /// @param stop true or false: True will send a stop message, releasing the bus after
+  /// transmission. False will send a restart, keeping the connection active.
+  /// @return an i2c::ErrorCode
+  /// @details This is a pure virtual method that must be implemented in the subclass.
+  virtual ErrorCode writev(uint8_t address, WriteBuffer *buffers, size_t count, bool stop) = 0;
 
  protected:
+  /// @brief Scans the I2C bus for devices. Devices presence is kept in an array of std::pair
+  /// that contains the address and the corresponding bool presence flag.
   void i2c_scan_() {
     for (uint8_t address = 8; address < 120; address++) {
       auto err = writev(address, nullptr, 0);
@@ -61,8 +104,8 @@ class I2CBus {
       }
     }
   }
-  std::vector<std::pair<uint8_t, bool>> scan_results_;
-  bool scan_{false};
+  std::vector<std::pair<uint8_t, bool>> scan_results_;  ///< array containing scan results
+  bool scan_{false};                                    ///< Should we scan ? Can be set in the yaml
 };
 
 }  // namespace i2c

From 977e0184a72880f917dbaa8cfe3d48b2cc862b52 Mon Sep 17 00:00:00 2001
From: mathieu-mp <mathieu-mp@users.noreply.github.com>
Date: Tue, 19 Dec 2023 04:29:00 +0100
Subject: [PATCH 25/34] Add deep sleep between updates for waveshare epaper
 1.54in and 1.54inv2 (#5961)

---
 .../waveshare_epaper/waveshare_epaper.cpp     | 33 +++++++++++++++++++
 .../waveshare_epaper/waveshare_epaper.h       | 25 ++++++++++----
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.cpp b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
index 53bfa57f4f..8fdb9a3ac0 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.cpp
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.cpp
@@ -167,6 +167,25 @@ void WaveshareEPaper::on_safe_shutdown() { this->deep_sleep(); }
 // ========================================================
 
 void WaveshareEPaperTypeA::initialize() {
+  // Achieve display intialization
+  this->init_display_();
+  // If a reset pin is configured, eligible displays can be set to deep sleep
+  // between updates, as recommended by the hardware provider
+  if (this->reset_pin_ != nullptr) {
+    switch (this->model_) {
+      // More models can be added here to enable deep sleep if eligible
+      case WAVESHARE_EPAPER_1_54_IN:
+      case WAVESHARE_EPAPER_1_54_IN_V2:
+        this->deep_sleep_between_updates_ = true;
+        ESP_LOGI(TAG, "Set the display to deep sleep");
+        this->deep_sleep();
+        break;
+      default:
+        break;
+    }
+  }
+}
+void WaveshareEPaperTypeA::init_display_() {
   if (this->model_ == TTGO_EPAPER_2_13_IN_B74) {
     this->reset_pin_->digital_write(false);
     delay(10);
@@ -261,6 +280,13 @@ void HOT WaveshareEPaperTypeA::display() {
   bool full_update = this->at_update_ == 0;
   bool prev_full_update = this->at_update_ == 1;
 
+  if (this->deep_sleep_between_updates_) {
+    ESP_LOGI(TAG, "Wake up the display");
+    this->reset_();
+    this->wait_until_idle_();
+    this->init_display_();
+  }
+
   if (!this->wait_until_idle_()) {
     this->status_set_warning();
     return;
@@ -384,6 +410,11 @@ void HOT WaveshareEPaperTypeA::display() {
   this->command(0xFF);
 
   this->status_clear_warning();
+
+  if (this->deep_sleep_between_updates_) {
+    ESP_LOGI(TAG, "Set the display back to deep sleep");
+    this->deep_sleep();
+  }
 }
 int WaveshareEPaperTypeA::get_width_internal() {
   switch (this->model_) {
@@ -445,6 +476,8 @@ void WaveshareEPaperTypeA::set_full_update_every(uint32_t full_update_every) {
 
 uint32_t WaveshareEPaperTypeA::idle_timeout_() {
   switch (this->model_) {
+    case WAVESHARE_EPAPER_1_54_IN:
+    case WAVESHARE_EPAPER_1_54_IN_V2:
     case TTGO_EPAPER_2_13_IN_B1:
       return 2500;
     default:
diff --git a/esphome/components/waveshare_epaper/waveshare_epaper.h b/esphome/components/waveshare_epaper/waveshare_epaper.h
index f6ccf90861..42e8a16829 100644
--- a/esphome/components/waveshare_epaper/waveshare_epaper.h
+++ b/esphome/components/waveshare_epaper/waveshare_epaper.h
@@ -92,13 +92,20 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
   void display() override;
 
   void deep_sleep() override {
-    if (this->model_ == WAVESHARE_EPAPER_2_9_IN_V2 || this->model_ == WAVESHARE_EPAPER_1_54_IN_V2) {
-      // COMMAND DEEP SLEEP MODE
-      this->command(0x10);
-      this->data(0x01);
-    } else {
-      // COMMAND DEEP SLEEP MODE
-      this->command(0x10);
+    switch (this->model_) {
+      // Models with specific deep sleep command and data
+      case WAVESHARE_EPAPER_1_54_IN:
+      case WAVESHARE_EPAPER_1_54_IN_V2:
+      case WAVESHARE_EPAPER_2_9_IN_V2:
+        // COMMAND DEEP SLEEP MODE
+        this->command(0x10);
+        this->data(0x01);
+        break;
+      // Other models default to simple deep sleep command
+      default:
+        // COMMAND DEEP SLEEP
+        this->command(0x10);
+        break;
     }
     this->wait_until_idle_();
   }
@@ -108,6 +115,8 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
  protected:
   void write_lut_(const uint8_t *lut, uint8_t size);
 
+  void init_display_();
+
   int get_width_internal() override;
 
   int get_height_internal() override;
@@ -118,6 +127,8 @@ class WaveshareEPaperTypeA : public WaveshareEPaper {
   uint32_t at_update_{0};
   WaveshareEPaperTypeAModel model_;
   uint32_t idle_timeout_() override;
+
+  bool deep_sleep_between_updates_{false};
 };
 
 enum WaveshareEPaperTypeBModel {

From e5414d70f5c9f3b6aab84b14a198611df3f0c34e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Mon, 18 Dec 2023 19:24:48 -1000
Subject: [PATCH 26/34] Speed up writing protobuf strings/bytes (#5828)

---
 esphome/components/api/proto.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h
index fea219ecb9..ccc6c0d52c 100644
--- a/esphome/components/api/proto.h
+++ b/esphome/components/api/proto.h
@@ -160,8 +160,7 @@ class ProtoWriteBuffer {
     this->encode_field_raw(field_id, 2);
     this->encode_varint_raw(len);
     auto *data = reinterpret_cast<const uint8_t *>(string);
-    for (size_t i = 0; i < len; i++)
-      this->write(data[i]);
+    this->buffer_->insert(this->buffer_->end(), data, data + len);
   }
   void encode_string(uint32_t field_id, const std::string &value, bool force = false) {
     this->encode_string(field_id, value.data(), value.size());

From 1b3068a4092e72263dc252b43848ae959f9028cd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 19 Dec 2023 13:41:26 -1000
Subject: [PATCH 27/34] Bump aioesphomeapi to 21.0.1 (#5969)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 20a5514e71..b9bcd5dae0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==21.0.0
+aioesphomeapi==21.0.1
 zeroconf==0.130.0
 python-magic==0.4.27
 

From e2a00f66b8c2150f49538af8282b80d6fd8aae30 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 09:59:58 +0900
Subject: [PATCH 28/34] Bump version to 2023.12.0b4

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 4153646aea..20218ab61d 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b3"
+__version__ = "2023.12.0b4"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From fbf3d03a33289aa8f47a664b11a724621f499142 Mon Sep 17 00:00:00 2001
From: NP v/d Spek <github_mail@lumensoft.nl>
Date: Mon, 18 Dec 2023 20:23:22 +0100
Subject: [PATCH 29/34] rename set_raw_touch_position_ to
 add_raw_touch_position_ (#5962)

---
 esphome/components/ektf2232/touchscreen/ektf2232.cpp          | 2 +-
 esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h    | 2 +-
 esphome/components/ft63x6/ft63x6.cpp                          | 4 ++--
 esphome/components/gt911/touchscreen/gt911_touchscreen.cpp    | 2 +-
 .../lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp     | 2 +-
 esphome/components/touchscreen/touchscreen.cpp                | 2 +-
 esphome/components/touchscreen/touchscreen.h                  | 2 +-
 esphome/components/tt21100/touchscreen/tt21100.cpp            | 2 +-
 esphome/components/xpt2046/touchscreen/xpt2046.cpp            | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/esphome/components/ektf2232/touchscreen/ektf2232.cpp b/esphome/components/ektf2232/touchscreen/ektf2232.cpp
index 1a2c0389af..00e00bc7e6 100644
--- a/esphome/components/ektf2232/touchscreen/ektf2232.cpp
+++ b/esphome/components/ektf2232/touchscreen/ektf2232.cpp
@@ -74,7 +74,7 @@ void EKTF2232Touchscreen::update_touches() {
     uint8_t *d = raw + 1 + (i * 3);
     x_raw = (d[0] & 0xF0) << 4 | d[1];
     y_raw = (d[0] & 0x0F) << 8 | d[2];
-    this->set_raw_touch_position_(i, x_raw, y_raw);
+    this->add_raw_touch_position_(i, x_raw, y_raw);
   }
 }
 
diff --git a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
index 497d6c906c..0b3a2c1b86 100644
--- a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
+++ b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.h
@@ -94,7 +94,7 @@ class FT5x06Touchscreen : public touchscreen::Touchscreen, public i2c::I2CDevice
 
       esph_log_d(TAG, "Read %X status, id: %d, pos %d/%d", status, id, x, y);
       if (status == 0 || status == 2) {
-        this->set_raw_touch_position_(id, x, y);
+        this->add_raw_touch_position_(id, x, y);
       }
     }
   }
diff --git a/esphome/components/ft63x6/ft63x6.cpp b/esphome/components/ft63x6/ft63x6.cpp
index 9198954253..b674ded22c 100644
--- a/esphome/components/ft63x6/ft63x6.cpp
+++ b/esphome/components/ft63x6/ft63x6.cpp
@@ -53,13 +53,13 @@ void FT63X6Touchscreen::update_touches() {
   uint8_t touch_id = this->read_touch_id_(FT63X6_ADDR_TOUCH1_ID);  // id1 = 0 or 1
   int16_t x = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH1_X);
   int16_t y = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH1_Y);
-  this->set_raw_touch_position_(touch_id, x, y);
+  this->add_raw_touch_position_(touch_id, x, y);
 
   if (touch_count >= 2) {
     touch_id = this->read_touch_id_(FT63X6_ADDR_TOUCH2_ID);  // id2 = 0 or 1(~id1 & 0x01)
     x = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH2_X);
     y = this->read_touch_coordinate_(FT63X6_ADDR_TOUCH2_Y);
-    this->set_raw_touch_position_(touch_id, x, y);
+    this->add_raw_touch_position_(touch_id, x, y);
   }
 }
 
diff --git a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
index adc577f5da..84854d5b0d 100644
--- a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
+++ b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp
@@ -92,7 +92,7 @@ void GT911Touchscreen::update_touches() {
     uint16_t id = data[i][0];
     uint16_t x = encode_uint16(data[i][2], data[i][1]);
     uint16_t y = encode_uint16(data[i][4], data[i][3]);
-    this->set_raw_touch_position_(id, x, y);
+    this->add_raw_touch_position_(id, x, y);
   }
   auto keys = data[num_of_touches][0];
   for (size_t i = 0; i != 4; i++) {
diff --git a/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp b/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
index eb61b6f31e..64cc7ad4d1 100644
--- a/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
+++ b/esphome/components/lilygo_t5_47/touchscreen/lilygo_t5_47_touchscreen.cpp
@@ -84,7 +84,7 @@ void LilygoT547Touchscreen::update_touches() {
     id = (buffer[i * 5] >> 4) & 0x0F;
     y_raw = (uint16_t) ((buffer[i * 5 + 1] << 4) | ((buffer[i * 5 + 3] >> 4) & 0x0F));
     x_raw = (uint16_t) ((buffer[i * 5 + 2] << 4) | (buffer[i * 5 + 3] & 0x0F));
-    this->set_raw_touch_position_(id, x_raw, y_raw);
+    this->add_raw_touch_position_(id, x_raw, y_raw);
   }
 
   this->status_clear_warning();
diff --git a/esphome/components/touchscreen/touchscreen.cpp b/esphome/components/touchscreen/touchscreen.cpp
index 140f46b6f6..f095c2af8c 100644
--- a/esphome/components/touchscreen/touchscreen.cpp
+++ b/esphome/components/touchscreen/touchscreen.cpp
@@ -51,7 +51,7 @@ void Touchscreen::loop() {
   }
 }
 
-void Touchscreen::set_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw) {
+void Touchscreen::add_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw) {
   TouchPoint tp;
   uint16_t x, y;
   if (this->touches_.count(id) == 0) {
diff --git a/esphome/components/touchscreen/touchscreen.h b/esphome/components/touchscreen/touchscreen.h
index 1fe304d967..74747c589c 100644
--- a/esphome/components/touchscreen/touchscreen.h
+++ b/esphome/components/touchscreen/touchscreen.h
@@ -87,7 +87,7 @@ class Touchscreen : public PollingComponent {
 
   void attach_interrupt_(InternalGPIOPin *irq_pin, esphome::gpio::InterruptType type);
 
-  void set_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw = 0);
+  void add_raw_touch_position_(uint8_t id, int16_t x_raw, int16_t y_raw, int16_t z_raw = 0);
 
   void send_touches_();
 
diff --git a/esphome/components/tt21100/touchscreen/tt21100.cpp b/esphome/components/tt21100/touchscreen/tt21100.cpp
index ff688fd0b0..6b5cba74cd 100644
--- a/esphome/components/tt21100/touchscreen/tt21100.cpp
+++ b/esphome/components/tt21100/touchscreen/tt21100.cpp
@@ -109,7 +109,7 @@ void TT21100Touchscreen::update_touches() {
                  i, touch->touch_type, touch->tip, touch->event_id, touch->touch_id, touch->x, touch->y,
                  touch->pressure, touch->major_axis_length, touch->orientation);
 
-        this->set_raw_touch_position_(touch->tip, touch->x, touch->y, touch->pressure);
+        this->add_raw_touch_position_(touch->tip, touch->x, touch->y, touch->pressure);
       }
     }
   }
diff --git a/esphome/components/xpt2046/touchscreen/xpt2046.cpp b/esphome/components/xpt2046/touchscreen/xpt2046.cpp
index 1a9c202af0..a268da06dd 100644
--- a/esphome/components/xpt2046/touchscreen/xpt2046.cpp
+++ b/esphome/components/xpt2046/touchscreen/xpt2046.cpp
@@ -55,7 +55,7 @@ void XPT2046Component::update_touches() {
 
     ESP_LOGV(TAG, "Touchscreen Update [%d, %d], z = %d", x_raw, y_raw, z_raw);
 
-    this->set_raw_touch_position_(0, x_raw, y_raw, z_raw);
+    this->add_raw_touch_position_(0, x_raw, y_raw, z_raw);
   }
 }
 

From a64e96e7adbca4879c5d3b239272f8052e5d91ff Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 13:43:43 +0900
Subject: [PATCH 30/34] Bump version to 2023.12.0b5

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 20218ab61d..0df9c27233 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b4"
+__version__ = "2023.12.0b5"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From ab25e32509798e8564bad1b556f8ad552f13880f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 20 Dec 2023 11:33:05 +0100
Subject: [PATCH 31/34] image: allow the image to by auto-loaded by animation
 (#5139)

---
 esphome/__main__.py                  |  4 ++--
 esphome/components/image/__init__.py |  1 +
 esphome/config.py                    | 20 ++++++++++++++++++--
 esphome/loader.py                    |  4 ++++
 esphome/writer.py                    |  8 ++++----
 tests/test8.yaml                     | 10 ++++++++++
 6 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/esphome/__main__.py b/esphome/__main__.py
index 0796dead43..baa5ecde47 100644
--- a/esphome/__main__.py
+++ b/esphome/__main__.py
@@ -12,7 +12,7 @@ import argcomplete
 
 from esphome import const, writer, yaml_util
 import esphome.codegen as cg
-from esphome.config import iter_components, read_config, strip_default_ids
+from esphome.config import iter_component_configs, read_config, strip_default_ids
 from esphome.const import (
     ALLOWED_NAME_CHARS,
     CONF_BAUD_RATE,
@@ -196,7 +196,7 @@ def write_cpp(config):
 def generate_cpp_contents(config):
     _LOGGER.info("Generating C++ source...")
 
-    for name, component, conf in iter_components(CORE.config):
+    for name, component, conf in iter_component_configs(CORE.config):
         if component.to_code is not None:
             coro = wrap_to_code(name, component)
             CORE.add_job(coro, conf)
diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py
index c11021fc9c..73dc73aa45 100644
--- a/esphome/components/image/__init__.py
+++ b/esphome/components/image/__init__.py
@@ -36,6 +36,7 @@ _LOGGER = logging.getLogger(__name__)
 DOMAIN = "image"
 DEPENDENCIES = ["display"]
 MULTI_CONF = True
+MULTI_CONF_NO_DEFAULT = True
 
 image_ns = cg.esphome_ns.namespace("image")
 
diff --git a/esphome/config.py b/esphome/config.py
index 745883c2ef..e9433d537e 100644
--- a/esphome/config.py
+++ b/esphome/config.py
@@ -39,6 +39,17 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def iter_components(config):
+    for domain, conf in config.items():
+        component = get_component(domain)
+        yield domain, component
+        if component.is_platform_component:
+            for p_config in conf:
+                p_name = f"{domain}.{p_config[CONF_PLATFORM]}"
+                platform = get_platform(domain, p_config[CONF_PLATFORM])
+                yield p_name, platform
+
+
+def iter_component_configs(config):
     for domain, conf in config.items():
         component = get_component(domain)
         if component.multi_conf:
@@ -303,8 +314,10 @@ class LoadValidationStep(ConfigValidationStep):
             # Ignore top-level keys starting with a dot
             return
         result.add_output_path([self.domain], self.domain)
-        result[self.domain] = self.conf
         component = get_component(self.domain)
+        if component.multi_conf_no_default and isinstance(self.conf, core.AutoLoad):
+            self.conf = []
+        result[self.domain] = self.conf
         path = [self.domain]
         if component is None:
             result.add_str_error(f"Component not found: {self.domain}", path)
@@ -424,7 +437,10 @@ class MetadataValidationStep(ConfigValidationStep):
 
     def run(self, result: Config) -> None:
         if self.conf is None:
-            result[self.domain] = self.conf = {}
+            if self.comp.multi_conf and self.comp.multi_conf_no_default:
+                result[self.domain] = self.conf = []
+            else:
+                result[self.domain] = self.conf = {}
 
         success = True
         for dependency in self.comp.dependencies:
diff --git a/esphome/loader.py b/esphome/loader.py
index cd21e5a509..40a38d0a14 100644
--- a/esphome/loader.py
+++ b/esphome/loader.py
@@ -57,6 +57,10 @@ class ComponentManifest:
     def multi_conf(self) -> bool:
         return getattr(self.module, "MULTI_CONF", False)
 
+    @property
+    def multi_conf_no_default(self) -> bool:
+        return getattr(self.module, "MULTI_CONF_NO_DEFAULT", False)
+
     @property
     def to_code(self) -> Optional[Callable[[Any], None]]:
         return getattr(self.module, "to_code", None)
diff --git a/esphome/writer.py b/esphome/writer.py
index 83e95614a6..3ad0e60d31 100644
--- a/esphome/writer.py
+++ b/esphome/writer.py
@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 from typing import Union
 
-from esphome.config import iter_components
+from esphome.config import iter_components, iter_component_configs
 from esphome.const import (
     HEADER_FILE_EXTENSIONS,
     SOURCE_FILE_EXTENSIONS,
@@ -70,14 +70,14 @@ UPLOAD_SPEED_OVERRIDE = {
 
 def get_flags(key):
     flags = set()
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         flags |= getattr(component, key)(conf)
     return flags
 
 
 def get_include_text():
     include_text = '#include "esphome.h"\nusing namespace esphome;\n'
-    for _, component, conf in iter_components(CORE.config):
+    for _, component, conf in iter_component_configs(CORE.config):
         if not hasattr(component, "includes"):
             continue
         includes = component.includes
@@ -232,7 +232,7 @@ the custom_components folder or the external_components feature.
 
 def copy_src_tree():
     source_files: list[loader.FileResource] = []
-    for _, component, _ in iter_components(CORE.config):
+    for _, component in iter_components(CORE.config):
         source_files += component.resources
     source_files_map = {
         Path(x.package.replace(".", "/") + "/" + x.resource): x for x in source_files
diff --git a/tests/test8.yaml b/tests/test8.yaml
index 558e86e1f9..fafdb76e12 100644
--- a/tests/test8.yaml
+++ b/tests/test8.yaml
@@ -92,3 +92,13 @@ sensor:
       name: "Loop Time"
     psram:
       name: "PSRAM Free"
+
+# Purposely test that `animation:` does auto-load `image:`
+# Keep the `image:` undefined.
+# image:
+
+animation:
+  - id: rgb565_animation
+    file: pnglogo.png
+    type: RGB565
+    use_transparency: no

From 6a9e85438f95b14be54bdf9a961897bdd78224b9 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 19:42:27 +0900
Subject: [PATCH 32/34] Fix pin reuse error with pin expanders (#5973)

Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>
---
 esphome/pins.py    | 20 +++++++++++---------
 tests/test1.yaml   |  4 ----
 tests/test3.1.yaml |  2 --
 3 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/esphome/pins.py b/esphome/pins.py
index e2fd8e98e2..87f7084d4f 100644
--- a/esphome/pins.py
+++ b/esphome/pins.py
@@ -1,7 +1,7 @@
 import operator
 from functools import reduce
 import esphome.config_validation as cv
-from esphome.core import CORE, ID
+from esphome.core import CORE
 
 from esphome.const import (
     CONF_INPUT,
@@ -25,15 +25,16 @@ class PinRegistry(dict):
     def reset(self):
         self.pins_used = {}
 
-    def get_count(self, key, number):
+    def get_count(self, key, id, number):
         """
         Get the number of places a given pin is used.
-        :param key: The ID of the defining component
+        :param key: The key of the registered pin schema.
+        :param id: The ID of the defining component
         :param number: The pin number
         :return: The number of places the pin is used.
         """
-        pin_key = (key, number)
-        return self.pins_used[pin_key] if pin_key in self.pins_used else 0
+        pin_key = (key, id, number)
+        return len(self.pins_used[pin_key]) if pin_key in self.pins_used else 0
 
     def register(self, name, schema, final_validate=None):
         """
@@ -65,9 +66,10 @@ class PinRegistry(dict):
         result = self[key][1](conf)
         if CONF_NUMBER in result:
             # key maps to the pin schema
-            if isinstance(key, ID):
-                key = key.id
-            pin_key = (key, result[CONF_NUMBER])
+            if key != CORE.target_platform:
+                pin_key = (key, conf[key], result[CONF_NUMBER])
+            else:
+                pin_key = (key, key, result[CONF_NUMBER])
             if pin_key not in self.pins_used:
                 self.pins_used[pin_key] = []
             # client_id identifies the instance of the providing component
@@ -101,7 +103,7 @@ class PinRegistry(dict):
         Run the final validation for all pins, and check for reuse
         :param fconf: The full config
         """
-        for (key, _), pin_list in self.pins_used.items():
+        for (key, _, _), pin_list in self.pins_used.items():
             count = len(pin_list)  # number of places same pin used.
             final_val_fun = self[key][2]  # final validation function
             for pin_path, client_id, pin_config in pin_list:
diff --git a/tests/test1.yaml b/tests/test1.yaml
index f7b433cce2..3f8ae0151f 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -1667,7 +1667,6 @@ binary_sensor:
       mcp23xxx: mcp23s08_hub
       # Use pin number 1
       number: 1
-      allow_other_uses: true
       # One of INPUT or INPUT_PULLUP
       mode: INPUT_PULLUP
       inverted: false
@@ -2149,7 +2148,6 @@ output:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2157,7 +2155,6 @@ output:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
@@ -2597,7 +2594,6 @@ switch:
       mcp23xxx: mcp23s08_hub
       # Use pin number 0
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
       inverted: false
   - platform: gpio
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 63ef4e8ce0..b5428abbfa 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -401,7 +401,6 @@ switch:
     pin:
       mcp23xxx: mcp23017_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: &interlock [gpio_switch1, gpio_switch2, gpio_switch3]
   - platform: gpio
@@ -409,7 +408,6 @@ switch:
     pin:
       mcp23xxx: mcp23008_hub
       number: 0
-      allow_other_uses: true
       mode: OUTPUT
     interlock: *interlock
   - platform: gpio

From bec1ad9396806086f61eae699ca6d3d8210682a7 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 20 Dec 2023 20:59:46 +0900
Subject: [PATCH 33/34] Bump version to 2023.12.0b6

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 0df9c27233..475f28c35e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b5"
+__version__ = "2023.12.0b6"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 867595561417deb035fcde921a33db72d1b8c3fa Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 21 Dec 2023 08:09:10 +0900
Subject: [PATCH 34/34] Bump version to 2023.12.0

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 475f28c35e..55be91a4f9 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b6"
+__version__ = "2023.12.0"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (