diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index c0e706305d..29fc664342 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -344,6 +344,7 @@ void VoiceAssistant::loop() {
   }
 }
 
+#ifdef USE_SPEAKER
 void VoiceAssistant::write_speaker_() {
   if (this->speaker_buffer_size_ > 0) {
     size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
@@ -357,6 +358,7 @@ void VoiceAssistant::write_speaker_() {
     }
   }
 }
+#endif
 
 void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) {
   if (!subscribe) {
diff --git a/esphome/const.py b/esphome/const.py
index 2e693d0ea0..6f8d83170e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.5"
+__version__ = "2023.11.6"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (