From 8e4b9c3c1e38f8aead90f19aa726143e86aad1a6 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:45:26 +1300 Subject: [PATCH 1/4] Voice Assistant improvements (#5827) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 2 + .../i2s_audio/speaker/i2s_audio_speaker.h | 2 + esphome/components/speaker/speaker.h | 2 + .../voice_assistant/voice_assistant.cpp | 108 +++++++++++------- .../voice_assistant/voice_assistant.h | 3 + 5 files changed, 78 insertions(+), 39 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index ed13e6b458..e729cdf954 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -220,6 +220,8 @@ size_t I2SAudioSpeaker::play(const uint8_t *data, size_t length) { return index; } +bool I2SAudioSpeaker::has_buffered_data() const { return uxQueueMessagesWaiting(this->buffer_queue_) > 0; } + } // namespace i2s_audio } // namespace esphome diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h index b075722e1b..20c36a69d3 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h @@ -56,6 +56,8 @@ class I2SAudioSpeaker : public Component, public speaker::Speaker, public I2SAud size_t play(const uint8_t *data, size_t length) override; + bool has_buffered_data() const override; + protected: void start_(); // void stop_(); diff --git a/esphome/components/speaker/speaker.h b/esphome/components/speaker/speaker.h index 3f520e3c5e..b494873160 100644 --- a/esphome/components/speaker/speaker.h +++ b/esphome/components/speaker/speaker.h @@ -18,6 +18,8 @@ class Speaker { virtual void start() = 0; virtual void stop() = 0; + virtual bool has_buffered_data() const = 0; + bool is_running() const { return this->state_ == STATE_RUNNING; } protected: diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 9b13a71039..c0e706305d 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -273,28 +273,27 @@ void VoiceAssistant::loop() { bool playing = false; #ifdef USE_SPEAKER if (this->speaker_ != nullptr) { + ssize_t received_len = 0; if (this->speaker_buffer_index_ + RECEIVE_SIZE < SPEAKER_BUFFER_SIZE) { - auto len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE); - if (len > 0) { - this->speaker_buffer_index_ += len; - this->speaker_buffer_size_ += len; + received_len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE); + if (received_len > 0) { + this->speaker_buffer_index_ += received_len; + this->speaker_buffer_size_ += received_len; + this->speaker_bytes_received_ += received_len; } } else { - ESP_LOGW(TAG, "Receive buffer full"); - } - if (this->speaker_buffer_size_ > 0) { - size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_); - if (written > 0) { - memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written); - this->speaker_buffer_size_ -= written; - this->speaker_buffer_index_ -= written; - this->set_timeout("speaker-timeout", 2000, [this]() { this->speaker_->stop(); }); - } else { - ESP_LOGW(TAG, "Speaker buffer full"); - } + ESP_LOGD(TAG, "Receive buffer full"); } + // Build a small buffer of audio before sending to the speaker + if (this->speaker_bytes_received_ > RECEIVE_SIZE * 4) + this->write_speaker_(); if (this->wait_for_stream_end_) { this->cancel_timeout("playing"); + if (this->stream_ended_ && received_len < 0) { + ESP_LOGD(TAG, "End of audio stream received"); + this->cancel_timeout("speaker-timeout"); + this->set_state_(State::RESPONSE_FINISHED, State::RESPONSE_FINISHED); + } break; // We dont want to timeout here as the STREAM_END event will take care of that. } playing = this->speaker_->is_running(); @@ -316,14 +315,26 @@ void VoiceAssistant::loop() { case State::RESPONSE_FINISHED: { #ifdef USE_SPEAKER if (this->speaker_ != nullptr) { + if (this->speaker_buffer_size_ > 0) { + this->write_speaker_(); + break; + } + if (this->speaker_->has_buffered_data() || this->speaker_->is_running()) { + break; + } + ESP_LOGD(TAG, "Speaker has finished outputting all audio"); this->speaker_->stop(); this->cancel_timeout("speaker-timeout"); this->cancel_timeout("playing"); this->speaker_buffer_size_ = 0; this->speaker_buffer_index_ = 0; + this->speaker_bytes_received_ = 0; memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); + this->wait_for_stream_end_ = false; + this->stream_ended_ = false; + + this->tts_stream_end_trigger_->trigger(); } - this->wait_for_stream_end_ = false; #endif this->set_state_(State::IDLE, State::IDLE); break; @@ -333,6 +344,20 @@ void VoiceAssistant::loop() { } } +void VoiceAssistant::write_speaker_() { + if (this->speaker_buffer_size_ > 0) { + size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_); + if (written > 0) { + memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written); + this->speaker_buffer_size_ -= written; + this->speaker_buffer_index_ -= written; + this->set_timeout("speaker-timeout", 5000, [this]() { this->speaker_->stop(); }); + } else { + ESP_LOGD(TAG, "Speaker buffer full, trying again next loop"); + } + } +} + void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscribe) { if (!subscribe) { if (this->api_client_ == nullptr || client != this->api_client_) { @@ -503,21 +528,20 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { switch (msg.event_type) { case api::enums::VOICE_ASSISTANT_RUN_START: ESP_LOGD(TAG, "Assist Pipeline running"); - this->start_trigger_->trigger(); + this->defer([this]() { this->start_trigger_->trigger(); }); break; case api::enums::VOICE_ASSISTANT_WAKE_WORD_START: break; case api::enums::VOICE_ASSISTANT_WAKE_WORD_END: { ESP_LOGD(TAG, "Wake word detected"); - this->wake_word_detected_trigger_->trigger(); + this->defer([this]() { this->wake_word_detected_trigger_->trigger(); }); break; } case api::enums::VOICE_ASSISTANT_STT_START: ESP_LOGD(TAG, "STT started"); - this->listening_trigger_->trigger(); + this->defer([this]() { this->listening_trigger_->trigger(); }); break; case api::enums::VOICE_ASSISTANT_STT_END: { - this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE); std::string text; for (auto arg : msg.data) { if (arg.name == "text") { @@ -529,12 +553,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { return; } ESP_LOGD(TAG, "Speech recognised as: \"%s\"", text.c_str()); - this->stt_end_trigger_->trigger(text); + this->defer([this, text]() { this->stt_end_trigger_->trigger(text); }); break; } case api::enums::VOICE_ASSISTANT_INTENT_START: ESP_LOGD(TAG, "Intent started"); - this->intent_start_trigger_->trigger(); + this->defer([this]() { this->intent_start_trigger_->trigger(); }); break; case api::enums::VOICE_ASSISTANT_INTENT_END: { for (auto arg : msg.data) { @@ -542,7 +566,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { this->conversation_id_ = std::move(arg.value); } } - this->intent_end_trigger_->trigger(); + this->defer([this]() { this->intent_end_trigger_->trigger(); }); break; } case api::enums::VOICE_ASSISTANT_TTS_START: { @@ -557,10 +581,12 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { return; } ESP_LOGD(TAG, "Response: \"%s\"", text.c_str()); - this->tts_start_trigger_->trigger(text); + this->defer([this, text]() { + this->tts_start_trigger_->trigger(text); #ifdef USE_SPEAKER - this->speaker_->start(); + this->speaker_->start(); #endif + }); break; } case api::enums::VOICE_ASSISTANT_TTS_END: { @@ -575,14 +601,16 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { return; } ESP_LOGD(TAG, "Response URL: \"%s\"", url.c_str()); + this->defer([this, url]() { #ifdef USE_MEDIA_PLAYER - if (this->media_player_ != nullptr) { - this->media_player_->make_call().set_media_url(url).perform(); - } + if (this->media_player_ != nullptr) { + this->media_player_->make_call().set_media_url(url).perform(); + } #endif + this->tts_end_trigger_->trigger(url); + }); State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE; this->set_state_(new_state, new_state); - this->tts_end_trigger_->trigger(url); break; } case api::enums::VOICE_ASSISTANT_RUN_END: { @@ -599,7 +627,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { this->set_state_(State::IDLE, State::IDLE); } } - this->end_trigger_->trigger(); + this->defer([this]() { this->end_trigger_->trigger(); }); break; } case api::enums::VOICE_ASSISTANT_ERROR: { @@ -617,8 +645,10 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { return; } else if (code == "wake-provider-missing" || code == "wake-engine-missing") { // Wake word is not set up or not ready on Home Assistant so stop and do not retry until user starts again. - this->request_stop(); - this->error_trigger_->trigger(code, message); + this->defer([this, code, message]() { + this->request_stop(); + this->error_trigger_->trigger(code, message); + }); return; } ESP_LOGE(TAG, "Error: %s - %s", code.c_str(), message.c_str()); @@ -626,32 +656,32 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { this->signal_stop_(); this->set_state_(State::STOP_MICROPHONE, State::IDLE); } - this->error_trigger_->trigger(code, message); + this->defer([this, code, message]() { this->error_trigger_->trigger(code, message); }); break; } case api::enums::VOICE_ASSISTANT_TTS_STREAM_START: { #ifdef USE_SPEAKER this->wait_for_stream_end_ = true; ESP_LOGD(TAG, "TTS stream start"); - this->tts_stream_start_trigger_->trigger(); + this->defer([this] { this->tts_stream_start_trigger_->trigger(); }); #endif break; } case api::enums::VOICE_ASSISTANT_TTS_STREAM_END: { - this->set_state_(State::RESPONSE_FINISHED, State::IDLE); #ifdef USE_SPEAKER + this->stream_ended_ = true; ESP_LOGD(TAG, "TTS stream end"); - this->tts_stream_end_trigger_->trigger(); #endif break; } case api::enums::VOICE_ASSISTANT_STT_VAD_START: ESP_LOGD(TAG, "Starting STT by VAD"); - this->stt_vad_start_trigger_->trigger(); + this->defer([this]() { this->stt_vad_start_trigger_->trigger(); }); break; case api::enums::VOICE_ASSISTANT_STT_VAD_END: ESP_LOGD(TAG, "STT by VAD end"); - this->stt_vad_end_trigger_->trigger(); + this->set_state_(State::STOP_MICROPHONE, State::AWAITING_RESPONSE); + this->defer([this]() { this->stt_vad_end_trigger_->trigger(); }); break; default: ESP_LOGD(TAG, "Unhandled event type: %d", msg.event_type); diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index f6dcd1c563..66bf4c3c57 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -156,11 +156,14 @@ class VoiceAssistant : public Component { microphone::Microphone *mic_{nullptr}; #ifdef USE_SPEAKER + void write_speaker_(); speaker::Speaker *speaker_{nullptr}; uint8_t *speaker_buffer_; size_t speaker_buffer_index_{0}; size_t speaker_buffer_size_{0}; + size_t speaker_bytes_received_{0}; bool wait_for_stream_end_{false}; + bool stream_ended_{false}; #endif #ifdef USE_MEDIA_PLAYER media_player::MediaPlayer *media_player_{nullptr}; From ff97639f791963bb9e5f35318be960a100f024b0 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:43:41 +1300 Subject: [PATCH 2/4] Fix missing include in remote_base (#5843) --- esphome/components/remote_base/remote_base.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp index 40c699e8ea..095f95053f 100644 --- a/esphome/components/remote_base/remote_base.cpp +++ b/esphome/components/remote_base/remote_base.cpp @@ -1,6 +1,8 @@ #include "remote_base.h" #include "esphome/core/log.h" +#include + namespace esphome { namespace remote_base { From 687f5ca6330c7b96e92823bd28f9d68731404152 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:57:40 +1300 Subject: [PATCH 3/4] Add 'voice_assistant.connected' condition (#5845) --- esphome/components/voice_assistant/__init__.py | 12 ++++++++++++ esphome/components/voice_assistant/voice_assistant.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index d05f39072c..59aef901f2 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -57,6 +57,9 @@ StopAction = voice_assistant_ns.class_( IsRunningCondition = voice_assistant_ns.class_( "IsRunningCondition", automation.Condition, cg.Parented.template(VoiceAssistant) ) +ConnectedCondition = voice_assistant_ns.class_( + "ConnectedCondition", automation.Condition, cg.Parented.template(VoiceAssistant) +) def tts_stream_validate(config): @@ -298,3 +301,12 @@ async def voice_assistant_is_running_to_code(config, condition_id, template_arg, var = cg.new_Pvariable(condition_id, template_arg) await cg.register_parented(var, config[CONF_ID]) return var + + +@register_condition( + "voice_assistant.connected", ConnectedCondition, VOICE_ASSISTANT_ACTION_SCHEMA +) +async def voice_assistant_connected_to_code(config, condition_id, template_arg, args): + var = cg.new_Pvariable(condition_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + return var diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index 66bf4c3c57..f9325dff54 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -222,6 +222,11 @@ template class IsRunningCondition : public Condition, pub bool check(Ts... x) override { return this->parent_->is_running() || this->parent_->is_continuous(); } }; +template class ConnectedCondition : public Condition, public Parented { + public: + bool check(Ts... x) override { return this->parent_->get_api_connection() != nullptr; } +}; + extern VoiceAssistant *global_voice_assistant; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace voice_assistant From 28a3cddde374a07c880622cd13095ce7dfbbec4d Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:14:26 +1300 Subject: [PATCH 4/4] Bump version to 2023.11.5 --- esphome/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/const.py b/esphome/const.py index 5596cbcd36..2e693d0ea0 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -1,6 +1,6 @@ """Constants used by esphome.""" -__version__ = "2023.11.4" +__version__ = "2023.11.5" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = (