From 863bee28d9649f3179b75b3a5ebb833d9c7232a5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 25 May 2024 07:42:24 +1200 Subject: [PATCH 1/3] [voice_assistant] Don't allocate buffers until starting the microphone for the first time (#6800) --- .../voice_assistant/voice_assistant.cpp | 96 +++++++++++++++---- .../voice_assistant/voice_assistant.h | 4 + 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 109e52f8eb..59ba39c527 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -71,6 +71,12 @@ void VoiceAssistant::setup() { ESP_LOGCONFIG(TAG, "Setting up Voice Assistant..."); global_voice_assistant = this; +} + +bool VoiceAssistant::allocate_buffers_() { + if (this->send_buffer_ != nullptr) { + return true; // Already allocated + } #ifdef USE_SPEAKER if (this->speaker_ != nullptr) { @@ -78,8 +84,7 @@ void VoiceAssistant::setup() { this->speaker_buffer_ = speaker_allocator.allocate(SPEAKER_BUFFER_SIZE); if (this->speaker_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate speaker buffer"); - this->mark_failed(); - return; + return false; } } #endif @@ -88,8 +93,7 @@ void VoiceAssistant::setup() { this->input_buffer_ = allocator.allocate(INPUT_BUFFER_SIZE); if (this->input_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate input buffer"); - this->mark_failed(); - return; + return false; } #ifdef USE_ESP_ADF @@ -99,17 +103,71 @@ void VoiceAssistant::setup() { this->ring_buffer_ = RingBuffer::create(BUFFER_SIZE * sizeof(int16_t)); if (this->ring_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate ring buffer"); - this->mark_failed(); - return; + return false; } ExternalRAMAllocator send_allocator(ExternalRAMAllocator::ALLOW_FAILURE); this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE); if (send_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate send buffer"); - this->mark_failed(); - return; + return false; } + + return true; +} + +void VoiceAssistant::clear_buffers_() { + if (this->send_buffer_ != nullptr) { + memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); + } + + if (this->input_buffer_ != nullptr) { + memset(this->input_buffer_, 0, INPUT_BUFFER_SIZE * sizeof(int16_t)); + } + + if (this->ring_buffer_ != nullptr) { + this->ring_buffer_->reset(); + } + +#ifdef USE_SPEAKER + if (this->speaker_buffer_ != nullptr) { + memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); + + this->speaker_buffer_size_ = 0; + this->speaker_buffer_index_ = 0; + this->speaker_bytes_received_ = 0; + } +#endif +} + +void VoiceAssistant::deallocate_buffers_() { + ExternalRAMAllocator send_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + send_deallocator.deallocate(this->send_buffer_, SEND_BUFFER_SIZE); + this->send_buffer_ = nullptr; + + if (this->ring_buffer_ != nullptr) { + this->ring_buffer_.reset(); + this->ring_buffer_ = nullptr; + } + +#ifdef USE_ESP_ADF + if (this->vad_instance_ != nullptr) { + vad_destroy(this->vad_instance_); + this->vad_instance_ = nullptr; + } +#endif + + ExternalRAMAllocator input_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + input_deallocator.deallocate(this->input_buffer_, INPUT_BUFFER_SIZE); + this->input_buffer_ = nullptr; + +#ifdef USE_SPEAKER + if (this->speaker_buffer_ != nullptr) { + ExternalRAMAllocator speaker_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + speaker_deallocator.deallocate(this->speaker_buffer_, SPEAKER_BUFFER_SIZE); + this->speaker_buffer_ = nullptr; + } +#endif } int VoiceAssistant::read_microphone_() { @@ -138,14 +196,13 @@ void VoiceAssistant::loop() { } this->continuous_ = false; this->signal_stop_(); + this->clear_buffers_(); return; } switch (this->state_) { case State::IDLE: { if (this->continuous_ && this->desired_state_ == State::IDLE) { this->idle_trigger_->trigger(); - - this->ring_buffer_->reset(); #ifdef USE_ESP_ADF if (this->use_wake_word_) { this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD); @@ -161,8 +218,15 @@ void VoiceAssistant::loop() { } case State::START_MICROPHONE: { ESP_LOGD(TAG, "Starting Microphone"); - memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); - memset(this->input_buffer_, 0, INPUT_BUFFER_SIZE * sizeof(int16_t)); + if (!this->allocate_buffers_()) { + this->status_set_error("Failed to allocate buffers"); + return; + } + if (this->status_has_error()) { + this->status_clear_error(); + } + this->clear_buffers_(); + this->mic_->start(); this->high_freq_.start(); this->set_state_(State::STARTING_MICROPHONE); @@ -343,10 +407,9 @@ void VoiceAssistant::loop() { this->speaker_->stop(); this->cancel_timeout("speaker-timeout"); this->cancel_timeout("playing"); - this->speaker_buffer_size_ = 0; - this->speaker_buffer_index_ = 0; - this->speaker_bytes_received_ = 0; - memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); + + this->clear_buffers_(); + this->wait_for_stream_end_ = false; this->stream_ended_ = false; @@ -507,7 +570,6 @@ void VoiceAssistant::request_start(bool continuous, bool silence_detection) { if (this->state_ == State::IDLE) { this->continuous_ = continuous; this->silence_detection_ = silence_detection; - this->ring_buffer_->reset(); #ifdef USE_ESP_ADF if (this->use_wake_word_) { this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD); diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index 1c0ea12f4f..17141365d4 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -151,6 +151,10 @@ class VoiceAssistant : public Component { void set_wake_word(const std::string &wake_word) { this->wake_word_ = wake_word; } protected: + bool allocate_buffers_(); + void clear_buffers_(); + void deallocate_buffers_(); + int read_microphone_(); void set_state_(State state); void set_state_(State state, State desired_state); From 04db724295e4e5a537e1532b426a9c64bc19a7aa Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 25 May 2024 07:42:24 +1200 Subject: [PATCH 2/3] [voice_assistant] Don't allocate buffers until starting the microphone for the first time (#6800) --- .../voice_assistant/voice_assistant.cpp | 96 +++++++++++++++---- .../voice_assistant/voice_assistant.h | 4 + 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 109e52f8eb..59ba39c527 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -71,6 +71,12 @@ void VoiceAssistant::setup() { ESP_LOGCONFIG(TAG, "Setting up Voice Assistant..."); global_voice_assistant = this; +} + +bool VoiceAssistant::allocate_buffers_() { + if (this->send_buffer_ != nullptr) { + return true; // Already allocated + } #ifdef USE_SPEAKER if (this->speaker_ != nullptr) { @@ -78,8 +84,7 @@ void VoiceAssistant::setup() { this->speaker_buffer_ = speaker_allocator.allocate(SPEAKER_BUFFER_SIZE); if (this->speaker_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate speaker buffer"); - this->mark_failed(); - return; + return false; } } #endif @@ -88,8 +93,7 @@ void VoiceAssistant::setup() { this->input_buffer_ = allocator.allocate(INPUT_BUFFER_SIZE); if (this->input_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate input buffer"); - this->mark_failed(); - return; + return false; } #ifdef USE_ESP_ADF @@ -99,17 +103,71 @@ void VoiceAssistant::setup() { this->ring_buffer_ = RingBuffer::create(BUFFER_SIZE * sizeof(int16_t)); if (this->ring_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate ring buffer"); - this->mark_failed(); - return; + return false; } ExternalRAMAllocator send_allocator(ExternalRAMAllocator::ALLOW_FAILURE); this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE); if (send_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate send buffer"); - this->mark_failed(); - return; + return false; } + + return true; +} + +void VoiceAssistant::clear_buffers_() { + if (this->send_buffer_ != nullptr) { + memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); + } + + if (this->input_buffer_ != nullptr) { + memset(this->input_buffer_, 0, INPUT_BUFFER_SIZE * sizeof(int16_t)); + } + + if (this->ring_buffer_ != nullptr) { + this->ring_buffer_->reset(); + } + +#ifdef USE_SPEAKER + if (this->speaker_buffer_ != nullptr) { + memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); + + this->speaker_buffer_size_ = 0; + this->speaker_buffer_index_ = 0; + this->speaker_bytes_received_ = 0; + } +#endif +} + +void VoiceAssistant::deallocate_buffers_() { + ExternalRAMAllocator send_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + send_deallocator.deallocate(this->send_buffer_, SEND_BUFFER_SIZE); + this->send_buffer_ = nullptr; + + if (this->ring_buffer_ != nullptr) { + this->ring_buffer_.reset(); + this->ring_buffer_ = nullptr; + } + +#ifdef USE_ESP_ADF + if (this->vad_instance_ != nullptr) { + vad_destroy(this->vad_instance_); + this->vad_instance_ = nullptr; + } +#endif + + ExternalRAMAllocator input_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + input_deallocator.deallocate(this->input_buffer_, INPUT_BUFFER_SIZE); + this->input_buffer_ = nullptr; + +#ifdef USE_SPEAKER + if (this->speaker_buffer_ != nullptr) { + ExternalRAMAllocator speaker_deallocator(ExternalRAMAllocator::ALLOW_FAILURE); + speaker_deallocator.deallocate(this->speaker_buffer_, SPEAKER_BUFFER_SIZE); + this->speaker_buffer_ = nullptr; + } +#endif } int VoiceAssistant::read_microphone_() { @@ -138,14 +196,13 @@ void VoiceAssistant::loop() { } this->continuous_ = false; this->signal_stop_(); + this->clear_buffers_(); return; } switch (this->state_) { case State::IDLE: { if (this->continuous_ && this->desired_state_ == State::IDLE) { this->idle_trigger_->trigger(); - - this->ring_buffer_->reset(); #ifdef USE_ESP_ADF if (this->use_wake_word_) { this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD); @@ -161,8 +218,15 @@ void VoiceAssistant::loop() { } case State::START_MICROPHONE: { ESP_LOGD(TAG, "Starting Microphone"); - memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); - memset(this->input_buffer_, 0, INPUT_BUFFER_SIZE * sizeof(int16_t)); + if (!this->allocate_buffers_()) { + this->status_set_error("Failed to allocate buffers"); + return; + } + if (this->status_has_error()) { + this->status_clear_error(); + } + this->clear_buffers_(); + this->mic_->start(); this->high_freq_.start(); this->set_state_(State::STARTING_MICROPHONE); @@ -343,10 +407,9 @@ void VoiceAssistant::loop() { this->speaker_->stop(); this->cancel_timeout("speaker-timeout"); this->cancel_timeout("playing"); - this->speaker_buffer_size_ = 0; - this->speaker_buffer_index_ = 0; - this->speaker_bytes_received_ = 0; - memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); + + this->clear_buffers_(); + this->wait_for_stream_end_ = false; this->stream_ended_ = false; @@ -507,7 +570,6 @@ void VoiceAssistant::request_start(bool continuous, bool silence_detection) { if (this->state_ == State::IDLE) { this->continuous_ = continuous; this->silence_detection_ = silence_detection; - this->ring_buffer_->reset(); #ifdef USE_ESP_ADF if (this->use_wake_word_) { this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD); diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index 1c0ea12f4f..17141365d4 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -151,6 +151,10 @@ class VoiceAssistant : public Component { void set_wake_word(const std::string &wake_word) { this->wake_word_ = wake_word; } protected: + bool allocate_buffers_(); + void clear_buffers_(); + void deallocate_buffers_(); + int read_microphone_(); void set_state_(State state); void set_state_(State state, State desired_state); From af755380b7e8f8b90cfd1f45c9dcbf09bb94c101 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 25 May 2024 08:14:39 +1200 Subject: [PATCH 3/3] Bump version to 2024.5.3 --- esphome/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/const.py b/esphome/const.py index 48b8aa8656..0c53287909 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -1,6 +1,6 @@ """Constants used by esphome.""" -__version__ = "2024.5.2" +__version__ = "2024.5.3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = (