diff --git a/CODEOWNERS b/CODEOWNERS index c630db7948..ccad2ed5b0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -121,6 +121,7 @@ esphome/components/esp32_improv/* @jesserockz esphome/components/esp32_rmt/* @jesserockz esphome/components/esp32_rmt_led_strip/* @jesserockz esphome/components/esp8266/* @esphome/core +esphome/components/esp_adf/* @jesserockz esphome/components/ethernet_info/* @gtjadsonsantos esphome/components/event/* @nohat esphome/components/exposure_notifications/* @OttoWinter diff --git a/esphome/components/esp_adf/__init__.py b/esphome/components/esp_adf/__init__.py new file mode 100644 index 0000000000..b4621f5950 --- /dev/null +++ b/esphome/components/esp_adf/__init__.py @@ -0,0 +1,103 @@ +import os + +import esphome.config_validation as cv +import esphome.codegen as cg +import esphome.final_validate as fv + +from esphome.components import esp32 + +from esphome.const import CONF_ID, CONF_BOARD + +CODEOWNERS = ["@jesserockz"] +DEPENDENCIES = ["esp32"] + +CONF_ESP_ADF_ID = "esp_adf_id" +CONF_ESP_ADF = "esp_adf" + +esp_adf_ns = cg.esphome_ns.namespace("esp_adf") +ESPADF = esp_adf_ns.class_("ESPADF", cg.Component) +ESPADFPipeline = esp_adf_ns.class_("ESPADFPipeline", cg.Parented.template(ESPADF)) + +SUPPORTED_BOARDS = { + "esp32s3box": "CONFIG_ESP32_S3_BOX_BOARD", + "esp32s3boxlite": "CONFIG_ESP32_S3_BOX_LITE_BOARD", + "esp32s3box3": "CONFIG_ESP32_S3_BOX_3_BOARD", + "esp32s3korvo1": "CONFIG_ESP32_S3_KORVO1_BOARD", +} + + +def _default_board(config): + config = config.copy() + if board := config.get(CONF_BOARD) is None: + board = esp32.get_board() + if board in SUPPORTED_BOARDS: + config[CONF_BOARD] = board + return config + + +def final_validate_usable_board(platform: str): + def _validate(adf_config): + board = adf_config.get(CONF_BOARD) + if board not in SUPPORTED_BOARDS: + raise cv.Invalid(f"Board {board} is not supported by esp-adf {platform}") + return adf_config + + return cv.Schema( + {cv.Required(CONF_ESP_ADF_ID): fv.id_declaration_match_schema(_validate)}, + extra=cv.ALLOW_EXTRA, + ) + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(ESPADF), + cv.Optional(CONF_BOARD): cv.string_strict, + } + ), + _default_board, + cv.only_with_esp_idf, +) + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + cg.add_define("USE_ESP_ADF") + + cg.add_platformio_option("build_unflags", "-Wl,--end-group") + + esp32.add_idf_component( + name="esp-adf", + repo="https://github.com/espressif/esp-adf", + path="components", + ref="v2.5", + components=["*"], + submodules=["components/esp-sr", "components/esp-adf-libs"], + ) + + esp32.add_idf_component( + name="esp-dsp", + repo="https://github.com/espressif/esp-dsp", + ref="v1.2.0", + ) + + cg.add_platformio_option( + "board_build.embed_txtfiles", "components/dueros_service/duer_profile" + ) + + if board := config.get(CONF_BOARD): + cg.add_define("USE_ESP_ADF_BOARD") + + esp32.add_idf_sdkconfig_option(SUPPORTED_BOARDS[board], True) + + esp32.add_extra_script( + "pre", + "apply_adf_patches.py", + os.path.join(os.path.dirname(__file__), "apply_adf_patches.py.script"), + ) + esp32.add_extra_build_file( + "esp_adf_patches/idf_v4.4_freertos.patch", + "https://github.com/espressif/esp-adf/raw/v2.5/idf_patches/idf_v4.4_freertos.patch", + ) diff --git a/esphome/components/esp_adf/apply_adf_patches.py.script b/esphome/components/esp_adf/apply_adf_patches.py.script new file mode 100644 index 0000000000..736a7fb219 --- /dev/null +++ b/esphome/components/esp_adf/apply_adf_patches.py.script @@ -0,0 +1,23 @@ +from os.path import join, isfile + +Import("env") + +FRAMEWORK_DIR = env.PioPlatform().get_package_dir("framework-espidf") +patchflag_path = join(FRAMEWORK_DIR, ".adf-patching-done") + +PROJECT_DIR = env.get('PROJECT_DIR') + +PATCH_FILE = join(PROJECT_DIR, "esp_adf_patches", "idf_v4.4_freertos.patch") + +# patch file only if we didn't do it before +if not isfile(patchflag_path): + print(PATCH_FILE) + assert isfile(PATCH_FILE) + + env.Execute("patch -p1 -d %s -i %s" % (FRAMEWORK_DIR, PATCH_FILE)) + + def _touch(path): + with open(path, "w") as fp: + fp.write("") + + env.Execute(lambda *args, **kwargs: _touch(patchflag_path)) diff --git a/esphome/components/esp_adf/esp_adf.cpp b/esphome/components/esp_adf/esp_adf.cpp new file mode 100644 index 0000000000..d6f3a1cc64 --- /dev/null +++ b/esphome/components/esp_adf/esp_adf.cpp @@ -0,0 +1,30 @@ +#include "esp_adf.h" +#include "esphome/core/defines.h" + +#ifdef USE_ESP_IDF + +#ifdef USE_ESP_ADF_BOARD +#include +#endif + +#include "esphome/core/log.h" + +namespace esphome { +namespace esp_adf { + +static const char *const TAG = "esp_adf"; + +void ESPADF::setup() { +#ifdef USE_ESP_ADF_BOARD + ESP_LOGI(TAG, "Start codec chip"); + audio_board_handle_t board_handle = audio_board_init(); + audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START); +#endif +} + +float ESPADF::get_setup_priority() const { return setup_priority::HARDWARE; } + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/esp_adf/esp_adf.h b/esphome/components/esp_adf/esp_adf.h new file mode 100644 index 0000000000..dfeeb14e2e --- /dev/null +++ b/esphome/components/esp_adf/esp_adf.h @@ -0,0 +1,58 @@ +#pragma once + +#ifdef USE_ESP_IDF + +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +namespace esphome { +namespace esp_adf { + +static const size_t BUFFER_SIZE = 1024; + +enum class TaskEventType : uint8_t { + STARTING = 0, + STARTED, + RUNNING, + STOPPING, + STOPPED, + WARNING = 255, +}; + +struct TaskEvent { + TaskEventType type; + esp_err_t err; +}; + +struct CommandEvent { + bool stop; +}; + +struct DataEvent { + bool stop; + size_t len; + uint8_t data[BUFFER_SIZE]; +}; + +class ESPADF; + +class ESPADFPipeline : public Parented {}; + +class ESPADF : public Component { + public: + void setup() override; + + float get_setup_priority() const override; + + void lock() { this->lock_.lock(); } + bool try_lock() { return this->lock_.try_lock(); } + void unlock() { this->lock_.unlock(); } + + protected: + Mutex lock_; +}; + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/esp_adf/microphone/__init__.py b/esphome/components/esp_adf/microphone/__init__.py new file mode 100644 index 0000000000..f173c91d8d --- /dev/null +++ b/esphome/components/esp_adf/microphone/__init__.py @@ -0,0 +1,42 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.components import microphone +from esphome.const import CONF_ID + +from .. import ( + CONF_ESP_ADF_ID, + ESPADF, + ESPADFPipeline, + esp_adf_ns, + final_validate_usable_board, +) + +AUTO_LOAD = ["esp_adf"] +CONFLICTS_WITH = ["i2s_audio"] +DEPENDENCIES = ["esp32"] + +ESPADFMicrophone = esp_adf_ns.class_( + "ESPADFMicrophone", ESPADFPipeline, microphone.Microphone, cg.Component +) + + +CONFIG_SCHEMA = cv.All( + microphone.MICROPHONE_SCHEMA.extend( + { + cv.GenerateID(): cv.declare_id(ESPADFMicrophone), + cv.GenerateID(CONF_ESP_ADF_ID): cv.use_id(ESPADF), + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_with_esp_idf, + cv.require_esphome_version(2023, 12, 7), +) + +FINAL_VALIDATE_SCHEMA = final_validate_usable_board("microphone") + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await cg.register_parented(var, config[CONF_ESP_ADF_ID]) + + await microphone.register_microphone(var, config) diff --git a/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp b/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp new file mode 100644 index 0000000000..59f4cb843c --- /dev/null +++ b/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp @@ -0,0 +1,315 @@ +#include "esp_adf_microphone.h" + +#ifdef USE_ESP_IDF + +#include + +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace esphome { +namespace esp_adf { + +static const char *const TAG = "esp_adf.microphone"; + +void ESPADFMicrophone::setup() { + ESP_LOGCONFIG(TAG, "Setting up ESP ADF Microphone..."); + this->ring_buffer_ = RingBuffer::create(8000 * sizeof(int16_t)); + if (this->ring_buffer_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate ring buffer"); + this->mark_failed(); + return; + } + + this->read_event_queue_ = xQueueCreate(20, sizeof(TaskEvent)); + if (this->read_event_queue_ == nullptr) { + ESP_LOGW(TAG, "Could not allocate event queue"); + this->mark_failed(); + return; + } + this->read_command_queue_ = xQueueCreate(20, sizeof(CommandEvent)); + if (this->read_command_queue_ == nullptr) { + ESP_LOGW(TAG, "Could not allocate command queue"); + this->mark_failed(); + return; + } + ESP_LOGCONFIG(TAG, "Successfully set up ESP ADF Microphone"); +} + +void ESPADFMicrophone::start() { + if (this->is_failed()) + return; + if (this->state_ == microphone::STATE_STOPPING) { + ESP_LOGW(TAG, "Microphone is stopping, cannot start."); + return; + } + this->state_ = microphone::STATE_STARTING; +} +void ESPADFMicrophone::start_() { + if (!this->parent_->try_lock()) { + return; + } + + xTaskCreate(ESPADFMicrophone::read_task, "read_task", 8192, (void *) this, 0, &this->read_task_handle_); +} + +void ESPADFMicrophone::read_task(void *params) { + ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params; + TaskEvent event; + + ExternalRAMAllocator allocator(ExternalRAMAllocator::ALLOW_FAILURE); + int16_t *buffer = allocator.allocate(BUFFER_SIZE / sizeof(int16_t)); + if (buffer == nullptr) { + event.type = TaskEventType::WARNING; + event.err = ESP_ERR_NO_MEM; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + event.type = TaskEventType::STOPPED; + event.err = ESP_OK; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + while (true) { + delay(10); + } + return; + } + + event.type = TaskEventType::STARTING; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + audio_pipeline_cfg_t pipeline_cfg = { + .rb_size = 8 * 1024, + }; + audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg); + + i2s_driver_config_t i2s_config = { + .mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX), + .sample_rate = 16000, + .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, + .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, + .communication_format = I2S_COMM_FORMAT_STAND_I2S, + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM, + .dma_buf_count = 8, + .dma_buf_len = 128, + .use_apll = false, + .tx_desc_auto_clear = true, + .fixed_mclk = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256, + .bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT, + }; + + i2s_stream_cfg_t i2s_cfg = { + .type = AUDIO_STREAM_READER, + .i2s_config = i2s_config, + .i2s_port = static_cast(CODEC_ADC_I2S_PORT), + .use_alc = false, + .volume = 0, + .out_rb_size = I2S_STREAM_RINGBUFFER_SIZE, + .task_stack = I2S_STREAM_TASK_STACK, + .task_core = I2S_STREAM_TASK_CORE, + .task_prio = I2S_STREAM_TASK_PRIO, + .stack_in_ext = false, + .multi_out_num = 0, + .uninstall_drv = true, + .need_expand = false, + .expand_src_bits = I2S_BITS_PER_SAMPLE_16BIT, + }; + audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg); + + rsp_filter_cfg_t rsp_cfg = { + .src_rate = 16000, + .src_ch = 2, + .dest_rate = 16000, + .dest_bits = 16, + .dest_ch = 1, + .src_bits = I2S_BITS_PER_SAMPLE_16BIT, + .mode = RESAMPLE_DECODE_MODE, + .max_indata_bytes = RSP_FILTER_BUFFER_BYTE, + .out_len_bytes = RSP_FILTER_BUFFER_BYTE, + .type = ESP_RESAMPLE_TYPE_AUTO, + .complexity = 2, + .down_ch_idx = 0, + .prefer_flag = ESP_RSP_PREFER_TYPE_SPEED, + .out_rb_size = RSP_FILTER_RINGBUFFER_SIZE, + .task_stack = RSP_FILTER_TASK_STACK, + .task_core = RSP_FILTER_TASK_CORE, + .task_prio = RSP_FILTER_TASK_PRIO, + .stack_in_ext = true, + }; + audio_element_handle_t filter = rsp_filter_init(&rsp_cfg); + + raw_stream_cfg_t raw_cfg = { + .type = AUDIO_STREAM_READER, + .out_rb_size = 8 * 1024, + }; + audio_element_handle_t raw_read = raw_stream_init(&raw_cfg); + + audio_pipeline_register(pipeline, i2s_stream_reader, "i2s"); + audio_pipeline_register(pipeline, filter, "filter"); + audio_pipeline_register(pipeline, raw_read, "raw"); + + const char *link_tag[3] = {"i2s", "filter", "raw"}; + audio_pipeline_link(pipeline, &link_tag[0], 3); + + audio_pipeline_run(pipeline); + + event.type = TaskEventType::STARTED; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + CommandEvent command_event; + + while (true) { + if (xQueueReceive(this_mic->read_command_queue_, &command_event, 0) == pdTRUE) { + if (command_event.stop) { + // Stop signal from main thread + break; + } + } + + int bytes_read = raw_stream_read(raw_read, (char *) buffer, BUFFER_SIZE); + + if (bytes_read == -2 || bytes_read == 0) { + // No data in buffers to read. + continue; + } else if (bytes_read < 0) { + event.type = TaskEventType::WARNING; + event.err = bytes_read; + xQueueSend(this_mic->read_event_queue_, &event, 0); + continue; + } + + size_t written = this_mic->ring_buffer_->write((void *) buffer, bytes_read); + + event.type = TaskEventType::RUNNING; + event.err = written; + xQueueSend(this_mic->read_event_queue_, &event, 0); + } + + allocator.deallocate(buffer, BUFFER_SIZE / sizeof(int16_t)); + + audio_pipeline_stop(pipeline); + audio_pipeline_wait_for_stop(pipeline); + audio_pipeline_terminate(pipeline); + + event.type = TaskEventType::STOPPING; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + audio_pipeline_unregister(pipeline, i2s_stream_reader); + audio_pipeline_unregister(pipeline, filter); + // audio_pipeline_unregister(pipeline, algo_stream); + audio_pipeline_unregister(pipeline, raw_read); + + audio_pipeline_deinit(pipeline); + audio_element_deinit(i2s_stream_reader); + audio_element_deinit(filter); + // audio_element_deinit(algo_stream); + audio_element_deinit(raw_read); + + event.type = TaskEventType::STOPPED; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + while (true) { + delay(10); + } +} + +void ESPADFMicrophone::stop() { + if (this->state_ == microphone::STATE_STOPPED || this->state_ == microphone::STATE_STOPPING || this->is_failed()) + return; + this->state_ = microphone::STATE_STOPPING; + CommandEvent command_event; + command_event.stop = true; + xQueueSendToFront(this->read_command_queue_, &command_event, portMAX_DELAY); + ESP_LOGD(TAG, "Stopping microphone"); +} + +size_t ESPADFMicrophone::read(int16_t *buf, size_t len) { + if (this->is_failed()) { + ESP_LOGE(TAG, "Microphone is failed, cannot read"); + return 0; + } + if (this->ring_buffer_->available() == 0) { + return 0; // No data + } + size_t bytes_read = this->ring_buffer_->read((void *) buf, len); + + if (bytes_read == 0) { + // No data in buffers to read. + this->status_set_warning(); + return 0; + } + this->status_clear_warning(); + + return bytes_read; +} + +void ESPADFMicrophone::read_() { + std::vector samples; + samples.resize(BUFFER_SIZE); + this->read(samples.data(), samples.size()); + + this->data_callbacks_.call(samples); +} + +void ESPADFMicrophone::watch_() { + TaskEvent event; + if (xQueueReceive(this->read_event_queue_, &event, 0) == pdTRUE) { + switch (event.type) { + case TaskEventType::STARTING: + case TaskEventType::STOPPING: + break; + case TaskEventType::STARTED: + ESP_LOGD(TAG, "Microphone started"); + this->state_ = microphone::STATE_RUNNING; + break; + case TaskEventType::RUNNING: + this->status_clear_warning(); + // ESP_LOGD(TAG, "Putting %d bytes into ring buffer", event.err); + break; + case TaskEventType::STOPPED: + this->parent_->unlock(); + this->state_ = microphone::STATE_STOPPED; + vTaskDelete(this->read_task_handle_); + this->read_task_handle_ = nullptr; + ESP_LOGD(TAG, "Microphone stopped"); + break; + case TaskEventType::WARNING: + ESP_LOGW(TAG, "Error writing to pipeline: %s", esp_err_to_name(event.err)); + this->status_set_warning(); + break; + } + } +} + +void ESPADFMicrophone::loop() { + this->watch_(); + switch (this->state_) { + case microphone::STATE_STOPPED: + case microphone::STATE_STOPPING: + break; + case microphone::STATE_STARTING: + this->start_(); + break; + case microphone::STATE_RUNNING: + if (this->data_callbacks_.size() > 0) { + this->read_(); + } + break; + } +} + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/esp_adf/microphone/esp_adf_microphone.h b/esphome/components/esp_adf/microphone/esp_adf_microphone.h new file mode 100644 index 0000000000..bb569e23e3 --- /dev/null +++ b/esphome/components/esp_adf/microphone/esp_adf_microphone.h @@ -0,0 +1,42 @@ +#pragma once + +#ifdef USE_ESP_IDF + +#include "../esp_adf.h" + +#include "esphome/core/component.h" +#include "esphome/core/ring_buffer.h" + +#include "esphome/components/microphone/microphone.h" + +namespace esphome { +namespace esp_adf { + +class ESPADFMicrophone : public ESPADFPipeline, public microphone::Microphone, public Component { + public: + void setup() override; + void start() override; + void stop() override; + + void loop() override; + + size_t read(int16_t *buf, size_t len) override; + + protected: + void start_(); + void read_(); + void watch_(); + + static void read_task(void *params); + + std::unique_ptr ring_buffer_; + + TaskHandle_t read_task_handle_{nullptr}; + QueueHandle_t read_event_queue_; + QueueHandle_t read_command_queue_; +}; + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/esp_adf/speaker/__init__.py b/esphome/components/esp_adf/speaker/__init__.py new file mode 100644 index 0000000000..1b30989fa2 --- /dev/null +++ b/esphome/components/esp_adf/speaker/__init__.py @@ -0,0 +1,41 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.components import speaker +from esphome.const import CONF_ID + +from .. import ( + CONF_ESP_ADF_ID, + ESPADF, + ESPADFPipeline, + esp_adf_ns, + final_validate_usable_board, +) + +AUTO_LOAD = ["esp_adf"] +CONFLICTS_WITH = ["i2s_audio"] +DEPENDENCIES = ["esp32"] + +ESPADFSpeaker = esp_adf_ns.class_( + "ESPADFSpeaker", ESPADFPipeline, speaker.Speaker, cg.Component +) + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(ESPADFSpeaker), + cv.GenerateID(CONF_ESP_ADF_ID): cv.use_id(ESPADF), + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_with_esp_idf, +) + +FINAL_VALIDATE_SCHEMA = final_validate_usable_board("speaker") + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await cg.register_parented(var, config[CONF_ESP_ADF_ID]) + + await speaker.register_speaker(var, config) diff --git a/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp b/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp new file mode 100644 index 0000000000..797e32b931 --- /dev/null +++ b/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp @@ -0,0 +1,295 @@ +#include "esp_adf_speaker.h" + +#ifdef USE_ESP_IDF + +#include + +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include + +namespace esphome { +namespace esp_adf { + +static const size_t BUFFER_COUNT = 50; + +static const char *const TAG = "esp_adf.speaker"; + +void ESPADFSpeaker::setup() { + ESP_LOGCONFIG(TAG, "Setting up ESP ADF Speaker..."); + + ExternalRAMAllocator allocator(ExternalRAMAllocator::ALLOW_FAILURE); + + this->buffer_queue_.storage = allocator.allocate(sizeof(StaticQueue_t) + (BUFFER_COUNT * sizeof(DataEvent))); + if (this->buffer_queue_.storage == nullptr) { + ESP_LOGE(TAG, "Failed to allocate buffer queue!"); + this->mark_failed(); + return; + } + + this->buffer_queue_.handle = + xQueueCreateStatic(BUFFER_COUNT, sizeof(DataEvent), this->buffer_queue_.storage + sizeof(StaticQueue_t), + (StaticQueue_t *) (this->buffer_queue_.storage)); + + this->event_queue_ = xQueueCreate(20, sizeof(TaskEvent)); + if (this->event_queue_ == nullptr) { + ESP_LOGW(TAG, "Could not allocate event queue."); + this->mark_failed(); + return; + } +} + +void ESPADFSpeaker::start() { this->state_ = speaker::STATE_STARTING; } +void ESPADFSpeaker::start_() { + if (!this->parent_->try_lock()) { + return; // Waiting for another i2s component to return lock + } + + xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_); +} + +void ESPADFSpeaker::player_task(void *params) { + ESPADFSpeaker *this_speaker = (ESPADFSpeaker *) params; + + TaskEvent event; + event.type = TaskEventType::STARTING; + xQueueSend(this_speaker->event_queue_, &event, portMAX_DELAY); + + i2s_driver_config_t i2s_config = { + .mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_TX), + .sample_rate = 16000, + .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, + .channel_format = I2S_CHANNEL_FMT_ONLY_RIGHT, + .communication_format = I2S_COMM_FORMAT_STAND_I2S, + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM, + .dma_buf_count = 8, + .dma_buf_len = 1024, + .use_apll = false, + .tx_desc_auto_clear = true, + .fixed_mclk = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256, + .bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT, + }; + + audio_pipeline_cfg_t pipeline_cfg = { + .rb_size = 8 * 1024, + }; + audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg); + + i2s_stream_cfg_t i2s_cfg = { + .type = AUDIO_STREAM_WRITER, + .i2s_config = i2s_config, + .i2s_port = I2S_NUM_0, + .use_alc = false, + .volume = 0, + .out_rb_size = I2S_STREAM_RINGBUFFER_SIZE, + .task_stack = I2S_STREAM_TASK_STACK, + .task_core = I2S_STREAM_TASK_CORE, + .task_prio = I2S_STREAM_TASK_PRIO, + .stack_in_ext = false, + .multi_out_num = 0, + .uninstall_drv = true, + .need_expand = false, + .expand_src_bits = I2S_BITS_PER_SAMPLE_16BIT, + }; + audio_element_handle_t i2s_stream_writer = i2s_stream_init(&i2s_cfg); + + rsp_filter_cfg_t rsp_cfg = { + .src_rate = 16000, + .src_ch = 1, + .dest_rate = 16000, + .dest_bits = 16, + .dest_ch = 2, + .src_bits = 16, + .mode = RESAMPLE_DECODE_MODE, + .max_indata_bytes = RSP_FILTER_BUFFER_BYTE, + .out_len_bytes = RSP_FILTER_BUFFER_BYTE, + .type = ESP_RESAMPLE_TYPE_AUTO, + .complexity = 2, + .down_ch_idx = 0, + .prefer_flag = ESP_RSP_PREFER_TYPE_SPEED, + .out_rb_size = RSP_FILTER_RINGBUFFER_SIZE, + .task_stack = RSP_FILTER_TASK_STACK, + .task_core = RSP_FILTER_TASK_CORE, + .task_prio = RSP_FILTER_TASK_PRIO, + .stack_in_ext = true, + }; + audio_element_handle_t filter = rsp_filter_init(&rsp_cfg); + + raw_stream_cfg_t raw_cfg = { + .type = AUDIO_STREAM_WRITER, + .out_rb_size = 8 * 1024, + }; + audio_element_handle_t raw_write = raw_stream_init(&raw_cfg); + + audio_pipeline_register(pipeline, raw_write, "raw"); + audio_pipeline_register(pipeline, filter, "filter"); + audio_pipeline_register(pipeline, i2s_stream_writer, "i2s"); + + const char *link_tag[3] = { + "raw", + // "filter", + "i2s", + }; + audio_pipeline_link(pipeline, &link_tag[0], 2); + + audio_pipeline_run(pipeline); + + DataEvent data_event; + + event.type = TaskEventType::STARTED; + xQueueSend(this_speaker->event_queue_, &event, 0); + + uint32_t last_received = millis(); + + while (true) { + if (xQueueReceive(this_speaker->buffer_queue_.handle, &data_event, 0) != pdTRUE) { + if (millis() - last_received > 500) { + // No audio for 500ms, stop + break; + } else { + continue; + } + } + if (data_event.stop) { + // Stop signal from main thread + while (xQueueReceive(this_speaker->buffer_queue_.handle, &data_event, 0) == pdTRUE) { + // Flush queue + } + break; + } + + size_t remaining = data_event.len; + size_t current = 0; + if (remaining > 0) + last_received = millis(); + + while (remaining > 0) { + int bytes_written = raw_stream_write(raw_write, (char *) data_event.data + current, remaining); + if (bytes_written == ESP_FAIL) { + event = {.type = TaskEventType::WARNING, .err = ESP_FAIL}; + xQueueSend(this_speaker->event_queue_, &event, 0); + continue; + } + + remaining -= bytes_written; + current += bytes_written; + } + + event.type = TaskEventType::RUNNING; + xQueueSend(this_speaker->event_queue_, &event, 0); + } + + audio_pipeline_stop(pipeline); + audio_pipeline_wait_for_stop(pipeline); + audio_pipeline_terminate(pipeline); + + event.type = TaskEventType::STOPPING; + xQueueSend(this_speaker->event_queue_, &event, portMAX_DELAY); + + audio_pipeline_unregister(pipeline, i2s_stream_writer); + audio_pipeline_unregister(pipeline, filter); + audio_pipeline_unregister(pipeline, raw_write); + + audio_pipeline_deinit(pipeline); + audio_element_deinit(i2s_stream_writer); + audio_element_deinit(filter); + audio_element_deinit(raw_write); + + event.type = TaskEventType::STOPPED; + xQueueSend(this_speaker->event_queue_, &event, portMAX_DELAY); + + while (true) { + delay(10); + } +} + +void ESPADFSpeaker::stop() { + if (this->state_ == speaker::STATE_STOPPED) + return; + if (this->state_ == speaker::STATE_STARTING) { + this->state_ = speaker::STATE_STOPPED; + return; + } + this->state_ = speaker::STATE_STOPPING; + DataEvent data; + data.stop = true; + xQueueSendToFront(this->buffer_queue_.handle, &data, portMAX_DELAY); +} + +void ESPADFSpeaker::watch_() { + TaskEvent event; + if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) { + switch (event.type) { + case TaskEventType::STARTING: + case TaskEventType::STOPPING: + break; + case TaskEventType::STARTED: + this->state_ = speaker::STATE_RUNNING; + break; + case TaskEventType::RUNNING: + this->status_clear_warning(); + break; + case TaskEventType::STOPPED: + this->parent_->unlock(); + this->state_ = speaker::STATE_STOPPED; + vTaskDelete(this->player_task_handle_); + this->player_task_handle_ = nullptr; + break; + case TaskEventType::WARNING: + ESP_LOGW(TAG, "Error writing to pipeline: %s", esp_err_to_name(event.err)); + this->status_set_warning(); + break; + } + } +} + +void ESPADFSpeaker::loop() { + this->watch_(); + switch (this->state_) { + case speaker::STATE_STARTING: + this->start_(); + break; + case speaker::STATE_RUNNING: + case speaker::STATE_STOPPING: + case speaker::STATE_STOPPED: + break; + } +} + +size_t ESPADFSpeaker::play(const uint8_t *data, size_t length) { + if (this->is_failed()) { + ESP_LOGE(TAG, "Failed to play audio, speaker is in failed state."); + return 0; + } + if (this->state_ != speaker::STATE_RUNNING && this->state_ != speaker::STATE_STARTING) { + this->start(); + } + size_t remaining = length; + size_t index = 0; + while (remaining > 0) { + DataEvent event; + event.stop = false; + size_t to_send_length = std::min(remaining, BUFFER_SIZE); + event.len = to_send_length; + memcpy(event.data, data + index, to_send_length); + if (xQueueSend(this->buffer_queue_.handle, &event, 0) != pdTRUE) { + return index; // Queue full + } + remaining -= to_send_length; + index += to_send_length; + } + return index; +} + +bool ESPADFSpeaker::has_buffered_data() const { return uxQueueMessagesWaiting(this->buffer_queue_.handle) > 0; } + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/esp_adf/speaker/esp_adf_speaker.h b/esphome/components/esp_adf/speaker/esp_adf_speaker.h new file mode 100644 index 0000000000..fb2a4c52c3 --- /dev/null +++ b/esphome/components/esp_adf/speaker/esp_adf_speaker.h @@ -0,0 +1,51 @@ +#pragma once + +#ifdef USE_ESP_IDF + +#include "../esp_adf.h" + +#include +#include + +#include "esphome/components/speaker/speaker.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include +#include + +namespace esphome { +namespace esp_adf { + +class ESPADFSpeaker : public ESPADFPipeline, public speaker::Speaker, public Component { + public: + float get_setup_priority() const override { return esphome::setup_priority::LATE; } + + void setup() override; + void loop() override; + + void start() override; + void stop() override; + + size_t play(const uint8_t *data, size_t length) override; + + bool has_buffered_data() const override; + + protected: + void start_(); + void watch_(); + + static void player_task(void *params); + + TaskHandle_t player_task_handle_{nullptr}; + struct { + QueueHandle_t handle; + uint8_t *storage; + } buffer_queue_; + QueueHandle_t event_queue_; +}; + +} // namespace esp_adf +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/core/defines.h b/esphome/core/defines.h index b09373bcde..5ee5609b17 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -74,6 +74,7 @@ // IDF-specific feature flags #ifdef USE_ESP_IDF #define USE_MQTT_IDF_ENQUEUE +#define USE_ESP_ADF #endif // ESP32-specific feature flags