From 854d3f2e4a919c27f723d940295f0adec1614168 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 30 May 2024 13:09:19 +1200 Subject: [PATCH] [voice_assistant] Timers (#6821) Co-authored-by: Keith Burzinski --- esphome/components/api/api.proto | 19 ++++ esphome/components/api/api_connection.cpp | 9 ++ esphome/components/api/api_connection.h | 1 + esphome/components/api/api_pb2.cpp | 92 +++++++++++++++++++ esphome/components/api/api_pb2.h | 23 +++++ esphome/components/api/api_pb2_service.cpp | 13 +++ esphome/components/api/api_pb2_service.h | 3 + .../components/voice_assistant/__init__.py | 66 +++++++++++++ .../voice_assistant/voice_assistant.cpp | 53 +++++++++++ .../voice_assistant/voice_assistant.h | 42 +++++++++ 10 files changed, 321 insertions(+) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 774ca7ed9b..0becec2348 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1517,6 +1517,25 @@ message VoiceAssistantAudio { bool end = 2; } +enum VoiceAssistantTimerEvent { + VOICE_ASSISTANT_TIMER_STARTED = 0; + VOICE_ASSISTANT_TIMER_UPDATED = 1; + VOICE_ASSISTANT_TIMER_CANCELLED = 2; + VOICE_ASSISTANT_TIMER_FINISHED = 3; +} + +message VoiceAssistantTimerEventResponse { + option (id) = 115; + option (source) = SOURCE_CLIENT; + option (ifdef) = "USE_VOICE_ASSISTANT"; + + VoiceAssistantTimerEvent event_type = 1; + string timer_id = 2; + string name = 3; + uint32 total_seconds = 4; + uint32 seconds_left = 5; + bool is_active = 6; +} // ==================== ALARM CONTROL PANEL ==================== enum AlarmControlPanelState { diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 2804dba31f..253f04aa39 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1193,6 +1193,15 @@ void APIConnection::on_voice_assistant_audio(const VoiceAssistantAudio &msg) { voice_assistant::global_voice_assistant->on_audio(msg); } }; +void APIConnection::on_voice_assistant_timer_event_response(const VoiceAssistantTimerEventResponse &msg) { + if (voice_assistant::global_voice_assistant != nullptr) { + if (voice_assistant::global_voice_assistant->get_api_connection() != this) { + return; + } + + voice_assistant::global_voice_assistant->on_timer_event(msg); + } +}; #endif diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index ee466c5d10..293da17fa4 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -150,6 +150,7 @@ class APIConnection : public APIServerConnection { void on_voice_assistant_response(const VoiceAssistantResponse &msg) override; void on_voice_assistant_event_response(const VoiceAssistantEventResponse &msg) override; void on_voice_assistant_audio(const VoiceAssistantAudio &msg) override; + void on_voice_assistant_timer_event_response(const VoiceAssistantTimerEventResponse &msg) override; #endif #ifdef USE_ALARM_CONTROL_PANEL diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index a48087e348..9db6482c49 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -475,6 +475,22 @@ template<> const char *proto_enum_to_string(enums::V } #endif #ifdef HAS_PROTO_MESSAGE_DUMP +template<> const char *proto_enum_to_string(enums::VoiceAssistantTimerEvent value) { + switch (value) { + case enums::VOICE_ASSISTANT_TIMER_STARTED: + return "VOICE_ASSISTANT_TIMER_STARTED"; + case enums::VOICE_ASSISTANT_TIMER_UPDATED: + return "VOICE_ASSISTANT_TIMER_UPDATED"; + case enums::VOICE_ASSISTANT_TIMER_CANCELLED: + return "VOICE_ASSISTANT_TIMER_CANCELLED"; + case enums::VOICE_ASSISTANT_TIMER_FINISHED: + return "VOICE_ASSISTANT_TIMER_FINISHED"; + default: + return "UNKNOWN"; + } +} +#endif +#ifdef HAS_PROTO_MESSAGE_DUMP template<> const char *proto_enum_to_string(enums::AlarmControlPanelState value) { switch (value) { case enums::ALARM_STATE_DISARMED: @@ -6857,6 +6873,82 @@ void VoiceAssistantAudio::dump_to(std::string &out) const { out.append("}"); } #endif +bool VoiceAssistantTimerEventResponse::decode_varint(uint32_t field_id, ProtoVarInt value) { + switch (field_id) { + case 1: { + this->event_type = value.as_enum(); + return true; + } + case 4: { + this->total_seconds = value.as_uint32(); + return true; + } + case 5: { + this->seconds_left = value.as_uint32(); + return true; + } + case 6: { + this->is_active = value.as_bool(); + return true; + } + default: + return false; + } +} +bool VoiceAssistantTimerEventResponse::decode_length(uint32_t field_id, ProtoLengthDelimited value) { + switch (field_id) { + case 2: { + this->timer_id = value.as_string(); + return true; + } + case 3: { + this->name = value.as_string(); + return true; + } + default: + return false; + } +} +void VoiceAssistantTimerEventResponse::encode(ProtoWriteBuffer buffer) const { + buffer.encode_enum(1, this->event_type); + buffer.encode_string(2, this->timer_id); + buffer.encode_string(3, this->name); + buffer.encode_uint32(4, this->total_seconds); + buffer.encode_uint32(5, this->seconds_left); + buffer.encode_bool(6, this->is_active); +} +#ifdef HAS_PROTO_MESSAGE_DUMP +void VoiceAssistantTimerEventResponse::dump_to(std::string &out) const { + __attribute__((unused)) char buffer[64]; + out.append("VoiceAssistantTimerEventResponse {\n"); + out.append(" event_type: "); + out.append(proto_enum_to_string(this->event_type)); + out.append("\n"); + + out.append(" timer_id: "); + out.append("'").append(this->timer_id).append("'"); + out.append("\n"); + + out.append(" name: "); + out.append("'").append(this->name).append("'"); + out.append("\n"); + + out.append(" total_seconds: "); + sprintf(buffer, "%" PRIu32, this->total_seconds); + out.append(buffer); + out.append("\n"); + + out.append(" seconds_left: "); + sprintf(buffer, "%" PRIu32, this->seconds_left); + out.append(buffer); + out.append("\n"); + + out.append(" is_active: "); + out.append(YESNO(this->is_active)); + out.append("\n"); + out.append("}"); +} +#endif bool ListEntitiesAlarmControlPanelResponse::decode_varint(uint32_t field_id, ProtoVarInt value) { switch (field_id) { case 6: { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 807b150d82..54cbd20559 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -191,6 +191,12 @@ enum VoiceAssistantEvent : uint32_t { VOICE_ASSISTANT_TTS_STREAM_START = 98, VOICE_ASSISTANT_TTS_STREAM_END = 99, }; +enum VoiceAssistantTimerEvent : uint32_t { + VOICE_ASSISTANT_TIMER_STARTED = 0, + VOICE_ASSISTANT_TIMER_UPDATED = 1, + VOICE_ASSISTANT_TIMER_CANCELLED = 2, + VOICE_ASSISTANT_TIMER_FINISHED = 3, +}; enum AlarmControlPanelState : uint32_t { ALARM_STATE_DISARMED = 0, ALARM_STATE_ARMED_HOME = 1, @@ -1775,6 +1781,23 @@ class VoiceAssistantAudio : public ProtoMessage { bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override; bool decode_varint(uint32_t field_id, ProtoVarInt value) override; }; +class VoiceAssistantTimerEventResponse : public ProtoMessage { + public: + enums::VoiceAssistantTimerEvent event_type{}; + std::string timer_id{}; + std::string name{}; + uint32_t total_seconds{0}; + uint32_t seconds_left{0}; + bool is_active{false}; + void encode(ProtoWriteBuffer buffer) const override; +#ifdef HAS_PROTO_MESSAGE_DUMP + void dump_to(std::string &out) const override; +#endif + + protected: + bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override; + bool decode_varint(uint32_t field_id, ProtoVarInt value) override; +}; class ListEntitiesAlarmControlPanelResponse : public ProtoMessage { public: std::string object_id{}; diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 093fe917e0..7c95bb03ad 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -484,6 +484,8 @@ bool APIServerConnectionBase::send_voice_assistant_audio(const VoiceAssistantAud return this->send_message_(msg, 106); } #endif +#ifdef USE_VOICE_ASSISTANT +#endif #ifdef USE_ALARM_CONTROL_PANEL bool APIServerConnectionBase::send_list_entities_alarm_control_panel_response( const ListEntitiesAlarmControlPanelResponse &msg) { @@ -1093,6 +1095,17 @@ bool APIServerConnectionBase::read_message(uint32_t msg_size, uint32_t msg_type, ESP_LOGVV(TAG, "on_date_time_command_request: %s", msg.dump().c_str()); #endif this->on_date_time_command_request(msg); +#endif + break; + } + case 115: { +#ifdef USE_VOICE_ASSISTANT + VoiceAssistantTimerEventResponse msg; + msg.decode(msg_data, msg_size); +#ifdef HAS_PROTO_MESSAGE_DUMP + ESP_LOGVV(TAG, "on_voice_assistant_timer_event_response: %s", msg.dump().c_str()); +#endif + this->on_voice_assistant_timer_event_response(msg); #endif break; } diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index 196d904aca..2f8a2b3def 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -244,6 +244,9 @@ class APIServerConnectionBase : public ProtoService { bool send_voice_assistant_audio(const VoiceAssistantAudio &msg); virtual void on_voice_assistant_audio(const VoiceAssistantAudio &value){}; #endif +#ifdef USE_VOICE_ASSISTANT + virtual void on_voice_assistant_timer_event_response(const VoiceAssistantTimerEventResponse &value){}; +#endif #ifdef USE_ALARM_CONTROL_PANEL bool send_list_entities_alarm_control_panel_response(const ListEntitiesAlarmControlPanelResponse &msg); #endif diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index 3ba0c58ce4..c18f0a6850 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -44,6 +44,12 @@ CONF_VOLUME_MULTIPLIER = "volume_multiplier" CONF_WAKE_WORD = "wake_word" +CONF_ON_TIMER_STARTED = "on_timer_started" +CONF_ON_TIMER_UPDATED = "on_timer_updated" +CONF_ON_TIMER_CANCELLED = "on_timer_cancelled" +CONF_ON_TIMER_FINISHED = "on_timer_finished" +CONF_ON_TIMER_TICK = "on_timer_tick" + voice_assistant_ns = cg.esphome_ns.namespace("voice_assistant") VoiceAssistant = voice_assistant_ns.class_("VoiceAssistant", cg.Component) @@ -64,6 +70,8 @@ ConnectedCondition = voice_assistant_ns.class_( "ConnectedCondition", automation.Condition, cg.Parented.template(VoiceAssistant) ) +Timer = voice_assistant_ns.struct("Timer") + def tts_stream_validate(config): if CONF_SPEAKER not in config and ( @@ -131,6 +139,21 @@ CONFIG_SCHEMA = cv.All( single=True ), cv.Optional(CONF_ON_IDLE): automation.validate_automation(single=True), + cv.Optional(CONF_ON_TIMER_STARTED): automation.validate_automation( + single=True + ), + cv.Optional(CONF_ON_TIMER_UPDATED): automation.validate_automation( + single=True + ), + cv.Optional(CONF_ON_TIMER_CANCELLED): automation.validate_automation( + single=True + ), + cv.Optional(CONF_ON_TIMER_FINISHED): automation.validate_automation( + single=True + ), + cv.Optional(CONF_ON_TIMER_TICK): automation.validate_automation( + single=True + ), } ).extend(cv.COMPONENT_SCHEMA), tts_stream_validate, @@ -270,6 +293,49 @@ async def to_code(config): config[CONF_ON_IDLE], ) + has_timers = False + if on_timer_started := config.get(CONF_ON_TIMER_STARTED): + await automation.build_automation( + var.get_timer_started_trigger(), + [(Timer, "timer")], + on_timer_started, + ) + has_timers = True + + if on_timer_updated := config.get(CONF_ON_TIMER_UPDATED): + await automation.build_automation( + var.get_timer_updated_trigger(), + [(Timer, "timer")], + on_timer_updated, + ) + has_timers = True + + if on_timer_cancelled := config.get(CONF_ON_TIMER_CANCELLED): + await automation.build_automation( + var.get_timer_cancelled_trigger(), + [(Timer, "timer")], + on_timer_cancelled, + ) + has_timers = True + + if on_timer_finished := config.get(CONF_ON_TIMER_FINISHED): + await automation.build_automation( + var.get_timer_finished_trigger(), + [(Timer, "timer")], + on_timer_finished, + ) + has_timers = True + + if on_timer_tick := config.get(CONF_ON_TIMER_TICK): + await automation.build_automation( + var.get_timer_tick_trigger(), + [(cg.std_vector.template(Timer), "timers")], + on_timer_tick, + ) + has_timers = True + + cg.add(var.set_has_timers(has_timers)) + cg.add_define("USE_VOICE_ASSISTANT") diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 712a0ab137..0bd8194190 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -798,12 +798,65 @@ void VoiceAssistant::on_audio(const api::VoiceAssistantAudio &msg) { this->speaker_buffer_index_ += msg.data.length(); this->speaker_buffer_size_ += msg.data.length(); this->speaker_bytes_received_ += msg.data.length(); + ESP_LOGD(TAG, "Received audio: %d bytes from API", msg.data.length()); } else { ESP_LOGE(TAG, "Cannot receive audio, buffer is full"); } #endif } +void VoiceAssistant::on_timer_event(const api::VoiceAssistantTimerEventResponse &msg) { + Timer timer = { + .id = msg.timer_id, + .name = msg.name, + .total_seconds = msg.total_seconds, + .seconds_left = msg.seconds_left, + .is_active = msg.is_active, + }; + this->timers_[timer.id] = timer; + ESP_LOGD(TAG, "Timer Event"); + ESP_LOGD(TAG, " Type: %d", msg.event_type); + ESP_LOGD(TAG, " %s", timer.to_string().c_str()); + + switch (msg.event_type) { + case api::enums::VOICE_ASSISTANT_TIMER_STARTED: + this->timer_started_trigger_->trigger(timer); + break; + case api::enums::VOICE_ASSISTANT_TIMER_UPDATED: + this->timer_updated_trigger_->trigger(timer); + break; + case api::enums::VOICE_ASSISTANT_TIMER_CANCELLED: + this->timer_cancelled_trigger_->trigger(timer); + this->timers_.erase(timer.id); + break; + case api::enums::VOICE_ASSISTANT_TIMER_FINISHED: + this->timer_finished_trigger_->trigger(timer); + this->timers_.erase(timer.id); + break; + } + + if (this->timers_.empty()) { + this->cancel_interval("timer-event"); + this->timer_tick_running_ = false; + } else if (!this->timer_tick_running_) { + this->set_interval("timer-event", 1000, [this]() { this->timer_tick_(); }); + this->timer_tick_running_ = true; + } +} + +void VoiceAssistant::timer_tick_() { + std::vector res; + res.reserve(this->timers_.size()); + for (auto &pair : this->timers_) { + auto &timer = pair.second; + if (timer.is_active && timer.seconds_left > 0) { + timer.seconds_left--; + } + res.push_back(timer); + } + this->timer_tick_trigger_->trigger(res); +} + VoiceAssistant *global_voice_assistant = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace voice_assistant diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index 17141365d4..a160972e22 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -24,6 +24,9 @@ #include #endif +#include +#include + namespace esphome { namespace voice_assistant { @@ -36,6 +39,7 @@ enum VoiceAssistantFeature : uint32_t { FEATURE_VOICE_ASSISTANT = 1 << 0, FEATURE_SPEAKER = 1 << 1, FEATURE_API_AUDIO = 1 << 2, + FEATURE_TIMERS = 1 << 3, }; enum class State { @@ -59,6 +63,20 @@ enum AudioMode : uint8_t { AUDIO_MODE_API, }; +struct Timer { + std::string id; + std::string name; + uint32_t total_seconds; + uint32_t seconds_left; + bool is_active; + + std::string to_string() const { + return str_sprintf("Timer(id=%s, name=%s, total_seconds=%" PRIu32 ", seconds_left=%" PRIu32 ", is_active=%s)", + this->id.c_str(), this->name.c_str(), this->total_seconds, this->seconds_left, + YESNO(this->is_active)); + } +}; + class VoiceAssistant : public Component { public: void setup() override; @@ -100,6 +118,11 @@ class VoiceAssistant : public Component { flags |= VoiceAssistantFeature::FEATURE_SPEAKER; } #endif + + if (this->has_timers_) { + flags |= VoiceAssistantFeature::FEATURE_TIMERS; + } + return flags; } @@ -108,6 +131,7 @@ class VoiceAssistant : public Component { void on_event(const api::VoiceAssistantEventResponse &msg); void on_audio(const api::VoiceAssistantAudio &msg); + void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg); bool is_running() const { return this->state_ != State::IDLE; } void set_continuous(bool continuous) { this->continuous_ = continuous; } @@ -150,6 +174,14 @@ class VoiceAssistant : public Component { void set_wake_word(const std::string &wake_word) { this->wake_word_ = wake_word; } + Trigger *get_timer_started_trigger() const { return this->timer_started_trigger_; } + Trigger *get_timer_updated_trigger() const { return this->timer_updated_trigger_; } + Trigger *get_timer_cancelled_trigger() const { return this->timer_cancelled_trigger_; } + Trigger *get_timer_finished_trigger() const { return this->timer_finished_trigger_; } + Trigger> *get_timer_tick_trigger() const { return this->timer_tick_trigger_; } + void set_has_timers(bool has_timers) { this->has_timers_ = has_timers; } + const std::unordered_map &get_timers() const { return this->timers_; } + protected: bool allocate_buffers_(); void clear_buffers_(); @@ -186,6 +218,16 @@ class VoiceAssistant : public Component { api::APIConnection *api_client_{nullptr}; + std::unordered_map timers_; + void timer_tick_(); + Trigger *timer_started_trigger_ = new Trigger(); + Trigger *timer_finished_trigger_ = new Trigger(); + Trigger *timer_updated_trigger_ = new Trigger(); + Trigger *timer_cancelled_trigger_ = new Trigger(); + Trigger> *timer_tick_trigger_ = new Trigger>(); + bool has_timers_{false}; + bool timer_tick_running_{false}; + microphone::Microphone *mic_{nullptr}; #ifdef USE_SPEAKER void write_speaker_();