diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index 372a318898..2d5927d3b1 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -22,13 +22,19 @@ void I2SAudioSpeaker::setup() {
   this->event_queue_ = xQueueCreate(BUFFER_COUNT, sizeof(TaskEvent));
 }
 
-void I2SAudioSpeaker::start() { this->state_ = speaker::STATE_WAITING_FOR_LOCK; }
+void I2SAudioSpeaker::start() {
+  this->task_created_ = false;
+  this->state_ = speaker::STATE_STARTING;
+}
 void I2SAudioSpeaker::start_() {
+  if (this->task_created_) {
+    return;
+  }
   if (!this->parent_->try_lock()) {
     return;  // Waiting for another i2s component to return lock
   }
-  this->state_ = speaker::STATE_STARTING;
   xTaskCreate(I2SAudioSpeaker::player_task, "speaker_task", 8192, (void *) this, 1, &this->player_task_handle_);
+  this->task_created_ = true;
 }
 
 void I2SAudioSpeaker::player_task(void *params) {
@@ -188,6 +194,7 @@ void I2SAudioSpeaker::watch_() {
       case TaskEventType::STOPPED:
         this->state_ = speaker::STATE_STOPPED;
         vTaskDelete(this->player_task_handle_);
+        this->task_created_ = false;
         this->player_task_handle_ = nullptr;
         this->parent_->unlock();
         xQueueReset(this->buffer_queue_);
@@ -203,10 +210,10 @@ void I2SAudioSpeaker::watch_() {
 
 void I2SAudioSpeaker::loop() {
   switch (this->state_) {
-    case speaker::STATE_WAITING_FOR_LOCK:
-      this->start_();
-      break;
     case speaker::STATE_STARTING:
+      this->start_();
+      this->watch_();
+      break;
     case speaker::STATE_RUNNING:
     case speaker::STATE_STOPPING:
       this->watch_();
diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
index 5625f1b7f0..bf1a42e300 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h
@@ -71,6 +71,7 @@ class I2SAudioSpeaker : public Component, public speaker::Speaker, public I2SAud
   QueueHandle_t event_queue_;
 
   uint8_t dout_pin_{0};
+  bool task_created_{false};
 
 #if SOC_I2S_SUPPORTS_DAC
   i2s_dac_mode_t internal_dac_mode_{I2S_DAC_CHANNEL_DISABLE};
diff --git a/esphome/components/speaker/speaker.h b/esphome/components/speaker/speaker.h
index 677740781f..a77e028874 100644
--- a/esphome/components/speaker/speaker.h
+++ b/esphome/components/speaker/speaker.h
@@ -5,7 +5,6 @@ namespace speaker {
 
 enum State : uint8_t {
   STATE_STOPPED = 0,
-  STATE_WAITING_FOR_LOCK,
   STATE_STARTING,
   STATE_RUNNING,
   STATE_STOPPING,