diff --git a/esphome/components/improv_serial/__init__.py b/esphome/components/improv_serial/__init__.py
index 21073a8ab3..67a0f7f4ed 100644
--- a/esphome/components/improv_serial/__init__.py
+++ b/esphome/components/improv_serial/__init__.py
@@ -1,6 +1,8 @@
-from esphome.const import CONF_BAUD_RATE, CONF_ID, CONF_LOGGER
+from esphome.components.logger import USB_CDC, USB_SERIAL_JTAG
+from esphome.const import CONF_BAUD_RATE, CONF_HARDWARE_UART, CONF_ID, CONF_LOGGER
 import esphome.codegen as cg
 import esphome.config_validation as cv
+from esphome.core import CORE
 import esphome.final_validate as fv
 
 CODEOWNERS = ["@esphome/core"]
@@ -17,14 +19,19 @@ CONFIG_SCHEMA = cv.Schema(
 ).extend(cv.COMPONENT_SCHEMA)
 
 
-def validate_logger_baud_rate(config):
+def validate_logger(config):
     logger_conf = fv.full_config.get()[CONF_LOGGER]
     if logger_conf[CONF_BAUD_RATE] == 0:
         raise cv.Invalid("improv_serial requires the logger baud_rate to be not 0")
+    if CORE.using_esp_idf:
+        if logger_conf[CONF_HARDWARE_UART] in [USB_SERIAL_JTAG, USB_CDC]:
+            raise cv.Invalid(
+                "improv_serial does not support the selected logger hardware_uart"
+            )
     return config
 
 
-FINAL_VALIDATE_SCHEMA = validate_logger_baud_rate
+FINAL_VALIDATE_SCHEMA = validate_logger
 
 
 async def to_code(config):