diff --git a/esphome/components/captive_portal/captive_portal.cpp b/esphome/components/captive_portal/captive_portal.cpp
index 74c6606fc0..cc78528e46 100644
--- a/esphome/components/captive_portal/captive_portal.cpp
+++ b/esphome/components/captive_portal/captive_portal.cpp
@@ -48,7 +48,7 @@ void CaptivePortal::start() {
   this->dns_server_ = make_unique<DNSServer>();
   this->dns_server_->setErrorReplyCode(DNSReplyCode::NoError);
   network::IPAddress ip = wifi::global_wifi_component->wifi_soft_ap_ip();
-  this->dns_server_->start(53, "*", (uint32_t) ip);
+  this->dns_server_->start(53, "*", IPAddress(ip));
 #endif
 
   this->base_->get_server()->onNotFound([this](AsyncWebServerRequest *req) {
diff --git a/esphome/components/e131/e131_packet.cpp b/esphome/components/e131/e131_packet.cpp
index ac8b72f6e7..e1ae41cbaf 100644
--- a/esphome/components/e131/e131_packet.cpp
+++ b/esphome/components/e131/e131_packet.cpp
@@ -67,8 +67,8 @@ bool E131Component::join_igmp_groups_() {
     if (!universe.second)
       continue;
 
-    ip4_addr_t multicast_addr = {static_cast<uint32_t>(
-        network::IPAddress(239, 255, ((universe.first >> 8) & 0xff), ((universe.first >> 0) & 0xff)))};
+    ip4_addr_t multicast_addr =
+        network::IPAddress(239, 255, ((universe.first >> 8) & 0xff), ((universe.first >> 0) & 0xff));
 
     auto err = igmp_joingroup(IP4_ADDR_ANY4, &multicast_addr);
 
@@ -101,8 +101,7 @@ void E131Component::leave_(int universe) {
   }
 
   if (listen_method_ == E131_MULTICAST) {
-    ip4_addr_t multicast_addr = {
-        static_cast<uint32_t>(network::IPAddress(239, 255, ((universe >> 8) & 0xff), ((universe >> 0) & 0xff)))};
+    ip4_addr_t multicast_addr = network::IPAddress(239, 255, ((universe >> 8) & 0xff), ((universe >> 0) & 0xff));
 
     igmp_leavegroup(IP4_ADDR_ANY4, &multicast_addr);
   }
diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp
index 59d2e4c4d6..03fdc49338 100644
--- a/esphome/components/ethernet/ethernet_component.cpp
+++ b/esphome/components/ethernet/ethernet_component.cpp
@@ -236,7 +236,7 @@ bool EthernetComponent::can_proceed() { return this->is_connected(); }
 network::IPAddress EthernetComponent::get_ip_address() {
   esp_netif_ip_info_t ip;
   esp_netif_get_ip_info(this->eth_netif_, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 
 void EthernetComponent::eth_event_handler(void *arg, esp_event_base_t event_base, int32_t event, void *event_data) {
@@ -293,9 +293,9 @@ void EthernetComponent::start_connect_() {
 
   esp_netif_ip_info_t info;
   if (this->manual_ip_.has_value()) {
-    info.ip.addr = static_cast<uint32_t>(this->manual_ip_->static_ip);
-    info.gw.addr = static_cast<uint32_t>(this->manual_ip_->gateway);
-    info.netmask.addr = static_cast<uint32_t>(this->manual_ip_->subnet);
+    info.ip = this->manual_ip_->static_ip;
+    info.gw = this->manual_ip_->gateway;
+    info.netmask = this->manual_ip_->subnet;
   } else {
     info.ip.addr = 0;
     info.gw.addr = 0;
@@ -318,24 +318,14 @@ void EthernetComponent::start_connect_() {
   ESPHL_ERROR_CHECK(err, "DHCPC set IP info error");
 
   if (this->manual_ip_.has_value()) {
-    if (uint32_t(this->manual_ip_->dns1) != 0) {
+    if (this->manual_ip_->dns1.is_set()) {
       ip_addr_t d;
-#if LWIP_IPV6
-      d.type = IPADDR_TYPE_V4;
-      d.u_addr.ip4.addr = static_cast<uint32_t>(this->manual_ip_->dns1);
-#else
-      d.addr = static_cast<uint32_t>(this->manual_ip_->dns1);
-#endif
+      d = this->manual_ip_->dns1;
       dns_setserver(0, &d);
     }
-    if (uint32_t(this->manual_ip_->dns2) != 0) {
+    if (this->manual_ip_->dns2.is_set()) {
       ip_addr_t d;
-#if LWIP_IPV6
-      d.type = IPADDR_TYPE_V4;
-      d.u_addr.ip4.addr = static_cast<uint32_t>(this->manual_ip_->dns2);
-#else
-      d.addr = static_cast<uint32_t>(this->manual_ip_->dns2);
-#endif
+      d = this->manual_ip_->dns2;
       dns_setserver(1, &d);
     }
   } else {
@@ -360,21 +350,16 @@ bool EthernetComponent::is_connected() { return this->state_ == EthernetComponen
 void EthernetComponent::dump_connect_params_() {
   esp_netif_ip_info_t ip;
   esp_netif_get_ip_info(this->eth_netif_, &ip);
-  ESP_LOGCONFIG(TAG, "  IP Address: %s", network::IPAddress(ip.ip.addr).str().c_str());
+  ESP_LOGCONFIG(TAG, "  IP Address: %s", network::IPAddress(&ip.ip).str().c_str());
   ESP_LOGCONFIG(TAG, "  Hostname: '%s'", App.get_name().c_str());
-  ESP_LOGCONFIG(TAG, "  Subnet: %s", network::IPAddress(ip.netmask.addr).str().c_str());
-  ESP_LOGCONFIG(TAG, "  Gateway: %s", network::IPAddress(ip.gw.addr).str().c_str());
+  ESP_LOGCONFIG(TAG, "  Subnet: %s", network::IPAddress(&ip.netmask).str().c_str());
+  ESP_LOGCONFIG(TAG, "  Gateway: %s", network::IPAddress(&ip.gw).str().c_str());
 
   const ip_addr_t *dns_ip1 = dns_getserver(0);
   const ip_addr_t *dns_ip2 = dns_getserver(1);
 
-#if LWIP_IPV6
-  ESP_LOGCONFIG(TAG, "  DNS1: %s", network::IPAddress(dns_ip1->u_addr.ip4.addr).str().c_str());
-  ESP_LOGCONFIG(TAG, "  DNS2: %s", network::IPAddress(dns_ip2->u_addr.ip4.addr).str().c_str());
-#else
-  ESP_LOGCONFIG(TAG, "  DNS1: %s", network::IPAddress(dns_ip1->addr).str().c_str());
-  ESP_LOGCONFIG(TAG, "  DNS2: %s", network::IPAddress(dns_ip2->addr).str().c_str());
-#endif
+  ESP_LOGCONFIG(TAG, "  DNS1: %s", network::IPAddress(dns_ip1).str().c_str());
+  ESP_LOGCONFIG(TAG, "  DNS2: %s", network::IPAddress(dns_ip2).str().c_str());
 
 #if ENABLE_IPV6
   if (this->ipv6_count_ > 0) {
diff --git a/esphome/components/mdns/mdns_rp2040.cpp b/esphome/components/mdns/mdns_rp2040.cpp
index b30a3a4ee7..56afd6f5e1 100644
--- a/esphome/components/mdns/mdns_rp2040.cpp
+++ b/esphome/components/mdns/mdns_rp2040.cpp
@@ -13,8 +13,7 @@ namespace mdns {
 void MDNSComponent::setup() {
   this->compile_records_();
 
-  network::IPAddress addr = network::get_ip_address();
-  MDNS.begin(this->hostname_.c_str(), (uint32_t) addr);
+  MDNS.begin(this->hostname_.c_str());
 
   for (const auto &service : this->services_) {
     // Strip the leading underscore from the proto and service_type. While it is
diff --git a/esphome/components/mqtt/mqtt_backend_esp8266.h b/esphome/components/mqtt/mqtt_backend_esp8266.h
index 2d91877e9d..981d27693f 100644
--- a/esphome/components/mqtt/mqtt_backend_esp8266.h
+++ b/esphome/components/mqtt/mqtt_backend_esp8266.h
@@ -19,9 +19,7 @@ class MQTTBackendESP8266 final : public MQTTBackend {
   void set_will(const char *topic, uint8_t qos, bool retain, const char *payload) final {
     mqtt_client_.setWill(topic, qos, retain, payload);
   }
-  void set_server(network::IPAddress ip, uint16_t port) final {
-    mqtt_client_.setServer(IPAddress(static_cast<uint32_t>(ip)), port);
-  }
+  void set_server(network::IPAddress ip, uint16_t port) final { mqtt_client_.setServer(IPAddress(ip), port); }
   void set_server(const char *host, uint16_t port) final { mqtt_client_.setServer(host, port); }
 #if ASYNC_TCP_SSL_ENABLED
   void set_secure(bool secure) { mqtt_client.setSecure(secure); }
diff --git a/esphome/components/mqtt/mqtt_client.cpp b/esphome/components/mqtt/mqtt_client.cpp
index fd5e13ecc7..6f63935e6e 100644
--- a/esphome/components/mqtt/mqtt_client.cpp
+++ b/esphome/components/mqtt/mqtt_client.cpp
@@ -171,11 +171,7 @@ void MQTTClientComponent::start_dnslookup_() {
     case ERR_OK: {
       // Got IP immediately
       this->dns_resolved_ = true;
-#if LWIP_IPV6
-      this->ip_ = addr.u_addr.ip4.addr;
-#else
-      this->ip_ = addr.addr;
-#endif
+      this->ip_ = network::IPAddress(&addr);
       this->start_connect_();
       return;
     }
@@ -226,11 +222,7 @@ void MQTTClientComponent::dns_found_callback(const char *name, const ip_addr_t *
   if (ipaddr == nullptr) {
     a_this->dns_resolve_error_ = true;
   } else {
-#if LWIP_IPV6
-    a_this->ip_ = ipaddr->u_addr.ip4.addr;
-#else
-    a_this->ip_ = ipaddr->addr;
-#endif
+    a_this->ip_ = network::IPAddress(ipaddr);
     a_this->dns_resolved_ = true;
   }
 }
diff --git a/esphome/components/network/ip_address.h b/esphome/components/network/ip_address.h
index af198179ba..8b05237c05 100644
--- a/esphome/components/network/ip_address.h
+++ b/esphome/components/network/ip_address.h
@@ -3,42 +3,104 @@
 #include <string>
 #include <cstdio>
 #include <array>
+#include <lwip/ip_addr.h>
+
+#if USE_ARDUINO
+#include <Arduino.h>
+#include <IPAddress.h>
+#endif /* USE_ADRDUINO */
+
+#if USE_ESP32_FRAMEWORK_ARDUINO
+#define arduino_ns Arduino_h
+#elif USE_LIBRETINY
+#define arduino_ns arduino
+#elif USE_ARDUINO
+#define arduino_ns
+#endif
+
+#ifdef USE_ESP32
+#include <cstring>
+#include <esp_netif.h>
+#endif
 
 namespace esphome {
 namespace network {
 
 struct IPAddress {
  public:
-  IPAddress() : addr_({0, 0, 0, 0}) {}
-  IPAddress(uint8_t first, uint8_t second, uint8_t third, uint8_t fourth) : addr_({first, second, third, fourth}) {}
-  IPAddress(uint32_t raw) {
-    addr_[0] = (uint8_t) (raw >> 0);
-    addr_[1] = (uint8_t) (raw >> 8);
-    addr_[2] = (uint8_t) (raw >> 16);
-    addr_[3] = (uint8_t) (raw >> 24);
+  IPAddress() { ip_addr_set_zero(&ip_addr_); }
+  IPAddress(uint8_t first, uint8_t second, uint8_t third, uint8_t fourth) {
+    IP_ADDR4(&ip_addr_, first, second, third, fourth);
   }
-  operator uint32_t() const {
-    uint32_t res = 0;
-    res |= ((uint32_t) addr_[0]) << 0;
-    res |= ((uint32_t) addr_[1]) << 8;
-    res |= ((uint32_t) addr_[2]) << 16;
-    res |= ((uint32_t) addr_[3]) << 24;
-    return res;
+  IPAddress(const ip_addr_t *other_ip) { ip_addr_copy(ip_addr_, *other_ip); }
+  IPAddress(const std::string &in_address) { ipaddr_aton(in_address.c_str(), &ip_addr_); }
+  IPAddress(ip4_addr_t *other_ip) { memcpy((void *) &ip_addr_, (void *) other_ip, sizeof(ip4_addr_t)); }
+#if USE_ARDUINO
+  IPAddress(const arduino_ns::IPAddress &other_ip) { ip_addr_set_ip4_u32(&ip_addr_, other_ip); }
+#endif
+#if LWIP_IPV6
+  IPAddress(ip6_addr_t *other_ip) {
+    memcpy((void *) &ip_addr_, (void *) other_ip, sizeof(ip6_addr_t));
+    ip_addr_.type = IPADDR_TYPE_V6;
   }
-  std::string str() const {
-    char buffer[24];
-    snprintf(buffer, sizeof(buffer), "%d.%d.%d.%d", addr_[0], addr_[1], addr_[2], addr_[3]);
-    return buffer;
+#endif /* LWIP_IPV6 */
+
+#ifdef USE_ESP32
+#if LWIP_IPV6
+  IPAddress(esp_ip6_addr_t *other_ip) {
+    memcpy((void *) &ip_addr_.u_addr.ip6, (void *) other_ip, sizeof(esp_ip6_addr_t));
+    ip_addr_.type = IPADDR_TYPE_V6;
   }
-  bool operator==(const IPAddress &other) const {
-    return addr_[0] == other.addr_[0] && addr_[1] == other.addr_[1] && addr_[2] == other.addr_[2] &&
-           addr_[3] == other.addr_[3];
+#endif /* LWIP_IPV6 */
+  IPAddress(esp_ip4_addr_t *other_ip) { memcpy((void *) &ip_addr_, (void *) other_ip, sizeof(esp_ip4_addr_t)); }
+  operator esp_ip_addr_t() const {
+    esp_ip_addr_t tmp;
+#if LWIP_IPV6
+    memcpy((void *) &tmp, (void *) &ip_addr_, sizeof(ip_addr_));
+#else
+    memcpy((void *) &tmp.u_addr.ip4, (void *) &ip_addr_, sizeof(ip_addr_));
+#endif /* LWIP_IPV6 */
+    return tmp;
+  }
+  operator esp_ip4_addr_t() const {
+    esp_ip4_addr_t tmp;
+#if LWIP_IPV6
+    memcpy((void *) &tmp, (void *) &ip_addr_.u_addr.ip4, sizeof(esp_ip4_addr_t));
+#else
+    memcpy((void *) &tmp, (void *) &ip_addr_, sizeof(ip_addr_));
+#endif /* LWIP_IPV6 */
+    return tmp;
+  }
+#endif /* USE_ESP32 */
+
+  operator ip_addr_t() const { return ip_addr_; }
+#if LWIP_IPV6
+  operator ip4_addr_t() const { return *ip_2_ip4(&ip_addr_); }
+#endif /* LWIP_IPV6 */
+
+#if USE_ARDUINO
+  operator arduino_ns::IPAddress() const { return ip_addr_get_ip4_u32(&ip_addr_); }
+#endif
+
+  bool is_set() { return !ip_addr_isany(&ip_addr_); }
+  bool is_ip4() { return IP_IS_V4(&ip_addr_); }
+  bool is_ip6() { return IP_IS_V6(&ip_addr_); }
+  std::string str() const { return ipaddr_ntoa(&ip_addr_); }
+  bool operator==(const IPAddress &other) const { return ip_addr_cmp(&ip_addr_, &other.ip_addr_); }
+  bool operator!=(const IPAddress &other) const { return !(&ip_addr_ == &other.ip_addr_); }
+  IPAddress &operator+=(uint8_t increase) {
+    if (IP_IS_V4(&ip_addr_)) {
+#if LWIP_IPV6
+      (((u8_t *) (&ip_addr_.u_addr.ip4))[3]) += increase;
+#else
+      (((u8_t *) (&ip_addr_.addr))[3]) += increase;
+#endif /* LWIP_IPV6 */
+    }
+    return *this;
   }
-  uint8_t operator[](int index) const { return addr_[index]; }
-  uint8_t &operator[](int index) { return addr_[index]; }
 
  protected:
-  std::array<uint8_t, 4> addr_;
+  ip_addr_t ip_addr_;
 };
 
 }  // namespace network
diff --git a/esphome/components/wake_on_lan/wake_on_lan.cpp b/esphome/components/wake_on_lan/wake_on_lan.cpp
index 893aa75895..a4dd0f3b6f 100644
--- a/esphome/components/wake_on_lan/wake_on_lan.cpp
+++ b/esphome/components/wake_on_lan/wake_on_lan.cpp
@@ -30,11 +30,10 @@ void WakeOnLanButton::press_action() {
   ESP_LOGI(TAG, "Sending Wake-on-LAN Packet...");
   bool begin_status = false;
   bool end_status = false;
-  uint32_t interface = esphome::network::get_ip_address();
-  IPAddress interface_ip = IPAddress(interface);
   IPAddress broadcast = IPAddress(255, 255, 255, 255);
 #ifdef USE_ESP8266
-  begin_status = this->udp_client_.beginPacketMulticast(broadcast, 9, interface_ip, 128);
+  begin_status = this->udp_client_.beginPacketMulticast(broadcast, 9,
+                                                        IPAddress((ip_addr_t) esphome::network::get_ip_address()), 128);
 #endif
 #ifdef USE_ESP32
   begin_status = this->udp_client_.beginPacket(broadcast, 9);
diff --git a/esphome/components/wifi/wifi_component_esp32_arduino.cpp b/esphome/components/wifi/wifi_component_esp32_arduino.cpp
index 5b147b20c6..17b15757ef 100644
--- a/esphome/components/wifi/wifi_component_esp32_arduino.cpp
+++ b/esphome/components/wifi/wifi_component_esp32_arduino.cpp
@@ -113,9 +113,9 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
 
   tcpip_adapter_ip_info_t info;
   memset(&info, 0, sizeof(info));
-  info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-  info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-  info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+  info.ip = manual_ip->static_ip;
+  info.gw = manual_ip->gateway;
+  info.netmask = manual_ip->subnet;
 
   esp_err_t dhcp_stop_ret = tcpip_adapter_dhcpc_stop(TCPIP_ADAPTER_IF_STA);
   if (dhcp_stop_ret != ESP_OK && dhcp_stop_ret != ESP_ERR_TCPIP_ADAPTER_DHCP_ALREADY_STOPPED) {
@@ -128,23 +128,16 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
   }
 
   ip_addr_t dns;
+// TODO: is this needed?
 #if LWIP_IPV6
   dns.type = IPADDR_TYPE_V4;
 #endif
-  if (uint32_t(manual_ip->dns1) != 0) {
-#if LWIP_IPV6
-    dns.u_addr.ip4.addr = static_cast<uint32_t>(manual_ip->dns1);
-#else
-    dns.addr = static_cast<uint32_t>(manual_ip->dns1);
-#endif
+  if (manual_ip->dns1.is_set()) {
+    dns = manual_ip->dns1;
     dns_setserver(0, &dns);
   }
-  if (uint32_t(manual_ip->dns2) != 0) {
-#if LWIP_IPV6
-    dns.u_addr.ip4.addr = static_cast<uint32_t>(manual_ip->dns2);
-#else
-    dns.addr = static_cast<uint32_t>(manual_ip->dns2);
-#endif
+  if (manual_ip->dns2.is_set()) {
+    dns = manual_ip->dns2;
     dns_setserver(1, &dns);
   }
 
@@ -156,7 +149,7 @@ network::IPAddress WiFiComponent::wifi_sta_ip() {
     return {};
   tcpip_adapter_ip_info_t ip;
   tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_STA, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 
 bool WiFiComponent::wifi_apply_hostname_() {
@@ -614,13 +607,13 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
   tcpip_adapter_ip_info_t info;
   memset(&info, 0, sizeof(info));
   if (manual_ip.has_value()) {
-    info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-    info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-    info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+    info.ip = manual_ip->static_ip;
+    info.gw = manual_ip->gateway;
+    info.netmask = manual_ip->subnet;
   } else {
-    info.ip.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.gw.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.netmask.addr = static_cast<uint32_t>(network::IPAddress(255, 255, 255, 0));
+    info.ip = network::IPAddress(192, 168, 4, 1);
+    info.gw = network::IPAddress(192, 168, 4, 1);
+    info.netmask = network::IPAddress(255, 255, 255, 0);
   }
   tcpip_adapter_dhcp_status_t dhcp_status;
   tcpip_adapter_dhcps_get_status(TCPIP_ADAPTER_IF_AP, &dhcp_status);
@@ -638,12 +631,12 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
 
   dhcps_lease_t lease;
   lease.enable = true;
-  network::IPAddress start_address = info.ip.addr;
-  start_address[3] += 99;
-  lease.start_ip.addr = static_cast<uint32_t>(start_address);
+  network::IPAddress start_address = network::IPAddress(&info.ip);
+  start_address += 99;
+  lease.start_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease start: %s", start_address.str().c_str());
-  start_address[3] += 100;
-  lease.end_ip.addr = static_cast<uint32_t>(start_address);
+  start_address += 100;
+  lease.end_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease end: %s", start_address.str().c_str());
   err = tcpip_adapter_dhcps_option(TCPIP_ADAPTER_OP_SET, TCPIP_ADAPTER_REQUESTED_IP_ADDRESS, &lease, sizeof(lease));
 
@@ -702,7 +695,7 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) {
 network::IPAddress WiFiComponent::wifi_soft_ap_ip() {
   tcpip_adapter_ip_info_t ip;
   tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_AP, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 bool WiFiComponent::wifi_disconnect_() { return esp_wifi_disconnect(); }
 
@@ -718,9 +711,9 @@ bssid_t WiFiComponent::wifi_bssid() {
 std::string WiFiComponent::wifi_ssid() { return WiFi.SSID().c_str(); }
 int8_t WiFiComponent::wifi_rssi() { return WiFi.RSSI(); }
 int32_t WiFiComponent::wifi_channel_() { return WiFi.channel(); }
-network::IPAddress WiFiComponent::wifi_subnet_mask_() { return {WiFi.subnetMask()}; }
-network::IPAddress WiFiComponent::wifi_gateway_ip_() { return {WiFi.gatewayIP()}; }
-network::IPAddress WiFiComponent::wifi_dns_ip_(int num) { return {WiFi.dnsIP(num)}; }
+network::IPAddress WiFiComponent::wifi_subnet_mask_() { return network::IPAddress(WiFi.subnetMask()); }
+network::IPAddress WiFiComponent::wifi_gateway_ip_() { return network::IPAddress(WiFi.gatewayIP()); }
+network::IPAddress WiFiComponent::wifi_dns_ip_(int num) { return network::IPAddress(WiFi.dnsIP(num)); }
 void WiFiComponent::wifi_loop_() {}
 
 }  // namespace wifi
diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp
index 6e7c491967..a48c6c711d 100644
--- a/esphome/components/wifi/wifi_component_esp8266.cpp
+++ b/esphome/components/wifi/wifi_component_esp8266.cpp
@@ -147,9 +147,9 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
 #endif
 
   struct ip_info info {};
-  info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-  info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-  info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+  info.ip = manual_ip->static_ip;
+  info.gw = manual_ip->gateway;
+  info.netmask = manual_ip->subnet;
 
   if (dhcp_status == DHCP_STARTED) {
     bool dhcp_stop_ret = wifi_station_dhcpc_stop();
@@ -165,12 +165,12 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
   }
 
   ip_addr_t dns;
-  if (uint32_t(manual_ip->dns1) != 0) {
-    ip_addr_set_ip4_u32_val(dns, static_cast<uint32_t>(manual_ip->dns1));
+  if (manual_ip->dns1.is_set()) {
+    dns = manual_ip->dns1;
     dns_setserver(0, &dns);
   }
-  if (uint32_t(manual_ip->dns2) != 0) {
-    ip_addr_set_ip4_u32_val(dns, static_cast<uint32_t>(manual_ip->dns2));
+  if (manual_ip->dns2.is_set()) {
+    dns = manual_ip->dns2;
     dns_setserver(1, &dns);
   }
 
@@ -190,7 +190,7 @@ network::IPAddress WiFiComponent::wifi_sta_ip() {
     return {};
   struct ip_info ip {};
   wifi_get_ip_info(STATION_IF, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 bool WiFiComponent::wifi_apply_hostname_() {
   const std::string &hostname = App.get_name();
@@ -695,13 +695,13 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
 
   struct ip_info info {};
   if (manual_ip.has_value()) {
-    info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-    info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-    info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+    info.ip = manual_ip->static_ip;
+    info.gw = manual_ip->gateway;
+    info.netmask = manual_ip->subnet;
   } else {
-    info.ip.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.gw.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.netmask.addr = static_cast<uint32_t>(network::IPAddress(255, 255, 255, 0));
+    info.ip = network::IPAddress(192, 168, 4, 1);
+    info.gw = network::IPAddress(192, 168, 4, 1);
+    info.netmask = network::IPAddress(255, 255, 255, 0);
   }
 
   if (wifi_softap_dhcps_status() == DHCP_STARTED) {
@@ -721,12 +721,12 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
 
   struct dhcps_lease lease {};
   lease.enable = true;
-  network::IPAddress start_address = info.ip.addr;
-  start_address[3] += 99;
-  lease.start_ip.addr = static_cast<uint32_t>(start_address);
+  network::IPAddress start_address = network::IPAddress(&info.ip);
+  start_address += 99;
+  lease.start_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease start: %s", start_address.str().c_str());
-  start_address[3] += 100;
-  lease.end_ip.addr = static_cast<uint32_t>(start_address);
+  start_address += 100;
+  lease.end_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease end: %s", start_address.str().c_str());
   if (!wifi_softap_set_dhcps_lease(&lease)) {
     ESP_LOGV(TAG, "Setting SoftAP DHCP lease failed!");
@@ -793,7 +793,7 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) {
 network::IPAddress WiFiComponent::wifi_soft_ap_ip() {
   struct ip_info ip {};
   wifi_get_ip_info(SOFTAP_IF, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 bssid_t WiFiComponent::wifi_bssid() {
   bssid_t bssid{};
@@ -807,9 +807,9 @@ bssid_t WiFiComponent::wifi_bssid() {
 std::string WiFiComponent::wifi_ssid() { return WiFi.SSID().c_str(); }
 int8_t WiFiComponent::wifi_rssi() { return WiFi.RSSI(); }
 int32_t WiFiComponent::wifi_channel_() { return WiFi.channel(); }
-network::IPAddress WiFiComponent::wifi_subnet_mask_() { return {WiFi.subnetMask()}; }
-network::IPAddress WiFiComponent::wifi_gateway_ip_() { return {WiFi.gatewayIP()}; }
-network::IPAddress WiFiComponent::wifi_dns_ip_(int num) { return {WiFi.dnsIP(num)}; }
+network::IPAddress WiFiComponent::wifi_subnet_mask_() { return {(const ip_addr_t *) WiFi.subnetMask()}; }
+network::IPAddress WiFiComponent::wifi_gateway_ip_() { return {(const ip_addr_t *) WiFi.gatewayIP()}; }
+network::IPAddress WiFiComponent::wifi_dns_ip_(int num) { return {(const ip_addr_t *) WiFi.dnsIP(num)}; }
 void WiFiComponent::wifi_loop_() {}
 
 }  // namespace wifi
diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp
index 0ff9e932b2..34ecaf887d 100644
--- a/esphome/components/wifi/wifi_component_esp_idf.cpp
+++ b/esphome/components/wifi/wifi_component_esp_idf.cpp
@@ -437,9 +437,9 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
   }
 
   esp_netif_ip_info_t info;  // struct of ip4_addr_t with ip, netmask, gw
-  info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-  info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-  info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+  info.ip = manual_ip->static_ip;
+  info.gw = manual_ip->gateway;
+  info.netmask = manual_ip->subnet;
   err = esp_netif_dhcpc_stop(s_sta_netif);
   if (err != ESP_OK && err != ESP_ERR_ESP_NETIF_DHCP_ALREADY_STOPPED) {
     ESP_LOGV(TAG, "esp_netif_dhcpc_stop failed: %s", esp_err_to_name(err));
@@ -452,12 +452,12 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
   }
 
   esp_netif_dns_info_t dns;
-  if (uint32_t(manual_ip->dns1) != 0) {
-    dns.ip.u_addr.ip4.addr = static_cast<uint32_t>(manual_ip->dns1);
+  if (manual_ip->dns1.is_set()) {
+    dns.ip = manual_ip->dns1;
     esp_netif_set_dns_info(s_sta_netif, ESP_NETIF_DNS_MAIN, &dns);
   }
-  if (uint32_t(manual_ip->dns2) != 0) {
-    dns.ip.u_addr.ip4.addr = static_cast<uint32_t>(manual_ip->dns2);
+  if (manual_ip->dns2.is_set()) {
+    dns.ip = manual_ip->dns2;
     esp_netif_set_dns_info(s_sta_netif, ESP_NETIF_DNS_BACKUP, &dns);
   }
 
@@ -471,9 +471,10 @@ network::IPAddress WiFiComponent::wifi_sta_ip() {
   esp_err_t err = esp_netif_get_ip_info(s_sta_netif, &ip);
   if (err != ESP_OK) {
     ESP_LOGV(TAG, "esp_netif_get_ip_info failed: %s", esp_err_to_name(err));
-    return false;
+    // TODO: do something smarter
+    // return false;
   }
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 
 bool WiFiComponent::wifi_apply_hostname_() {
@@ -769,13 +770,13 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
 
   esp_netif_ip_info_t info;
   if (manual_ip.has_value()) {
-    info.ip.addr = static_cast<uint32_t>(manual_ip->static_ip);
-    info.gw.addr = static_cast<uint32_t>(manual_ip->gateway);
-    info.netmask.addr = static_cast<uint32_t>(manual_ip->subnet);
+    info.ip = manual_ip->static_ip;
+    info.gw = manual_ip->gateway;
+    info.netmask = manual_ip->subnet;
   } else {
-    info.ip.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.gw.addr = static_cast<uint32_t>(network::IPAddress(192, 168, 4, 1));
-    info.netmask.addr = static_cast<uint32_t>(network::IPAddress(255, 255, 255, 0));
+    info.ip = network::IPAddress(192, 168, 4, 1);
+    info.gw = network::IPAddress(192, 168, 4, 1);
+    info.netmask = network::IPAddress(255, 255, 255, 0);
   }
 
   err = esp_netif_dhcpc_stop(s_sta_netif);
@@ -792,12 +793,12 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
 
   dhcps_lease_t lease;
   lease.enable = true;
-  network::IPAddress start_address = info.ip.addr;
-  start_address[3] += 99;
-  lease.start_ip.addr = static_cast<uint32_t>(start_address);
+  network::IPAddress start_address = network::IPAddress(&info.ip);
+  start_address += 99;
+  lease.start_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease start: %s", start_address.str().c_str());
-  start_address[3] += 100;
-  lease.end_ip.addr = static_cast<uint32_t>(start_address);
+  start_address += 100;
+  lease.end_ip = start_address;
   ESP_LOGV(TAG, "DHCP server IP lease end: %s", start_address.str().c_str());
   err = esp_netif_dhcps_option(s_sta_netif, ESP_NETIF_OP_SET, ESP_NETIF_REQUESTED_IP_ADDRESS, &lease, sizeof(lease));
 
@@ -855,7 +856,7 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) {
 network::IPAddress WiFiComponent::wifi_soft_ap_ip() {
   esp_netif_ip_info_t ip;
   esp_netif_get_ip_info(s_sta_netif, &ip);
-  return {ip.ip.addr};
+  return network::IPAddress(&ip.ip);
 }
 bool WiFiComponent::wifi_disconnect_() { return esp_wifi_disconnect(); }
 
@@ -907,7 +908,7 @@ network::IPAddress WiFiComponent::wifi_subnet_mask_() {
     ESP_LOGW(TAG, "esp_netif_get_ip_info failed: %s", esp_err_to_name(err));
     return {};
   }
-  return {ip.netmask.addr};
+  return network::IPAddress(&ip.netmask);
 }
 network::IPAddress WiFiComponent::wifi_gateway_ip_() {
   esp_netif_ip_info_t ip;
@@ -916,15 +917,11 @@ network::IPAddress WiFiComponent::wifi_gateway_ip_() {
     ESP_LOGW(TAG, "esp_netif_get_ip_info failed: %s", esp_err_to_name(err));
     return {};
   }
-  return {ip.gw.addr};
+  return network::IPAddress(&ip.gw);
 }
 network::IPAddress WiFiComponent::wifi_dns_ip_(int num) {
   const ip_addr_t *dns_ip = dns_getserver(num);
-#if LWIP_IPV6
-  return {dns_ip->u_addr.ip4.addr};
-#else
-  return {dns_ip->addr};
-#endif
+  return network::IPAddress(dns_ip);
 }
 
 }  // namespace wifi
diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp
index abad5aca9c..d7f4406540 100644
--- a/esphome/components/wifi/wifi_component_libretiny.cpp
+++ b/esphome/components/wifi/wifi_component_libretiny.cpp
@@ -76,9 +76,7 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
     return true;
   }
 
-  WiFi.config(static_cast<uint32_t>(manual_ip->static_ip), static_cast<uint32_t>(manual_ip->gateway),
-              static_cast<uint32_t>(manual_ip->subnet), static_cast<uint32_t>(manual_ip->dns1),
-              static_cast<uint32_t>(manual_ip->dns2));
+  WiFi.config(manual_ip->static_ip, manual_ip->gateway, manual_ip->subnet, manual_ip->dns1, manual_ip->dns2);
 
   return true;
 }
@@ -420,8 +418,7 @@ bool WiFiComponent::wifi_ap_ip_config_(optional<ManualIP> manual_ip) {
     return false;
 
   if (manual_ip.has_value()) {
-    return WiFi.softAPConfig(static_cast<uint32_t>(manual_ip->static_ip), static_cast<uint32_t>(manual_ip->gateway),
-                             static_cast<uint32_t>(manual_ip->subnet));
+    return WiFi.softAPConfig(manual_ip->static_ip, manual_ip->gateway, manual_ip->subnet);
   } else {
     return WiFi.softAPConfig(IPAddress(192, 168, 4, 1), IPAddress(192, 168, 4, 1), IPAddress(255, 255, 255, 0));
   }
diff --git a/esphome/components/wifi/wifi_component_pico_w.cpp b/esphome/components/wifi/wifi_component_pico_w.cpp
index 149ca61cd5..d67b466d6c 100644
--- a/esphome/components/wifi/wifi_component_pico_w.cpp
+++ b/esphome/components/wifi/wifi_component_pico_w.cpp
@@ -70,11 +70,11 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
     return true;
   }
 
-  IPAddress ip_address = IPAddress(manual_ip->static_ip);
-  IPAddress gateway = IPAddress(manual_ip->gateway);
-  IPAddress subnet = IPAddress(manual_ip->subnet);
+  IPAddress ip_address = manual_ip->static_ip;
+  IPAddress gateway = manual_ip->gateway;
+  IPAddress subnet = manual_ip->subnet;
 
-  IPAddress dns = IPAddress(manual_ip->dns1);
+  IPAddress dns = manual_ip->dns1;
 
   WiFi.config(ip_address, dns, gateway, subnet);
   return true;
@@ -151,7 +151,7 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) {
 
   return true;
 }
-network::IPAddress WiFiComponent::wifi_soft_ap_ip() { return {WiFi.localIP()}; }
+network::IPAddress WiFiComponent::wifi_soft_ap_ip() { return {(const ip_addr_t *) WiFi.localIP()}; }
 
 bool WiFiComponent::wifi_disconnect_() {
   int err = cyw43_wifi_leave(&cyw43_state, CYW43_ITF_STA);
@@ -170,16 +170,12 @@ std::string WiFiComponent::wifi_ssid() { return WiFi.SSID().c_str(); }
 int8_t WiFiComponent::wifi_rssi() { return WiFi.RSSI(); }
 int32_t WiFiComponent::wifi_channel_() { return WiFi.channel(); }
 
-network::IPAddress WiFiComponent::wifi_sta_ip() { return {WiFi.localIP()}; }
-network::IPAddress WiFiComponent::wifi_subnet_mask_() { return {WiFi.subnetMask()}; }
-network::IPAddress WiFiComponent::wifi_gateway_ip_() { return {WiFi.gatewayIP()}; }
+network::IPAddress WiFiComponent::wifi_sta_ip() { return {(const ip_addr_t *) WiFi.localIP()}; }
+network::IPAddress WiFiComponent::wifi_subnet_mask_() { return {(const ip_addr_t *) WiFi.subnetMask()}; }
+network::IPAddress WiFiComponent::wifi_gateway_ip_() { return {(const ip_addr_t *) WiFi.gatewayIP()}; }
 network::IPAddress WiFiComponent::wifi_dns_ip_(int num) {
   const ip_addr_t *dns_ip = dns_getserver(num);
-#ifdef PIO_FRAMEWORK_ARDUINO_ENABLE_IPV6
-  return {dns_ip->u_addr.ip4.addr};
-#else
-  return {dns_ip->addr};
-#endif
+  return network::IPAddress(dns_ip);
 }
 
 void WiFiComponent::wifi_loop_() {
diff --git a/tests/test1.yaml b/tests/test1.yaml
index 350057e3cc..96dda707b6 100644
--- a/tests/test1.yaml
+++ b/tests/test1.yaml
@@ -82,6 +82,9 @@ wifi:
   reboot_timeout: 120s
   power_save_mode: light
 
+network:
+  enable_ipv6: true
+
 mdns:
   disabled: false
 
diff --git a/tests/test10.yaml b/tests/test10.yaml
index 0470e37e6c..fc74d95d84 100644
--- a/tests/test10.yaml
+++ b/tests/test10.yaml
@@ -14,6 +14,9 @@ wifi:
   reboot_timeout: 3min
   power_save_mode: high
 
+network:
+  enable_ipv6: true
+
 logger:
   level: VERBOSE
 
diff --git a/tests/test11.5.yaml b/tests/test11.5.yaml
index 544dc10930..06985611e7 100644
--- a/tests/test11.5.yaml
+++ b/tests/test11.5.yaml
@@ -25,6 +25,9 @@ wifi:
         gateway: 192.168.1.1
         subnet: 255.255.255.0
 
+network:
+  enable_ipv6: true
+
 api:
 
 ota:
diff --git a/tests/test2.yaml b/tests/test2.yaml
index c04e6726b1..5485711c2e 100644
--- a/tests/test2.yaml
+++ b/tests/test2.yaml
@@ -28,6 +28,9 @@ ethernet:
     subnet: 255.255.255.0
   domain: .local
 
+network:
+  enable_ipv6: true
+
 mdns:
   disabled: true
 
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 16f31409d8..8884479f61 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -21,6 +21,9 @@ wifi:
   ssid: "MySSID"
   password: "password1"
 
+network:
+  enable_ipv6: true
+
 web_server:
   port: 80
   version: 2
diff --git a/tests/test3.yaml b/tests/test3.yaml
index 5d30e415fb..e7cf24a95a 100644
--- a/tests/test3.yaml
+++ b/tests/test3.yaml
@@ -215,6 +215,9 @@ wifi:
   ssid: "MySSID"
   password: "password1"
 
+network:
+  enable_ipv6: true
+
 uart:
   - id: uart_1
     tx_pin:
diff --git a/tests/test4.yaml b/tests/test4.yaml
index 341e613785..1b809256e7 100644
--- a/tests/test4.yaml
+++ b/tests/test4.yaml
@@ -21,6 +21,9 @@ ethernet:
     subnet: 255.255.255.0
   domain: .local
 
+network:
+  enable_ipv6: true
+
 api:
 
 i2c:
diff --git a/tests/test5.yaml b/tests/test5.yaml
index 5727d30e61..d87a7f50d4 100644
--- a/tests/test5.yaml
+++ b/tests/test5.yaml
@@ -22,6 +22,9 @@ wifi:
         gateway: 192.168.1.1
         subnet: 255.255.255.0
 
+network:
+  enable_ipv6: true
+
 api:
 
 ota:
diff --git a/tests/test6.yaml b/tests/test6.yaml
index 3d6a1ceb1f..7c72151860 100644
--- a/tests/test6.yaml
+++ b/tests/test6.yaml
@@ -16,6 +16,9 @@ wifi:
     - ssid: "MySSID"
       password: "password1"
 
+network:
+  enable_ipv6: true
+
 api:
 
 ota:
diff --git a/tests/test7.yaml b/tests/test7.yaml
index 2355dd6feb..b22fbfbcb4 100644
--- a/tests/test7.yaml
+++ b/tests/test7.yaml
@@ -3,6 +3,9 @@
 wifi:
   ssid: 'ssid'
 
+network:
+  enable_ipv6: true
+
 esp32:
   board: lolin_c3_mini
   framework:
diff --git a/tests/test8.yaml b/tests/test8.yaml
index 01d12ea330..3ba8aa19ef 100644
--- a/tests/test8.yaml
+++ b/tests/test8.yaml
@@ -3,6 +3,9 @@
 wifi:
   ssid: "ssid"
 
+network:
+  enable_ipv6: true
+
 esp32:
   board: esp32s3box
   variant: ESP32S3