Initial bluetooth_proxy support (#3736)

CODEOWNERS

@@ -34,6 +34,7 @@ esphome/components/binary_sensor/* @esphome/core
 esphome/components/bl0939/* @ziceva
 esphome/components/bl0940/* @tobias-
 esphome/components/ble_client/* @buxtronix
+esphome/components/bluetooth_proxy/* @jesserockz
 esphome/components/bme680_bsec/* @trvrnrth
 esphome/components/bmp3xx/* @martgras
 esphome/components/button/* @esphome/core

esphome/components/api/api.proto

@@ -27,6 +27,7 @@ service APIConnection {
   rpc subscribe_logs (SubscribeLogsRequest) returns (void) {}
   rpc subscribe_homeassistant_services (SubscribeHomeassistantServicesRequest) returns (void) {}
   rpc subscribe_home_assistant_states (SubscribeHomeAssistantStatesRequest) returns (void) {}
+  rpc subscribe_bluetooth_le_advertisements (SubscribeBluetoothLEAdvertisementsRequest) returns (void) {}
   rpc get_time (GetTimeRequest) returns (GetTimeResponse) {
     option (needs_authentication) = false;
   }
@@ -190,6 +191,8 @@ message DeviceInfoResponse {
   string project_version = 9;
 
   uint32 webserver_port = 10;
+
+  bool has_bluetooth_proxy = 11;
 }
 
 message ListEntitiesRequest {
@@ -1099,3 +1102,28 @@ message MediaPlayerCommandRequest {
   bool has_media_url = 6;
   string media_url = 7;
 }
+
+// ==================== BLUETOOTH ====================
+message SubscribeBluetoothLEAdvertisementsRequest {
+  option (id) = 66;
+  option (source) = SOURCE_CLIENT;
+}
+
+message BluetoothServiceData {
+  string uuid = 1;
+  repeated uint32 data = 2 [packed=false];
+}
+message BluetoothLEAdvertisementResponse {
+  option (id) = 67;
+  option (source) = SOURCE_SERVER;
+  option (ifdef) = "USE_BLUETOOTH_PROXY";
+  option (no_delay) = true;
+
+  uint64 address = 1;
+  string name = 2;
+  sint32 rssi = 3;
+
+  repeated string service_uuids = 4;
+  repeated BluetoothServiceData service_data = 5;
+  repeated BluetoothServiceData manufacturer_data = 6;
+}

esphome/components/api/api_connection.cpp

@@ -886,6 +886,9 @@ DeviceInfoResponse APIConnection::device_info(const DeviceInfoRequest &msg) {
 #endif
 #ifdef USE_WEBSERVER
   resp.webserver_port = USE_WEBSERVER_PORT;
+#endif
+#ifdef USE_BLUETOOTH_PROXY
+  resp.has_bluetooth_proxy = true;
 #endif
   return resp;
 }

esphome/components/api/api_connection.h

@@ -7,6 +7,10 @@
 #include "api_server.h"
 #include "api_frame_helper.h"
 
+#ifdef USE_BLUETOOTH_PROXY
+#include "esphome/components/bluetooth_proxy/bluetooth_proxy.h"
+#endif
+
 namespace esphome {
 namespace api {
 
@@ -94,6 +98,13 @@ class APIConnection : public APIServerConnection {
       return;
     this->send_homeassistant_service_response(call);
   }
+#ifdef USE_BLUETOOTH_PROXY
+  bool send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &call) {
+    if (!this->bluetooth_le_advertisement_subscription_)
+      return false;
+    return this->send_bluetooth_le_advertisement_response(call);
+  }
+#endif
 #ifdef USE_HOMEASSISTANT_TIME
   void send_time_request() {
     GetTimeRequest req;
@@ -134,6 +145,9 @@ class APIConnection : public APIServerConnection {
     return {};
   }
   void execute_service(const ExecuteServiceRequest &msg) override;
+  void subscribe_bluetooth_le_advertisements(const SubscribeBluetoothLEAdvertisementsRequest &msg) override {
+    this->bluetooth_le_advertisement_subscription_ = true;
+  }
   bool is_authenticated() override { return this->connection_state_ == ConnectionState::AUTHENTICATED; }
   bool is_connection_setup() override {
     return this->connection_state_ == ConnectionState ::CONNECTED || this->is_authenticated();
@@ -176,6 +190,7 @@ class APIConnection : public APIServerConnection {
   uint32_t last_traffic_;
   bool sent_ping_{false};
   bool service_call_subscription_{false};
+  bool bluetooth_le_advertisement_subscription_{true};
   bool next_close_ = false;
   APIServer *parent_;
   InitialStateIterator initial_state_iterator_;

esphome/components/api/api_pb2.cpp

@@ -495,6 +495,10 @@ bool DeviceInfoResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
       this->webserver_port = value.as_uint32();
       return true;
     }
+    case 11: {
+      this->has_bluetooth_proxy = value.as_bool();
+      return true;
+    }
     default:
       return false;
   }
@@ -544,6 +548,7 @@ void DeviceInfoResponse::encode(ProtoWriteBuffer buffer) const {
   buffer.encode_string(8, this->project_name);
   buffer.encode_string(9, this->project_version);
   buffer.encode_uint32(10, this->webserver_port);
+  buffer.encode_bool(11, this->has_bluetooth_proxy);
 }
 #ifdef HAS_PROTO_MESSAGE_DUMP
 void DeviceInfoResponse::dump_to(std::string &out) const {
@@ -589,6 +594,10 @@ void DeviceInfoResponse::dump_to(std::string &out) const {
   sprintf(buffer, "%u", this->webserver_port);
   out.append(buffer);
   out.append("\n");
+
+  out.append("  has_bluetooth_proxy: ");
+  out.append(YESNO(this->has_bluetooth_proxy));
+  out.append("\n");
   out.append("}");
 }
 #endif
@@ -4854,6 +4863,143 @@ void MediaPlayerCommandRequest::dump_to(std::string &out) const {
   out.append("}");
 }
 #endif
+void SubscribeBluetoothLEAdvertisementsRequest::encode(ProtoWriteBuffer buffer) const {}
+#ifdef HAS_PROTO_MESSAGE_DUMP
+void SubscribeBluetoothLEAdvertisementsRequest::dump_to(std::string &out) const {
+  out.append("SubscribeBluetoothLEAdvertisementsRequest {}");
+}
+#endif
+bool BluetoothServiceData::decode_varint(uint32_t field_id, ProtoVarInt value) {
+  switch (field_id) {
+    case 2: {
+      this->data.push_back(value.as_uint32());
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+bool BluetoothServiceData::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
+  switch (field_id) {
+    case 1: {
+      this->uuid = value.as_string();
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+void BluetoothServiceData::encode(ProtoWriteBuffer buffer) const {
+  buffer.encode_string(1, this->uuid);
+  for (auto &it : this->data) {
+    buffer.encode_uint32(2, it, true);
+  }
+}
+#ifdef HAS_PROTO_MESSAGE_DUMP
+void BluetoothServiceData::dump_to(std::string &out) const {
+  __attribute__((unused)) char buffer[64];
+  out.append("BluetoothServiceData {\n");
+  out.append("  uuid: ");
+  out.append("'").append(this->uuid).append("'");
+  out.append("\n");
+
+  for (const auto &it : this->data) {
+    out.append("  data: ");
+    sprintf(buffer, "%u", it);
+    out.append(buffer);
+    out.append("\n");
+  }
+  out.append("}");
+}
+#endif
+bool BluetoothLEAdvertisementResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
+  switch (field_id) {
+    case 1: {
+      this->address = value.as_uint64();
+      return true;
+    }
+    case 3: {
+      this->rssi = value.as_sint32();
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+bool BluetoothLEAdvertisementResponse::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
+  switch (field_id) {
+    case 2: {
+      this->name = value.as_string();
+      return true;
+    }
+    case 4: {
+      this->service_uuids.push_back(value.as_string());
+      return true;
+    }
+    case 5: {
+      this->service_data.push_back(value.as_message<BluetoothServiceData>());
+      return true;
+    }
+    case 6: {
+      this->manufacturer_data.push_back(value.as_message<BluetoothServiceData>());
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+void BluetoothLEAdvertisementResponse::encode(ProtoWriteBuffer buffer) const {
+  buffer.encode_uint64(1, this->address);
+  buffer.encode_string(2, this->name);
+  buffer.encode_sint32(3, this->rssi);
+  for (auto &it : this->service_uuids) {
+    buffer.encode_string(4, it, true);
+  }
+  for (auto &it : this->service_data) {
+    buffer.encode_message<BluetoothServiceData>(5, it, true);
+  }
+  for (auto &it : this->manufacturer_data) {
+    buffer.encode_message<BluetoothServiceData>(6, it, true);
+  }
+}
+#ifdef HAS_PROTO_MESSAGE_DUMP
+void BluetoothLEAdvertisementResponse::dump_to(std::string &out) const {
+  __attribute__((unused)) char buffer[64];
+  out.append("BluetoothLEAdvertisementResponse {\n");
+  out.append("  address: ");
+  sprintf(buffer, "%llu", this->address);
+  out.append(buffer);
+  out.append("\n");
+
+  out.append("  name: ");
+  out.append("'").append(this->name).append("'");
+  out.append("\n");
+
+  out.append("  rssi: ");
+  sprintf(buffer, "%d", this->rssi);
+  out.append(buffer);
+  out.append("\n");
+
+  for (const auto &it : this->service_uuids) {
+    out.append("  service_uuids: ");
+    out.append("'").append(it).append("'");
+    out.append("\n");
+  }
+
+  for (const auto &it : this->service_data) {
+    out.append("  service_data: ");
+    it.dump_to(out);
+    out.append("\n");
+  }
+
+  for (const auto &it : this->manufacturer_data) {
+    out.append("  manufacturer_data: ");
+    it.dump_to(out);
+    out.append("\n");
+  }
+  out.append("}");
+}
+#endif
 
 }  // namespace api
 }  // namespace esphome

esphome/components/api/api_pb2.h

@@ -263,6 +263,7 @@ class DeviceInfoResponse : public ProtoMessage {
   std::string project_name{};
   std::string project_version{};
   uint32_t webserver_port{0};
+  bool has_bluetooth_proxy{false};
   void encode(ProtoWriteBuffer buffer) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   void dump_to(std::string &out) const override;
@@ -1214,6 +1215,45 @@ class MediaPlayerCommandRequest : public ProtoMessage {
   bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override;
   bool decode_varint(uint32_t field_id, ProtoVarInt value) override;
 };
+class SubscribeBluetoothLEAdvertisementsRequest : public ProtoMessage {
+ public:
+  void encode(ProtoWriteBuffer buffer) const override;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  void dump_to(std::string &out) const override;
+#endif
+
+ protected:
+};
+class BluetoothServiceData : public ProtoMessage {
+ public:
+  std::string uuid{};
+  std::vector<uint32_t> data{};
+  void encode(ProtoWriteBuffer buffer) const override;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  void dump_to(std::string &out) const override;
+#endif
+
+ protected:
+  bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override;
+  bool decode_varint(uint32_t field_id, ProtoVarInt value) override;
+};
+class BluetoothLEAdvertisementResponse : public ProtoMessage {
+ public:
+  uint64_t address{0};
+  std::string name{};
+  int32_t rssi{0};
+  std::vector<std::string> service_uuids{};
+  std::vector<BluetoothServiceData> service_data{};
+  std::vector<BluetoothServiceData> manufacturer_data{};
+  void encode(ProtoWriteBuffer buffer) const override;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  void dump_to(std::string &out) const override;
+#endif
+
+ protected:
+  bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override;
+  bool decode_varint(uint32_t field_id, ProtoVarInt value) override;
+};
 
 }  // namespace api
 }  // namespace esphome

esphome/components/api/api_pb2_service.cpp

@@ -328,6 +328,14 @@ bool APIServerConnectionBase::send_media_player_state_response(const MediaPlayer
 #endif
 #ifdef USE_MEDIA_PLAYER
 #endif
+#ifdef USE_BLUETOOTH_PROXY
+bool APIServerConnectionBase::send_bluetooth_le_advertisement_response(const BluetoothLEAdvertisementResponse &msg) {
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  ESP_LOGVV(TAG, "send_bluetooth_le_advertisement_response: %s", msg.dump().c_str());
+#endif
+  return this->send_message_<BluetoothLEAdvertisementResponse>(msg, 67);
+}
+#endif
 bool APIServerConnectionBase::read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) {
   switch (msg_type) {
     case 1: {
@@ -595,6 +603,15 @@ bool APIServerConnectionBase::read_message(uint32_t msg_size, uint32_t msg_type,
 #endif
       break;
     }
+    case 66: {
+      SubscribeBluetoothLEAdvertisementsRequest msg;
+      msg.decode(msg_data, msg_size);
+#ifdef HAS_PROTO_MESSAGE_DUMP
+      ESP_LOGVV(TAG, "on_subscribe_bluetooth_le_advertisements_request: %s", msg.dump().c_str());
+#endif
+      this->on_subscribe_bluetooth_le_advertisements_request(msg);
+      break;
+    }
     default:
       return false;
   }
@@ -691,6 +708,18 @@ void APIServerConnection::on_subscribe_home_assistant_states_request(const Subsc
   }
   this->subscribe_home_assistant_states(msg);
 }
+void APIServerConnection::on_subscribe_bluetooth_le_advertisements_request(
+    const SubscribeBluetoothLEAdvertisementsRequest &msg) {
+  if (!this->is_connection_setup()) {
+    this->on_no_setup_connection();
+    return;
+  }
+  if (!this->is_authenticated()) {
+    this->on_unauthenticated_access();
+    return;
+  }
+  this->subscribe_bluetooth_le_advertisements(msg);
+}
 void APIServerConnection::on_get_time_request(const GetTimeRequest &msg) {
   if (!this->is_connection_setup()) {
     this->on_no_setup_connection();

esphome/components/api/api_pb2_service.h

@@ -153,6 +153,11 @@ class APIServerConnectionBase : public ProtoService {
 #endif
 #ifdef USE_MEDIA_PLAYER
   virtual void on_media_player_command_request(const MediaPlayerCommandRequest &value){};
+#endif
+  virtual void on_subscribe_bluetooth_le_advertisements_request(
+      const SubscribeBluetoothLEAdvertisementsRequest &value){};
+#ifdef USE_BLUETOOTH_PROXY
+  bool send_bluetooth_le_advertisement_response(const BluetoothLEAdvertisementResponse &msg);
 #endif
  protected:
   bool read_message(uint32_t msg_size, uint32_t msg_type, uint8_t *msg_data) override;
@@ -170,6 +175,7 @@ class APIServerConnection : public APIServerConnectionBase {
   virtual void subscribe_logs(const SubscribeLogsRequest &msg) = 0;
   virtual void subscribe_homeassistant_services(const SubscribeHomeassistantServicesRequest &msg) = 0;
   virtual void subscribe_home_assistant_states(const SubscribeHomeAssistantStatesRequest &msg) = 0;
+  virtual void subscribe_bluetooth_le_advertisements(const SubscribeBluetoothLEAdvertisementsRequest &msg) = 0;
   virtual GetTimeResponse get_time(const GetTimeRequest &msg) = 0;
   virtual void execute_service(const ExecuteServiceRequest &msg) = 0;
 #ifdef USE_COVER
@@ -216,6 +222,7 @@ class APIServerConnection : public APIServerConnectionBase {
   void on_subscribe_logs_request(const SubscribeLogsRequest &msg) override;
   void on_subscribe_homeassistant_services_request(const SubscribeHomeassistantServicesRequest &msg) override;
   void on_subscribe_home_assistant_states_request(const SubscribeHomeAssistantStatesRequest &msg) override;
+  void on_subscribe_bluetooth_le_advertisements_request(const SubscribeBluetoothLEAdvertisementsRequest &msg) override;
   void on_get_time_request(const GetTimeRequest &msg) override;
   void on_execute_service_request(const ExecuteServiceRequest &msg) override;
 #ifdef USE_COVER

esphome/components/api/api_server.cpp

@@ -291,6 +291,13 @@ void APIServer::send_homeassistant_service_call(const HomeassistantServiceRespon
     client->send_homeassistant_service_call(call);
   }
 }
+#ifdef USE_BLUETOOTH_PROXY
+void APIServer::send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &call) {
+  for (auto &client : this->clients_) {
+    client->send_bluetooth_le_advertisement(call);
+  }
+}
+#endif
 APIServer::APIServer() { global_api_server = this; }
 void APIServer::subscribe_home_assistant_state(std::string entity_id, optional<std::string> attribute,
                                                std::function<void(std::string)> f) {

esphome/components/api/api_server.h

@@ -73,6 +73,9 @@ class APIServer : public Component, public Controller {
   void on_media_player_update(media_player::MediaPlayer *obj) override;
 #endif
   void send_homeassistant_service_call(const HomeassistantServiceResponse &call);
+#ifdef USE_BLUETOOTH_PROXY
+  void send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &call);
+#endif
   void register_user_service(UserServiceDescriptor *descriptor) { this->user_services_.push_back(descriptor); }
 #ifdef USE_HOMEASSISTANT_TIME
   void request_time();

esphome/components/api/proto.h

@@ -70,7 +70,7 @@ class ProtoVarInt {
     }
   }
   void encode(std::vector<uint8_t> &out) {
-    uint32_t val = this->value_;
+    uint64_t val = this->value_;
     if (val <= 0x7F) {
       out.push_back(val);
       return;

esphome/components/bluetooth_proxy/__init__.py

@@ -0,0 +1,27 @@
+from esphome.components import esp32_ble_tracker
+import esphome.config_validation as cv
+import esphome.codegen as cg
+from esphome.const import CONF_ID
+
+DEPENDENCIES = ["esp32", "esp32_ble_tracker"]
+CODEOWNERS = ["@jesserockz"]
+
+
+bluetooth_proxy_ns = cg.esphome_ns.namespace("bluetooth_proxy")
+
+BluetoothProxy = bluetooth_proxy_ns.class_("BluetoothProxy", cg.Component)
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(BluetoothProxy),
+    }
+).extend(esp32_ble_tracker.ESP_BLE_DEVICE_SCHEMA)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+
+    await esp32_ble_tracker.register_ble_device(var, config)
+
+    cg.add_define("USE_BLUETOOTH_PROXY")

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -0,0 +1,58 @@
+#include "bluetooth_proxy.h"
+
+#ifdef USE_API
+#include "esphome/components/api/api_pb2.h"
+#include "esphome/components/api/api_server.h"
+#endif  // USE_API
+#include "esphome/core/log.h"
+
+#ifdef USE_ESP32
+
+namespace esphome {
+namespace bluetooth_proxy {
+
+static const char *const TAG = "bluetooth_proxy";
+
+bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) {
+  ESP_LOGV(TAG, "Proxying packet from %s - %s. RSSI: %d dB", device.get_name().c_str(), device.address_str().c_str(),
+           device.get_rssi());
+  this->send_api_packet_(device);
+  return true;
+}
+
+void BluetoothProxy::send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device) {
+#ifndef USE_API
+  return;
+#else
+  api::BluetoothLEAdvertisementResponse resp;
+  resp.address = device.address_uint64();
+  if (!device.get_name().empty())
+    resp.name = device.get_name();
+  resp.rssi = device.get_rssi();
+  for (auto uuid : device.get_service_uuids()) {
+    resp.service_uuids.push_back(uuid.to_string());
+  }
+  for (auto &data : device.get_service_datas()) {
+    api::BluetoothServiceData service_data;
+    service_data.uuid = data.uuid.to_string();
+    for (auto d : data.data)
+      service_data.data.push_back(d);
+    resp.service_data.push_back(service_data);
+  }
+  for (auto &data : device.get_manufacturer_datas()) {
+    api::BluetoothServiceData manufacturer_data;
+    manufacturer_data.uuid = data.uuid.to_string();
+    for (auto d : data.data)
+      manufacturer_data.data.push_back(d);
+    resp.manufacturer_data.push_back(manufacturer_data);
+  }
+  api::global_api_server->send_bluetooth_le_advertisement(resp);
+#endif
+}
+
+void BluetoothProxy::dump_config() { ESP_LOGCONFIG(TAG, "Bluetooth Proxy:"); }
+
+}  // namespace bluetooth_proxy
+}  // namespace esphome
+
+#endif  // USE_ESP32

esphome/components/bluetooth_proxy/bluetooth_proxy.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#ifdef USE_ESP32
+
+#include <map>
+
+#include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h"
+#include "esphome/core/automation.h"
+#include "esphome/core/component.h"
+
+namespace esphome {
+namespace bluetooth_proxy {
+
+class BluetoothProxy : public Component, public esp32_ble_tracker::ESPBTDeviceListener {
+ public:
+  bool parse_device(const esp32_ble_tracker::ESPBTDevice &device) override;
+  void dump_config() override;
+
+ protected:
+  void send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device);
+};
+
+}  // namespace bluetooth_proxy
+}  // namespace esphome
+
+#endif  // USE_ESP32

esphome/core/defines.h

@@ -70,6 +70,7 @@
 #define USE_ESP32_IGNORE_EFUSE_MAC_CRC
 #define USE_IMPROV
 #define USE_SOCKET_IMPL_BSD_SOCKETS
+#define USE_BLUETOOTH_PROXY
 
 #ifdef USE_ARDUINO
 #define USE_ARDUINO_VERSION_CODE VERSION_CODE(1, 0, 6)

tests/test1.yaml

@@ -288,6 +288,8 @@ adalight:
 
 esp32_ble_tracker:
 
+bluetooth_proxy:
+
 ble_client:
   - mac_address: AA:BB:CC:DD:EE:FF
     id: ble_foo