From 78bad658168ae306d23a27056f6f00aae00e4d07 Mon Sep 17 00:00:00 2001 From: Liu Chao Date: Mon, 5 Aug 2024 15:48:29 +0800 Subject: [PATCH] feat: Ch_vtap_port support chost and pod_node --- server/controller/db/mysql/ch_model.go | 26 ++++++---- .../db/mysql/migration/rawsql/init.sql | 4 ++ .../mysql/migration/rawsql/issu/6.6.2.0.sql | 25 +++++++++ .../controller/db/mysql/migration/version.go | 2 +- .../http/service/vtap/vtap_interface.go | 16 ++++++ server/controller/model/model.go | 4 ++ server/controller/tagrecorder/ch_vtap_port.go | 52 +++++++++++-------- server/controller/tagrecorder/const.go | 4 ++ .../engine/clickhouse/tag/translation.go | 2 +- 9 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 server/controller/db/mysql/migration/rawsql/issu/6.6.2.0.sql diff --git a/server/controller/db/mysql/ch_model.go b/server/controller/db/mysql/ch_model.go index 5423df7c42b1..49c5ba23f847 100644 --- a/server/controller/db/mysql/ch_model.go +++ b/server/controller/db/mysql/ch_model.go @@ -57,17 +57,21 @@ type ChDevice struct { } type ChVTapPort struct { - VTapID int `gorm:"primaryKey;column:vtap_id;type:int;not null" json:"VTAP_ID"` - TapPort int64 `gorm:"primaryKey;column:tap_port;type:bigint;not null" json:"TAP_PORT"` - MacType int `gorm:"column:mac_type;type:int;default:null" json:"MAC_TYPE"` - HostID int `gorm:"column:host_id;type:int;default:null" json:"HOST_ID"` - Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"` - HostName string `gorm:"column:host_name;type:varchar(256);default:null" json:"HOST_NAME"` - DeviceType int `gorm:"column:device_type;type:int;not null" json:"DEVICE_TYPE"` - DeviceID int `gorm:"column:device_id;type:int;not null" json:"DEVICE_ID"` - DeviceName string `gorm:"column:device_name;type:varchar(256);not null" json:"DEVICE_NAME"` - IconID int `gorm:"column:icon_id;type:int;default:null" json:"ICON_ID"` - TeamID int `gorm:"column:team_id;type:int;not null" json:"TEAM_ID"` + VTapID int `gorm:"primaryKey;column:vtap_id;type:int;not null" json:"VTAP_ID"` + TapPort int64 `gorm:"primaryKey;column:tap_port;type:bigint;not null" json:"TAP_PORT"` + MacType int `gorm:"column:mac_type;type:int;default:null" json:"MAC_TYPE"` + HostID int `gorm:"column:host_id;type:int;default:null" json:"HOST_ID"` + Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"` + HostName string `gorm:"column:host_name;type:varchar(256);default:null" json:"HOST_NAME"` + DeviceType int `gorm:"column:device_type;type:int;not null" json:"DEVICE_TYPE"` + DeviceID int `gorm:"column:device_id;type:int;not null" json:"DEVICE_ID"` + DeviceName string `gorm:"column:device_name;type:varchar(256);not null" json:"DEVICE_NAME"` + IconID int `gorm:"column:icon_id;type:int;default:null" json:"ICON_ID"` + TeamID int `gorm:"column:team_id;type:int;not null" json:"TEAM_ID"` + CHostID int `gorm:"column:chost_id;type:int;default:null" json:"CHOST_ID"` + CHostName string `gorm:"column:chost_name;type:varchar(256);default:null" json:"CHOST_NAME"` + PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" json:"POD_NODE_ID"` + PodNodeName string `gorm:"column:pod_node_name;type:varchar(256);default:null" json:"POD_NODE_NAME"` } func (ChVTapPort) TableName() string { diff --git a/server/controller/db/mysql/migration/rawsql/init.sql b/server/controller/db/mysql/migration/rawsql/init.sql index c7643e321ea0..bb796ff146dc 100644 --- a/server/controller/db/mysql/migration/rawsql/init.sql +++ b/server/controller/db/mysql/migration/rawsql/init.sql @@ -2013,6 +2013,10 @@ CREATE TABLE IF NOT EXISTS ch_vtap_port ( mac_type INTEGER DEFAULT 1 COMMENT '1:tap_mac,2:mac', host_id INTEGER, host_name VARCHAR(256), + chost_id INTEGER, + chost_name VARCHAR(256), + pod_node_id INTEGER, + pod_node_name VARCHAR(256), device_type INTEGER, device_id INTEGER, device_name VARCHAR(256), diff --git a/server/controller/db/mysql/migration/rawsql/issu/6.6.2.0.sql b/server/controller/db/mysql/migration/rawsql/issu/6.6.2.0.sql new file mode 100644 index 000000000000..c1969c4d69ec --- /dev/null +++ b/server/controller/db/mysql/migration/rawsql/issu/6.6.2.0.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS ch_vtap_port; + +CREATE TABLE IF NOT EXISTS ch_vtap_port ( + vtap_id INTEGER NOT NULL, + tap_port BIGINT NOT NULL, + name VARCHAR(256), + mac_type INTEGER DEFAULT 1 COMMENT '1:tap_mac,2:mac', + host_id INTEGER, + host_name VARCHAR(256), + chost_id INTEGER, + chost_name VARCHAR(256), + pod_node_id INTEGER, + pod_node_name VARCHAR(256), + device_type INTEGER, + device_id INTEGER, + device_name VARCHAR(256), + icon_id INTEGER, + team_id INTEGER, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (vtap_id, tap_port) +)ENGINE=innodb DEFAULT CHARSET=utf8; + +-- whether default db or not, update db_version to latest, remember update DB_VERSION_EXPECT in migrate/init.go +UPDATE db_version SET version='6.6.2.0'; +-- modify end diff --git a/server/controller/db/mysql/migration/version.go b/server/controller/db/mysql/migration/version.go index e59495ce1cfa..644c9033e0fa 100644 --- a/server/controller/db/mysql/migration/version.go +++ b/server/controller/db/mysql/migration/version.go @@ -18,5 +18,5 @@ package migration const ( DB_VERSION_TABLE = "db_version" - DB_VERSION_EXPECTED = "6.6.1.5" + DB_VERSION_EXPECTED = "6.6.2.0" ) diff --git a/server/controller/http/service/vtap/vtap_interface.go b/server/controller/http/service/vtap/vtap_interface.go index ae7b4ce83d52..05c3741471fa 100644 --- a/server/controller/http/service/vtap/vtap_interface.go +++ b/server/controller/http/service/vtap/vtap_interface.go @@ -234,13 +234,21 @@ func (v *VTapInterface) formatVTapVInterfaces(vifs *simplejson.Json, filter map[ switch vtapVIF.DeviceType { case common.VIF_DEVICE_TYPE_HOST: vtapVIF.DeviceName = toolDS.hostIDToName[vtapVIF.DeviceID] + vtapVIF.DeviceHostID = vtapVIF.DeviceID + vtapVIF.DeviceHostName = vtapVIF.DeviceName case common.VIF_DEVICE_TYPE_VM: if podNodeID, ok := toolDS.vmIDToPodNodeID[vtapVIF.DeviceID]; ok { vtapVIF.DeviceType = common.VIF_DEVICE_TYPE_POD_NODE vtapVIF.DeviceID = podNodeID vtapVIF.DeviceName = toolDS.podNodeIDToName[podNodeID] + vtapVIF.DeviceCHostID = toolDS.podNodeIDToVMID[podNodeID] + vtapVIF.DeviceCHostName = toolDS.vmIDToName[vtapVIF.DeviceCHostID] + vtapVIF.DevicePodNodeID = podNodeID + vtapVIF.DevicePodNodeName = toolDS.podNodeIDToName[podNodeID] } else { vtapVIF.DeviceName = toolDS.vmIDToName[vtapVIF.DeviceID] + vtapVIF.DeviceCHostID = vtapVIF.DeviceID + vtapVIF.DeviceCHostName = toolDS.vmIDToName[vtapVIF.DeviceID] } vtapVIF.DeviceHostID = toolDS.hostIPToID[toolDS.vmIDToLaunchServer[vtapVIF.DeviceID]] vtapVIF.DeviceHostName = toolDS.hostIDToName[vtapVIF.DeviceHostID] @@ -248,6 +256,10 @@ func (v *VTapInterface) formatVTapVInterfaces(vifs *simplejson.Json, filter map[ vtapVIF.DeviceName = toolDS.podNodeIDToName[vtapVIF.DeviceID] vtapVIF.DeviceHostID = toolDS.hostIPToID[toolDS.vmIDToLaunchServer[toolDS.podNodeIDToVMID[vtapVIF.DeviceID]]] vtapVIF.DeviceHostName = toolDS.hostIDToName[vtapVIF.DeviceHostID] + vtapVIF.DeviceCHostID = toolDS.podNodeIDToVMID[vtapVIF.DeviceID] + vtapVIF.DeviceCHostName = toolDS.vmIDToName[vtapVIF.DeviceCHostID] + vtapVIF.DevicePodNodeID = vtapVIF.DeviceID + vtapVIF.DevicePodNodeName = toolDS.podNodeIDToName[vtapVIF.DeviceID] case common.VIF_DEVICE_TYPE_VROUTER: vtapVIF.DeviceName = toolDS.vrouterIDToName[vtapVIF.DeviceID] case common.VIF_DEVICE_TYPE_DHCP_PORT: @@ -266,6 +278,10 @@ func (v *VTapInterface) formatVTapVInterfaces(vifs *simplejson.Json, filter map[ vtapVIF.DeviceName = toolDS.podIDToName[vtapVIF.DeviceID] vtapVIF.DeviceHostID = toolDS.hostIPToID[toolDS.vmIDToLaunchServer[toolDS.podNodeIDToVMID[toolDS.podIDToPodNodeID[vtapVIF.DeviceID]]]] vtapVIF.DeviceHostName = toolDS.hostIDToName[vtapVIF.DeviceHostID] + vtapVIF.DeviceCHostID = toolDS.podNodeIDToVMID[toolDS.podIDToPodNodeID[vtapVIF.DeviceID]] + vtapVIF.DeviceCHostName = toolDS.vmIDToName[vtapVIF.DeviceCHostID] + vtapVIF.DevicePodNodeID = toolDS.podIDToPodNodeID[vtapVIF.DeviceID] + vtapVIF.DevicePodNodeName = toolDS.podNodeIDToName[vtapVIF.DevicePodNodeID] } } } else if vtapID != 0 { diff --git a/server/controller/model/model.go b/server/controller/model/model.go index 7c21c0c38cb6..b76a0f16551f 100644 --- a/server/controller/model/model.go +++ b/server/controller/model/model.go @@ -513,6 +513,10 @@ type VTapInterface struct { HostIP string `json:"HOST_IP"` NodeIP string `json:"NODE_IP"` LastSeen string `json:"LAST_SEEN"` + DeviceCHostID int `json:"DEVICE_CHOST_ID"` + DeviceCHostName string `json:"DEVICE_CHOST_NAME"` + DevicePodNodeID int `json:"DEVICE_POD_NODE_ID"` + DevicePodNodeName string `json:"DEVICE_POD_NODE_NAME"` } type GenesisHost struct { diff --git a/server/controller/tagrecorder/ch_vtap_port.go b/server/controller/tagrecorder/ch_vtap_port.go index 385744ac5c25..9af5181d7b22 100644 --- a/server/controller/tagrecorder/ch_vtap_port.go +++ b/server/controller/tagrecorder/ch_vtap_port.go @@ -168,17 +168,21 @@ func (v *ChVTapPort) generateNewData(db *mysql.DB) (map[VtapPortKey]mysql.ChVTap } else { if data.VTapID != 0 || tapPort != 0 { keyToItem[tapMacKey] = mysql.ChVTapPort{ - VTapID: data.VTapID, - TapPort: tapPort, - MacType: CH_VTAP_PORT_TYPE_TAP_MAC, - Name: data.TapName, - DeviceID: data.DeviceID, - HostID: data.DeviceHostID, - DeviceType: data.DeviceType, - DeviceName: data.DeviceName, - HostName: data.DeviceHostName, - IconID: deviceKeyToIconID[DeviceKey{DeviceID: data.DeviceID, DeviceType: data.DeviceType}], - TeamID: VTapIDToTeamID[data.VTapID], + VTapID: data.VTapID, + TapPort: tapPort, + MacType: CH_VTAP_PORT_TYPE_TAP_MAC, + Name: data.TapName, + DeviceID: data.DeviceID, + HostID: data.DeviceHostID, + DeviceType: data.DeviceType, + DeviceName: data.DeviceName, + HostName: data.DeviceHostName, + IconID: deviceKeyToIconID[DeviceKey{DeviceID: data.DeviceID, DeviceType: data.DeviceType}], + TeamID: VTapIDToTeamID[data.VTapID], + CHostID: data.DeviceCHostID, + CHostName: data.DeviceCHostName, + PodNodeID: data.DevicePodNodeID, + PodNodeName: data.DevicePodNodeName, } log.Debugf("add new: %+v", keyToItem[tapMacKey], db.LogPrefixORGID) } @@ -221,17 +225,21 @@ func (v *ChVTapPort) generateNewData(db *mysql.DB) (map[VtapPortKey]mysql.ChVTap log.Debugf("update: %+v", vTapPort, db.LogPrefixORGID) } else { keyToItem[macKey] = mysql.ChVTapPort{ - VTapID: data.VTapID, - TapPort: macPort, - MacType: CH_VTAP_PORT_TYPE_TAP_MAC, - Name: data.Name, - DeviceID: data.DeviceID, - HostID: data.DeviceHostID, - DeviceType: data.DeviceType, - DeviceName: data.DeviceName, - HostName: data.DeviceHostName, - IconID: deviceKeyToIconID[DeviceKey{DeviceID: data.DeviceID, DeviceType: data.DeviceType}], - TeamID: VTapIDToTeamID[data.VTapID], + VTapID: data.VTapID, + TapPort: macPort, + MacType: CH_VTAP_PORT_TYPE_TAP_MAC, + Name: data.Name, + DeviceID: data.DeviceID, + HostID: data.DeviceHostID, + DeviceType: data.DeviceType, + DeviceName: data.DeviceName, + HostName: data.DeviceHostName, + IconID: deviceKeyToIconID[DeviceKey{DeviceID: data.DeviceID, DeviceType: data.DeviceType}], + TeamID: VTapIDToTeamID[data.VTapID], + CHostID: data.DeviceCHostID, + CHostName: data.DeviceCHostName, + PodNodeID: data.DevicePodNodeID, + PodNodeName: data.DevicePodNodeName, } log.Debugf("add new: %+v", keyToItem[tapMacKey], db.LogPrefixORGID) } diff --git a/server/controller/tagrecorder/const.go b/server/controller/tagrecorder/const.go index 16ce27d58931..7b26155e9451 100644 --- a/server/controller/tagrecorder/const.go +++ b/server/controller/tagrecorder/const.go @@ -317,6 +317,10 @@ const ( " `name` String,\n" + " `host_id` Int64,\n" + " `host_name` String,\n" + + " `chost_id` Int64,\n" + + " `chost_name` String,\n" + + " `pod_node_id` Int64,\n" + + " `pod_node_name` String,\n" + " `device_type` UInt64,\n" + " `device_id` UInt64,\n" + " `device_name` String,\n" + diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index e2d47b7f980f..c85d6546b760 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -601,7 +601,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { } // vtap对应资源 - vtapResource := "'device_type','device_id','device_name','icon_id','host_id','host_name'" + vtapResource := "'device_type','device_id','device_name','icon_id','host_id','host_name','chost_id','chost_name','pod_node_id','pod_node_name'" gwDictGet := fmt.Sprintf("dictGet(flow_tag.vtap_port_map, (%s),(toUInt64(agent_id),toUInt64(capture_nic)))", vtapResource) tagResourceMap["resource_from_vtap"] = map[string]*Tag{ "default": NewTag(