Merge branch 'main' into support_pcap_compress

deepflowio · Jan 17, 2025 · b5108c7 · b5108c7
2 parents f0df2e2 + 0d5a86f
commit b5108c7
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 36 deletions.
diff --git a/server/controller/common/const.go b/server/controller/common/const.go
@@ -262,6 +262,11 @@ const (
 	NPB_POLICY_FLOW_DISTRIBUTE = 1
 )
 
+const (
+	POLICY_VTAP_TYPE_VTAP       = 1
+	POLICY_VTAP_TYPE_VTAP_GROUP = 2
+)
+
 const (
 	DEFAULT_ENCRYPTION_PASSWORD = "******"
 	DEFAULT_ALL_MATCH_REGEX     = ".*"

diff --git a/server/controller/db/metadb/migrator/schema/const.go b/server/controller/db/metadb/migrator/schema/const.go
@@ -20,5 +20,5 @@ const (
 	RAW_SQL_ROOT_DIR = "/etc/metadb/schema/rawsql"
 
 	DB_VERSION_TABLE    = "db_version"
-	DB_VERSION_EXPECTED = "7.0.1.2"
+	DB_VERSION_EXPECTED = "7.0.1.3"
 )
diff --git a/server/controller/db/metadb/migrator/schema/rawsql/mysql/init.sql b/server/controller/db/metadb/migrator/schema/rawsql/mysql/init.sql
@@ -1078,7 +1078,9 @@ CREATE TABLE IF NOT EXISTS npb_policy (
     payload_slice          INTEGER DEFAULT NULL,
     acl_id                 INTEGER,
     policy_acl_group_id    INTEGER,
+    vtap_type              TINYINT(1) DEFAULT 1 COMMENT '1: vtap; 2: vtap_group',
     vtap_ids               TEXT COMMENT 'separated by ,',
+    vtap_group_ids         TEXT COMMENT 'separated by ,',
     created_at             TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at             TIMESTAMP NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     lcuuid                 CHAR(64)

diff --git a/server/controller/db/metadb/migrator/schema/rawsql/mysql/issu/7.0.1.3.sql b/server/controller/db/metadb/migrator/schema/rawsql/mysql/issu/7.0.1.3.sql
@@ -0,0 +1,30 @@
+DROP PROCEDURE IF EXISTS AddColumnIfNotExists;
+
+CREATE PROCEDURE AddColumnIfNotExists(
+    IN tableName VARCHAR(255),
+    IN colName VARCHAR(255),
+    IN colType VARCHAR(255),
+    IN afterCol VARCHAR(255)
+)
+BEGIN
+    DECLARE column_count INT;
+
+    SELECT COUNT(*)
+    INTO column_count
+    FROM information_schema.columns
+    WHERE TABLE_SCHEMA = DATABASE()
+    AND TABLE_NAME = tableName
+    AND column_name = colName;
+
+    IF column_count = 0 THEN
+        SET @sql = CONCAT('ALTER TABLE ', tableName, ' ADD COLUMN ', colName, ' ', colType, ' AFTER ', afterCol);
+        PREPARE stmt FROM @sql;
+        EXECUTE stmt;
+        DEALLOCATE PREPARE stmt;
+    END IF;
+END;
+
+CALL AddColumnIfNotExists('npb_policy', 'vtap_type', "TINYINT(1) DEFAULT 1 COMMENT '1: vtap; 2: vtap_group'", 'policy_acl_group_id');
+CALL AddColumnIfNotExists('npb_policy', 'vtap_group_ids', "TEXT COMMENT 'separated by ,'", 'vtap_ids');
+
+UPDATE db_version SET version='7.0.1.3';
diff --git a/server/controller/db/metadb/model/model.go b/server/controller/db/metadb/model/model.go
@@ -357,7 +357,9 @@ type NpbPolicy struct {
 	PayloadSlice     *int      `gorm:"column:payload_slice;type:int;default:null" json:"PAYLOAD_SLICE"`
 	ACLID            int       `gorm:"column:acl_id;type:int;default:null" json:"ACL_ID"`
 	PolicyACLGroupID int       `gorm:"column:policy_acl_group_id;type:int;default:null" json:"POLICY_ACL_GROUP_ID"`
-	VtapIDs          string    `gorm:"column:vtap_ids;type:text;default:null" json:"VTAP_IDS"` // separated by ,
+	VtapType         int       `gorm:"column:vtap_type;type:type:tinyint(1);default:1" json:"VTAP_TYPE"` // 1: vtap; 2: vtap_group
+	VtapIDs          string    `gorm:"column:vtap_ids;type:text" json:"VTAP_IDS"`                        // separated by ,
+	VtapGroupIDs     string    `gorm:"column:vtap_group_ids;type:text" json:"VTAP_GROUP_IDS"`            // separated by ,
 	CreatedAt        time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"CREATED_AT"`
 	UpdatedAt        time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"UPDATED_AT"`
 	Lcuuid           string    `gorm:"column:lcuuid;type:char(64);default:null" json:"LCUUID"`

diff --git a/server/controller/trisolaris/dbcache/db_data.go b/server/controller/trisolaris/dbcache/db_data.go
@@ -77,9 +77,10 @@ type DBDataCache struct {
 	processes               []*models.Process
 	vips                    []*models.VIP
 
-	podNSs    []*models.PodNamespace
-	vtaps     []*models.VTap
-	chDevices []*models.ChDevice
+	podNSs     []*models.PodNamespace
+	vtaps      []*models.VTap
+	vtapGroups []*models.VTapGroup
+	chDevices  []*models.ChDevice
 
 	config *config.Config
 
@@ -286,6 +287,10 @@ func (d *DBDataCache) GetVTapsIDAndName() []*models.VTap {
 	return d.vtaps
 }
 
+func (d *DBDataCache) GetVTapGroupsIDAndLcuuid() []*models.VTapGroup {
+	return d.vtapGroups
+}
+
 func (d *DBDataCache) GetChDevicesIDTypeAndName() []*models.ChDevice {
 	return d.chDevices
 }
@@ -623,13 +628,20 @@ func (d *DBDataCache) GetDataCacheFromDB(db *gorm.DB) {
 		log.Error(d.Log(err.Error()))
 	}
 
-	vtaps, err := dbmgr.DBMgr[models.VTap](db).GetFields([]string{"id", "name", "launch_server_id", "type"})
+	vtaps, err := dbmgr.DBMgr[models.VTap](db).GetFields([]string{"id", "name", "launch_server_id", "type", "vtap_group_lcuuid"})
 	if err == nil {
 		d.vtaps = vtaps
 	} else {
 		log.Error(d.Log(err.Error()))
 	}
 
+	vtapGroups, err := dbmgr.DBMgr[models.VTapGroup](db).GetFields([]string{"id", "lcuuid"})
+	if err == nil {
+		d.vtapGroups = vtapGroups
+	} else {
+		log.Error(d.Log(err.Error()))
+	}
+
 	chDevices, err := dbmgr.DBMgr[models.ChDevice](db).GetFields([]string{"devicetype", "deviceid", "name"})
 	if err == nil {
 		d.chDevices = chDevices

diff --git a/server/controller/trisolaris/metadata/agentmetadata/policy.go b/server/controller/trisolaris/metadata/agentmetadata/policy.go
@@ -35,22 +35,24 @@ import (
 )
 
 type PolicyRawData struct {
-	idToNpbTunnel      map[int]*models.NpbTunnel
-	idToACL            map[int]*models.ACL
-	aclIDToNpbPolices  map[int][]*models.NpbPolicy
-	aclIDToPcapPolices map[int][]*models.PcapPolicy
-	idToNpbPolicy      map[int]*models.NpbPolicy
-	idToPcapPolicy     map[int]*models.PcapPolicy
+	vtapGroupIDToAgentIDs map[int][]int
+	idToNpbTunnel         map[int]*models.NpbTunnel
+	idToACL               map[int]*models.ACL
+	aclIDToNpbPolices     map[int][]*models.NpbPolicy
+	aclIDToPcapPolices    map[int][]*models.PcapPolicy
+	idToNpbPolicy         map[int]*models.NpbPolicy
+	idToPcapPolicy        map[int]*models.PcapPolicy
 }
 
 func newPolicyRawData() *PolicyRawData {
 	return &PolicyRawData{
-		idToNpbTunnel:      make(map[int]*models.NpbTunnel),
-		idToACL:            make(map[int]*models.ACL),
-		aclIDToNpbPolices:  make(map[int][]*models.NpbPolicy),
-		aclIDToPcapPolices: make(map[int][]*models.PcapPolicy),
-		idToNpbPolicy:      make(map[int]*models.NpbPolicy),
-		idToPcapPolicy:     make(map[int]*models.PcapPolicy),
+		vtapGroupIDToAgentIDs: make(map[int][]int),
+		idToNpbTunnel:         make(map[int]*models.NpbTunnel),
+		idToACL:               make(map[int]*models.ACL),
+		aclIDToNpbPolices:     make(map[int][]*models.NpbPolicy),
+		aclIDToPcapPolices:    make(map[int][]*models.PcapPolicy),
+		idToNpbPolicy:         make(map[int]*models.NpbPolicy),
+		idToPcapPolicy:        make(map[int]*models.PcapPolicy),
 	}
 }
 
@@ -351,8 +353,23 @@ func (op *PolicyDataOP) generateRawData() {
 	acls := dbDataCache.GetACLs()
 	npbPolicies := dbDataCache.GetNpbPolicies()
 	pcapPolicies := dbDataCache.GetPcapPolicies()
+	vtaps := dbDataCache.GetVTapsIDAndName()
+	vtapGroups := dbDataCache.GetVTapGroupsIDAndLcuuid()
 
 	rawData := newPolicyRawData()
+	vtapGroupLcuuidToID := map[string]int{}
+	for _, vtapGroup := range vtapGroups {
+		vtapGroupLcuuidToID[vtapGroup.Lcuuid] = vtapGroup.ID
+	}
+	for _, vtap := range vtaps {
+		vtapGroupID, ok := vtapGroupLcuuidToID[vtap.VtapGroupLcuuid]
+		if !ok {
+			log.Warning(op.Logf("agent(%s) group lcuuid(%s) not found group id", vtap.Name, vtap.VtapGroupLcuuid))
+			continue
+		}
+		rawData.vtapGroupIDToAgentIDs[vtapGroupID] = append(rawData.vtapGroupIDToAgentIDs[vtapGroupID], vtap.ID)
+	}
+
 	for _, npbTunnel := range npbTunnels {
 		rawData.idToNpbTunnel[npbTunnel.ID] = npbTunnel
 	}
@@ -600,9 +617,8 @@ func (op *PolicyDataOP) generateProtoActions(acl *models.ACL) (map[int][]*agent.
 				NpbAclGroupId:     proto.Uint32(uint32(npbPolicy.PolicyACLGroupID)),
 				Direction:         &direction,
 			}
-			if len(npbPolicy.VtapIDs) == 0 {
-				allAgentNpbActions = append(allAgentNpbActions, npbAction)
-			} else {
+			switch npbPolicy.VtapType {
+			case POLICY_VTAP_TYPE_VTAP:
 				for _, agentIDStr := range strings.Split(npbPolicy.VtapIDs, ",") {
 					agentIDInt, err := strconv.Atoi(agentIDStr)
 					if err != nil {
@@ -611,6 +627,24 @@ func (op *PolicyDataOP) generateProtoActions(acl *models.ACL) (map[int][]*agent.
 					}
 					agentIDToNpbActions[agentIDInt] = append(agentIDToNpbActions[agentIDInt], npbAction)
 				}
+			case POLICY_VTAP_TYPE_VTAP_GROUP:
+				for _, vtapGroupIDStr := range strings.Split(npbPolicy.VtapGroupIDs, ",") {
+					vtapGroupIDInt, err := strconv.Atoi(vtapGroupIDStr)
+					if err != nil {
+						log.Errorf(op.Logf("err: %s, vtapGroupIDs: %s", err, npbPolicy.VtapGroupIDs))
+						continue
+					}
+					agentIDs, ok := rawData.vtapGroupIDToAgentIDs[vtapGroupIDInt]
+					if !ok {
+						log.Errorf(op.Logf("not found agent in vtap group id(%d)", vtapGroupIDInt))
+						continue
+					}
+					for agentID := range agentIDs {
+						agentIDToNpbActions[agentID] = append(agentIDToNpbActions[agentID], npbAction)
+					}
+				}
+			default:
+				allAgentNpbActions = append(allAgentNpbActions, npbAction)
 			}
 		}
 	case APPLICATION_PCAP:

diff --git a/server/controller/trisolaris/metadata/policy.go b/server/controller/trisolaris/metadata/policy.go
@@ -35,22 +35,24 @@ import (
 )
 
 type PolicyRawData struct {
-	idToNpbTunnel      map[int]*models.NpbTunnel
-	idToACL            map[int]*models.ACL
-	aclIDToNpbPolices  map[int][]*models.NpbPolicy
-	aclIDToPcapPolices map[int][]*models.PcapPolicy
-	idToNpbPolicy      map[int]*models.NpbPolicy
-	idToPcapPolicy     map[int]*models.PcapPolicy
+	vtapGroupIDToVtapIDs map[int][]int
+	idToNpbTunnel        map[int]*models.NpbTunnel
+	idToACL              map[int]*models.ACL
+	aclIDToNpbPolices    map[int][]*models.NpbPolicy
+	aclIDToPcapPolices   map[int][]*models.PcapPolicy
+	idToNpbPolicy        map[int]*models.NpbPolicy
+	idToPcapPolicy       map[int]*models.PcapPolicy
 }
 
 func newPolicyRawData() *PolicyRawData {
 	return &PolicyRawData{
-		idToNpbTunnel:      make(map[int]*models.NpbTunnel),
-		idToACL:            make(map[int]*models.ACL),
-		aclIDToNpbPolices:  make(map[int][]*models.NpbPolicy),
-		aclIDToPcapPolices: make(map[int][]*models.PcapPolicy),
-		idToNpbPolicy:      make(map[int]*models.NpbPolicy),
-		idToPcapPolicy:     make(map[int]*models.PcapPolicy),
+		vtapGroupIDToVtapIDs: make(map[int][]int),
+		idToNpbTunnel:        make(map[int]*models.NpbTunnel),
+		idToACL:              make(map[int]*models.ACL),
+		aclIDToNpbPolices:    make(map[int][]*models.NpbPolicy),
+		aclIDToPcapPolices:   make(map[int][]*models.PcapPolicy),
+		idToNpbPolicy:        make(map[int]*models.NpbPolicy),
+		idToPcapPolicy:       make(map[int]*models.PcapPolicy),
 	}
 }
 
@@ -374,8 +376,23 @@ func (op *PolicyDataOP) generateRawData() {
 	acls := dbDataCache.GetACLs()
 	npbPolicies := dbDataCache.GetNpbPolicies()
 	pcapPolicies := dbDataCache.GetPcapPolicies()
+	vtaps := dbDataCache.GetVTapsIDAndName()
+	vtapGroups := dbDataCache.GetVTapGroupsIDAndLcuuid()
 
 	rawData := newPolicyRawData()
+	vtapGroupLcuuidToID := map[string]int{}
+	for _, vtapGroup := range vtapGroups {
+		vtapGroupLcuuidToID[vtapGroup.Lcuuid] = vtapGroup.ID
+	}
+	for _, vtap := range vtaps {
+		vtapGroupID, ok := vtapGroupLcuuidToID[vtap.VtapGroupLcuuid]
+		if !ok {
+			log.Warning(op.Logf("vtap(%s) group lcuuid(%s) not found group id", vtap.Name, vtap.VtapGroupLcuuid))
+			continue
+		}
+		rawData.vtapGroupIDToVtapIDs[vtapGroupID] = append(rawData.vtapGroupIDToVtapIDs[vtapGroupID], vtap.ID)
+	}
+
 	for _, npbTunnel := range npbTunnels {
 		rawData.idToNpbTunnel[npbTunnel.ID] = npbTunnel
 	}
@@ -623,9 +640,9 @@ func (op *PolicyDataOP) generateProtoActions(acl *models.ACL) (map[int][]*triden
 				NpbAclGroupId: proto.Uint32(uint32(npbPolicy.PolicyACLGroupID)),
 				Direction:     &direction,
 			}
-			if len(npbPolicy.VtapIDs) == 0 {
-				allVTapNpbActions = append(allVTapNpbActions, npbAction)
-			} else {
+
+			switch npbPolicy.VtapType {
+			case POLICY_VTAP_TYPE_VTAP:
 				for _, vtapIDStr := range strings.Split(npbPolicy.VtapIDs, ",") {
 					vtapIDInt, err := strconv.Atoi(vtapIDStr)
 					if err != nil {
@@ -634,6 +651,24 @@ func (op *PolicyDataOP) generateProtoActions(acl *models.ACL) (map[int][]*triden
 					}
 					vtapIDToNpbActions[vtapIDInt] = append(vtapIDToNpbActions[vtapIDInt], npbAction)
 				}
+			case POLICY_VTAP_TYPE_VTAP_GROUP:
+				for _, vtapGroupIDStr := range strings.Split(npbPolicy.VtapGroupIDs, ",") {
+					vtapGroupIDInt, err := strconv.Atoi(vtapGroupIDStr)
+					if err != nil {
+						log.Errorf(op.Logf("err: %s, vtapGroupIDs: %s", err, npbPolicy.VtapGroupIDs))
+						continue
+					}
+					vtapIDs, ok := rawData.vtapGroupIDToVtapIDs[vtapGroupIDInt]
+					if !ok {
+						log.Errorf(op.Logf("not found vtap in vtap group id(%d)", vtapGroupIDInt))
+						continue
+					}
+					for vtapID := range vtapIDs {
+						vtapIDToNpbActions[vtapID] = append(vtapIDToNpbActions[vtapID], npbAction)
+					}
+				}
+			default:
+				allVTapNpbActions = append(allVTapNpbActions, npbAction)
 			}
 		}
 	case APPLICATION_PCAP: