Merge pull request #68 from untangle/MFW-2299-clean-device

MFW-2299: Prune the in-memory device data in reportd

services/discovery/devices_list_test.go

@@ -23,6 +23,7 @@ type DeviceListTestSuite struct {
 	mac2         string
 	mac3         string
 	mac4         string
+	mac5         string
 	devicesTable map[string]*DeviceEntry
 	now          time.Time
 	oneHourAgo   time.Time
@@ -42,6 +43,7 @@ func (suite *DeviceListTestSuite) SetupTest() {
 	suite.mac2 = "00:11:22:33:44:66"
 	suite.mac3 = "00:11:33:44:55:66"
 	suite.mac4 = "00:aa:bb:cc:dd:ee"
+	suite.mac5 = "00:aa:bb:cc:dd:ef"
 	suite.devicesTable = map[string]*DeviceEntry{
 		suite.mac1: {DiscoveryEntry: disco.DiscoveryEntry{
 			MacAddress:  suite.mac1,
@@ -62,14 +64,19 @@ func (suite *DeviceListTestSuite) SetupTest() {
 			MacAddress: suite.mac4,
 			LastUpdate: suite.aDayago.Unix(),
 		}},
+		suite.mac5: {DiscoveryEntry: disco.DiscoveryEntry{
+			MacAddress:  suite.mac5,
+			LastUpdate:  suite.aDayago.Unix(),
+			IPv4Address: "192.168.56.4",
+		}},
 	}
 	suite.deviceList = NewDevicesList()
 	for _, entry := range suite.devicesTable {
 		suite.deviceList.PutDevice(entry)
 	}
 }
 
-//TestListing tests that applying various predicates to the list
+// TestListing tests that applying various predicates to the list
 // works.
 func (suite *DeviceListTestSuite) TestListing() {
 
@@ -89,7 +96,7 @@ func (suite *DeviceListTestSuite) TestListing() {
 	tests := []testSpec{
 		{
 			predicates:       []ListPredicate{},
-			expectedListMacs: []string{suite.mac1, suite.mac2, suite.mac3, suite.mac4},
+			expectedListMacs: []string{suite.mac1, suite.mac2, suite.mac3, suite.mac4, suite.mac5},
 			description:      "Empty predicate list should return all MACs.",
 		},
 		{
@@ -104,6 +111,13 @@ func (suite *DeviceListTestSuite) TestListing() {
 			expectedListMacs: []string{},
 			description:      "There are no devices in the list that have an update time within 20 minutes and are not local.",
 		},
+		{
+			predicates: []ListPredicate{
+				LastUpdateOlderThanDuration(time.Hour * 24),
+			},
+			expectedListMacs: []string{suite.mac4, suite.mac5},
+			description:      "List of macs with an update time older than 24 hours should be mac4",
+		},
 	}
 	// duplicate the tests for concurrent testing.
 	testsDuplicated := append(tests, tests...)
@@ -142,8 +156,10 @@ func (suite *DeviceListTestSuite) TestListing() {
 			}
 			wg.Done()
 		}()
+
 	}
 	wg.Wait()
+
 }
 
 // TestMerge tests the merge and add funcionality. We test that
@@ -252,6 +268,43 @@ func (suite *DeviceListTestSuite) TestBroadcastInsertion() {
 	suite.Equal(suite.devicesTable, deviceList.Devices, "Adding broadcast discovery entry.")
 }
 
+// Test clean device entry
+func (suite *DeviceListTestSuite) TestCleanDeviceEntry() {
+
+	var deviceList DevicesList
+	var device_count, device_ip_count uint32
+	deviceList.Devices = map[string]*DeviceEntry{}
+	deviceList.devicesByIP = map[string]*DeviceEntry{}
+
+	for _, entry := range suite.devicesTable {
+		deviceList.PutDevice(entry)
+
+		device_count++
+
+	}
+	//device_ip_count = device_count - 1, because one device entry didn't have IPV4address.
+	//So it should not be added to devicesByIP map
+	device_ip_count = device_count - 1
+
+	//Clean entries which are 48 hours older
+	predicates1 := []ListPredicate{
+		LastUpdateOlderThanDuration(time.Hour * 48),
+	}
+	deviceList.CleanOldDeviceEntry(predicates1...)
+	//No entries should be deleted from the list, because no entries with LastUpdate older than 48 hours
+	suite.EqualValues(device_count, len(deviceList.Devices), "Cleaned 48 hours older entry")
+	suite.EqualValues(device_ip_count, len(deviceList.devicesByIP), "Cleaned 48 hours older entry")
+
+	//Clean entries which are 24 hours older
+	predicates2 := []ListPredicate{
+		LastUpdateOlderThanDuration(time.Hour * 24),
+	}
+	deviceList.CleanOldDeviceEntry(predicates2...)
+	//One device entry should be deleted from the device list which entry has LastUpdate with >24 hours
+	suite.EqualValues(device_count-2, len(deviceList.Devices), "Cleaned 24 hours older entry")
+	suite.EqualValues(device_ip_count-1, len(deviceList.devicesByIP), "Cleaned 24 hours older entry")
+}
+
 // TestMarshallingList tests that we can marshal a list of devices
 // obtained via the ApplyToDeviceList function to JSON without getting
 // an exception.

services/discovery/types.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/untangle/golang-shared/services/logger"
 	"github.com/untangle/golang-shared/structs/protocolbuffers/ActiveSessions"
 	disco "github.com/untangle/golang-shared/structs/protocolbuffers/Discoverd"
 	"google.golang.org/protobuf/proto"
@@ -66,6 +67,35 @@ func (list *DevicesList) PutDevice(entry *DeviceEntry) {
 	list.putDeviceUnsafe(entry)
 }
 
+// Get 24hours older device discovery entry from device list and delete the entry from device list
+func (list *DevicesList) CleanOldDeviceEntry(preds ...ListPredicate) {
+	list.Lock.Lock()
+	defer list.Lock.Unlock()
+	listOfDevs := list.listDevices(preds...)
+	list.CleanDevices(listOfDevs)
+}
+
+// Clean device discovery entry from devices list if the entry lastUpdate is 24 hours older
+func (list *DevicesList) CleanDevices(devices []*DeviceEntry) {
+
+	for _, device := range devices {
+		delete(list.Devices, device.MacAddress)
+		if device.IPv4Address != "" {
+			delete(list.devicesByIP, device.IPv4Address)
+		}
+		logger.Debug("Deleted entry %s:%s\n", device.MacAddress, device.IPv4Address)
+	}
+}
+
+// Get the device discovery entries lastUpdate time is older than the duration(24 hours)
+func LastUpdateOlderThanDuration(period time.Duration) ListPredicate {
+	return func(entry *DeviceEntry) bool {
+		lastUpdated := time.Unix(entry.LastUpdate, 0)
+		now := time.Now()
+		return (now.Sub(lastUpdated) >= period)
+	}
+}
+
 // listDevices returns a list of devices matching all predicates. It
 // doesn't do anything with locks so without the outer function
 // locking appropriately is unsafe.