diff --git a/go.mod b/go.mod index 3b4237d1..2454da8a 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.12 require ( github.com/golang/protobuf v1.5.2 // indirect + github.com/oschwald/geoip2-golang v1.8.0 github.com/pebbe/zmq4 v1.2.9 github.com/r3labs/diff/v2 v2.15.1 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.8.0 golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 diff --git a/go.sum b/go.sum index e3f7c1df..5becaf44 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,17 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs= +github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw= +github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg= +github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0= github.com/pebbe/zmq4 v1.2.9 h1:JlHcdgq6zpppNR1tH0wXJq0XK03pRUc4lBlHTD7aj/4= github.com/pebbe/zmq4 v1.2.9/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -13,8 +19,13 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg= github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -24,6 +35,8 @@ golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME= +golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -42,3 +55,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/geoip/geoip.go b/services/geoip/geoip.go new file mode 100644 index 00000000..a3a0c391 --- /dev/null +++ b/services/geoip/geoip.go @@ -0,0 +1,345 @@ +package geoip + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "sync" + "time" + + "github.com/oschwald/geoip2-golang" + "github.com/untangle/golang-shared/services/logger" + "github.com/untangle/golang-shared/services/settings" + "github.com/untangle/golang-shared/services/uritranslations" + "github.com/untangle/golang-shared/util/cache/cacher" +) + +// The full path of the database filename. We look for it here and +// also download it to here if it doesn't exist or is out of date. +const DbFilename = "/usr/share/geoip/" + MaxMindDbFileName + +// the name of the database file we look for in tarballs we download. +const MaxMindDbFileName = "GeoLite2-Country.mmdb" + +// how long a downloaded database is valid for before we should +// download a new one. Thirty days. We figure out how old it is by +// looking at the file timestamp. +const validityOfDbDuration = time.Hour * 24 * 30 + +// The name of the cache used by geoip, only used for debugging/logging purposes +const cacheName = "geoIpCache" + +// Capacity of cache used by GeoIPManager for country code lookups. +const cacheCapacity = 500 + +// GeoIPDB is an interface that GeoIP databases conform to. +type GeoIPDB interface { + // LookupCountryCodeOfIP will look up the country code of a + // given IP address. If it is found, it returns the code and + // true. If it is not found, it returns "", and false. + LookupCountryCodeOfIP(ip net.IP) (string, bool) + + // Refresh() will 'refresh' the database -- downloading it + // from a remote source if necessary. + Refresh() error +} + +// GeoIPClassifier encapsulates all of the logic and responsibilities of +// the GEO IP plugin. It handles: +// 1. Downloading a geoIP database periodically. +// +// 2. Identifying the country code of a dispatch.Session object, when +// called from the main packetd dispatching code as a plugin. +// +// 3. Notifying all Listeners when it determines (2). See the +// Observable object. + +// MaxMindGeoIPManager is a GeoIPDB implementation of the geo IP +// database that uses a MaxMind database. +type MaxMindGeoIPManager struct { + databaseFilename string + + geoDatabaseReader *geoip2.Reader + databaseCache cacher.Cacher + cacheLocker sync.RWMutex +} + +// LockingGeoIPManager is a wrapper object for a GeoIPManager +// (specifically MaxMindGeoIPManager) that wraps all calls to +// LookupCountryCodeOfIP and Refresh with an RWLock. Refresh() will be +// called with a write lock and LookupCountryCodeOfIP has a read lock. +type LockingGeoIPManager struct { + lock sync.RWMutex + IpDB GeoIPDB +} + +// NewMaxMindGeoIPManager creates a new NewMaxMindGeoIPManager that +// 'points at' the database given by filename. filename need not +// exist, if it does not, you will need to downloadAndExtractDb(). +func NewMaxMindGeoIPManager(filename string) *MaxMindGeoIPManager { + return &MaxMindGeoIPManager{ + databaseCache: cacher.NewLruCache(cacheCapacity, cacheName), + databaseFilename: filename} +} + +// downloadAndExtractDB will download the MaxMind geoIP database from +// downloads.untangle.com and extract the database itself into the +// filesystem it by calling extractDBFile(). It returns an error if +// not successful. If successful, you will probably want to call +// openDBFile(). +func (db *MaxMindGeoIPManager) downloadAndExtractDB() error { + uid, err := settings.GetUIDOpenwrt() + if err != nil { + uid = "00000000-0000-0000-0000-000000000000" + logger.Warn("Unable to read UID: %s - Using all zeros\n", err.Error()) + } + target := fmt.Sprintf( + "https://downloads.untangle.com/download.php?resource=geoipCountry&uid=%s", + uid) + translatedTarget, err := uritranslations.GetURI(target) + if err == nil { + target = translatedTarget + } + resp, err := http.Get(target) + if err != nil { + return fmt.Errorf("HTTP GET failure: %w", err) + } + defer resp.Body.Close() + + // Check server response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP bad return code: %v, %v", + resp.StatusCode, + resp.Status) + } + return db.extractDBFile(resp.Body) + +} +func (db *MaxMindGeoIPManager) extractDBFile(reader io.ReadCloser) error { + defer reader.Close() + + logger.Info("Starting GeoIP database extraction: %s\n", db.databaseFilename) + + // Make sure the target directory exists + marker := strings.LastIndex(db.databaseFilename, "/") + + // Get the index of the last slash so we can isolate the path and create the directory + if marker > 0 { + os.MkdirAll(db.databaseFilename[0:marker], 0755) + } + + // Create a reader for the compressed data + zipReader, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("error calling gzip.NewReader(): %w", err) + } + defer zipReader.Close() + + // Create a tar reader using the uncompressed data stream + tarReader := tar.NewReader(zipReader) + + // Create the file where we'll store the extracted database + writer, err := os.Create(db.databaseFilename) + + if err != nil { + return fmt.Errorf( + "unable to write database file: %s", + db.databaseFilename) + } + + readError := fmt.Errorf( + "couldn't extract expected db file: %s from tar", + MaxMindDbFileName) + + for { + // get the next entry in the archive + header, err := tarReader.Next() + + // break out of the loop on end of file + if errors.Is(err, io.EOF) { + break + } else if err != nil { + // log any other errors and break out of the loop + readError = fmt.Errorf( + "error while reading database tar archive: %w", + err) + break + } + + // ignore everything that is not a regular file + if header.Typeflag != tar.TypeReg { + continue + } + + // ignore everything except the actual database file + if !strings.HasSuffix(header.Name, MaxMindDbFileName) { + continue + } + + // found the database so write to the output file, set the goodfile flag, and break + if _, err := io.Copy(writer, tarReader); err != nil { + readError = fmt.Errorf( + "Error writing found DB file to disk: %w", + err) + } else { + readError = nil + logger.Info("Finished GeoIP database download\n") + + } + break + } + writer.Close() + // If we had an error, delete the created file. + if readError != nil { + os.Remove(db.databaseFilename) + return readError + } + return nil +} + +// checkForDBFile checks if the MaxMind geoIP database file exists and +// is current. 'current' is defined by the value of +// validityOfDbDuration. +func (db *MaxMindGeoIPManager) checkForDBFile() bool { + filename := db.databaseFilename + if fileinfo, err := os.Stat(filename); err != nil { + return false + } else if fileinfo.Size() == 0 { + return false + } else { + + filetime := fileinfo.ModTime() + currtime := time.Now() + + // Return true if the time since the file modification time to + // now is less than the validity period (i.e. return true if + // the file is not stale yet). + return currtime.Sub(filetime) < validityOfDbDuration + } +} + +// openDBFile will open the database file, calling the underlying +// MaxMind implementation package. If the database is already open, it +// closes it and re-opens it. +func (db *MaxMindGeoIPManager) openDBFile() error { + if db.geoDatabaseReader != nil { + db.geoDatabaseReader.Close() + db.geoDatabaseReader = nil + } + + mmDB, err := geoip2.Open(db.databaseFilename) + + if err != nil { + logger.Warn("Unable to load GeoIP Database: %s\n", err) + db.geoDatabaseReader = nil + return fmt.Errorf("couldn't open GeoIP db: %w", err) + } + logger.Info("Loading GeoIP Database: %s\n", db.databaseFilename) + db.geoDatabaseReader = mmDB + return nil +} + +// LookupCountryCodeOfIP looks up the country code of ip. A cache of +// previously looked up countries is checked first. If not in the cache, +// the code is looked up in the database and added to the cache. If the +// country code is found in the cache/database, it +// returns the code and the value true. If not found, it returns "" +// and false. +func (db *MaxMindGeoIPManager) LookupCountryCodeOfIP(ip net.IP) (string, bool) { + db.cacheLocker.Lock() + defer db.cacheLocker.Unlock() + retCountryCode := "" + retOk := false + + if db.geoDatabaseReader != nil { + if countryFromCache, ok := db.databaseCache.Get(ip.String()); ok { + retCountryCode = countryFromCache.(*geoip2.Country).Country.IsoCode + retOk = true + + // Lookup country code in the database if a cache miss occurs. Update cache + // with the retrieved value + } else if countryFromDb, err := db.geoDatabaseReader.Country(ip); err == nil { + if len(countryFromDb.Country.IsoCode) != 0 { + db.databaseCache.Put(ip.String(), countryFromDb) + + retCountryCode = countryFromDb.Country.IsoCode + retOk = true + } + } + } else { + logger.Warn( + "LookupCountryCodeOfIP() called with nil MaxMind DB reader!\n") + retOk = false + } + + return retCountryCode, retOk +} + +// Refresh will: +// +// 1. Check to see if the database file exists and is current. If so, +// it opens it if it hasn't been opened, and exits. +// +// 2. If the database file is not current or does not exist as +// determined by checkForDBFile, it will download it and open it. +// +// 3. Clear the cache storing previously looked up country codes +// +// So unless something goes wrong (i.e. a database file doesn't exist +// on the filesystem and we can't download a new one), at the end of +// this call the database should always be opened in the +// MaxMindGeoIPManager object. +func (db *MaxMindGeoIPManager) Refresh() error { + if !db.checkForDBFile() { + err := db.downloadAndExtractDB() + if err != nil { + return err + } + err = db.openDBFile() + if err != nil { + return err + } + } else if db.geoDatabaseReader == nil { + if err := db.openDBFile(); err != nil { + return err + } + } + + db.cacheLocker.Lock() + db.databaseCache.Clear() + db.cacheLocker.Unlock() + + return nil +} + +// NewLockingGeoIPManager creates a new LockingGeoIPManager, which +// wraps the db object given. +func NewLockingGeoIPManager(db GeoIPDB) *LockingGeoIPManager { + return &LockingGeoIPManager{ + IpDB: db, + } +} + +// LookupCountryCodeOfIP calls the underlying ipDB of the +// LockingGeoIPManager's LookupCountryCodeOfIP after taking out an +// RLock(). It returns whatever the underlying database returns. +func (db *LockingGeoIPManager) LookupCountryCodeOfIP(ip net.IP) (string, bool) { + db.lock.RLock() + defer db.lock.RUnlock() + return db.IpDB.LookupCountryCodeOfIP(ip) +} + +// Refresh calls the underling ipDB's Refresh method after calling +// Lock() (i.e. taking out a write lock). It returns whatever the +// Refresh method of the underlying object returns. +func (db *LockingGeoIPManager) Refresh() error { + db.lock.Lock() + defer db.lock.Unlock() + return db.IpDB.Refresh() +} diff --git a/services/geoip/geoip_test.go b/services/geoip/geoip_test.go new file mode 100644 index 00000000..13c2fb65 --- /dev/null +++ b/services/geoip/geoip_test.go @@ -0,0 +1,270 @@ +package geoip + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/untangle/golang-shared/testing/data" +) + +// TestGeoIP is a Test suite for testing the geoip plugin. We use the +// testify suite package for this. +type TestGeoIP struct { + suite.Suite + + // mock geo ip database used in some tests. + mockDB *MockGeoIPDB + + // geoip plugin under test. + //GeoIPClassifier *GeoIPClassifier + + // list of files to delete after test cases. + deleteFiles []string +} + +// MockGeoIPDB is a mock geoIP database so we can test plugin logic +// with a mock database. +type MockGeoIPDB struct { + // ipToCountryMap is a map of string representations of IPs to + // country code strings. + ipToCountryMap map[string]string +} + +// get a filename to extract the database to. This filename will be in +// a temporary directory created for the test case. It will contain a +// non-existent directory inside that directory. +func (suite *TestGeoIP) getDBFilename() string { + tmpDir, err := ioutil.TempDir("", "GeoIPUnitTest") + suite.failIferror(err, "Can't open tmpDir") + fullFileName := path.Join(tmpDir, "extraComponent", MaxMindDbFileName) + suite.addDeleteFile(tmpDir) + return fullFileName +} + +// addDeleteFile adds file to a list of files that will be deleted (as +// if with rm -r) after the end of each test case. +func (suite *TestGeoIP) addDeleteFile(file string) { + suite.deleteFiles = append(suite.deleteFiles, file) +} + +// generic function for testing that we fail gracefully for various +// types of bad/invalid database files. +// +// filename -- filename to try to extract with extractDBFile +// +// description -- string description of the expected error so we can +// log it for visibility. +// +// shouldExtractFail -- if set to true we assert extractDBFile returns +// an error, else we don't. +func (suite *TestGeoIP) testFailure(filename string, description string, shouldExtractFail bool) { + extractedDbFileName := suite.getDBFilename() + readerCloser := suite.getReaderCloserForFile(filename) + geoIP := NewMaxMindGeoIPManager(extractedDbFileName) + result := geoIP.extractDBFile(readerCloser) + if shouldExtractFail { + suite.NotNil(result) + } else { + suite.Nil(result) + } + + result = geoIP.openDBFile() + suite.NotNil(result) + log.Printf("Caught error from %s (expected): %v", + description, + result) + countryResult, found := geoIP.LookupCountryCodeOfIP(net.ParseIP("2000:ff::1")) + suite.Equal(countryResult, "") + suite.False(found) +} + +// Check that Refresh() works with a pre-existing file. +func (suite *TestGeoIP) TestRefreshWithGoodDbFile() { + fullFileName := suite.getDBFilename() + geoIP := NewMaxMindGeoIPManager(fullFileName) + result, found := geoIP.LookupCountryCodeOfIP(net.ParseIP("3.3.3.3")) + suite.False(found) + suite.Equal(result, "") + suite.False(geoIP.checkForDBFile()) + suite.Nil(geoIP.extractDBFile(suite.getReaderCloserForTestTarball())) + suite.True(geoIP.checkForDBFile()) + suite.Nil(geoIP.geoDatabaseReader) + suite.Nil(geoIP.Refresh()) + suite.True(geoIP.checkForDBFile()) + suite.NotNil(geoIP.geoDatabaseReader) + result, found = geoIP.LookupCountryCodeOfIP(net.ParseIP("3.3.3.3")) + suite.True(found) + suite.NotEqual(result, "") +} +func (suite *TestGeoIP) TestDBExtract() { + fullFileName := suite.getDBFilename() + geoIP := NewMaxMindGeoIPManager(fullFileName) + + result := geoIP.extractDBFile(suite.getReaderCloserForTestTarball()) + suite.Nil(result) + if _, err := os.Stat(fullFileName); err != nil { + suite.Failf("File wasn't created", fullFileName) + } + + // Manually computed from terminal. + expectedFileSha256Hex := + "85f9fef478a5366daac20c71b5d6784b90bce61fafc90502ff974f083c09563c" + openedFile, err := os.Open(fullFileName) + suite.failIferror(err, "Can't open produced file") + bytes, err := ioutil.ReadAll(openedFile) + suite.failIferror(err, "Couldn't read produced DB file") + + hash := sha256.Sum256(bytes) + suite.Equal( + expectedFileSha256Hex, + hex.EncodeToString(hash[:])) +} + +// Check that the checkForDBFile method works: +// 1. It returns false if the file doesn't exist. +// +// 2. It returns false if the file is old. +// +// 3. It returns true if the file exists and is not too old (as +// determined by the value of validityOfDbDuration). +func (suite *TestGeoIP) TestDBStatusChecker() { + fullFileName := suite.getDBFilename() + geoIP := NewMaxMindGeoIPManager(fullFileName) + suite.False(geoIP.checkForDBFile()) + extractResult := geoIP.extractDBFile( + suite.getReaderCloserForTestTarball()) + suite.Nil(extractResult) + suite.True(geoIP.checkForDBFile()) + + // Here we use os.Chtimes to backdate the file's timestamps + // and make it appear old. It is one second older than + // validityOfDbDuration, so we expect checkForDBFile to return + // false. + now := time.Now() + invalidTime := now.Add(-(validityOfDbDuration + time.Second)) + err := os.Chtimes( + fullFileName, + invalidTime, + invalidTime) + suite.failIferror(err, "Couldn't backdate file timestamp") + suite.False(geoIP.checkForDBFile()) +} + +// Test that we call the MaxMind database reader correctly and do not +// get errors. Currently we do not make assertions about returned +// country codes, since we do not 'own' the databas, but merely query +// it. This is to make sure that we don't get unexpected errors, and +// that the basic sequence of functions for extracting and opening the +// file work as expected. +func (suite *TestGeoIP) TestDBReader() { + fullFileName := suite.getDBFilename() + geoIP := NewMaxMindGeoIPManager(fullFileName) + extractResult := geoIP.extractDBFile( + suite.getReaderCloserForTestTarball()) + suite.Nil(extractResult) + suite.failIferror(geoIP.openDBFile(), "Couldn't open DB file") + cc, didSucceed := geoIP.LookupCountryCodeOfIP( + net.IPv4(3, 3, 3, 3)) + suite.True(didSucceed) + + // In this case, don't make assertions on a database we don't + // own. We just want to make sure that we can look things up + // with no errors. + fmt.Printf("Country Code of 3.3.3.3: %v\n", cc) + + // Look up a google IP. + googleIP := "2001:4860:4860::8888" + cc, didSucceed = geoIP.LookupCountryCodeOfIP( + net.ParseIP(googleIP)) + suite.True(didSucceed) + fmt.Printf("Country Code of %s: %v\n", googleIP, cc) + + // Look up a google IP. + googleIP = "2001:4860:4860::8888" + cc, didSucceed = geoIP.LookupCountryCodeOfIP( + net.ParseIP(googleIP)) + suite.True(didSucceed) + fmt.Printf("Country Code of %s: %v\n", googleIP, cc) +} + +// Test that the downloadAndExtractDB() method works -- this will do a +// 'real' download of the MaxMind geoIP country database from +// downloads.untangle.com with an all-zero UID. +func (suite *TestGeoIP) TestDownload() { + fullFileName := suite.getDBFilename() + geoIP := NewMaxMindGeoIPManager(fullFileName) + suite.Nil(geoIP.downloadAndExtractDB()) + suite.failIferror(geoIP.openDBFile(), + "Couldn't open DB file after download.") + // Test that we can re-open, which involves closing. + suite.failIferror(geoIP.openDBFile(), + "Couldn't open DB file after download (second open).") +} + +// Test that if we have the geoIP manager object in a bad state, it +// doesn't panic or throw errors but just acts as if it can't find the +// IP. +func (suite *TestGeoIP) TestNilLookup() { + fullFileName := "/garbage/garbage2/" + geoIP := NewMaxMindGeoIPManager(fullFileName) + result, found := geoIP.LookupCountryCodeOfIP(net.ParseIP("111.111.111.111")) + suite.Equal(result, "") + suite.False(found) +} + +// Test that we fail gracefully if the tarball is completely +// bad/wrong. +func (suite *TestGeoIP) TestBadTar() { + suite.testFailure( + "FakeGEOIP.tar.gz", "bad tar file", true) +} + +// Test that we fail gracefully if the file isn't in the tarball at +// all. +func (suite *TestGeoIP) TestMissingFile() { + suite.testFailure( + "GeoIP2Missing.tar.gz", + "tar had missing database file", + true) +} + +// Test that we fail gracefully if we have a database file that exists +// in the tar but it is bad. +func (suite *TestGeoIP) TestBadDBFile() { + suite.testFailure("GeoIP2BadDB.tar.gz", "tar bad database file", false) +} + +// Fail the test if err != nil with msg as a message. +func (suite *TestGeoIP) failIferror(err error, msg string) { + if err != nil { + suite.Fail(msg, err) + } +} + +// Get a io.ReadCloser object that wraps a valid test tarball with a +// real database. +func (suite *TestGeoIP) getReaderCloserForTestTarball() io.ReadCloser { + return suite.getReaderCloserForFile("GeoIP2.tar.gz") +} + +// Get a io.ReadCloser for a given named file in test data. +func (suite *TestGeoIP) getReaderCloserForFile(fname string) io.ReadCloser { + goodTarFile := data.GetTestFileLocation(fname) + file, err := os.Open(goodTarFile) + suite.failIferror(err, "Can't open GEOIP tar file.") + return ioutil.NopCloser(file) + +} +func TestGeoIPSuite(t *testing.T) { + suite.Run(t, &TestGeoIP{}) +} diff --git a/testing/data/data.go b/testing/data/data.go new file mode 100644 index 00000000..83148d53 --- /dev/null +++ b/testing/data/data.go @@ -0,0 +1,17 @@ +package data + +import ( + "path" + "runtime" +) + +// GetCommonTestDataDirectory returns data directory for tests. +func GetCommonTestDataDirectory() string { + _, ourPath, _, _ := runtime.Caller(0) + return path.Join(path.Dir(ourPath), "testdata") +} + +// GetTestFileLocation returns the full path of a specific test file. +func GetTestFileLocation(testFile string) string { + return path.Join(GetCommonTestDataDirectory(), testFile) +} diff --git a/testing/data/testdata/FakeGEOIP.tar.gz b/testing/data/testdata/FakeGEOIP.tar.gz new file mode 100644 index 00000000..c2c5c6f6 --- /dev/null +++ b/testing/data/testdata/FakeGEOIP.tar.gz @@ -0,0 +1 @@ +FAKE diff --git a/testing/data/testdata/GeoIP2.tar.gz b/testing/data/testdata/GeoIP2.tar.gz new file mode 100644 index 00000000..a6295f78 Binary files /dev/null and b/testing/data/testdata/GeoIP2.tar.gz differ diff --git a/testing/data/testdata/GeoIP2BadDB.tar.gz b/testing/data/testdata/GeoIP2BadDB.tar.gz new file mode 100644 index 00000000..7577983b Binary files /dev/null and b/testing/data/testdata/GeoIP2BadDB.tar.gz differ diff --git a/testing/data/testdata/GeoIP2Missing.tar.gz b/testing/data/testdata/GeoIP2Missing.tar.gz new file mode 100644 index 00000000..5df416cd Binary files /dev/null and b/testing/data/testdata/GeoIP2Missing.tar.gz differ