From bba4df6be8686ad60e9d55fbe86fe10406f10634 Mon Sep 17 00:00:00 2001 From: Greg Linton Date: Tue, 16 Aug 2016 12:37:33 -0600 Subject: [PATCH 1/2] Add postgresql as backend --- README.md | 1 + api/README.md | 6 +- api/records.go | 2 +- build.sh | 3 +- cache/cache.go | 16 ++-- cache/cache_test.go | 1 + cache/postgres.go | 204 +++++++++++++++++++++++++++++++++++++++++ cache/postgres_test.go | 93 +++++++++++++++++++ cache/scribble.go | 12 +-- config/config.go | 2 +- core/common/common.go | 2 +- core/shaman.go | 6 +- server/dns.go | 2 +- 13 files changed, 327 insertions(+), 23 deletions(-) create mode 100644 cache/postgres.go create mode 100644 cache/postgres_test.go diff --git a/README.md b/README.md index 24b44b0..db9c7a1 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,7 @@ Fields: ## Todo - atomic local cache updates - export in hosts file format +- improve scribble add (adding before stored in cache overwrites) ## Changelog diff --git a/api/README.md b/api/README.md index e2ad045..998202f 100644 --- a/api/README.md +++ b/api/README.md @@ -12,9 +12,9 @@ Small, lightweight, api-driven dns server. | **POST** /records | Adds the domain and full record | json domain object | json domain object | | **PUT** /records | Update all domains and records (replaces all) | json array of domain objects | json array of domain objects | | **GET** /records | Returns a list of domains we have records for | nil | string array of domains | -| **PUT** /records/{id} | Update domain's records (replaces all) | json domain object | json domain object | -| **GET** /records/{id} | Returns the records for that domain | nil | json domain object | -| **DELETE** /records/{id} | Delete a domain | nil | success message | +| **PUT** /records/{domain} | Update domain's records (replaces all) | json domain object | json domain object | +| **GET** /records/{domain} | Returns the records for that domain | nil | json domain object | +| **DELETE** /records/{domain} | Delete a domain | nil | success message | ## Usage Example: diff --git a/api/records.go b/api/records.go index e4ffcce..8c7dd47 100644 --- a/api/records.go +++ b/api/records.go @@ -69,7 +69,7 @@ func updateRecord(rw http.ResponseWriter, req *http.Request) { return } - // "MUST reply 201"(https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html) + // "MUST reply 201" (https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html) writeBody(rw, req, resource, http.StatusCreated) return } diff --git a/build.sh b/build.sh index 7b6969a..f15e363 100755 --- a/build.sh +++ b/build.sh @@ -26,7 +26,8 @@ MD5=$(which md5 || which md5sum) # build shaman echo "Building SHAMAN and uploading it to 's3://tools.nanopack.io/shaman'" -gox -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" -osarch "linux/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman" +gox -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" \ + -osarch "linux/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman" # look through each os/arch/file and generate an md5 for each echo "Generating md5s..." diff --git a/cache/cache.go b/cache/cache.go index 4213261..6fe4e34 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -18,11 +18,11 @@ var ( // The cacher interface is what all the backends [will] implement type cacher interface { initialize() error - addRecord(resource *shaman.Resource) error + addRecord(resource shaman.Resource) error getRecord(domain string) (*shaman.Resource, error) - updateRecord(domain string, resource *shaman.Resource) error + updateRecord(domain string, resource shaman.Resource) error deleteRecord(domain string) error - resetRecords(resources *[]shaman.Resource) error + resetRecords(resources []shaman.Resource) error listRecords() ([]shaman.Resource, error) } @@ -36,6 +36,10 @@ func Initialize() error { switch u.Scheme { case "scribble": storage = &scribbleDb{} + case "postgres": + storage = &postgresDb{} + case "postgresql": + storage = &postgresDb{} case "none": storage = nil default: @@ -60,7 +64,7 @@ func AddRecord(resource *shaman.Resource) error { return nil } resource.Validate() - return storage.addRecord(resource) + return storage.addRecord(*resource) } // GetRecord gets a record to the persistent cache @@ -80,7 +84,7 @@ func UpdateRecord(domain string, resource *shaman.Resource) error { } shaman.SanitizeDomain(&domain) resource.Validate() - return storage.updateRecord(domain, resource) + return storage.updateRecord(domain, *resource) } // DeleteRecord removes a record from the persistent cache @@ -101,7 +105,7 @@ func ResetRecords(resources *[]shaman.Resource) error { (*resources)[i].Validate() } - return storage.resetRecords(resources) + return storage.resetRecords(*resources) } // ListRecords lists all records in the persistent cache diff --git a/cache/cache_test.go b/cache/cache_test.go index 3b27766..79aa14d 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -19,6 +19,7 @@ var ( func TestMain(m *testing.M) { // manually configure + // config.Log = lumber.NewConsoleLogger(lumber.LvlInt("trace")) config.Log = lumber.NewConsoleLogger(lumber.LvlInt("FATAL")) // run tests diff --git a/cache/postgres.go b/cache/postgres.go new file mode 100644 index 0000000..b738237 --- /dev/null +++ b/cache/postgres.go @@ -0,0 +1,204 @@ +package cache + +import ( + "database/sql" + "fmt" + + _ "github.com/lib/pq" + + "github.com/nanopack/shaman/config" + shaman "github.com/nanopack/shaman/core/common" +) + +type postgresDb struct { + pg *sql.DB +} + +func (p *postgresDb) connect() error { + // todo: example: config.DatabaseConnection = "postgres://postgres@127.0.0.1?sslmode=disable" + db, err := sql.Open("postgres", config.L2Connect) + if err != nil { + return fmt.Errorf("Failed to connect to postgres - %v", err) + } + err = db.Ping() + if err != nil { + return fmt.Errorf("Failed to ping postgres on connect - %v", err) + } + + p.pg = db + return nil +} + +func (p postgresDb) createTables() error { + // create records table + _, err := p.pg.Exec(` +CREATE TABLE IF NOT EXISTS records ( + recordId SERIAL PRIMARY KEY NOT NULL, + domain TEXT NOT NULL, + address TEXT NOT NULL, + ttl INTEGER, + class TEXT, + type TEXT +)`) + if err != nil { + return fmt.Errorf("Failed to create records table - %v", err) + } + + return nil +} + +func (p *postgresDb) initialize() error { + err := p.connect() + if err != nil { + return fmt.Errorf("Failed to create new connection - %v", err) + } + + // create tables + err = p.createTables() + if err != nil { + return fmt.Errorf("Failed to create tables - %v", err) + } + + return nil +} + +func (p postgresDb) addRecord(resource shaman.Resource) error { + resources, err := p.listRecords() + if err != nil { + return err + } + + for i := range resources { + if resources[i].Domain == resource.Domain { + // if domains match, check address + for k := range resources[i].Records { + next: + for j := range resource.Records { + // check if the record exists... + if resource.Records[j].RType == resources[i].Records[k].RType && + resource.Records[j].Address == resources[i].Records[k].Address && + resource.Records[j].Class == resources[i].Records[k].Class { + // if so, skip + config.Log.Trace("Record exists in persistent, skipping...") + resource.Records = append(resource.Records[:i], resource.Records[i+1:]...) + goto next + } + } + } + } + } + + // add records + for i := range resource.Records { + config.Log.Trace("Adding record to database...") + _, err = p.pg.Exec(fmt.Sprintf(` +INSERT INTO records(domain, address, ttl, class, type) +VALUES('%v', '%v', '%v', '%v', '%v')`, + resource.Domain, resource.Records[i].Address, resource.Records[i].TTL, + resource.Records[i].Class, resource.Records[i].RType)) + if err != nil { + return fmt.Errorf("Failed to insert into records table - %v", err) + } + } + + return nil +} + +func (p postgresDb) getRecord(domain string) (*shaman.Resource, error) { + // read from records table + rows, err := p.pg.Query(fmt.Sprintf("SELECT address, ttl, class, type FROM records WHERE domain = '%v'", domain)) + if err != nil { + return nil, fmt.Errorf("Failed to select from records table - %v", err) + } + defer rows.Close() + + records := make([]shaman.Record, 0, 0) + + // get data + for rows.Next() { + rcrd := shaman.Record{} + err = rows.Scan(&rcrd.Address, &rcrd.TTL, &rcrd.Class, &rcrd.RType) + if err != nil { + return nil, fmt.Errorf("Failed to save results into record - %v", err) + } + + records = append(records, rcrd) + } + + // check for errors + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("Error with results - %v", err) + } + + if len(records) == 0 { + return nil, errNoRecordError + } + + return &shaman.Resource{Domain: domain, Records: records}, nil +} + +func (p postgresDb) updateRecord(domain string, resource shaman.Resource) error { + // delete old from records + err := p.deleteRecord(domain) + if err != nil { + return fmt.Errorf("Failed to clean old records - %v", err) + } + + return p.addRecord(resource) +} + +func (p postgresDb) deleteRecord(domain string) error { + _, err := p.pg.Exec(fmt.Sprintf(`DELETE FROM records WHERE domain = '%v'`, domain)) + if err != nil { + return fmt.Errorf("Failed to delete from records table - %v", err) + } + + return nil +} + +func (p postgresDb) resetRecords(resources []shaman.Resource) error { + // truncate records table + _, err := p.pg.Exec("TRUNCATE records") + if err != nil { + return fmt.Errorf("Failed to truncate records table - %v", err) + } + for i := range resources { + err = p.addRecord(resources[i]) // prevents duplicates + if err != nil { + return fmt.Errorf("Failed to save records - %v", err) + } + } + return nil +} + +func (p postgresDb) listRecords() ([]shaman.Resource, error) { + // read from records table + rows, err := p.pg.Query("SELECT DISTINCT domain FROM records") + if err != nil { + return nil, fmt.Errorf("Failed to select from records table - %v", err) + } + defer rows.Close() + + resources := make([]shaman.Resource, 0) + + // get data + for rows.Next() { + var domain string + err = rows.Scan(&domain) + if err != nil { + return nil, fmt.Errorf("Failed to save domain - %v", err) + } + resource, err := p.getRecord(domain) + if err != nil { + return nil, fmt.Errorf("Failed to get record for domain - %v", err) + } + + resources = append(resources, *resource) + } + + // check for errors + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("Error with results - %v", err) + } + return resources, nil +} diff --git a/cache/postgres_test.go b/cache/postgres_test.go new file mode 100644 index 0000000..ab28d7d --- /dev/null +++ b/cache/postgres_test.go @@ -0,0 +1,93 @@ +package cache_test + +import ( + "testing" + + "github.com/nanopack/shaman/cache" + "github.com/nanopack/shaman/config" + shaman "github.com/nanopack/shaman/core/common" +) + +// test postgres cache init +func TestPostgresInitialize(t *testing.T) { + config.L2Connect = "postgres://postgres@127.0.0.1?sslmode=disable" // default + err := cache.Initialize() + config.L2Connect = "postgresql://postgres@127.0.0.1:9999?sslmode=disable" // unable to init? + err2 := cache.Initialize() + if err != nil || err2 != nil { + t.Errorf("Failed to initalize postgres cacher - %v%v", err, err2) + } +} + +// test postgres cache addRecord +func TestPostgresAddRecord(t *testing.T) { + postgresReset() + err := cache.AddRecord(&nanopack) + if err != nil { + t.Errorf("Failed to add record to postgres cacher - %v", err) + } + + err = cache.AddRecord(&nanopack) + if err != nil { + t.Errorf("Failed to add record to postgres cacher - %v", err) + } +} + +// test postgres cache getRecord +func TestPostgresGetRecord(t *testing.T) { + postgresReset() + cache.AddRecord(&nanopack) + _, err := cache.GetRecord("nanobox.io.") + _, err2 := cache.GetRecord("nanopack.io") + if err == nil || err2 != nil { + t.Errorf("Failed to get record from postgres cacher - %v%v", err, err2) + } +} + +// test postgres cache updateRecord +func TestPostgresUpdateRecord(t *testing.T) { + postgresReset() + err := cache.UpdateRecord("nanobox.io", &nanopack) + err2 := cache.UpdateRecord("nanopack.io", &nanopack) + if err != nil || err2 != nil { + t.Errorf("Failed to update record in postgres cacher - %v%v", err, err2) + } +} + +// test postgres cache deleteRecord +func TestPostgresDeleteRecord(t *testing.T) { + postgresReset() + err := cache.DeleteRecord("nanobox.io") + cache.AddRecord(&nanopack) + err2 := cache.DeleteRecord("nanopack.io") + if err != nil || err2 != nil { + t.Errorf("Failed to delete record from postgres cacher - %v%v", err, err2) + } +} + +// test postgres cache resetRecords +func TestPostgresResetRecords(t *testing.T) { + postgresReset() + err := cache.ResetRecords(&nanoBoth) + if err != nil { + t.Errorf("Failed to reset records in postgres cacher - %v", err) + } +} + +// test postgres cache listRecords +func TestPostgresListRecords(t *testing.T) { + postgresReset() + _, err := cache.ListRecords() + cache.ResetRecords(&nanoBoth) + _, err2 := cache.ListRecords() + if err != nil || err2 != nil { + t.Errorf("Failed to list records in postgres cacher - %v%v", err, err2) + } +} + +func postgresReset() { + config.L2Connect = "postgres://postgres@127.0.0.1?sslmode=disable" + cache.Initialize() + blank := make([]shaman.Resource, 0, 0) + cache.ResetRecords(&blank) +} diff --git a/cache/scribble.go b/cache/scribble.go index 8799b07..36cf25e 100644 --- a/cache/scribble.go +++ b/cache/scribble.go @@ -36,8 +36,8 @@ func (self *scribbleDb) initialize() error { return nil } -func (self scribbleDb) addRecord(resource *shaman.Resource) error { - err := self.db.Write("hosts", resource.Domain, *resource) +func (self scribbleDb) addRecord(resource shaman.Resource) error { + err := self.db.Write("hosts", resource.Domain, resource) if err != nil { err = fmt.Errorf("Failed to save record - %v", err) } @@ -56,7 +56,7 @@ func (self scribbleDb) getRecord(domain string) (*shaman.Resource, error) { return &resource, nil } -func (self scribbleDb) updateRecord(domain string, resource *shaman.Resource) error { +func (self scribbleDb) updateRecord(domain string, resource shaman.Resource) error { if domain != resource.Domain { err := self.deleteRecord(domain) if err != nil { @@ -79,10 +79,10 @@ func (self scribbleDb) deleteRecord(domain string) error { return err } -func (self scribbleDb) resetRecords(resources *[]shaman.Resource) (err error) { +func (self scribbleDb) resetRecords(resources []shaman.Resource) (err error) { self.db.Delete("hosts", "") - for i := range *resources { - err = self.db.Write("hosts", (*resources)[i].Domain, (*resources)[i]) + for i := range resources { + err = self.db.Write("hosts", resources[i].Domain, resources[i]) if err != nil { err = fmt.Errorf("Failed to save records - %v", err) } diff --git a/config/config.go b/config/config.go index 8499b28..c712ff3 100644 --- a/config/config.go +++ b/config/config.go @@ -1,4 +1,4 @@ -// Package "config" is a central location for configuration options. It also contains +// Package config is a central location for configuration options. It also contains // config file parsing logic. package config diff --git a/core/common/common.go b/core/common/common.go index 1882f96..fbfa27d 100644 --- a/core/common/common.go +++ b/core/common/common.go @@ -1,4 +1,4 @@ -// Package "common" contains common structs used in shaman +// Package common contains common structs used in shaman package common import ( diff --git a/core/shaman.go b/core/shaman.go index b8f5b37..f74ae2d 100644 --- a/core/shaman.go +++ b/core/shaman.go @@ -1,4 +1,4 @@ -// Package "shaman" contains the logic to add/remove DNS entries. +// Package shaman contains the logic to add/remove DNS entries. package shaman // todo: atomic C.U.D. @@ -104,12 +104,12 @@ func AddRecord(resource *sham.Resource) error { resource.Records[j].Address == Answers[domain].Records[k].Address && resource.Records[j].Class == Answers[domain].Records[k].Class { // if so, skip... - config.Log.Trace("Record exists in local cache, skipping") + config.Log.Trace("Record exists in local cache, skipping...") goto next } } // otherwise, add the record - config.Log.Trace("Record not in local cache, adding") + config.Log.Trace("Record not in local cache, adding...") resource.Records = append(resource.Records, Answers[domain].Records[k]) next: } diff --git a/server/dns.go b/server/dns.go index 83ce2e2..ed3dd73 100644 --- a/server/dns.go +++ b/server/dns.go @@ -1,4 +1,4 @@ -// Package "server" contains logic to handle DNS requests. +// Package server contains logic to handle DNS requests. package server import ( From 7f770f4e092564b58236e143777e625127d59cf3 Mon Sep 17 00:00:00 2001 From: Greg Linton Date: Tue, 16 Aug 2016 16:29:47 -0600 Subject: [PATCH 2/2] Add scripts and clean up versioning --- README.md | 2 ++ build.sh | 52 ----------------------------------------------- main.go | 7 ++++++- scripts/build.sh | 35 +++++++++++++++++++++++++++++++ scripts/upload.sh | 6 ++++++ version.go | 9 -------- 6 files changed, 49 insertions(+), 62 deletions(-) delete mode 100755 build.sh create mode 100755 scripts/build.sh create mode 100755 scripts/upload.sh delete mode 100644 version.go diff --git a/README.md b/README.md index db9c7a1..71b9b47 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,8 @@ Fields: - v0.0.3 (May 12, 2016) - Tests for DNS server - Start Server Insecure +- v0.0.4 (Aug 16, 2016) + - Postgresql as a backend ## Contributing diff --git a/build.sh b/build.sh deleted file mode 100755 index f15e363..0000000 --- a/build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -e - -# for versioning -getCurrCommit() { - echo `git rev-parse HEAD | tr -d "[ \r\n\']"` -} - -# for versioning -getCurrTag() { - echo `git describe --always --tags --abbrev=0 | tr -d "[v\r\n]"` -} - -# for versioning -getCurrBranch() { - echo `git rev-parse --abbrev-ref HEAD | tr -d "[\r\n ]"` -} - -# for versioning -commit=$(getCurrCommit) -branch=$(getCurrBranch) -tag=$(getCurrTag) - -# try and use the correct MD5 lib (depending on user OS darwin/linux) -MD5=$(which md5 || which md5sum) - -# build shaman -echo "Building SHAMAN and uploading it to 's3://tools.nanopack.io/shaman'" -gox -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" \ - -osarch "linux/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman" - -# look through each os/arch/file and generate an md5 for each -echo "Generating md5s..." -for os in $(ls ./build); do - for arch in $(ls ./build/${os}); do - for file in $(ls ./build/${os}/${arch}); do - cat "./build/${os}/${arch}/${file}" | ${MD5} | awk '{print $1}' >> "./build/${os}/${arch}/${file}.md5" - done - done -done - -# upload to AWS S3 -echo "Uploading builds to S3..." -aws s3 sync ./build/ s3://tools.nanopack.io/shaman --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --region us-east-1 - -# -echo "Cleaning up..." - -# remove build -[ -e "./build" ] && \ - echo "Removing build files..." && \ - rm -rf "./build" diff --git a/main.go b/main.go index 56a80a5..73652ef 100644 --- a/main.go +++ b/main.go @@ -62,6 +62,11 @@ var ( SilenceErrors: true, SilenceUsage: true, } + + // shaman version information (populated by go linker) + // -ldflags="-X main.version=${tag} -X main.commit=${commit}" + version string + commit string ) // add supported cli commands/flags @@ -90,7 +95,7 @@ func readConfig(ccmd *cobra.Command, args []string) error { func preFlight(ccmd *cobra.Command, args []string) error { if config.Version { - fmt.Printf("shaman %s (git: %s %s)\n", version, branch, commit) + fmt.Printf("shaman %s (%s)\n", version, commit) return fmt.Errorf("") } diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..443e1ae --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +# try and use the correct MD5 lib (depending on user OS darwin/linux) +MD5=$(which md5 || which md5sum) + +# for versioning +getCurrCommit() { + echo `git rev-parse --short HEAD| tr -d "[ \r\n\']"` +} + +# for versioning +getCurrTag() { + echo `git describe --always --tags --abbrev=0 | tr -d "[v\r\n]"` +} + +# remove any previous builds that may have failed +[ -e "./build" ] && \ + echo "Cleaning up old builds..." && \ + rm -rf "./build" + +# build shaman +echo "Building shaman..." +gox -ldflags="-X main.version=$(getCurrTag) -X main.commit=$(getCurrCommit)" \ + -osarch "darwin/amd64 linux/amd64 windows/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman" + +# look through each os/arch/file and generate an md5 for each +echo "Generating md5s..." +for os in $(ls ./build); do + for arch in $(ls ./build/${os}); do + for file in $(ls ./build/${os}/${arch}); do + cat "./build/${os}/${arch}/${file}" | ${MD5} | awk '{print $1}' >> "./build/${os}/${arch}/${file}.md5" + done + done +done diff --git a/scripts/upload.sh b/scripts/upload.sh new file mode 100755 index 0000000..5dcdd34 --- /dev/null +++ b/scripts/upload.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +# upload to AWS S3 +echo "Uploading builds to S3..." +aws s3 sync ./build/ s3://tools.nanopack.io/shaman --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --region us-east-1 diff --git a/version.go b/version.go deleted file mode 100644 index e8d6cfc..0000000 --- a/version.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -// shaman version information (populated by go linker) -// -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" -var ( - version string - branch string - commit string -)