Merge pull request #29 from renproject/feat/zcash-canopy

Zcash mainnet canopy upgrade

api/server_test.go

@@ -2,7 +2,6 @@ package api_test
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -11,10 +10,9 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/renproject/mercury/api"
-
 	"github.com/renproject/kv"
 	"github.com/renproject/mercury/api"
+	. "github.com/renproject/mercury/api"
 	"github.com/renproject/mercury/cache"
 	"github.com/renproject/mercury/proxy"
 	"github.com/renproject/mercury/rpc"
@@ -34,11 +32,8 @@ var _ = Describe("Server", func() {
 			logger := logrus.StandardLogger()
 			btcTestnetNodeClient := rpc.NewClient(btcTestnetURL, btcTestnetUser, btcTestnetPassword)
 			btcTestnetProxy := proxy.NewProxy(btcTestnetNodeClient)
-
-			db := kv.NewMemDB(kv.JSONCodec)
-			store := kv.NewTable(db, "test")
-			ttl := kv.NewTTLCache(context.Background(), db, "ttl", time.Second)
-			btcCache := cache.New(store, ttl, logger)
+			store := kv.NewTable(kv.NewMemDB(kv.JSONCodec), "test")
+			btcCache := cache.New(store, logger)
 			btcTestnetAPI := api.NewApi(btctypes.BtcTestnet, btcTestnetProxy, btcCache, logger)
 			server := NewServer(logrus.StandardLogger(), "5000", btcTestnetAPI)
 			go server.Run()

cache/cache.go

@@ -20,16 +20,14 @@ var (
 
 type Cache struct {
 	locks  sync.Map
-	ttl    kv.Table
 	store  kv.Table
 	logger logrus.FieldLogger
 }
 
 // New returns a new Cache.
-func New(store, ttl kv.Table, logger logrus.FieldLogger) *Cache {
+func New(store kv.Table, logger logrus.FieldLogger) *Cache {
 	return &Cache{
 		locks:  sync.Map{},
-		ttl:    ttl,
 		store:  store,
 		logger: logger,
 	}
@@ -39,21 +37,8 @@ func New(store, ttl kv.Table, logger logrus.FieldLogger) *Cache {
 // requests that are sent while the result is being retrieved, wait until the first function call returns. This prevents
 // the function f() from being called multiple times for the same request.
 func (cache *Cache) Get(level types.AccessLevel, hash string, f func() ([]byte, error)) ([]byte, error) {
-	if level == types.FullAccess {
-		if cache.ttl == nil {
-			return f()
-		}
-		var data []byte
-		err := cache.ttl.Get(hash, &data)
-		if err == kv.ErrKeyNotFound {
-			value, err := f()
-			if err != nil {
-				return nil, err
-			}
-
-			return value, cache.ttl.Insert(hash, value)
-		}
-		return data, err
+	if level == 2 {
+		return f()
 	}
 
 	// Check if the result already exists in the store.

cache/cache_test.go

@@ -2,8 +2,11 @@ package cache_test
 
 import (
 	"bytes"
-	"context"
+	"encoding/json"
+	"io/ioutil"
 	"math/rand"
+	"net/http"
+	"net/http/httptest"
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -19,92 +22,130 @@ var _ = Describe("Cache", func() {
 	rand.Seed(0)
 
 	Context("when sending multiple identical requests", func() {
-		Context("when the request is of CachedAccess level", func() {
-			It("should only forward a single request", func() {
-				db := kv.NewMemDB(kv.JSONCodec)
-				store := kv.NewTable(db, "test")
-				ttl := kv.NewTTLCache(context.Background(), db, "ttl", time.Second)
-				logger := logrus.StandardLogger()
-				cache := New(store, ttl, logger)
-
-				counter := 0
-				f := func() ([]byte, error) {
-					counter++
-					return []byte(time.Now().String()), nil
-				}
-
-				phi.ParForAll(10, func(i int) {
-					time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
-					_, err := cache.Get(1, "hash", f)
-					Expect(err).ToNot(HaveOccurred())
-				})
-
-				Expect(counter).To(Equal(1))
-			})
+		It("should only forward a single request", func() {
+			store := kv.NewTable(kv.NewMemDB(kv.JSONCodec), "test")
+			logger := logrus.StandardLogger()
+			cache := New(store, logger)
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte{})
+			}))
+			defer server.Close()
 
-			It("should return the same result for each request", func() {
-				db := kv.NewMemDB(kv.JSONCodec)
-				store := kv.NewTable(db, "test")
-				ttl := kv.NewTTLCache(context.Background(), db, "ttl", time.Second)
-				logger := logrus.StandardLogger()
-				cache := New(store, ttl, logger)
-
-				counter := 0
-				f := func() ([]byte, error) {
-					counter++
-					return []byte(time.Now().String()), nil
-				}
-
-				numRequests := 10
-				responses := make([][]byte, numRequests)
-				phi.ParForAll(numRequests, func(i int) {
-					time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
-					resp, err := cache.Get(1, "hash", f)
-					Expect(err).ToNot(HaveOccurred())
-					responses[i] = resp
-				})
-				for _, resp := range responses {
-					Expect(bytes.Equal(resp, responses[0])).Should(BeTrue())
-				}
+			numRequests := 0
+			phi.ParBegin(func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				_, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
 			})
+
+			Expect(numRequests).To(Equal(1))
 		})
 
-		Context("when the request is FullAccess level", func() {
-			It("should return the same result for each request and only be forwards once", func() {
-				db := kv.NewMemDB(kv.JSONCodec)
-				store := kv.NewTable(db, "test")
-				ttl := kv.NewTTLCache(context.Background(), db, "ttl", time.Second)
-				logger := logrus.StandardLogger()
-				cache := New(store, ttl, logger)
-
-				expire := time.After(4 * time.Second)
-				counter := 0
-				f := func() ([]byte, error) {
-					counter++
-					return []byte(time.Now().String()), nil
-				}
-
-				// Expect result to be cached and only one request will be forwarded
-				numRequests := 10
-				responses := make([][]byte, numRequests)
-				phi.ParForAll(numRequests, func(i int) {
-					time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
-					resp, err := cache.Get(2, "hash", f)
-					Expect(err).ToNot(HaveOccurred())
-					responses[i] = resp
-				})
-				Expect(counter).Should(Equal(1))
-				for _, resp := range responses {
-					Expect(bytes.Equal(resp, responses[0])).Should(BeTrue())
-				}
-
-				// Expect the result to be expired and the request be forwarded again.
-				<-expire
-				resp, err := cache.Get(2, "hash", f)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(counter).Should(Equal(2))
-				Expect(bytes.Equal(resp, responses[0])).ShouldNot(BeTrue())
+		It("should return the same result for each request", func() {
+			store := kv.NewTable(kv.NewMemDB(kv.JSONCodec), "test")
+			logger := logrus.StandardLogger()
+			cache := New(store, logger)
+
+			response := []byte("response")
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(response)
+			}))
+			defer server.Close()
+
+			numRequests := 0
+			phi.ParBegin(func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
+			}, func() {
+				time.Sleep(time.Duration(rand.Intn(5000)) * time.Millisecond)
+				resp, err := cache.Get(1, "hash", getResponse(server.URL, &numRequests))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(resp).To(Equal(response))
 			})
 		})
 	})
 })
+
+func getResponse(url string, numRequests *int) func() ([]byte, error) {
+	return func() ([]byte, error) {
+		*numRequests++
+
+		req := map[string]string{"": ""}
+		reqBytes, err := json.Marshal(req)
+		Expect(err).ToNot(HaveOccurred())
+
+		resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBytes))
+		Expect(err).ToNot(HaveOccurred())
+
+		data, err := ioutil.ReadAll(resp.Body)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Add 3 second timeout to simulate latency.
+		time.Sleep(3 * time.Second)
+
+		return data, nil
+	}
+}

cmd/mercury/main.go

@@ -1,9 +1,7 @@
 package main
 
 import (
-	"context"
 	"os"
-	"time"
 
 	"github.com/renproject/kv"
 	"github.com/renproject/mercury/api"
@@ -22,25 +20,15 @@ func main() {
 	db := kv.NewMemDB(kv.JSONCodec)
 
 	// Initialise stores.
-	rinkebyTTL := kv.NewTTLCache(context.Background(), db, "ethRinkebyCache", 5*time.Second)
-	kovanTTL := kv.NewTTLCache(context.Background(), db, "ethKovanCache", 5*time.Second)
-	// btcTestTTL := kv.NewTTLCache(context.Background(), db, "btcTestCache", 5*time.Second)
-	// zecTestTTL := kv.NewTTLCache(context.Background(), db, "zecTestCache", 5*time.Second)
-	// bchTestTTL := kv.NewTTLCache(context.Background(), db, "bchTestCache", 5*time.Second)
-	// ethTTL := kv.NewTTLCache(context.Background(), db, "ethCache", 5*time.Second)
-	// btcTTL := kv.NewTTLCache(context.Background(), db, "btcCache", 5*time.Second)
-	// zecTTL := kv.NewTTLCache(context.Background(), db, "zecCache", 5*time.Second)
-	// bchTTL := kv.NewTTLCache(context.Background(), db, "bchCache", 5*time.Second)
-
-	ethRinkebyCache := cache.New(kv.NewTable(db, "ethRinkeby"), rinkebyTTL, logger)
-	ethKovanCache := cache.New(kv.NewTable(db, "ethKovan"), kovanTTL, logger)
-	btcTestCache := cache.New(kv.NewTable(db, "btcTest"), nil, logger)
-	zecTestCache := cache.New(kv.NewTable(db, "zecTest"), nil, logger)
-	bchTestCache := cache.New(kv.NewTable(db, "bchTest"), nil, logger)
-	ethCache := cache.New(kv.NewTable(db, "eth"), nil, logger)
-	btcCache := cache.New(kv.NewTable(db, "btc"), nil, logger)
-	zecCache := cache.New(kv.NewTable(db, "zec"), nil, logger)
-	bchCache := cache.New(kv.NewTable(db, "bch"), nil, logger)
+	ethRinkebyCache := cache.New(kv.NewTable(db, "ethRinkeby"), logger)
+	ethKovanCache := cache.New(kv.NewTable(db, "ethKovan"), logger)
+	btcTestCache := cache.New(kv.NewTable(db, "btcTest"), logger)
+	zecTestCache := cache.New(kv.NewTable(db, "zecTest"), logger)
+	bchTestCache := cache.New(kv.NewTable(db, "bchTest"), logger)
+	ethCache := cache.New(kv.NewTable(db, "eth"), logger)
+	btcCache := cache.New(kv.NewTable(db, "btc"), logger)
+	zecCache := cache.New(kv.NewTable(db, "zec"), logger)
+	bchCache := cache.New(kv.NewTable(db, "bch"), logger)
 
 	// Initialise Bitcoin API.
 	btcTestnetURL := os.Getenv("BITCOIN_TESTNET_RPC_URL")

go.mod

@@ -3,31 +3,25 @@ module github.com/renproject/mercury
 go 1.12
 
 require (
-	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/allegro/bigcache v1.2.1 // indirect
 	github.com/aristanetworks/goarista v0.0.0-20200310212843-2da4c1f5881b // indirect
 	github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/cespare/cp v1.1.1 // indirect
 	github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf
 	github.com/deckarep/golang-set v1.7.1 // indirect
-	github.com/dgraph-io/badger v1.6.1 // indirect
-	github.com/dgraph-io/ristretto v0.0.3 // indirect
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/elastic/gosigar v0.10.5 // indirect
 	github.com/ethereum/go-ethereum v1.9.5
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/huin/goupnp v1.0.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect
-	github.com/mattn/go-colorable v0.1.7 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/miguelmota/go-ethereum-hdwallet v0.0.0-20190720004541-5f6b3168e4a0
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
 	github.com/onsi/ginkgo v1.10.1
@@ -45,10 +39,7 @@ require (
 	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.2
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
-	golang.org/x/sys v0.0.0-20200722175500-76b94024e4b6 // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
+	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0 // indirect