bump substreams, fix compression on connectweb http requests

CHANGELOG.md

@@ -33,6 +33,11 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s
 * Add support for ConnectWeb firehose requests.
 * Always use gzip compression on firehose requests for clients that support it (instead of always answering with the same compression as the request).
 
+### Substreams
+
+* Properly accept and compress responses with `gzip` for browser HTTP clients using ConnectWeb with `Accept-Encoding` header
+* Allow setting subscription channel max capacity via `SOURCE_CHAN_SIZE` env var (default: 100)
+
 ## v1.6.9
 
 ### Substreams
@@ -45,11 +50,7 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s
 ## v1.6.8
 
 > [!NOTE]
-<<<<<<< HEAD
-> This release will reject connections from clients that don't support GZIP compression. Use `--substreams-tier1-enforce-compression=false` to keep previous behavior, then check the logs for `incoming Substreams Blocks request` logs with the value `compressed: false` to track users who are not using compressed HTTP connections.
-=======
 > This release will reject substreams connections from clients that don't support GZIP compression. Use `--substreams-tier1-enforce-compression=false` to keep previous behavior, then check the logs for `incoming Substreams Blocks request` logs with the value `compressed: false` to track users who are not using compressed HTTP connections.
->>>>>>> 380546c (switch firehose to connectweb server, disable proxy for firehoseV1, add --firehose-enforce-compression true by default)
 
 * Substreams: add `--substreams-tier1-enforce-compression` to reject connections from clients that do not support GZIP compression
 * Substreams performance: reduced the number of `mallocs` (patching some third-party libraries)

firehose/server/blocks.go

@@ -82,7 +82,7 @@ func (s *Server) Blocks(ctx context.Context, request *connect.Request[pbfirehose
 
 	logger := logging.Logger(ctx, s.logger)
 
-	if !matchHeader(request.Header(), acceptedCompressionValues) {
+	if !matchHeader(request.Header()) {
 		if s.enforceCompression {
 			return status.Error(codes.InvalidArgument, "client does not support compression")
 		}
@@ -302,15 +302,18 @@ func stepToProto(step bstream.StepType, finalBlocksOnly bool) (outStep pbfirehos
 }
 
 // must be lowercase
-var compressionHeader = map[string]bool{"grpc-accept-encoding": true, "connect-accept-encoding": true}
-var acceptedCompressionValues = map[string]bool{"gzip": true, "zstd": true}
+var compressionHeader = map[string]map[string]bool{
+	"grpc-accept-encoding":    {"gzip": true, "zstd": true},
+	"connect-accept-encoding": {"gzip": true, "zstd": true},
+	"accept-encoding":         {"gzip": true}, // HTTP encoding for connect+proto in browser
+}
 
-func matchHeader(headers http.Header, expected map[string]bool) bool {
-	for k, v := range headers {
-		if compressionHeader[strings.ToLower(k)] {
+func matchHeader(header http.Header) bool {
+	for k, v := range header {
+		if validEncodings, ok := compressionHeader[strings.ToLower(k)]; ok {
 			for _, vv := range v {
 				for _, vvv := range strings.Split(vv, ",") {
-					if expected[strings.TrimSpace(strings.ToLower(vvv))] {
+					if validEncodings[strings.TrimSpace(strings.ToLower(vvv))] {
 						return true
 					}
 				}

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/streamingfast/dauth v0.0.0-20240222213226-519afc16cf84
 	github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c
 	github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1
-	github.com/streamingfast/dgrpc v0.0.0-20250109212433-ae21a7f7a01a
+	github.com/streamingfast/dgrpc v0.0.0-20250115215805-6f4ad2be7eef
 	github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4
 	github.com/streamingfast/dmetering v0.0.0-20241101155221-489f5a9d9139
 	github.com/streamingfast/dmetrics v0.0.0-20230919161904-206fa8ebd545
@@ -31,7 +31,7 @@ require (
 	github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2
 	github.com/streamingfast/pbgo v0.0.6-0.20250114182320-0b43084f4000
 	github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0
-	github.com/streamingfast/substreams v1.11.4-0.20250113142113-36c2750be692
+	github.com/streamingfast/substreams v1.11.4-0.20250116013259-ebf5362125de
 	github.com/stretchr/testify v1.9.0
 	github.com/test-go/testify v1.1.4
 	go.uber.org/multierr v1.10.0

go.sum

@@ -2146,8 +2146,8 @@ github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c h1:6WjE2yInE+
 github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c/go.mod h1:dbfiy9ORrL8c6ldSq+L0H9pg8TOqqu/FsghsgUEWK54=
 github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1 h1:xJB7rXnOHLesosMjfwWsEL2i/40mFSkzenEb3M0qTyM=
 github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1/go.mod h1:QSm/AfaDsE0k1xBYi0lW580YJ/WDV/FKZI628tkZR0Y=
-github.com/streamingfast/dgrpc v0.0.0-20250109212433-ae21a7f7a01a h1:yrxCZ7Py0FdMu80cWPv/EpDvBLyumPlfhehD7iJ5VJM=
-github.com/streamingfast/dgrpc v0.0.0-20250109212433-ae21a7f7a01a/go.mod h1:bxRfCxRKQ0ZH2BGi6UcYdlH0nkj8yERm3kpP1jPLQLY=
+github.com/streamingfast/dgrpc v0.0.0-20250115215805-6f4ad2be7eef h1:He9qXjmnDtxVrJcHAOfFiWFA6An48zTezpU5iMnNHuY=
+github.com/streamingfast/dgrpc v0.0.0-20250115215805-6f4ad2be7eef/go.mod h1:bxRfCxRKQ0ZH2BGi6UcYdlH0nkj8yERm3kpP1jPLQLY=
 github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4 h1:HKi8AIkLBzxZWmbCRUo1RxoOLK33iXO6gZprfsE9rf4=
 github.com/streamingfast/dhammer v0.0.0-20230125192823-c34bbd561bd4/go.mod h1:ehPytv7E4rI65iLcrwTes4rNGGqPPiugnH+20nDQyp4=
 github.com/streamingfast/dmetering v0.0.0-20241101155221-489f5a9d9139 h1:a22XzjeY7n9Xv+0yJMV2pzuPptALtOu6jdg69pOwuO4=
@@ -2181,8 +2181,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt
 github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 h1:Y15G1Z4fpEdm2b+/70owI7TLuXadlqBtGM7rk4Hxrzk=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0/go.mod h1:/Rnz2TJvaShjUct0scZ9kKV2Jr9/+KBAoWy4UMYxgv4=
-github.com/streamingfast/substreams v1.11.4-0.20250113142113-36c2750be692 h1:YrIj24iHkkdhzWHVNqdjdL76BqEOs9PxMjW8ejbEGnk=
-github.com/streamingfast/substreams v1.11.4-0.20250113142113-36c2750be692/go.mod h1:gl4g6eqMV3tAvir2J+3tY/JfXwm3TThHe7VL53glywE=
+github.com/streamingfast/substreams v1.11.4-0.20250116013259-ebf5362125de h1:4MH2yq5LqUZJZTR217lD+bZln5+q6FDkD70zwO3nbks=
+github.com/streamingfast/substreams v1.11.4-0.20250116013259-ebf5362125de/go.mod h1:Dgbt37alWqMyahFQ4rdhX8iFLZHn2qD8TBhcP3NIuW8=
 github.com/streamingfast/wazero v0.0.0-20241202185309-91287c3640ed h1:LU6/c376zP1cMAo9L6rFLyjo0W7RU+hIh7BegH8Zo5M=
 github.com/streamingfast/wazero v0.0.0-20241202185309-91287c3640ed/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

test/integration_test.go

@@ -75,7 +75,7 @@ func TestIntegration(t *testing.T) {
 		meteringServer.Run()
 	}()
 
-	clientConfig := client.NewSubstreamsClientConfig("localhost:9003", "", 0, false, true)
+	clientConfig := client.NewSubstreamsClientConfig("localhost:9003", "", 0, false, true, "firecore_test")
 	substreamsClient, _, _, _, err := client.NewSubstreamsClient(clientConfig)
 	require.NoError(t, err)