Rewrite Remote Output Service on top of Google's protocol

Google has merged a change for adding the Remote Output Service protocol to their code base: bazelbuild/bazel#21140. They did however make a couple of changes to it. For example: - The protocol has been made REv2 agnostic. All explicit coupling to REv2 has been moved into a helper protocol. - BatchCreate() has been renamed to StageArtifacts(). It can only be used to create files and directories. Not symlinks. It also doesn't provide options to clean directories. This is likely going to hurt runfiles directory creation, but we'll see whether that is actually a problem in practice. - BatchStat() no longer provides follow_symlinks and include_file_digests. Symlinks are no longer followed, and file digests should always be included. - There is a FinalizeArtifacts() function. This function can be used to reliably implement file modification tracking. As we don't implement that yet, we can simply let it be a stub for the time being.
buildbarn · Mar 6, 2024 · ad659bd · ad659bd
1 parent c3ea196
commit ad659bd
Show file tree

Hide file tree

Showing 13 changed files with 444 additions and 644 deletions.
diff --git a/README.md b/README.md
@@ -242,7 +242,7 @@ disadvantages:
   of the build, meaning that Bazel can't continue where it left off.
 
 To solve these issues, bb\_clientd implements a gRPC API named the
-Remote Output Service. Bazel can use this API to store the entire
+Bazel Output Service. Bazel can use this API to store the entire
 `bazel-out/` directory inside the FUSE/NFSv4 file system. Every time
 Bazel needs to download a file from the Remote Execution service, it
 calls into a `BatchCreate()` gRPC method. bb\_clientd implements this
@@ -251,13 +251,7 @@ directory. This means that you can enjoy the performance improvements of
 "Remote Builds without the Bytes", but not with the restrictions that it
 currently imposes.
 
-The Remote Output Service can be used by building a copy of Bazel that
-includes [upstream PR#12823](https://github.com/bazelbuild/bazel/pull/12823).
-You may then run Bazel with the following additional command line flags:
-
-```
---remote_output_service unix:${HOME}/.cache/bb_clientd/grpc --remote_output_service_output_path_prefix ${HOME}/bb_clientd/outputs
-```
-
-We hope that upstream versions of Bazel ship with integrated support for
-the Remote Output Service going forward.
+The Bazel Output Service protocol was added to the Bazel repository in
+[PR#21140](https://github.com/bazelbuild/bazel/pull/21140). It is
+expected that a client for the Bazel Output Service protocol is added to
+Bazel 7.2.
diff --git a/cmd/bb_clientd/BUILD.bazel b/cmd/bb_clientd/BUILD.bazel
@@ -17,7 +17,7 @@ go_library(
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/virtual",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/virtual/configuration",
-        "@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/configuration",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/grpcservers",
         "@com_github_buildbarn_bb_storage//pkg/builder",

diff --git a/cmd/bb_clientd/main.go b/cmd/bb_clientd/main.go
@@ -17,7 +17,7 @@ import (
 	re_filesystem "github.com/buildbarn/bb-remote-execution/pkg/filesystem"
 	re_vfs "github.com/buildbarn/bb-remote-execution/pkg/filesystem/virtual"
 	virtual_configuration "github.com/buildbarn/bb-remote-execution/pkg/filesystem/virtual/configuration"
-	"github.com/buildbarn/bb-remote-execution/pkg/proto/remoteoutputservice"
+	"github.com/buildbarn/bb-remote-execution/pkg/proto/bazeloutputservice"
 	blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration"
 	"github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers"
 	"github.com/buildbarn/bb-storage/pkg/builder"
@@ -204,10 +204,10 @@ func main() {
 			return allocateHandle().AsStatelessDirectory(re_vfs.NewStaticDirectory(blobsDirectoryContents))
 		}
 
-		// Implementation of the Remote Output Service. The Remote
+		// Implementation of the Bazel Output Service. The Bazel
 		// Output Service allows Bazel to place its bazel-out/
-		// directories on a virtual file system, thereby allowing data
-		// to be loaded lazily.
+		// directories on a virtual file system, thereby
+		// allowing data to be loaded lazily.
 		symlinkFactory := re_vfs.NewHandleAllocatingSymlinkFactory(
 			re_vfs.BaseSymlinkFactory,
 			rootHandleAllocator.New())
@@ -247,7 +247,7 @@ func main() {
 				symlinkFactory)
 		}
 
-		outputsDirectory := cd_vfs.NewRemoteOutputServiceDirectory(
+		outputsDirectory := cd_vfs.NewBazelOutputServiceDirectory(
 			rootHandleAllocator,
 			outputPathFactory,
 			bareContentAddressableStorage,
@@ -319,7 +319,7 @@ func main() {
 						})))
 				remoteexecution.RegisterExecutionServer(s, buildQueue)
 
-				remoteoutputservice.RegisterRemoteOutputServiceServer(s, outputsDirectory)
+				bazeloutputservice.RegisterBazelOutputServiceServer(s, outputsDirectory)
 			},
 			siblingsGroup,
 		); err != nil {

diff --git a/go.mod b/go.mod
@@ -7,7 +7,7 @@ replace github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/g
 
 require (
 	github.com/bazelbuild/remote-apis v0.0.0-20240215191509-9ff14cecffe5
-	github.com/buildbarn/bb-remote-execution v0.0.0-20240222085313-f5b199467dd6
+	github.com/buildbarn/bb-remote-execution v0.0.0-20240306174844-94b3776d5dba
 	github.com/buildbarn/bb-storage v0.0.0-20240227100204-0aa40dfdbead
 	golang.org/x/sync v0.6.0
 	google.golang.org/genproto/googleapis/bytestream v0.0.0-20240228224816-df926f6c8641

diff --git a/go.sum b/go.sum
@@ -57,6 +57,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/buildbarn/bb-remote-execution v0.0.0-20240222085313-f5b199467dd6 h1:dgXMsgYgc48Sxb8gcrd+C2ze8Rtn1UvaDMxrUGti50g=
 github.com/buildbarn/bb-remote-execution v0.0.0-20240222085313-f5b199467dd6/go.mod h1:qwNvc1PxPWzRzcKiQ/Hq1MQmxB8fjWQ+Lv0h9r8aysU=
+github.com/buildbarn/bb-remote-execution v0.0.0-20240306174844-94b3776d5dba h1:pe9k/EHyywQK/OYEQkmHY7cYycG41kDEd6Ae+lHynkM=
+github.com/buildbarn/bb-remote-execution v0.0.0-20240306174844-94b3776d5dba/go.mod h1:qwNvc1PxPWzRzcKiQ/Hq1MQmxB8fjWQ+Lv0h9r8aysU=
 github.com/buildbarn/bb-storage v0.0.0-20240227100204-0aa40dfdbead h1:fHapKnQQLgJaMxGiBAUCPVHNfD5vV1LDfXqmyClJ6Lc=
 github.com/buildbarn/bb-storage v0.0.0-20240227100204-0aa40dfdbead/go.mod h1:gHT0PInDFOV/JZjeeNwvqmn33MKHHyk3V18e4/Cs/jM=
 github.com/buildbarn/go-xdr v0.0.0-20231115101217-a9e2aa4cf64b h1:/sKWC0Fs5fXNo/t72BRZRLERg4v2gFoEeg2Mk+a8xak=

diff --git a/go_dependencies.bzl b/go_dependencies.bzl
@@ -173,8 +173,8 @@ def go_dependencies():
     go_repository(
         name = "com_github_buildbarn_bb_remote_execution",
         importpath = "github.com/buildbarn/bb-remote-execution",
-        sum = "h1:dgXMsgYgc48Sxb8gcrd+C2ze8Rtn1UvaDMxrUGti50g=",
-        version = "v0.0.0-20240222085313-f5b199467dd6",
+        sum = "h1:pe9k/EHyywQK/OYEQkmHY7cYycG41kDEd6Ae+lHynkM=",
+        version = "v0.0.0-20240306174844-94b3776d5dba",
     )
     go_repository(
         name = "com_github_buildbarn_bb_storage",

diff --git a/internal/mock/BUILD.bazel b/internal/mock/BUILD.bazel
@@ -169,8 +169,8 @@ go_library(
         "//pkg/outputpathpersistency",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/virtual",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice",
         "@com_github_buildbarn_bb_remote_execution//pkg/proto/outputpathpersistency",
-        "@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice",
         "@com_github_buildbarn_bb_storage//pkg/blobstore",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/buffer",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/slicing",

diff --git a/pkg/filesystem/virtual/BUILD.bazel b/pkg/filesystem/virtual/BUILD.bazel
@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "virtual",
     srcs = [
+        "bazel_output_service_directory.go",
         "blob_access_command_file_factory.go",
         "cas_directory.go",
         "cas_directory_factory.go",
@@ -16,7 +17,6 @@ go_library(
         "non_iterable_directory.go",
         "output_path_factory.go",
         "persistent_output_path_factory.go",
-        "remote_output_service_directory.go",
         "tree_cas_directory_factory.go",
     ],
     importpath = "github.com/buildbarn/bb-clientd/pkg/filesystem/virtual",
@@ -30,8 +30,9 @@ go_library(
         "@com_github_buildbarn_bb_remote_execution//pkg/cas",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/virtual",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice/rev2",
         "@com_github_buildbarn_bb_remote_execution//pkg/proto/outputpathpersistency",
-        "@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice",
         "@com_github_buildbarn_bb_storage//pkg/blobstore",
         "@com_github_buildbarn_bb_storage//pkg/clock",
         "@com_github_buildbarn_bb_storage//pkg/digest",
@@ -40,7 +41,6 @@ go_library(
         "@com_github_buildbarn_bb_storage//pkg/util",
         "@org_golang_google_grpc//codes",
         "@org_golang_google_grpc//status",
-        "@org_golang_google_protobuf//types/known/emptypb",
         "@org_golang_google_protobuf//types/known/timestamppb",
         "@org_golang_x_sync//semaphore",
     ],
@@ -49,23 +49,24 @@ go_library(
 go_test(
     name = "virtual_test",
     srcs = [
+        "bazel_output_service_directory_test.go",
         "cas_directory_test.go",
         "decomposed_cas_directory_factory_test.go",
         "digest_parsing_directory_test.go",
         "in_memory_output_path_factory_test.go",
         "instance_name_parsing_directory_test.go",
         "local_file_uploading_output_path_factory_test.go",
         "persistent_output_path_factory_test.go",
-        "remote_output_service_directory_test.go",
         "tree_cas_directory_factory_test.go",
     ],
     deps = [
         ":virtual",
         "//internal/mock",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
         "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/virtual",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/bazeloutputservice/rev2",
         "@com_github_buildbarn_bb_remote_execution//pkg/proto/outputpathpersistency",
-        "@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice",
         "@com_github_buildbarn_bb_storage//pkg/blobstore",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/buffer",
         "@com_github_buildbarn_bb_storage//pkg/digest",
@@ -76,7 +77,7 @@ go_test(
         "@com_github_stretchr_testify//require",
         "@org_golang_google_grpc//codes",
         "@org_golang_google_grpc//status",
-        "@org_golang_google_protobuf//types/known/timestamppb",
+        "@org_golang_google_protobuf//types/known/anypb",
         "@org_golang_x_sync//semaphore",
     ],
 )