diff --git a/pom.xml b/pom.xml index 42ddea0f..b2df5590 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,7 @@ 3.26.3 2.10.1 + 2.16.1 1.3.2 1.0.0 5.11.3 @@ -152,6 +153,14 @@ + + com.fasterxml.jackson + jackson-bom + ${jackson-bom.version} + import + pom + + com.github.marschall memoryfilesystem diff --git a/protobuf-maven-plugin/pom.xml b/protobuf-maven-plugin/pom.xml index eda408f1..44205234 100644 --- a/protobuf-maven-plugin/pom.xml +++ b/protobuf-maven-plugin/pom.xml @@ -34,27 +34,21 @@ 3.8.2 - - - - org.junit - junit-bom - ${junit.version} - import - pom - - - - org.mockito - mockito-bom - ${mockito.version} - import - pom - - - - + + + com.fasterxml.jackson.core + jackson-databind + compile + + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + compile + + javax.annotation javax.annotation-api diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/invoker.properties b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/invoker.properties new file mode 100644 index 00000000..13f987b7 --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/invoker.properties @@ -0,0 +1,19 @@ +# +# Copyright (C) 2023 - 2024, Ashley Scopes. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +invoker.goals.0=clean +invoker.goals.1=package +invoker.goals.2=package diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/pom.xml b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/pom.xml new file mode 100644 index 00000000..082cc527 --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/pom.xml @@ -0,0 +1,62 @@ + + + + 4.0.0 + + + @project.groupId@.it + integration-test-parent + @project.version@ + ../setup/pom.xml + + + gh-438-incremental-compilation-no-changes + gh-438-incremental-compilation-no-changes + + + + com.google.protobuf + protobuf-java + compile + + + + + + + @project.groupId@ + @project.artifactId@ + + + true + + + + + + generate + + + + + + + diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/avatar.proto b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/avatar.proto new file mode 100644 index 00000000..81a4b82d --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/avatar.proto @@ -0,0 +1,33 @@ +// +// Copyright (C) 2023 - 2024, Ashley Scopes. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package org.example; + +option java_multiple_files = true; +option java_package = "org.example.users"; + +enum AvatarFormat { + PNG = 0; + WEBP = 1; + SVG = 2; +} + +message Avatar { + string binary_data = 1; + AvatarFormat format = 2; +} diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/channel.proto b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/channel.proto new file mode 100644 index 00000000..bbe3954f --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/channel.proto @@ -0,0 +1,27 @@ +// +// Copyright (C) 2023 - 2024, Ashley Scopes. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package org.example; + +option java_multiple_files = true; +option java_package = "org.example"; + +message Channel { + string id = 1; + string name = 2; +} diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/message.proto b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/message.proto new file mode 100644 index 00000000..2117bec7 --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/message.proto @@ -0,0 +1,32 @@ +// +// Copyright (C) 2023 - 2024, Ashley Scopes. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package org.example; + +option java_multiple_files = true; +option java_package = "org.example"; + +import "org/example/user.proto"; +import "org/example/channel.proto"; + +message Message { + string id = 1; + string content = 2; + org.example.User author = 3; + org.example.Channel channel = 4; +} diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/user.proto b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/user.proto new file mode 100644 index 00000000..87929c9f --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/src/main/protobuf/org/example/user.proto @@ -0,0 +1,31 @@ +// +// Copyright (C) 2023 - 2024, Ashley Scopes. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package org.example; + +option java_multiple_files = true; +option java_package = "org.example"; + +import "org/example/avatar.proto"; + +message User { + string id = 1; + string name = 2; + optional string nickname = 3; + optional org.example.Avatar avatar = 4; +} diff --git a/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/test.groovy b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/test.groovy new file mode 100644 index 00000000..0ce43f73 --- /dev/null +++ b/protobuf-maven-plugin/src/it/gh-438-incremental-compilation-no-changes/test.groovy @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023 - 2024, Ashley Scopes. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.file.Path + +Path baseDirectory = basedir.toPath().toAbsolutePath() + +return true diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/generation/ProtobufBuildOrchestrator.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/generation/ProtobufBuildOrchestrator.java index 357b5553..935f8a31 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/generation/ProtobufBuildOrchestrator.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/generation/ProtobufBuildOrchestrator.java @@ -16,18 +16,24 @@ package io.github.ascopes.protobufmavenplugin.generation; +import static io.github.ascopes.protobufmavenplugin.sources.SourceListing.flattenSourceProtoFiles; + import io.github.ascopes.protobufmavenplugin.plugins.ProjectPluginResolver; import io.github.ascopes.protobufmavenplugin.protoc.ArgLineBuilder; import io.github.ascopes.protobufmavenplugin.protoc.CommandLineExecutor; import io.github.ascopes.protobufmavenplugin.protoc.ProtocResolver; +import io.github.ascopes.protobufmavenplugin.sources.ProjectInputListing; import io.github.ascopes.protobufmavenplugin.sources.ProjectInputResolver; import io.github.ascopes.protobufmavenplugin.sources.SourceListing; +import io.github.ascopes.protobufmavenplugin.sources.incremental.IncrementalCacheManager; import io.github.ascopes.protobufmavenplugin.utils.FileUtils; import io.github.ascopes.protobufmavenplugin.utils.ResolutionException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Named; import org.apache.maven.execution.MavenSession; @@ -51,6 +57,7 @@ public final class ProtobufBuildOrchestrator { private final ProtocResolver protocResolver; private final ProjectInputResolver projectInputResolver; private final ProjectPluginResolver projectPluginResolver; + private final IncrementalCacheManager incrementalCacheManager; private final CommandLineExecutor commandLineExecutor; @Inject @@ -59,12 +66,14 @@ public ProtobufBuildOrchestrator( ProtocResolver protocResolver, ProjectInputResolver projectInputResolver, ProjectPluginResolver projectPluginResolver, + IncrementalCacheManager incrementalCacheManager, CommandLineExecutor commandLineExecutor ) { this.mavenSession = mavenSession; this.protocResolver = protocResolver; this.projectInputResolver = projectInputResolver; this.projectPluginResolver = projectPluginResolver; + this.incrementalCacheManager = incrementalCacheManager; this.commandLineExecutor = commandLineExecutor; } @@ -102,8 +111,14 @@ public boolean generate(GenerationRequest request) throws ResolutionException, I var argLineBuilder = new ArgLineBuilder(protocPath) .fatalWarnings(request.isFatalWarnings()) - .importPaths(projectInputs.getCompilableSources()) - .importPaths(projectInputs.getDependencySources()); + .importPaths(projectInputs.getCompilableSources() + .stream() + .map(SourceListing::getSourceRoot) + .collect(Collectors.toUnmodifiableList())) + .importPaths(projectInputs.getDependencySources() + .stream() + .map(SourceListing::getSourceRoot) + .collect(Collectors.toUnmodifiableList())); request.getEnabledLanguages() .forEach(language -> argLineBuilder.generateCodeFor( @@ -115,12 +130,23 @@ public boolean generate(GenerationRequest request) throws ResolutionException, I // GH-269: Add the plugins after the enabled languages to support generated code injection argLineBuilder.plugins(resolvedPlugins, request.getOutputDirectory()); - var argLine = argLineBuilder.compile(projectInputs.getCompilableSources()); + var compilableSources = computeActualSourcesToCompile(request, projectInputs); + if (compilableSources.isEmpty()) { + // Nothing to compile. If we hit here, then we likely received inputs but were using + // incremental compilation and nothing changed since the last build. + return true; + } + + var argLine = argLineBuilder.compile(compilableSources); if (!commandLineExecutor.execute(argLine)) { return false; } + // Since we've succeeded in the codegen phase, we can replace the old incremental cache + // with the new one. + incrementalCacheManager.updateIncrementalCache(); + registerSourceRoots(request); if (request.isEmbedSourcesInClassOutputs()) { @@ -175,6 +201,36 @@ private void registerSourceRoots(GenerationRequest request) { } } + private Collection computeActualSourcesToCompile( + GenerationRequest request, + ProjectInputListing projectInputs + ) throws IOException { + var totalSourceFileCount = projectInputs.getCompilableSources().stream() + .mapToInt(sourcePath -> sourcePath.getSourceProtoFiles().size()) + .sum(); + + var sourcesToCompile = request.isIncrementalCompilationEnabled() + ? incrementalCacheManager.determineSourcesToCompile(projectInputs) + : flattenSourceProtoFiles(projectInputs.getCompilableSources()); + + if (sourcesToCompile.isEmpty()) { + log.info( + "Found {} source files, but none have any changes, so there is nothing to do", + totalSourceFileCount + ); + return List.of(); + } + + log.info( + "Generating source code from {} (discovered within {}, from a total of {})", + pluralize(sourcesToCompile.size(), "source file"), + pluralize(projectInputs.getCompilableSources().size(), "source root"), + pluralize(projectInputs.getCompilableSources().size(), "candidate source file") + ); + + return sourcesToCompile; + } + private void embedSourcesInClassOutputs( SourceRootRegistrar registrar, Collection listings @@ -190,4 +246,10 @@ private void embedSourcesInClassOutputs( } } } + + private static String pluralize(int count, String name) { + return count == 1 + ? "1 " + name + : count + " " + name + "s"; + } } diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/mojo/AbstractGenerateMojo.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/mojo/AbstractGenerateMojo.java index f264f06a..eb30508c 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/mojo/AbstractGenerateMojo.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/mojo/AbstractGenerateMojo.java @@ -773,6 +773,12 @@ public void execute() throws MojoExecutionException, MojoFailureException { return; } + if (incrementalCompilation) { + // TODO(ascopes): remove this warning once we're happy with the stability. + log.warn("You have enabled incremental compilation. This is highly experimental and subject " + + "to change between minor versions. Please report any bugs you encounter on GitHub."); + } + var enabledLanguages = Language.languageSet() .addIf(cppEnabled, Language.CPP) .addIf(csharpEnabled, Language.C_SHARP) diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/package-info.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/package-info.java index bb287bdb..61981ffe 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/package-info.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/package-info.java @@ -26,7 +26,6 @@ defaultAsDefault = true, deferCollectionAllocation = true, headerComments = true, - jacksonIntegration = false, jdkOnly = true, jdk9Collections = true, nullableAnnotation = "org.jspecify.annotations.Nullable", diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/protoc/ArgLineBuilder.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/protoc/ArgLineBuilder.java index b57dfbe8..60d84233 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/protoc/ArgLineBuilder.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/protoc/ArgLineBuilder.java @@ -45,7 +45,7 @@ public ArgLineBuilder(Path protocPath) { importPaths = new ArrayList<>(); } - public List compile(Collection sourceListings) { + public List compile(Collection sources) { if (targets.isEmpty()) { throw new IllegalStateException("No output target operations were provided"); } @@ -61,10 +61,8 @@ public List compile(Collection sourceListings) { target.addArgsTo(args); } - for (var sourceListing : sourceListings) { - for (var sourcePath : sourceListing.getSourceProtoFiles()) { - args.add(sourcePath.toString()); - } + for (var source : sources) { + args.add(source.toString()); } for (var importPath : importPaths) { @@ -85,9 +83,9 @@ public ArgLineBuilder generateCodeFor(Language language, Path outputPath, boolea return this; } - public ArgLineBuilder importPaths(Collection importPathListings) { - for (var importPathListing : importPathListings) { - importPaths.add(importPathListing.getSourceRoot()); + public ArgLineBuilder importPaths(Collection importRootPaths) { + for (var importRootPath : importRootPaths) { + importPaths.add(importRootPath); } return this; diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputListing.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputListing.java index 1e90bf3a..2fe4ecc3 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputListing.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputListing.java @@ -27,6 +27,7 @@ */ @Immutable public interface ProjectInputListing { + Collection getCompilableSources(); Collection getDependencySources(); diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputResolver.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputResolver.java index c21c8bb3..e2522b6c 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputResolver.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/ProjectInputResolver.java @@ -85,21 +85,9 @@ private Collection resolveCompilableSources( filter ); - var sourcePaths = Stream + return Stream .concat(sourcePathsListings.stream(), sourceDependencyListings.stream()) .collect(Collectors.toUnmodifiableList()); - - var sourceFileCount = sourcePaths.stream() - .mapToInt(sourcePath -> sourcePath.getSourceProtoFiles().size()) - .sum(); - - log.info( - "Generating source code for {} from {}", - pluralize(sourceFileCount, "protobuf file"), - pluralize(sourcePaths.size(), "source file tree") - ); - - return sourcePaths; } private Collection resolveDependencySources( @@ -121,10 +109,4 @@ private Collection resolveDependencySources( return sourceResolver.resolveSources(importPaths, filter); } - - private static String pluralize(int count, String name) { - return count == 1 - ? "1 " + name - : count + " " + name + "s"; - } } diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/SourceListing.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/SourceListing.java index 2203fc68..a7e568e6 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/SourceListing.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/SourceListing.java @@ -17,7 +17,9 @@ package io.github.ascopes.protobufmavenplugin.sources; import java.nio.file.Path; +import java.util.Collection; import java.util.Set; +import java.util.stream.Collectors; import org.immutables.value.Value.Immutable; /** @@ -31,4 +33,11 @@ public interface SourceListing { Path getSourceRoot(); Set getSourceProtoFiles(); + + static Collection flattenSourceProtoFiles(Collection listings) { + return listings.stream() + .map(SourceListing::getSourceProtoFiles) + .flatMap(Collection::stream) + .collect(Collectors.toUnmodifiableList()); + } } diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/IncrementalCacheManager.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/IncrementalCacheManager.java new file mode 100644 index 00000000..fcb4ae32 --- /dev/null +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/IncrementalCacheManager.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2023 - 2024, Ashley Scopes. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.ascopes.protobufmavenplugin.sources.incremental; + +import static io.github.ascopes.protobufmavenplugin.sources.SourceListing.flattenSourceProtoFiles; +import static java.util.function.Predicate.not; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.ascopes.protobufmavenplugin.sources.ProjectInputListing; +import io.github.ascopes.protobufmavenplugin.sources.SourceListing; +import io.github.ascopes.protobufmavenplugin.utils.ConcurrentExecutor; +import io.github.ascopes.protobufmavenplugin.utils.Digests; +import io.github.ascopes.protobufmavenplugin.utils.TemporarySpace; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.OffsetDateTime; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Collection; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.FutureTask; +import java.util.stream.Collectors; +import javax.inject.Inject; +import javax.inject.Named; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manager that detects situations where incremental compilation may be faster on large codebases. + * + * @author Ashley Scopes + * @since 2.7.0 + */ +@Named +public class IncrementalCacheManager { + + private static final Logger log = LoggerFactory.getLogger(IncrementalCacheManager.class); + + private final ConcurrentExecutor concurrentExecutor; + private final TemporarySpace temporarySpace; + private final ObjectMapper objectMapper; + + @Inject + public IncrementalCacheManager( + ConcurrentExecutor concurrentExecutor, + TemporarySpace temporarySpace + ) { + this.concurrentExecutor = concurrentExecutor; + this.temporarySpace = temporarySpace; + + objectMapper = new JsonMapper() + .configure(SerializationFeature.INDENT_OUTPUT, true) + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .registerModule(new JavaTimeModule()) + .registerModule(new SerializedIncrementalCacheModule()); + } + + public void updateIncrementalCache() throws IOException { + var previousCache = getPreviousIncrementalCachePath(); + var newCache = getNewIncrementalCachePath(); + if (Files.exists(newCache)) { + log.debug("Overwriting incremental compilation cache at {} with {}", previousCache, newCache); + Files.move(newCache, previousCache, StandardCopyOption.REPLACE_EXISTING); + } else { + log.debug("No new incremental cache was created, so nothing will be updated..."); + } + } + + public Collection determineSourcesToCompile( + ProjectInputListing listing + ) throws IOException { + final var startTime = System.nanoTime(); + + // Always update the cache to catch changes in the next builds. + var newBuildCache = buildIncrementalCache(listing); + writeIncrementalCache(getNewIncrementalCachePath(), newBuildCache); + var maybePreviousBuildCache = readIncrementalCache(getPreviousIncrementalCachePath()); + + // If we lack a cache from a previous build, then we cannot determine what we should compile + // and what we should ignore, so we'll have to rebuild everything anyway. + if (maybePreviousBuildCache.isEmpty()) { + log.info("All sources will be compiled, as no previous build data was detected"); + return flattenSourceProtoFiles(listing.getCompilableSources()); + } + + var previousBuildCache = maybePreviousBuildCache.get(); + + // If dependencies change, we should recompile everything so that we can spot any compilation + // failures that have been created by changes to imported messages. + if (!previousBuildCache.getDependencies().equals(newBuildCache.getDependencies())) { + log.info("Detected a change in dependencies, all sources will be recompiled"); + return flattenSourceProtoFiles(listing.getCompilableSources()); + } + + var filesDeletedSinceLastBuild = previousBuildCache.getSources().keySet() + .stream() + .anyMatch(not(newBuildCache.getSources().keySet()::contains)); + + // If any sources were deleted, we should rebuild everything, as those files being deleted may + // have caused a compilation failure. + if (filesDeletedSinceLastBuild) { + log.info("Detected that source files have been deleted, all sources will be recompiled"); + return flattenSourceProtoFiles(listing.getCompilableSources()); + } + + var sourcesToCompile = newBuildCache.getSources().keySet() + .stream() + .filter(file -> !Objects.equals( + newBuildCache.getSources().get(file), + previousBuildCache.getSources().get(file) + )) + .collect(Collectors.toUnmodifiableSet()); + + var timeTaken = (System.nanoTime() - startTime) / 1_000_000L; + log.info("Detected {} sources to compile in {}ms", sourcesToCompile, timeTaken); + return sourcesToCompile; + } + + private Optional readIncrementalCache(Path path) throws IOException { + log.debug("Reading incremental cache in from {}", path); + + try (var inputStream = Files.newInputStream(path)) { + return Optional.of(objectMapper.readValue(inputStream, SerializedIncrementalCache.class)); + } catch (NoSuchFileException ex) { + log.debug("No file found at {}", path); + return Optional.empty(); + } + } + + private void writeIncrementalCache( + Path path, + SerializedIncrementalCache cache + ) throws IOException { + log.debug("Writing incremental cache out to {}", path); + + try (var outputStream = Files.newOutputStream(path)) { + objectMapper.writeValue(outputStream, cache); + } + } + + private SerializedIncrementalCache buildIncrementalCache(ProjectInputListing listing) { + var dependencyDigests = listing.getDependencySources().stream() + .map(SourceListing::getSourceProtoFiles) + .flatMap(Collection::stream) + .map(this::createSerializedFileDigestAsync) + .collect(concurrentExecutor.awaiting()) + .stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + + var sourceDigests = listing.getCompilableSources().stream() + .map(SourceListing::getSourceProtoFiles) + .flatMap(Collection::stream) + .map(this::createSerializedFileDigestAsync) + .collect(concurrentExecutor.awaiting()) + .stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + + return ImmutableSerializedIncrementalCache.builder() + .generatedAt(OffsetDateTime.now()) + .dependencies(dependencyDigests) + .sources(sourceDigests) + .build(); + } + + private FutureTask> createSerializedFileDigestAsync(Path file) { + return concurrentExecutor.submit(() -> { + log.trace("Generating digest for {}", file); + try (var inputStream = Files.newInputStream(file)) { + var digest = Digests.sha512ForStream(inputStream); + return new SimpleImmutableEntry<>(file, digest); + } + }); + } + + private Path getIncrementalCacheRoot() { + return temporarySpace.createTemporarySpace("incremental"); + } + + private Path getPreviousIncrementalCachePath() { + return getIncrementalCacheRoot().resolve("previous-build-cache.json"); + } + + private Path getNewIncrementalCachePath() { + return getIncrementalCacheRoot().resolve("current-build-cache.json"); + } +} diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCache.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCache.java new file mode 100644 index 00000000..93e473ae --- /dev/null +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCache.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 - 2024, Ashley Scopes. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.ascopes.protobufmavenplugin.sources.incremental; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.util.Map; +import org.immutables.value.Value.Immutable; + +@Immutable +@JsonDeserialize(builder = ImmutableSerializedIncrementalCache.Builder.class) +@JsonSerialize(as = ImmutableSerializedIncrementalCache.class) +interface SerializedIncrementalCache { + + OffsetDateTime getGeneratedAt(); + + Map getDependencies(); + + Map getSources(); +} diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCacheModule.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCacheModule.java new file mode 100644 index 00000000..dce9a33a --- /dev/null +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/sources/incremental/SerializedIncrementalCacheModule.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2023 - 2024, Ashley Scopes. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.ascopes.protobufmavenplugin.sources.incremental; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.KeyDeserializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; + +/** + * Custom serializer/deserializer support for the incremental cache serialization in Jackson. + * + * @author Ashley Scopes + * @since 2.7.0 + */ +final class SerializedIncrementalCacheModule extends SimpleModule { + SerializedIncrementalCacheModule() { + addKeyDeserializer(Path.class, new PathKeyDeserializer()); + addKeySerializer(Path.class, new PathKeySerializer()); + } + + private static final class PathKeyDeserializer extends KeyDeserializer { + @Override + public Object deserializeKey(String key, DeserializationContext context) { + return Path.of(URI.create(key)); + } + } + + private static final class PathKeySerializer extends StdSerializer { + private PathKeySerializer() { + super(Path.class); + } + + @Override + public void serialize( + Path path, + JsonGenerator generator, + SerializerProvider provider + ) throws IOException { + generator.writeFieldName(path.toUri().toASCIIString()); + } + } +} diff --git a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/utils/Digests.java b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/utils/Digests.java index 9ec88f2b..eb71d889 100644 --- a/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/utils/Digests.java +++ b/protobuf-maven-plugin/src/main/java/io/github/ascopes/protobufmavenplugin/utils/Digests.java @@ -16,6 +16,8 @@ package io.github.ascopes.protobufmavenplugin.utils; +import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.Base64; @@ -32,10 +34,31 @@ private Digests() { } public static String sha1(String string) { + var messageDigest = createMessageDigest("SHA-1"); + var bytes = string.getBytes(StandardCharsets.UTF_8); + return base64Encode(messageDigest.digest(bytes)); + } + + public static String sha512ForStream(InputStream inputStream) throws IOException { + var messageDigest = createMessageDigest("SHA-512"); + var buff = new byte[4_096]; + int offset; + + while ((offset = inputStream.read(buff)) != -1) { + messageDigest.update(buff, 0, offset); + } + + return base64Encode(messageDigest.digest()); + } + + private static String base64Encode(byte[] digest) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(digest); + } + + @SuppressWarnings("SameParameterValue") + private static MessageDigest createMessageDigest(String algorithm) { try { - var bytes = string.getBytes(StandardCharsets.UTF_8); - var digest = MessageDigest.getInstance("SHA-1").digest(bytes); - return Base64.getUrlEncoder().withoutPadding().encodeToString(digest); + return MessageDigest.getInstance(algorithm); } catch (Exception ex) { throw new IllegalArgumentException(ex.getMessage(), ex); } diff --git a/protobuf-maven-plugin/src/test/java/io/github/ascopes/protobufmavenplugin/utils/DigestsTest.java b/protobuf-maven-plugin/src/test/java/io/github/ascopes/protobufmavenplugin/utils/DigestsTest.java index a87680a1..99ddfefd 100644 --- a/protobuf-maven-plugin/src/test/java/io/github/ascopes/protobufmavenplugin/utils/DigestsTest.java +++ b/protobuf-maven-plugin/src/test/java/io/github/ascopes/protobufmavenplugin/utils/DigestsTest.java @@ -17,8 +17,13 @@ package io.github.ascopes.protobufmavenplugin.utils; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -31,7 +36,9 @@ @DisplayName("Digests tests") class DigestsTest { - @DisplayName("sha1(String) returns the expected un-padded url-safe base64 string") + @DisplayName( + ".sha1(String) returns the expected SHA-1 digest in an un-padded url-safe base64 string" + ) @CsvSource({ " foobarbaz, X1UT-IIv2-UUWvM7ZNjZcNz5XG4", " /src/main/java, R4yHFYpaVvKy0Ckbl9gOpNRrw4I", @@ -41,19 +48,25 @@ class DigestsTest { // found. "EsWoyIWuIcpMltIOJJAv, zYwfI-X_k4pk__DriohLNCpAHbU", }) - @ParameterizedTest(name = "sha1(\"{0}\") returns \"{1}\"") - void sha1ReturnsExpectedUnPaddedUrlSafeBase64String(String input, String expected) { + @ParameterizedTest(name = ".sha1(\"{0}\") returns \"{1}\"") + void sha1ReturnsExpectedSha1DigestInUnPaddedUrlSafeBase64String(String input, String expected) { // When var actual = Digests.sha1(input); // Then assertThat(actual).isEqualTo(expected); } - @DisplayName("sha1(String) raises an IllegalArgumentException on error") + @DisplayName(".sha1(String) raises an IllegalArgumentException if unsupported") @Test - void sha1RaisesIllegalArgumentExceptionOnError() { - // Then - assertThatThrownBy(() -> Digests.sha1(null)) - .isInstanceOf(IllegalArgumentException.class); + void sha1RaisesAnIllegalArgumentExceptionIfUnsupported() { + try (var mockMessageDigestCls = mockStatic(MessageDigest.class)) { + // Given + mockMessageDigestCls.when(() -> MessageDigest.getInstance(any())) + .thenThrow(new NoSuchAlgorithmException("that doesn't exist!")); + + // Then + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> Digests.sha1("foobar")); + } } }