-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Shared Caching for NeoForm task outputs
- Loading branch information
Showing
25 changed files
with
892 additions
and
226 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
42 changes: 42 additions & 0 deletions
42
common/src/main/java/net/neoforged/gradle/common/caching/CacheKey.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package net.neoforged.gradle.common.caching; | ||
|
|
||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
|
|
||
| public final class CacheKey { | ||
| private static final String CACHE_DOMAIN_ALL = "all"; | ||
| @Nullable | ||
| private final String cacheDomain; | ||
| private final String hashCode; | ||
| private final String sourceMaterial; | ||
|
|
||
| CacheKey(@Nullable String cacheDomain, String hashCode, String sourceMaterial) { | ||
| this.cacheDomain = cacheDomain; | ||
| this.hashCode = hashCode; | ||
| this.sourceMaterial = sourceMaterial; | ||
| } | ||
|
|
||
| @Nullable | ||
| String getCacheDomain() { | ||
| return cacheDomain; | ||
| } | ||
|
|
||
| String getHashCode() { | ||
| return hashCode; | ||
| } | ||
|
|
||
| String getSourceMaterial() { | ||
| return sourceMaterial; | ||
| } | ||
|
|
||
| Path asPath(@Nullable String extension) { | ||
| String filename = hashCode; | ||
| if (extension != null) { | ||
| filename += "." + extension; | ||
| } | ||
|
|
||
| return Paths.get(cacheDomain != null ? cacheDomain : CACHE_DOMAIN_ALL, filename); | ||
| } | ||
| } |
65 changes: 65 additions & 0 deletions
65
common/src/main/java/net/neoforged/gradle/common/caching/FileHashing.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package net.neoforged.gradle.common.caching; | ||
|
|
||
| import net.neoforged.gradle.util.HashFunction; | ||
| import org.apache.commons.io.output.NullOutputStream; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.UncheckedIOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.attribute.BasicFileAttributeView; | ||
| import java.nio.file.attribute.BasicFileAttributes; | ||
| import java.security.DigestOutputStream; | ||
| import java.security.MessageDigest; | ||
| import java.util.Map; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
|
|
||
| /** | ||
| * Provides in-memory caching of file-hashes based on last-modification time and size. | ||
| */ | ||
| class FileHashing { | ||
| private final Map<Path, CachedHash> cachedHashes = new ConcurrentHashMap<>(); | ||
|
|
||
| public byte[] getMd5Hash(Path path) { | ||
| return cachedHashes.compute(path, CachedHash::compute).hashValue; | ||
| } | ||
|
|
||
| private static final class CachedHash { | ||
| private final long lastModified; | ||
| private final long fileSize; | ||
| private final byte[] hashValue; | ||
|
|
||
| public static CachedHash compute(Path path, @Nullable CachedHash cachedHash) { | ||
| try { | ||
| // Instead of reading size + last modified separately, we use this function to make race conditions | ||
| // less likely. We still don't know if the underlying OS APIs return this information atomically, | ||
| // but if they do, we at least make use of that fact. | ||
| BasicFileAttributes attributes = Files.getFileAttributeView(path, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| long lastModified = attributes.lastModifiedTime().toMillis(); | ||
| long fileSize = attributes.size(); | ||
|
|
||
| if (cachedHash != null && cachedHash.lastModified == lastModified && cachedHash.fileSize == fileSize) { | ||
| return cachedHash; | ||
| } | ||
|
|
||
| // Compute the digest in a streaming fashion without reading the full file into memory | ||
| MessageDigest digest = HashFunction.MD5.get(); | ||
| try (DigestOutputStream out = new DigestOutputStream(NullOutputStream.NULL_OUTPUT_STREAM, digest)) { | ||
| Files.copy(path, out); | ||
| } | ||
|
|
||
| return new CachedHash(digest.digest(), lastModified, fileSize); | ||
| } catch (IOException e) { | ||
| throw new UncheckedIOException(e); | ||
| } | ||
| } | ||
|
|
||
| public CachedHash(byte[] hashValue, long lastModified, long fileSize) { | ||
| this.hashValue = hashValue; | ||
| this.lastModified = lastModified; | ||
| this.fileSize = fileSize; | ||
| } | ||
| } | ||
| } |
48 changes: 48 additions & 0 deletions
48
common/src/main/java/net/neoforged/gradle/common/caching/HashCodeBuilder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package net.neoforged.gradle.common.caching; | ||
|
|
||
| import org.apache.commons.codec.binary.Hex; | ||
|
|
||
| import java.nio.charset.StandardCharsets; | ||
| import java.nio.file.Path; | ||
| import java.security.MessageDigest; | ||
| import java.security.NoSuchAlgorithmException; | ||
|
|
||
| final class HashCodeBuilder { | ||
| private final FileHashing fileHashing; | ||
| private final StringBuilder sourceMaterial = new StringBuilder(); | ||
| private final MessageDigest digest; | ||
|
|
||
| public HashCodeBuilder(FileHashing fileHashing) { | ||
| this.fileHashing = fileHashing; | ||
| // Relativize any path to gradle home or project root, | ||
| // which will work for anything but maven local dependencies | ||
| try { | ||
| digest = MessageDigest.getInstance("MD5"); | ||
| } catch (NoSuchAlgorithmException e) { | ||
| throw new RuntimeException("Standard algorithm MD5 is missing.", e); | ||
| } | ||
| } | ||
|
|
||
| public void add(Path path) { | ||
| byte[] fileHash = fileHashing.getMd5Hash(path); | ||
| String fileHashHex = Hex.encodeHexString(fileHash); | ||
| add(fileHash, "HASHED-CONTENT(" + path + ") = " + fileHashHex); | ||
| } | ||
|
|
||
| public void add(String data) { | ||
| add(data.getBytes(StandardCharsets.UTF_8), "STRING(" + data + ")"); | ||
| } | ||
|
|
||
| public void add(byte[] data, String sourceMaterial) { | ||
| digest.update(data); | ||
| this.sourceMaterial.append(sourceMaterial).append('\n'); | ||
| } | ||
|
|
||
| public String buildHashCode() { | ||
| return Hex.encodeHexString(digest.digest()); | ||
| } | ||
|
|
||
| public String buildSourceMaterial() { | ||
| return sourceMaterial.toString(); | ||
| } | ||
| } |
Oops, something went wrong.