diff --git a/.gitignore b/.gitignore index 646e97241..c0a592162 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ src/docs/.vuepress/dist/ jd-gui.cfg bin/ .vscode/ +/package-lock.json \ No newline at end of file diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 728779924..9e571db7a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,6 +4,7 @@ dependencies { shadow 'org.codehaus.groovy:groovy-backports-compat23:3.0.8' implementation 'org.jdom:jdom2:2.0.6.1' + implementation 'com.google.code.gson:gson:2.10.1' implementation 'org.ow2.asm:asm:9.4' implementation 'org.ow2.asm:asm-commons:9.4' implementation 'commons-io:commons-io:2.11.0' diff --git a/src/docs/changes/README.md b/src/docs/changes/README.md index 666cae4ca..42473e0e1 100644 --- a/src/docs/changes/README.md +++ b/src/docs/changes/README.md @@ -3,6 +3,7 @@ ## v8.1.1 (2023-03-20) **NOTE: ** As of this version, the Github repository has migrated to the `main` branch as the default branch for releases. +* Added collision logging from chapmajs. [Release Notes](https://github.com/johnrengelman/shadow/releases/tag/8.1.1) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 4c3f07142..ca7077fc5 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -5,8 +5,10 @@ import com.github.jengelman.gradle.plugins.shadow.impl.RelocatorRemapper import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator +import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext +import groovy.util.logging.Log import groovy.util.logging.Slf4j import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils @@ -15,6 +17,7 @@ import org.apache.tools.zip.Zip64RequiredException import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipFile import org.apache.tools.zip.ZipOutputStream +import org.codehaus.groovy.transform.LogASTTransformation import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException @@ -37,13 +40,18 @@ import org.objectweb.asm.ClassReader import org.objectweb.asm.ClassVisitor import org.objectweb.asm.ClassWriter import org.objectweb.asm.commons.ClassRemapper +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import javax.annotation.Nullable import java.util.zip.ZipException -@Slf4j + class ShadowCopyAction implements CopyAction { static final long CONSTANT_TIME_FOR_ZIP_ENTRIES = (new GregorianCalendar(1980, 1, 1, 0, 0, 0)).getTimeInMillis() + final static Logger log = LoggerFactory.getLogger(ShadowCopyAction.class); + private final File zipFile private final ZipCompressor compressor private final DocumentationRegistry documentationRegistry @@ -57,9 +65,9 @@ class ShadowCopyAction implements CopyAction { private final UnusedTracker unusedTracker ShadowCopyAction(File zipFile, ZipCompressor compressor, DocumentationRegistry documentationRegistry, - String encoding, List transformers, List relocators, - PatternSet patternSet, ShadowStats stats, - boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker) { + String encoding, List transformers, List relocators, + PatternSet patternSet, ShadowStats stats, + boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker) { this.zipFile = zipFile this.compressor = compressor @@ -148,7 +156,7 @@ class ShadowCopyAction implements CopyAction { private static void withResource(T resource, Action action) { try { action.execute(resource) - } catch(Throwable t) { + } catch (Throwable t) { try { resource.close() } catch (IOException ignored) { @@ -197,11 +205,11 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Set visitedFiles = new HashSet() + private Map visitedFiles = new HashMap<>() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, - List relocators, PatternSet patternSet, Set unused, - ShadowStats stats) { + List relocators, PatternSet patternSet, Set unused, + ShadowStats stats) { this.zipOutStr = zipOutStr this.transformers = transformers this.relocators = relocators @@ -209,13 +217,38 @@ class ShadowCopyAction implements CopyAction { this.patternSet = patternSet this.unused = unused this.stats = stats - if(encoding != null) { + if (encoding != null) { this.zipOutStr.setEncoding(encoding) } } - private boolean recordVisit(RelativePath path) { - return visitedFiles.add(path.pathString) + /** + * Record visit and return true if visited for the first time. + * + * @param path Visited path. + * @param size Size. + * @param originJar JAR it originated from. + * @return True if wasn't visited already. + */ + private boolean recordVisit(String path, long size, @Nullable RelativePath originJar) { + if (visitedFiles.containsKey(path)) { + return false + } + + if (originJar == null) { + originJar = new RelativePath(false) + } + + visitedFiles.put(path.toString(), [size: size, originJar: originJar]) + return true + } + + private boolean recordVisit(path) { + return recordVisit(path.toString(), 0, null) + } + + private boolean recordVisit(FileCopyDetails fileCopyDetails) { + return recordVisit(fileCopyDetails.relativePath.toString(), fileCopyDetails.size, null) } @Override @@ -238,7 +271,7 @@ class ShadowCopyAction implements CopyAction { } else if (isClass && !isUnused(fileDetails.path)) { remapClass(fileDetails) } - recordVisit(fileDetails.relativePath) + recordVisit(fileDetails) } catch (Exception e) { throw new GradleException(String.format("Could not add %s to ZIP '%s'.", fileDetails, zipFile), e) } @@ -260,7 +293,7 @@ class ShadowCopyAction implements CopyAction { } filteredArchiveElements.each { ArchiveFileTreeElement archiveElement -> if (archiveElement.relativePath.file) { - visitArchiveFile(archiveElement, archive) + visitArchiveFile(archiveElement, archive, fileDetails) } } } finally { @@ -270,21 +303,50 @@ class ShadowCopyAction implements CopyAction { } private void visitArchiveDirectory(RelativeArchivePath archiveDir) { - if (recordVisit(archiveDir)) { + if (recordVisit(archiveDir.toString())) { zipOutStr.putNextEntry(archiveDir.entry) zipOutStr.closeEntry() } } - private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive) { - def archiveFilePath = archiveFile.relativePath + private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive, FileCopyDetails fileDetails) { + RelativeArchivePath archiveFilePath = archiveFile.relativePath + long archiveFileSize = archiveFile.size + if (archiveFile.classFile || !isTransformable(archiveFile)) { - if (recordVisit(archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { + String path = archiveFilePath.toString() + if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { remapClass(archiveFilePath, archive) } + } else { + def archiveFileInVisitedFiles = visitedFiles.get(path) + if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { + // Give of only a debug-level warning for this file: + final String lowLevelWarningFile = "META-INF/MANIFEST.MF" + + final logDebug = (String msg) -> { log.debug(msg) } + final logWarn = (String msg) -> { log.warn(msg) } + + final Closure logger + if (archiveFilePath.toString() == lowLevelWarningFile) { + logger = logDebug + } else { + logger = logWarn + } + logger("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}," + + " size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") + if (archiveFileInVisitedFiles.originJar) { + logger("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") + } else { + logger("\t--> file originated from project sourcecode") + } + if (new StandardFilesMergeTransformer().canTransformResource(archiveFile)) { + logger("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) + } + } } } else { transform(archiveFile, archive) @@ -377,6 +439,12 @@ class ShadowCopyAction implements CopyAction { } } + /** + * Copy archive entry. + * + * @param archiveFile Source archive entry. + * @param archive Source archive. + */ private void copyArchiveEntry(RelativeArchivePath archiveFile, ZipFile archive) { String mappedPath = remapper.map(archiveFile.entry.name) ZipEntry entry = new ZipEntry(mappedPath) @@ -410,19 +478,20 @@ class ShadowCopyAction implements CopyAction { } private void transform(ArchiveFileTreeElement element, ZipFile archive) { - transformAndClose(element, archive.getInputStream(element.relativePath.entry)) + transformAndClose(element, archive, archive.getInputStream(element.relativePath.entry)) } private void transform(FileCopyDetails details) { - transformAndClose(details, details.file.newInputStream()) + transformAndClose(details, null, details.file.newInputStream()) } - private void transformAndClose(FileTreeElement element, InputStream is) { + private void transformAndClose(FileTreeElement element, @Nullable ZipFile archive, InputStream is) { try { String mappedPath = remapper.map(element.relativePath.pathString) transformers.find { it.canTransformResource(element) }.transform( TransformerContext.builder() .path(mappedPath) + .origin(archive) .is(is) .relocators(relocators) .stats(stats) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 067294854..043b929b0 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -1,11 +1,31 @@ package com.github.jengelman.gradle.plugins.shadow.tasks; +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; + +import javax.annotation.Nonnull; + import com.github.jengelman.gradle.plugins.shadow.ShadowStats; -import com.github.jengelman.gradle.plugins.shadow.internal.*; +import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.DependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.GradleVersionUtil; +import com.github.jengelman.gradle.plugins.shadow.internal.MinimizeDependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.RelocationUtil; +import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker; +import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor; import com.github.jengelman.gradle.plugins.shadow.relocation.CacheableRelocator; import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator; import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator; -import com.github.jengelman.gradle.plugins.shadow.transformers.*; +import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.CacheableTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.GroovyExtensionModuleTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.ServiceFileTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import org.gradle.api.Action; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DuplicatesStrategy; @@ -13,16 +33,20 @@ import org.gradle.api.internal.DocumentationRegistry; import org.gradle.api.internal.file.FileResolver; import org.gradle.api.internal.file.copy.CopyAction; -import org.gradle.api.tasks.*; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Nested; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.util.PatternSet; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; @CacheableTask public class ShadowJar extends Jar implements ShadowSpec { @@ -55,7 +79,17 @@ public ShadowJar() { dependencyFilter = new DefaultDependencyFilter(getProject()); dependencyFilterForMinimize = new MinimizeDependencyFilter(getProject()); setManifest(new DefaultInheritManifest(getProject(), getServices().get(FileResolver.class))); - transformers = new ArrayList<>(); + /* + Add as default the StandardFilesMergeTransformer, remove it with "removeDefaultTransformers()". + This is added by default, because otherwise: + a) In projects with many dependencies the user gets flooded with information about duplicated entries + like "META-INF/notice.txt", "META-INF/license.txt"... + b) Important licensing information written in META-INF/license.txt and other files may be lost. + c) Helpful information written in readme files may be lost. + d) The merging of plain text files is safe, there is no important logic to follow. Not like MANIFEST.MF, + property files, xml files, etc. Merged HTML may not look that good, but it works. + */ + transformers = new ArrayList<>(Collections.singletonList(new StandardFilesMergeTransformer())); relocators = new ArrayList<>(); configurations = new ArrayList<>(); @@ -100,10 +134,11 @@ public InheritManifest getManifest() { } @Override - @NotNull + @Nonnull protected CopyAction createCopyAction() { DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class); - final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), getSourceSetsClassesDirs().getFiles(), getToMinimize()) : null; + final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), + getSourceSetsClassesDirs().getFiles(), getToMinimize()) : null; return new ShadowCopyAction(getArchiveFile().get().getAsFile(), getInternalCompressor(), documentationRegistry, this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), shadowStats, isPreserveFileTimestamps(), minimizeJar, unusedTracker); @@ -119,7 +154,6 @@ FileCollection getToMinimize() { return toMinimize; } - @Classpath FileCollection getApiJars() { if (apiJars == null) { @@ -130,7 +164,6 @@ FileCollection getApiJars() { return apiJars; } - @InputFiles @PathSensitive(PathSensitivity.RELATIVE) FileCollection getSourceSetsClassesDirs() { @@ -207,12 +240,28 @@ public ShadowJar transform(Class clazz) throws Instantiat * @param c the configuration for the transformer * @return this */ - public ShadowJar transform(Class clazz, Action c) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar transform(Class clazz, Action c) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { T transformer = clazz.getDeclaredConstructor().newInstance(); addTransform(transformer, c); return this; } + /** + * Removes all default transformers. + *
Right now only {@link StandardFilesMergeTransformer} is added as default transformer, this method removes + * it. + * + * @return this + */ + public ShadowJar removeDefaultTransformers() { + final java.util.Optional standardFilesMergeTransformer = transformers.stream() // + .filter(StandardFilesMergeTransformer.class::isInstance) // + .findAny(); + standardFilesMergeTransformer.ifPresent(transformer -> transformers.remove(transformer)); + return this; + } + private boolean isCacheableTransform(Class clazz) { return clazz.isAnnotationPresent(CacheableTransformer.class); } @@ -348,7 +397,8 @@ public ShadowJar relocate(Relocator relocator) { * @param relocatorClass the relocator class to add. Must have a no-arg constructor. * @return this */ - public ShadowJar relocate(Class relocatorClass) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar relocate(Class relocatorClass) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { return relocate(relocatorClass, null); } @@ -367,7 +417,8 @@ private void addRelocator(R relocator, Action configure * @param configure the configuration for the relocator * @return this */ - public ShadowJar relocate(Class relocatorClass, Action configure) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar relocate(Class relocatorClass, Action configure) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { R relocator = relocatorClass.getDeclaredConstructor().newInstance(); addRelocator(relocator, configure); return this; diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy new file mode 100644 index 000000000..810a34988 --- /dev/null +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy @@ -0,0 +1,158 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import org.gradle.api.file.FileTreeElement +import org.gradle.api.logging.Logging +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Optional + +import com.google.gson.JsonArray +import com.google.gson.JsonNull +import com.google.gson.JsonObject +import com.google.gson.JsonElement +import com.google.gson.JsonPrimitive +import com.google.gson.JsonParser +import com.google.gson.Gson + +import org.apache.tools.zip.ZipOutputStream +import org.apache.tools.zip.ZipEntry + +/** + * Merge multiple occurrence of JSON files. + * + * @author Logic Fan, extended to process an array of files by Jan-Hendrik Diederich + */ +@CacheableTransformer +class JsonTransformer implements Transformer { + private static final GSON = new Gson() + private static final LOGGER = Logging.getLogger(JsonTransformer.class) + + @Optional + @Input + List paths + + private Map matchedPath = [:] + + @Override + boolean canTransformResource(FileTreeElement element) { + String path = element.relativePath.pathString + for (p in paths) { + if (path.equalsIgnoreCase(p)) { + matchedPath[path] = null + return true + } + } + return false + } + + @Override + void transform(TransformerContext context) { + String path = context.getPath() + final JsonElement j + try { + j = JsonParser.parseReader(new InputStreamReader(context.is, "UTF-8")) + } catch (Exception e) { + throw new RuntimeException("error on processing json", e) + } + + matchedPath[path] = (matchedPath[path] == null) ? j : mergeJson(matchedPath[path], j) + } + + @Override + boolean hasTransformedResource() { + return !matchedPath.isEmpty() + } + + @Override + void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) { + if (paths == null) { + throw new IllegalArgumentException("\"paths\" is null and not set") + } + for (Map.Entry entrySet in matchedPath) { + ZipEntry entry = new ZipEntry(entrySet.key) + entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time) + os.putNextEntry(entry) + os.write(GSON.toJson(entrySet.value).getBytes()) + } + matchedPath = [:] + } + + /** + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
{@code lhs} {@code rhs} {@code return}
Any {@code JsonNull} {@code lhs}
{@code JsonNull} Any {@code rhs}
{@code JsonArray} {@code JsonArray} concatenation
{@code JsonObject} {@code JsonObject} merge for each key
{@code JsonPrimitive} {@code JsonPrimitive}return lhs if {@code lhs.equals(rhs)}, error otherwise
Other error
+ * @param lhs a {@code JsonElement} + * @param rhs a {@code JsonElement} + * @param id used for logging purpose only + * @return the merged {@code JsonElement} + */ + private static JsonElement mergeJson(JsonElement lhs, JsonElement rhs, String id = "") { + if (rhs == null || rhs instanceof JsonNull) { + return lhs + } else if (lhs == null || lhs instanceof JsonNull) { + return rhs + } else if (lhs instanceof JsonArray && rhs instanceof JsonArray) { + return mergeJsonArray(lhs as JsonArray, rhs as JsonArray) + } else if (lhs instanceof JsonObject && rhs instanceof JsonObject) { + return mergeJsonObject(lhs as JsonObject, rhs as JsonObject, id) + } else if (lhs instanceof JsonPrimitive && rhs instanceof JsonPrimitive) { + return mergeJsonPrimitive(lhs as JsonPrimitive, rhs as JsonPrimitive, id) + } else { + LOGGER.warn("conflicts for property {} detected, {} & {}", + id, lhs.toString(), rhs.toString()) + return lhs + } + } + + private static JsonPrimitive mergeJsonPrimitive(JsonPrimitive lhs, JsonPrimitive rhs, String id) { + // In Groovy, {@code a == b} is equivalent to {@code a.equals(b)} + if (lhs != rhs) { + LOGGER.warn("conflicts for property {} detected, {} & {}", + id, lhs.toString(), rhs.toString()) + } + return lhs + } + + private static JsonObject mergeJsonObject(JsonObject lhs, JsonObject rhs, String id) { + JsonObject object = new JsonObject() + + Set properties = new HashSet<>() + properties.addAll(lhs.keySet()) + properties.addAll(rhs.keySet()) + for (String property : properties) { + object.add(property, + mergeJson(lhs.get(property), rhs.get(property), id + ":" + property)) + } + + return object + } + + private static JsonArray mergeJsonArray(JsonArray lhs, JsonArray rhs) { + JsonArray array = new JsonArray() + + array.addAll(lhs) + array.addAll(rhs) + + return array + } +} \ No newline at end of file diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy new file mode 100644 index 000000000..72c1639d2 --- /dev/null +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + + +import org.apache.commons.io.FilenameUtils +import org.apache.commons.io.IOUtils +import org.apache.tools.zip.ZipEntry +import org.apache.tools.zip.ZipOutputStream +import org.codehaus.plexus.util.IOUtil +import org.gradle.api.file.FileTreeElement + +/** + * Merges standard files, like "META-INF/license.txt", "META-INF/notice.txt", "readme.txt" into one, + * writing as prefix where the content comes from, so no license information or important hints gets lost. + * + * @author Jan-Hendrik Diederich + */ +@CacheableTransformer +class StandardFilesMergeTransformer implements Transformer { + private class StandardFile implements Serializable { + List origins = new ArrayList<>(); + String content; + + StandardFile(String origin, String content) { + this.origins.add(origin) + this.content = content + } + } + + private final List mergedFiles = [ + "META-INF/license", // + "META-INF/notice", // + "META-INF/readme", // + "readme", // + ] + + private final List fileExtensions = [ + "txt", "md", "htm", "html" + ] + + // Can't use normal HashMap, ...notice.txt and ...NOTICE.txt would otherwise be different entries. + private Map> fileEntries = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) + + @Override + boolean canTransformResource(FileTreeElement element) { + String path = element.relativePath.pathString + mergedFiles.stream() // + .anyMatch(mergeFile -> { + if (path.equalsIgnoreCase(mergeFile)) { + return true + } else { + for (extension in fileExtensions) { + if (path.equalsIgnoreCase(mergeFile + "." + extension)) { + return true + } + } + return false + } + }) + } + + @Override + void transform(TransformerContext context) { + List files = fileEntries.computeIfAbsent(context.path, key -> new ArrayList<>()) + + OutputStream outputStream = new ByteArrayOutputStream() + IOUtils.copyLarge(context.is, outputStream) + + def fileContent = outputStream.toString() + // Remove leading and trailing newlines. Don't trim whitespaces, so centered headers stay centered. + def trimmedFileContent = fileContent.replaceAll("^[\\r\\n]+|[\\r\\n]+\$", "") + + var standardFile = files.stream() // + .filter(entry -> trimmedFileContent.equalsIgnoreCase(entry.content)) // + .findAny() + String originName = context.origin != null + ? FilenameUtils.getName(context.origin.name) + : "Sourcecode" + if (standardFile.isPresent()) { + standardFile.get().origins.add(originName) + } else { + files.add(new StandardFile(originName, trimmedFileContent)) + } + } + + @Override + boolean hasTransformedResource() { + return fileEntries.size() > 0 + } + + @Override + void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) { + fileEntries.each { String path, List files -> + ZipEntry entry = new ZipEntry(path) + entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time) + os.putNextEntry(entry) + IOUtil.copy(toInputStream(files), os) + os.closeEntry() + } + } + + private static InputStream toInputStream(List entries) { + String joined = entries.stream() // + .map(entry -> "Origins: " + entry.origins.sort().join(", ") // + + "\n\n" + entry.content) // + .collect() // + .join("\n" + "=".repeat(80) + "\n") + new ByteArrayInputStream(joined.getBytes()) + } +} \ No newline at end of file diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy index b951dfa66..d5dfc956b 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy @@ -5,7 +5,9 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowCopyAction import groovy.transform.Canonical import groovy.transform.builder.Builder +import org.apache.tools.zip.ZipFile +import javax.annotation.Nullable @Canonical @Builder @@ -16,6 +18,9 @@ class TransformerContext { List relocators ShadowStats stats + @Nullable + ZipFile origin + static long getEntryTimestamp(boolean preserveFileTimestamps, long entryTime) { preserveFileTimestamps ? entryTime : ShadowCopyAction.CONSTANT_TIME_FOR_ZIP_ENTRIES } diff --git a/src/main/resources/shadow-version.txt b/src/main/resources/shadow-version.txt index 8104cabd3..fbb9ea12d 100644 --- a/src/main/resources/shadow-version.txt +++ b/src/main/resources/shadow-version.txt @@ -1 +1 @@ -8.1.0 +8.2.0 diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index c268ffd9a..b8e0a78ef 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -132,6 +132,27 @@ class ShadowPluginSpec extends PluginSpecification { assert output.exists() } + def 'warns when a file is masked by a previously shadowed resource'() { + given: + URL artifact = this.class.classLoader.getResource('test-artifact-1.0-SNAPSHOT.jar') + URL project = this.class.classLoader.getResource('test-project-1.0-SNAPSHOT.jar') + + buildFile << """ + shadowJar { + | destinationDirectory = buildDir + | archiveBaseName = 'shadow' + | from('${artifact.path}') + | from('${project.path}') + |} + """.stripMargin() + + when: + BuildResult result = run('shadowJar') + + then: + assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar, size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ + } + def 'include project sources'() { given: file('src/main/java/shadow/Passed.java') << ''' diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy new file mode 100644 index 000000000..f6d373af2 --- /dev/null +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + +import com.github.jengelman.gradle.plugins.shadow.ShadowStats +import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import org.apache.tools.zip.ZipOutputStream +import org.junit.Before +import org.junit.Test + +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipInputStream + +import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +/** + * Test for {@link JsonTransformer}. + * + * @author Jan-Hendrik Diederich + * + * Modified from com.github.jengelman.gradle.plugins.shadow.transformers.XmlAppendingTransformerTest.java + */ +class JsonAppendingTransformerTest extends TransformerTestSupport { + + JsonTransformer transformer + + static final String TEST_ARTIFACT_JAR = 'test-artifact-1.0-SNAPSHOT.jar' + static final String TEST_PROJECT_JAR = 'test-project-1.0-SNAPSHOT.jar' + + static final String TEST_JSON = 'test.json' + static final String TEST2_JSON = 'test2.json' + + @Before + void setUp() { + transformer = new JsonTransformer() + } + + @Test + void testCanTransformResource() { + transformer.paths = ["test.json"] + + assertTrue(this.transformer.canTransformResource(getFileElement("test.json"))) + assertFalse(this.transformer.canTransformResource(getFileElement("META-INF/MANIFEST.MF"))) + } + + @Test + void transformResource() { + transformer.transform(new TransformerContext(TEST_JSON, readFromTestJar(TEST_ARTIFACT_JAR, TEST_JSON), + Collections. emptyList(), new ShadowStats())) + transformer.transform(new TransformerContext(TEST2_JSON, readFromTestJar(TEST_ARTIFACT_JAR, TEST2_JSON), + Collections. emptyList(), new ShadowStats())) + + transformer.transform(new TransformerContext(TEST_JSON, readFromTestJar(TEST_PROJECT_JAR, TEST_JSON), + Collections. emptyList(), new ShadowStats())) + transformer.transform(new TransformerContext(TEST2_JSON, readFromTestJar(TEST_PROJECT_JAR, TEST2_JSON), + Collections. emptyList(), new ShadowStats())) + + def zipFileName = "testable-zip-file-" + def zipFileSuffix = ".jar" + def testableZipFile = File.createTempFile(zipFileName, zipFileSuffix) + def fileOutputStream = new FileOutputStream(testableZipFile) + def bufferedOutputStream = new BufferedOutputStream(fileOutputStream) + def zipOutputStream = new ZipOutputStream(bufferedOutputStream) + + transformer.paths = [TEST_JSON, TEST2_JSON] + try { + transformer.modifyOutputStream(zipOutputStream, false) + } finally { + zipOutputStream.close() + bufferedOutputStream.close() + fileOutputStream.close() + } + // Read 1st file. + String targetJson = readFromZipFile(testableZipFile.absolutePath, TEST_JSON) + println("Target JSON: \"" + targetJson + "\"") + + assertFalse(targetJson.isEmpty()) + assertTrue(targetJson.contains("\"C: Only here\"")) + + JsonElement jsonElement = JsonParser.parseString(targetJson) + JsonObject jsonObject = jsonElement.getAsJsonObject() + + JsonElement subAA = jsonObject.get("a.a") + + JsonElement subAA1 = subAA.getAsJsonObject().get("a.sub1") + assertEquals("A Sub 1", subAA1.asString) + + JsonElement subAA2 = subAA.getAsJsonObject().get("a.sub2") + assertEquals("A Sub 2", subAA2.asString) + + // Read 2nd file. + String target2Json = readFromZipFile(testableZipFile.absolutePath, TEST2_JSON) + assertFalse(target2Json.isEmpty()) + JsonElement jsonElement2 = JsonParser.parseString(target2Json) + JsonObject jsonObject2 = jsonElement2.getAsJsonObject() + JsonArray jsonArray2 = jsonObject2.get("Array").asJsonArray + assertEquals(List.of("A", "B", "C", "C", "D", "E"), + (List) jsonArray2.collect({ it -> it.getAsString() })) + } + + static InputStream readFromTestJar(String resourceName, String fileName) { + try (ZipInputStream inputStream = new ZipInputStream(getResourceStream(resourceName))) { + while (true) { + ZipEntry entry = inputStream.nextEntry + if (entry == null) { + break + } else if (entry.name == fileName) { + // Read the content of the entry + byte[] buffer = new byte[entry.size] + inputStream.read(buffer) + return new ByteArrayInputStream(buffer) + } + } + } + throw new IllegalArgumentException("Missing entry " + fileName) + } + + static String readFromZipFile(String resourceName, String fileName) { + def zip = new ZipFile(resourceName) + try { + ZipEntry entry = zip.getEntry(fileName) + if (!entry) { + throw new IllegalArgumentException("Missing entry " + fileName + " in " + resourceName) + } + return new String(zip.getInputStream(entry).readAllBytes()) + } finally { + zip.close() + } + } + + private static InputStream getResourceStream(String resource) { + JsonAppendingTransformerTest.class.classLoader.getResourceAsStream(resource) + } +} diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy new file mode 100644 index 000000000..3dba6bf28 --- /dev/null +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +/** + * Test for {@link StandardFilesMergeTransformer}. + * + * @author Benjamin Bentmann + * @version $Id: ApacheNoticeResourceTransformerTest.java 673906 2008-07-04 05:03:20Z brett $ + * + * Modified from org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformerTest.java + */ +class StandardFilesMergeTransformerTest extends TransformerTestSupport { + + private StandardFilesMergeTransformer transformer + + static { + /* + * NOTE: The Turkish locale has an usual case transformation for the letters "I" and "i", making it a prime + * choice to test for improper case-less string comparisons. + */ + Locale.setDefault(new Locale("tr")) + } + + @Before + void setUp() { + this.transformer = new StandardFilesMergeTransformer() + } + + @Test + void testCanTransformResource() { + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/NOTICE"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/NOTICE.TXT"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/Notice.txt"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/Notice.hTml"))) + assertFalse(this.transformer.canTransformResource(getFileElement("META-INF/MANIFEST.MF"))) + } +} \ No newline at end of file diff --git a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar index 009abad49..dbe60ba23 100644 Binary files a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar and b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar differ diff --git a/src/test/resources/test-project-1.0-SNAPSHOT.jar b/src/test/resources/test-project-1.0-SNAPSHOT.jar index f80e03f90..be69613a5 100644 Binary files a/src/test/resources/test-project-1.0-SNAPSHOT.jar and b/src/test/resources/test-project-1.0-SNAPSHOT.jar differ