From 8c7c1af0bce3dcc1b270bfe80c2bfe5d0e64b9a7 Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 2 Feb 2017 21:25:44 +0100 Subject: [PATCH] Implemented JAR minimization This commit adds two new configuration options to the 'shadowJar' task. When 'minimizeJar' is set the shadow JAR would only include the classes for the project the task operates on and its dependencies. The user can protect other classes from minimization by specifying them as entry points. Here's an example configuration shadowJar { minimizeJar = true entryPoint 'foo.Foo' entryPoint 'foo.Bar' } N.B. imported but unused classes are not considered as dependencies. --- gradle/dependencies.gradle | 1 + .../shadow/ShadowApplicationPlugin.groovy | 1 + .../shadow/internal/UnusedTracker.groovy | 56 +++++++++++ .../shadow/tasks/ShadowCopyAction.groovy | 98 ++++++++++++++----- .../plugins/shadow/tasks/ShadowJar.java | 22 ++++- .../plugins/shadow/tasks/ShadowSpec.java | 5 + .../plugins/shadow/ApplicationSpec.groovy | 12 +-- .../plugins/shadow/ShadowPluginSpec.groovy | 83 +++++++++++++--- .../shadow/util/PluginSpecification.groovy | 2 +- 9 files changed, 232 insertions(+), 48 deletions(-) create mode 100644 src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/UnusedTracker.groovy diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 50fdd7c68..e8b2255a6 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -10,6 +10,7 @@ dependencies { compile 'commons-io:commons-io:2.5' compile 'org.apache.ant:ant:1.9.7' compile 'org.codehaus.plexus:plexus-utils:3.0.24' + compile 'org.vafer:jdependency:1.1' testCompile gradleTestKit() testCompile("org.spockframework:spock-core:1.0-groovy-2.4") { diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowApplicationPlugin.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowApplicationPlugin.groovy index 6c1b2d79f..341a149dd 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowApplicationPlugin.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowApplicationPlugin.groovy @@ -47,6 +47,7 @@ class ShadowApplicationPlugin implements Plugin { jar.doFirst { manifest.attributes 'Main-Class': pluginConvention.mainClassName } + jar.entryPoint(pluginConvention.mainClassName) } protected void addRunTask(Project project) { diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/UnusedTracker.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/UnusedTracker.groovy new file mode 100644 index 000000000..e1dd983f3 --- /dev/null +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/UnusedTracker.groovy @@ -0,0 +1,56 @@ +package com.github.jengelman.gradle.plugins.shadow.internal + +import org.gradle.api.Project +import org.gradle.api.tasks.SourceSet +import org.vafer.jdependency.Clazz +import org.vafer.jdependency.Clazzpath +import org.vafer.jdependency.ClazzpathUnit + +/** Tracks unused classes in the project classpath. */ +class UnusedTracker { + private final List entryPoints + private final List projectUnits + private final Clazzpath cp = new Clazzpath() + + private UnusedTracker(List classDirs, List entryPoints) { + this.entryPoints = entryPoints + projectUnits = classDirs.collect { cp.addClazzpathUnit(it) } + } + + Set findUnused() { + Set unused = cp.clazzes + + for (cpu in projectUnits) { + unused.removeAll(cpu.clazzes) + unused.removeAll(cpu.transitiveDependencies) + } + + for (entryPoint in entryPoints) { + Clazz clazz = cp.getClazz(entryPoint) + if (clazz == null) { + throw new RuntimeException("Entry point not found: " + className); + } + + unused.remove(clazz) + unused.removeAll(clazz.transitiveDependencies) + } + + return unused.collect { it.name }.toSet() + } + + void addDependency(File jarOrDir) { + cp.addClazzpathUnit(jarOrDir) + } + + static UnusedTracker forProject(Project project, List entryPoints) { + final List classDirs = new ArrayList<>() + for (SourceSet sourceSet in project.sourceSets) { + File classDir = sourceSet.output.classesDir + if (classDir.isDirectory()) { + classDirs.add(classDir) + } + } + + new UnusedTracker(classDirs, entryPoints) + } +} diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 5cc327954..e9877dad6 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -2,6 +2,7 @@ package com.github.jengelman.gradle.plugins.shadow.tasks import com.github.jengelman.gradle.plugins.shadow.ShadowStats import com.github.jengelman.gradle.plugins.shadow.impl.RelocatorRemapper +import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer @@ -9,11 +10,7 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContex import groovy.util.logging.Slf4j import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils -import org.apache.tools.zip.UnixStat -import org.apache.tools.zip.Zip64RequiredException -import org.apache.tools.zip.ZipEntry -import org.apache.tools.zip.ZipFile -import org.apache.tools.zip.ZipOutputStream +import org.apache.tools.zip.* import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException @@ -49,10 +46,13 @@ public class ShadowCopyAction implements CopyAction { private final PatternSet patternSet private final ShadowStats stats private final String encoding + private final boolean minimizeJar + private final UnusedTracker unusedTracker public ShadowCopyAction(File zipFile, ZipCompressor compressor, DocumentationRegistry documentationRegistry, String encoding, List transformers, List relocators, - PatternSet patternSet, ShadowStats stats) { + PatternSet patternSet, boolean minimizeJar, + UnusedTracker unusedTracker, ShadowStats stats) { this.zipFile = zipFile this.compressor = compressor @@ -62,10 +62,30 @@ public class ShadowCopyAction implements CopyAction { this.patternSet = patternSet this.stats = stats this.encoding = encoding + this.minimizeJar = minimizeJar + this.unusedTracker = unusedTracker } @Override WorkResult execute(CopyActionProcessingStream stream) { + Set unusedClasses + if (minimizeJar) { + stream.process(new BaseStreamAction() { + @Override + void visitFile(FileCopyDetails fileDetails) { + // All project sources are already present, we just need + // to deal with JAR dependencies. + if (isArchive(fileDetails)) { + unusedTracker.addDependency(fileDetails.file) + } + } + }) + + unusedClasses = unusedTracker.findUnused() + } else { + unusedClasses = Collections.emptySet() + } + final ZipOutputStream zipOutStr try { @@ -79,7 +99,7 @@ public class ShadowCopyAction implements CopyAction { public void execute(ZipOutputStream outputStream) { try { stream.process(new StreamAction(outputStream, encoding, transformers, relocators, patternSet, - stats)) + unusedClasses, stats)) processTransformers(outputStream) } catch (Exception e) { log.error('ex', e) @@ -126,50 +146,65 @@ public class ShadowCopyAction implements CopyAction { } } - class StreamAction implements CopyActionProcessingStreamAction { + abstract class BaseStreamAction implements CopyActionProcessingStreamAction { + protected boolean isArchive(FileCopyDetails fileDetails) { + return fileDetails.relativePath.pathString.endsWith('.jar') + } + + protected boolean isClass(FileCopyDetails fileDetails) { + return FilenameUtils.getExtension(fileDetails.path) == 'class' + } + + @Override + void processFile(FileCopyDetailsInternal details) { + if (details.directory) { + visitDir(details) + } else { + visitFile(details) + } + } + + protected void visitDir(FileCopyDetails dirDetails) {} + + protected void visitFile(FileCopyDetails fileDetails) {} + } + + private class StreamAction extends BaseStreamAction { private final ZipOutputStream zipOutStr private final List transformers private final List relocators private final RelocatorRemapper remapper private final PatternSet patternSet + private final Set unused private final ShadowStats stats private Set visitedFiles = new HashSet() public StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, - List relocators, PatternSet patternSet, ShadowStats stats) { + List relocators, PatternSet patternSet, Set unused, + ShadowStats stats) { this.zipOutStr = zipOutStr this.transformers = transformers this.relocators = relocators this.remapper = new RelocatorRemapper(relocators, stats) this.patternSet = patternSet + this.unused = unused this.stats = stats if(encoding != null) { this.zipOutStr.setEncoding(encoding); } } - public void processFile(FileCopyDetailsInternal details) { - if (details.directory) { - visitDir(details) - } else { - visitFile(details) - } - } - - private boolean isArchive(FileCopyDetails fileDetails) { - return fileDetails.relativePath.pathString.endsWith('.jar') - } - private boolean recordVisit(RelativePath path) { return visitedFiles.add(path.pathString) } - private void visitFile(FileCopyDetails fileDetails) { + @Override + void visitFile(FileCopyDetails fileDetails) { if (!isArchive(fileDetails)) { try { - boolean isClass = (FilenameUtils.getExtension(fileDetails.path) == 'class') + boolean isClass = isClass(fileDetails) if (!remapper.hasRelocators() || !isClass) { if (!isTransformable(fileDetails)) { String mappedPath = remapper.map(fileDetails.relativePath.pathString) @@ -182,7 +217,7 @@ public class ShadowCopyAction implements CopyAction { } else { transform(fileDetails) } - } else if (isClass) { + } else if (isClass && !isUnused(fileDetails.path)) { remapClass(fileDetails) } recordVisit(fileDetails.relativePath) @@ -223,7 +258,7 @@ public class ShadowCopyAction implements CopyAction { private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive) { def archiveFilePath = archiveFile.relativePath if (archiveFile.classFile || !isTransformable(archiveFile)) { - if (recordVisit(archiveFilePath)) { + if (recordVisit(archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { @@ -244,6 +279,16 @@ public class ShadowCopyAction implements CopyAction { } } + private boolean isUnused(String classPath) { + final String className = FilenameUtils.removeExtension(classPath) + .replace(File.separatorChar, '.' as char) + final boolean result = unused.contains(className) + if (result) { + log.info("Dropping unused class: $className") + } + return result + } + private void remapClass(RelativeArchivePath file, ZipFile archive) { if (file.classFile) { addParentDirectories(new RelativeArchivePath(new ZipEntry(remapper.mapPath(file) + '.class'), null)) @@ -304,7 +349,8 @@ public class ShadowCopyAction implements CopyAction { zipOutStr.closeEntry() } - private void visitDir(FileCopyDetails dirDetails) { + @Override + protected void visitDir(FileCopyDetails dirDetails) { try { // Trailing slash in name indicates that entry is a directory String path = dirDetails.relativePath.pathString + '/' diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 7fe30275e..b696967e3 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -31,6 +31,8 @@ public class ShadowJar extends Jar implements ShadowSpec { private List relocators; private List configurations; private DependencyFilter dependencyFilter; + private boolean minimizeJar = false; + private List entryPoints = new ArrayList<>(1); private final ShadowStats shadowStats = new ShadowStats(); private final GradleVersionUtil versionUtil; @@ -45,6 +47,22 @@ public ShadowJar() { configurations = new ArrayList(); } + @Override + public void setMinimizeJar(boolean enabled) { + this.minimizeJar = enabled; + } + + @Override + public boolean isMinimizeJar() { + return minimizeJar; + } + + @Override + public ShadowSpec entryPoint(String className) { + entryPoints.add(className); + return this; + } + @Override public ShadowStats getStats() { return shadowStats; @@ -58,8 +76,10 @@ public InheritManifest getManifest() { @Override protected CopyAction createCopyAction() { DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class); + final UnusedTracker unusedTracker = UnusedTracker.forProject(getProject(), entryPoints); return new ShadowCopyAction(getArchivePath(), getInternalCompressor(), documentationRegistry, - this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), shadowStats); + this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), + minimizeJar, unusedTracker, shadowStats); } protected ZipCompressor getInternalCompressor() { diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java index c4aa001b9..8e5ba582d 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java @@ -10,6 +10,11 @@ import org.gradle.api.file.CopySpec; interface ShadowSpec extends CopySpec { + void setMinimizeJar(boolean enabled); + + boolean isMinimizeJar(); + + ShadowSpec entryPoint(String className); // or reuse include? ShadowSpec dependencies(Action configure); diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ApplicationSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ApplicationSpec.groovy index 3975df16c..e095c571a 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ApplicationSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ApplicationSpec.groovy @@ -39,11 +39,11 @@ class ApplicationSpec extends PluginSpecification { apply plugin: 'application' mainClassName = 'myapp.Main' - + dependencies { compile 'shadow:a:1.0' } - + runShadow { args 'foo' } @@ -100,11 +100,11 @@ class ApplicationSpec extends PluginSpecification { apply plugin: 'application' mainClassName = 'myapp.Main' - + dependencies { shadow 'shadow:a:1.0' } - + runShadow { args 'foo' } @@ -150,11 +150,11 @@ class ApplicationSpec extends PluginSpecification { apply plugin: 'application' mainClassName = 'myapp.Main' - + dependencies { compile 'shadow:a:1.0' } - + runShadow { args 'foo' } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index 9076ea6ec..73c57ab5e 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -45,6 +45,7 @@ class ShadowPluginSpec extends PluginSpecification { assert shadow.version == version assert shadow.classifier == 'all' assert shadow.extension == 'jar' + assert !shadow.minimizeJar and: Configuration shadowConfig = project.configurations.findByName('shadow') @@ -76,7 +77,7 @@ class ShadowPluginSpec extends PluginSpecification { compile 'junit:junit:3.8.2' compile files('${escapedPath(one)}') } - + shadowJar { mergeServiceFiles() } @@ -144,7 +145,7 @@ class ShadowPluginSpec extends PluginSpecification { buildFile << """ dependencies { compile 'junit:junit:3.8.2' } - + // tag::rename[] shadowJar { baseName = 'shadow' @@ -183,16 +184,16 @@ class ShadowPluginSpec extends PluginSpecification { file('server/src/main/java/server/Server.java') << """ package server; - + import client.Client; - + public class Server {} """.stripIndent() file('server/build.gradle') << """ apply plugin: 'java' apply plugin: 'com.github.johnrengelman.shadow' - + repositories { maven { url "${repo.uri}" } } dependencies { compile project(':client') } @@ -211,6 +212,60 @@ class ShadowPluginSpec extends PluginSpecification { ]) } + def 'include minimized project dependencies'() { + given: + file('settings.gradle') << """ + include 'client', 'server' + """.stripIndent() + + file('client/src/main/java/client/Client.java') << """ + package client; + public class Client {} + """.stripIndent() + + file('client/build.gradle') << """ + apply plugin: 'java' + repositories { maven { url "${repo.uri}" } } + dependencies { compile 'junit:junit:3.8.2' } + """.stripIndent() + + file('server/src/main/java/server/Server.java') << """ + package server; + + import client.Client; + + public class Server { + private final String client = Client.class.getName(); + } + """.stripIndent() + + file('server/build.gradle') << """ + apply plugin: 'java' + apply plugin: 'com.github.johnrengelman.shadow' + + shadowJar { + minimizeJar = true + entryPoint 'server.Server' + } + + repositories { maven { url "${repo.uri}" } } + dependencies { compile project(':client') } + + """.stripIndent() + + File serverOutput = file('server/build/libs/server-all.jar') + + when: + runner.withArguments(':server:shadowJar').withDebug(true).build() + + then: + contains(serverOutput, [ + 'client/Client.class', + 'server/Server.class' + ]) + doesNotContain(serverOutput, ['junit/framework/Test.class']) + } + def 'depend on project shadow jar'() { given: file('settings.gradle') << """ @@ -227,7 +282,7 @@ class ShadowPluginSpec extends PluginSpecification { apply plugin: 'com.github.johnrengelman.shadow' repositories { maven { url "${repo.uri}" } } dependencies { compile 'junit:junit:3.8.2' } - + shadowJar { relocate 'junit.framework', 'client.junit.framework' } @@ -235,16 +290,16 @@ class ShadowPluginSpec extends PluginSpecification { file('server/src/main/java/server/Server.java') << """ package server; - + import client.Client; import client.junit.framework.Test; - + public class Server {} """.stripIndent() file('server/build.gradle') << """ apply plugin: 'java' - + repositories { maven { url "${repo.uri}" } } dependencies { compile project(path: ':client', configuration: 'shadow') } """.stripIndent() @@ -283,7 +338,7 @@ class ShadowPluginSpec extends PluginSpecification { apply plugin: 'com.github.johnrengelman.shadow' repositories { maven { url "${repo.uri}" } } dependencies { compile 'junit:junit:3.8.2' } - + shadowJar { relocate 'junit.framework', 'client.junit.framework' } @@ -291,17 +346,17 @@ class ShadowPluginSpec extends PluginSpecification { file('server/src/main/java/server/Server.java') << """ package server; - + import client.Client; import client.junit.framework.Test; - + public class Server {} """.stripIndent() file('server/build.gradle') << """ apply plugin: 'java' apply plugin: 'com.github.johnrengelman.shadow' - + repositories { maven { url "${repo.uri}" } } dependencies { compile project(path: ':client', configuration: 'shadow') } """.stripIndent() @@ -462,7 +517,7 @@ class ShadowPluginSpec extends PluginSpecification { shadow 'junit:junit:3.8.2' } // end::shadowConfig[] - + // tag::jarManifest[] jar { manifest { diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy index a51b54654..9f485a515 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy @@ -56,7 +56,7 @@ class PluginSpecification extends Specification { GradleRunner.create() .withProjectDir(dir.root) .forwardOutput() -// .withDebug(true) + .withDebug(true) .withTestKitDir(getTestKitDir()) }