[RFC] Implemented JAR minimization

gradle/dependencies.gradle

@@ -10,6 +10,7 @@ dependencies {
     compile 'commons-io:commons-io:2.5'
     compile 'org.apache.ant:ant:1.9.7'
     compile 'org.codehaus.plexus:plexus-utils:3.0.24'
+    compile 'org.vafer:jdependency:1.1'
 
     testCompile gradleTestKit()
     testCompile("org.spockframework:spock-core:1.0-groovy-2.4") {

src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowApplicationPlugin.groovy

@@ -47,6 +47,7 @@ class ShadowApplicationPlugin implements Plugin<Project> {
         jar.doFirst {
             manifest.attributes 'Main-Class': pluginConvention.mainClassName
         }
+        jar.entryPoint(pluginConvention.mainClassName)
     }
 
     protected void addRunTask(Project project) {

src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/UnusedTracker.groovy

@@ -0,0 +1,56 @@
+package com.github.jengelman.gradle.plugins.shadow.internal
+
+import org.gradle.api.Project
+import org.gradle.api.tasks.SourceSet
+import org.vafer.jdependency.Clazz
+import org.vafer.jdependency.Clazzpath
+import org.vafer.jdependency.ClazzpathUnit
+
+/** Tracks unused classes in the project classpath. */
+class UnusedTracker {
+    private final List<String> entryPoints
+    private final List<ClazzpathUnit> projectUnits
+    private final Clazzpath cp = new Clazzpath()
+
+    private UnusedTracker(List<File> classDirs, List<String> entryPoints) {
+        this.entryPoints = entryPoints
+        projectUnits = classDirs.collect { cp.addClazzpathUnit(it) }
+    }
+
+    Set<String> findUnused() {
+        Set<Clazz> unused = cp.clazzes
+
+        for (cpu in projectUnits) {
+            unused.removeAll(cpu.clazzes)
+            unused.removeAll(cpu.transitiveDependencies)
+        }
+
+        for (entryPoint in entryPoints) {
+            Clazz clazz = cp.getClazz(entryPoint)
+            if (clazz == null) {
+                throw new RuntimeException("Entry point not found: " + className);
+            }
+
+            unused.remove(clazz)
+            unused.removeAll(clazz.transitiveDependencies)
+        }
+
+        return unused.collect { it.name }.toSet()
+    }
+
+    void addDependency(File jarOrDir) {
+        cp.addClazzpathUnit(jarOrDir)
+    }
+
+    static UnusedTracker forProject(Project project, List<String> entryPoints) {
+        final List<File> classDirs = new ArrayList<>()
+        for (SourceSet sourceSet in project.sourceSets) {
+            File classDir = sourceSet.output.classesDir
+            if (classDir.isDirectory()) {
+                classDirs.add(classDir)
+            }
+        }
+
+        new UnusedTracker(classDirs, entryPoints)
+    }
+}

src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy

@@ -2,18 +2,15 @@ package com.github.jengelman.gradle.plugins.shadow.tasks
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowStats
 import com.github.jengelman.gradle.plugins.shadow.impl.RelocatorRemapper
+import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker
 import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor
 import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator
 import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
 import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
 import groovy.util.logging.Slf4j
 import org.apache.commons.io.FilenameUtils
 import org.apache.commons.io.IOUtils
-import org.apache.tools.zip.UnixStat
-import org.apache.tools.zip.Zip64RequiredException
-import org.apache.tools.zip.ZipEntry
-import org.apache.tools.zip.ZipFile
-import org.apache.tools.zip.ZipOutputStream
+import org.apache.tools.zip.*
 import org.gradle.api.Action
 import org.gradle.api.GradleException
 import org.gradle.api.UncheckedIOException
@@ -49,10 +46,13 @@ public class ShadowCopyAction implements CopyAction {
     private final PatternSet patternSet
     private final ShadowStats stats
     private final String encoding
+    private final boolean minimizeJar
+    private final UnusedTracker unusedTracker
 
     public ShadowCopyAction(File zipFile, ZipCompressor compressor, DocumentationRegistry documentationRegistry,
                             String encoding, List<Transformer> transformers, List<Relocator> relocators,
-                            PatternSet patternSet, ShadowStats stats) {
+                            PatternSet patternSet, boolean minimizeJar,
+                            UnusedTracker unusedTracker, ShadowStats stats) {
 
         this.zipFile = zipFile
         this.compressor = compressor
@@ -62,10 +62,30 @@ public class ShadowCopyAction implements CopyAction {
         this.patternSet = patternSet
         this.stats = stats
         this.encoding = encoding
+        this.minimizeJar = minimizeJar
+        this.unusedTracker = unusedTracker
     }
 
     @Override
     WorkResult execute(CopyActionProcessingStream stream) {
+        Set<String> unusedClasses
+        if (minimizeJar) {
+            stream.process(new BaseStreamAction() {
+                @Override
+                void visitFile(FileCopyDetails fileDetails) {
+                    // All project sources are already present, we just need
+                    // to deal with JAR dependencies.
+                    if (isArchive(fileDetails)) {
+                        unusedTracker.addDependency(fileDetails.file)
+                    }
+                }
+            })
+
+            unusedClasses = unusedTracker.findUnused()
+        } else {
+            unusedClasses = Collections.emptySet()
+        }
+
         final ZipOutputStream zipOutStr
 
         try {
@@ -79,7 +99,7 @@ public class ShadowCopyAction implements CopyAction {
                 public void execute(ZipOutputStream outputStream) {
                     try {
                         stream.process(new StreamAction(outputStream, encoding, transformers, relocators, patternSet,
-                                stats))
+                                unusedClasses, stats))
                         processTransformers(outputStream)
                     } catch (Exception e) {
                         log.error('ex', e)
@@ -126,50 +146,65 @@ public class ShadowCopyAction implements CopyAction {
         }
     }
 
-    class StreamAction implements CopyActionProcessingStreamAction {
+    abstract class BaseStreamAction implements CopyActionProcessingStreamAction {
+        protected boolean isArchive(FileCopyDetails fileDetails) {
+            return fileDetails.relativePath.pathString.endsWith('.jar')
+        }
+
+        protected boolean isClass(FileCopyDetails fileDetails) {
+            return FilenameUtils.getExtension(fileDetails.path) == 'class'
+        }
+
+        @Override
+        void processFile(FileCopyDetailsInternal details) {
+            if (details.directory) {
+                visitDir(details)
+            } else {
+                visitFile(details)
+            }
+        }
+
+        protected void visitDir(FileCopyDetails dirDetails) {}
+
+        protected void visitFile(FileCopyDetails fileDetails) {}
+    }
+
+    private class StreamAction extends BaseStreamAction {
 
         private final ZipOutputStream zipOutStr
         private final List<Transformer> transformers
         private final List<Relocator> relocators
         private final RelocatorRemapper remapper
         private final PatternSet patternSet
+        private final Set<String> unused
         private final ShadowStats stats
 
         private Set<String> visitedFiles = new HashSet<String>()
 
         public StreamAction(ZipOutputStream zipOutStr, String encoding, List<Transformer> transformers,
-                            List<Relocator> relocators, PatternSet patternSet, ShadowStats stats) {
+                            List<Relocator> relocators, PatternSet patternSet, Set<String> unused,
+                            ShadowStats stats) {
             this.zipOutStr = zipOutStr
             this.transformers = transformers
             this.relocators = relocators
             this.remapper = new RelocatorRemapper(relocators, stats)
             this.patternSet = patternSet
+            this.unused = unused
             this.stats = stats
             if(encoding != null) {
                 this.zipOutStr.setEncoding(encoding);
             }
         }
 
-        public void processFile(FileCopyDetailsInternal details) {
-            if (details.directory) {
-                visitDir(details)
-            } else {
-                visitFile(details)
-            }
-        }
-
-        private boolean isArchive(FileCopyDetails fileDetails) {
-            return fileDetails.relativePath.pathString.endsWith('.jar')
-        }
-
         private boolean recordVisit(RelativePath path) {
             return visitedFiles.add(path.pathString)
         }
 
-        private void visitFile(FileCopyDetails fileDetails) {
+        @Override
+        void visitFile(FileCopyDetails fileDetails) {
             if (!isArchive(fileDetails)) {
                 try {
-                    boolean isClass = (FilenameUtils.getExtension(fileDetails.path) == 'class')
+                    boolean isClass = isClass(fileDetails)
                     if (!remapper.hasRelocators() || !isClass) {
                         if (!isTransformable(fileDetails)) {
                             String mappedPath = remapper.map(fileDetails.relativePath.pathString)
@@ -182,7 +217,7 @@ public class ShadowCopyAction implements CopyAction {
                         } else {
                             transform(fileDetails)
                         }
-                    } else if (isClass) {
+                    } else if (isClass && !isUnused(fileDetails.path)) {
                         remapClass(fileDetails)
                     }
                     recordVisit(fileDetails.relativePath)
@@ -223,7 +258,7 @@ public class ShadowCopyAction implements CopyAction {
         private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive) {
             def archiveFilePath = archiveFile.relativePath
             if (archiveFile.classFile || !isTransformable(archiveFile)) {
-                if (recordVisit(archiveFilePath)) {
+                if (recordVisit(archiveFilePath) && !isUnused(archiveFilePath.entry.name)) {
                     if (!remapper.hasRelocators() || !archiveFile.classFile) {
                         copyArchiveEntry(archiveFilePath, archive)
                     } else {
@@ -244,6 +279,16 @@ public class ShadowCopyAction implements CopyAction {
             }
         }
 
+        private boolean isUnused(String classPath) {
+            final String className = FilenameUtils.removeExtension(classPath)
+                    .replace(File.separatorChar, '.' as char)
+            final boolean result = unused.contains(className)
+            if (result) {
+                log.info("Dropping unused class: $className")
+            }
+            return result
+        }
+
         private void remapClass(RelativeArchivePath file, ZipFile archive) {
             if (file.classFile) {
                 addParentDirectories(new RelativeArchivePath(new ZipEntry(remapper.mapPath(file) + '.class'), null))
@@ -304,7 +349,8 @@ public class ShadowCopyAction implements CopyAction {
             zipOutStr.closeEntry()
         }
 
-        private void visitDir(FileCopyDetails dirDetails) {
+        @Override
+        protected void visitDir(FileCopyDetails dirDetails) {
             try {
                 // Trailing slash in name indicates that entry is a directory
                 String path = dirDetails.relativePath.pathString + '/'

src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java

@@ -31,6 +31,8 @@ public class ShadowJar extends Jar implements ShadowSpec {
     private List<Relocator> relocators;
     private List<Configuration> configurations;
     private DependencyFilter dependencyFilter;
+    private boolean minimizeJar = false;
+    private List<String> entryPoints = new ArrayList<>(1);
 
     private final ShadowStats shadowStats = new ShadowStats();
     private final GradleVersionUtil versionUtil;
@@ -45,6 +47,22 @@ public ShadowJar() {
         configurations = new ArrayList<Configuration>();
     }
 
+    @Override
+    public void setMinimizeJar(boolean enabled) {
+        this.minimizeJar = enabled;
+    }
+
+    @Override
+    public boolean isMinimizeJar() {
+        return minimizeJar;
+    }
+
+    @Override
+    public ShadowSpec entryPoint(String className) {
+        entryPoints.add(className);
+        return this;
+    }
+
     @Override
     public ShadowStats getStats() {
         return shadowStats;
@@ -58,8 +76,10 @@ public InheritManifest getManifest() {
     @Override
     protected CopyAction createCopyAction() {
         DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class);
+        final UnusedTracker unusedTracker = UnusedTracker.forProject(getProject(), entryPoints);
         return new ShadowCopyAction(getArchivePath(), getInternalCompressor(), documentationRegistry,
-                this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), shadowStats);
+                this.getMetadataCharset(), transformers, relocators, getRootPatternSet(),
+                minimizeJar, unusedTracker, shadowStats);
     }
 
     protected ZipCompressor getInternalCompressor() {

src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java

@@ -10,6 +10,11 @@
 import org.gradle.api.file.CopySpec;
 
 interface ShadowSpec extends CopySpec {
+    void setMinimizeJar(boolean enabled);
+
+    boolean isMinimizeJar();
+
+    ShadowSpec entryPoint(String className);  // or reuse include?
 
     ShadowSpec dependencies(Action<DependencyFilter> configure);
 

src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ApplicationSpec.groovy

@@ -39,11 +39,11 @@ class ApplicationSpec extends PluginSpecification {
             apply plugin: 'application'
 
             mainClassName = 'myapp.Main'
-            
+
             dependencies {
                compile 'shadow:a:1.0'
             }
-            
+
             runShadow {
                args 'foo'
             }
@@ -100,11 +100,11 @@ class ApplicationSpec extends PluginSpecification {
             apply plugin: 'application'
 
             mainClassName = 'myapp.Main'
-            
+
             dependencies {
                shadow 'shadow:a:1.0'
             }
-            
+
             runShadow {
                args 'foo'
             }
@@ -150,11 +150,11 @@ class ApplicationSpec extends PluginSpecification {
             apply plugin: 'application'
 
             mainClassName = 'myapp.Main'
-            
+
             dependencies {
                compile 'shadow:a:1.0'
             }
-            
+
             runShadow {
                args 'foo'
             }