From 7fca241222c812de8134b40c97f6c1b31556ad48 Mon Sep 17 00:00:00 2001 From: Pratham Gahlout Date: Mon, 18 Mar 2024 13:20:00 +0530 Subject: [PATCH] License detection now shows 3 most probable licenses with match perctng (#37) --- .gitignore | 1 + .../licensedetector/LicenseDetector.java | 26 ++++-- .../interfaces/LicenseDetectorInterface.java | 7 +- .../utils/models/LicenseDetectionResult.java | 85 +++++++++++++++++++ .../phsyberdome/plugin/cargo/PluginCargo.java | 8 +- .../phsyberdome/plugin/maven/PluginMaven.java | 6 +- plugin.npm/pom.xml | 18 ++++ .../plugin/npm/NPMVersionHelper.java | 2 +- .../plugin/npm/NPMVersionHelperV2.java | 10 ++- .../com/phsyberdome/plugin/npm/PluginNpm.java | 21 ++--- 10 files changed, 155 insertions(+), 29 deletions(-) create mode 100644 common-utils/src/main/java/com/phsyberdome/common/utils/models/LicenseDetectionResult.java diff --git a/.gitignore b/.gitignore index b4fe7dc..1adfd55 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ jshell.history nbactions.xml nb-configuration.xml /plugin.cargo/nbproject/ +/plugin.npm/nbproject/ diff --git a/LicenseDetector/src/main/java/com/phsyberdome/licensedetector/LicenseDetector.java b/LicenseDetector/src/main/java/com/phsyberdome/licensedetector/LicenseDetector.java index 1179e83..255431a 100644 --- a/LicenseDetector/src/main/java/com/phsyberdome/licensedetector/LicenseDetector.java +++ b/LicenseDetector/src/main/java/com/phsyberdome/licensedetector/LicenseDetector.java @@ -6,6 +6,7 @@ import com.phsyberdome.common.utils.CLIHelper; import com.phsyberdome.common.utils.models.Pair; import com.phsyberdome.common.utils.FileUtil; +import com.phsyberdome.common.utils.models.LicenseDetectionResult; import com.phsyberdome.common.utils.NetworkHelper; import java.io.File; import java.nio.file.Path; @@ -96,13 +97,13 @@ private Map analyze(String licenseContent) { return queryResult; } - public Pair detect(String path){ + public LicenseDetectionResult detect(String path){ // Check if it is valid url if(NetworkHelper.isValidURL(path)){ Path rootPath = FileUtil.getFilePathFromURL(path,cloneLocation.toString()); - Pair result = filePathDetection(rootPath.toString()); + LicenseDetectionResult result = filePathDetection(rootPath.toString()); if(rootPath.toFile().exists()){ FileUtil.deleteDirectory(rootPath.toFile()); } @@ -114,8 +115,9 @@ public Pair detect(String path){ } - private Pair filePathDetection(String path) { + private LicenseDetectionResult filePathDetection(String path) { Path pathToFile = searchLicenseFile(path); + var results = new LicenseDetectionResult(); totalScanned++; if(pathToFile == null) { CLIHelper.updateCurrentLine("Couldn't get license file at "+path,Ansi.Color.RED); @@ -124,19 +126,29 @@ private Pair filePathDetection(String path) { // Maybe handover this responsiblity to the cloud service. pathToFile = searchReadmeFile(path); if(pathToFile == null){ - return new Pair<>("null","null"); + results.addProbableLicense(new Pair<>("null",0.0)); + results.setAnalyzedContent(""); + return results; } String content = FileUtil.readFile(pathToFile); - return new Pair<>("null",content); + results.addProbableLicense(new Pair<>("null",0.0)); + results.setAnalyzedContent(""); + return results; } String content = readLicense(pathToFile); Map res = analyze(content); if(res.isEmpty()) { - return new Pair<>("null","null"); + results.addProbableLicense(new Pair<>("null",0.0)); + results.setAnalyzedContent(content); + return results; }else{ - return new Pair<>(res.entrySet().iterator().next().getKey(),content); + res.entrySet().forEach(possibility -> { + results.addProbableLicense(new Pair<>(possibility.getKey(), possibility.getValue())); + }); + results.setAnalyzedContent(content); + return results; } } diff --git a/common-utils/src/main/java/com/phsyberdome/common/interfaces/LicenseDetectorInterface.java b/common-utils/src/main/java/com/phsyberdome/common/interfaces/LicenseDetectorInterface.java index 84a7f05..0d7cde6 100644 --- a/common-utils/src/main/java/com/phsyberdome/common/interfaces/LicenseDetectorInterface.java +++ b/common-utils/src/main/java/com/phsyberdome/common/interfaces/LicenseDetectorInterface.java @@ -2,6 +2,7 @@ package com.phsyberdome.common.interfaces; import com.phsyberdome.common.utils.CLIHelper; +import com.phsyberdome.common.utils.models.LicenseDetectionResult; import com.phsyberdome.common.utils.models.Pair; import org.fusesource.jansi.Ansi; @@ -12,8 +13,10 @@ */ public interface LicenseDetectorInterface { - default public Pair detect(String content) { - return new Pair<>("Detector not initialized!","1.0"); + default public LicenseDetectionResult detect(String content) { + var result = new LicenseDetectionResult(); + result.addProbableLicense(new Pair<>("No dependencies scanned!",1.0)); + return result; } default public void printScanStats() { diff --git a/common-utils/src/main/java/com/phsyberdome/common/utils/models/LicenseDetectionResult.java b/common-utils/src/main/java/com/phsyberdome/common/utils/models/LicenseDetectionResult.java new file mode 100644 index 0000000..2d86904 --- /dev/null +++ b/common-utils/src/main/java/com/phsyberdome/common/utils/models/LicenseDetectionResult.java @@ -0,0 +1,85 @@ + +package com.phsyberdome.common.utils.models; + +import java.util.ArrayList; +import java.util.List; + +/** + * + * @author Pratham Gahlout + */ +public class LicenseDetectionResult { + + private static final double DIFFERENTIATING_TOLERANCE = 0.05; // 5% + + private List> licenses; + private String content; + + public LicenseDetectionResult(List> licenses, String content) { + this.licenses = licenses; + this.content = content; + } + + public LicenseDetectionResult(String content) { + this.content = content; + this.licenses = new ArrayList<>(); + } + + public LicenseDetectionResult() { + this.licenses = new ArrayList<>(); + this.content = ""; + } + + public void addProbableLicense(Pair license) { + if(licenses==null) { + this.licenses = new ArrayList<>(); + } + this.licenses.add(license); + } + + private void sortLicensesByProbability() { + List> modifiable = new ArrayList<>(this.licenses); + modifiable.sort((Pair o1, Pair o2) -> { + if(o1.second > o2.second){ + return -1; + }else if(o1.second < o2.second) { + return 1; + }else{ + return 0; + } + }); + this.setLicenses(modifiable); + } + + public List> getLicenses(){ + return this.licenses; + } + + public String getAnalyzedContent() { + return this.content; + } + + public void setAnalyzedContent(String content){ + this.content = content; + } + + public void setLicenses(List> licenses) { + this.licenses = licenses; + } + + public LicenseDetectionResult getResultWithMostProbableLicenses() { + var newResult = new LicenseDetectionResult(); + newResult.setAnalyzedContent(this.content); + var highest = this.licenses.getFirst().second; + newResult.setLicenses(this.licenses.stream().filter(item -> (highest - item.second) < DIFFERENTIATING_TOLERANCE) + .map(item -> { + return new Pair<>(item.first,Math.floor(item.second * 1000)/1000); + }).limit(3).toList()); + newResult.sortLicensesByProbability(); + return newResult; + } + + public String getLicensesAsString() { + return String.join(" | ", licenses.stream().map(item -> item.first + " " + item.second).toList()); + } +} diff --git a/plugin.cargo/src/main/java/com/phsyberdome/plugin/cargo/PluginCargo.java b/plugin.cargo/src/main/java/com/phsyberdome/plugin/cargo/PluginCargo.java index c60f745..62a89b5 100644 --- a/plugin.cargo/src/main/java/com/phsyberdome/plugin/cargo/PluginCargo.java +++ b/plugin.cargo/src/main/java/com/phsyberdome/plugin/cargo/PluginCargo.java @@ -84,6 +84,8 @@ public ArrayList readModules() { String version = toml.getString("version","null"); Toml dependencies = toml.getTable("dependencies"); + if(dependencies==null) + dependencies = toml.getTable("workspace.dependencies"); Module root = new Module(moduleTitle, version); if(dependencies != null) { @@ -109,17 +111,21 @@ public ArrayList readModules() { private void resolveDependencyTree(Module module) { String license = RegistryHelper.getLicenseFromRegistry(module.getName(), module.getVersion()); + String analyzedContent = "{Registry-Metadata}"; if(license.isEmpty()) { // Try to see if there is an open source url? If yes then analyze that String repoUrl = RegistryHelper.getSourceCodeRepoLink(module.getName()); if(!repoUrl.isEmpty()) { Path path = FileUtil.getFilePathFromURL(repoUrl,this.cloneLocation.toString()); if(path!=null){ - license = licenseDetector.detect(path.toString()).first; + var detectionResults = licenseDetector.detect(path.toString()); + license = detectionResults.getResultWithMostProbableLicenses().getLicensesAsString(); + analyzedContent = detectionResults.getAnalyzedContent(); } } } module.setLicense(license); + module.setAnalyzedContent(analyzedContent); // Get dependencies declared in the registry against this version List> depData = RegistryHelper.getDependenciesOfCrate(module.getName(),module.getVersion()); for(var dep: depData) { diff --git a/plugin.maven/src/main/java/com/phsyberdome/plugin/maven/PluginMaven.java b/plugin.maven/src/main/java/com/phsyberdome/plugin/maven/PluginMaven.java index a0c6e8a..7d84067 100644 --- a/plugin.maven/src/main/java/com/phsyberdome/plugin/maven/PluginMaven.java +++ b/plugin.maven/src/main/java/com/phsyberdome/plugin/maven/PluginMaven.java @@ -178,9 +178,9 @@ private void getLicenseAndTransitiveDependenciesForModule(Module root) { return; } buildDependencyTree(root,path); - Pair detectionResult = licenseDetector.detect(path.toString()); - root.setLicense(detectionResult.first); - root.setAnalyzedContent(detectionResult.second); + var detectionResult = licenseDetector.detect(path.toString()); + root.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString()); + root.setAnalyzedContent(detectionResult.getAnalyzedContent()); } private void buildDependencyTree(Module root, Path pathToModule) { diff --git a/plugin.npm/pom.xml b/plugin.npm/pom.xml index 058f15f..7297b34 100644 --- a/plugin.npm/pom.xml +++ b/plugin.npm/pom.xml @@ -11,6 +11,24 @@ common-utils ${project.version} + + org.junit.jupiter + junit-jupiter-api + 5.6.0 + test + + + org.junit.jupiter + junit-jupiter-params + 5.6.0 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.6.0 + test + UTF-8 diff --git a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelper.java b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelper.java index 1349220..a54e45d 100644 --- a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelper.java +++ b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelper.java @@ -354,7 +354,7 @@ public static List fetchAllVersions(String name){ temp.append(line); } List> versionObjs = JSONHelper.getValues("/versions", temp.toString()); - return (List) versionObjs.stream().map(pair -> pair.second); + return versionObjs.stream().map(pair -> pair.first).toList(); } catch (MalformedURLException ex) { Logger.getLogger(NPMVersionHelper.class.getName()).log(Level.SEVERE, null, ex); diff --git a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelperV2.java b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelperV2.java index 55f72c4..44d5649 100644 --- a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelperV2.java +++ b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/NPMVersionHelperV2.java @@ -4,6 +4,7 @@ import static com.phsyberdome.plugin.npm.NPMVersionHelper.fetchAllVersions; import com.vdurmont.semver4j.Semver; +import com.vdurmont.semver4j.SemverException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -27,9 +28,12 @@ public static String pinpointPackageVersion(String name,String version){ List allVersions = fetchAllVersions(name); List candidates = new ArrayList<>(); for(String ver:allVersions){ - Semver semver = new Semver(ver,Semver.SemverType.NPM); - if(semver.satisfies(version)){ - candidates.add(semver); + try { + Semver semver = new Semver(ver,Semver.SemverType.NPM); + if(semver.satisfies(version)){ + candidates.add(semver); + } + } catch(SemverException ex) { } } diff --git a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/PluginNpm.java b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/PluginNpm.java index dfafa79..0012e75 100644 --- a/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/PluginNpm.java +++ b/plugin.npm/src/main/java/com/phsyberdome/plugin/npm/PluginNpm.java @@ -174,10 +174,9 @@ private void getRootModuleWithDependenciesFromLockFile(Module root,JsonNode node modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString()); } - Pair detectionResult = licenseDetector.detect(modulePath.toString()); - String license = detectionResult.first; - m.setLicense(license); - m.setAnalyzedContent(detectionResult.second); + var detectionResult = licenseDetector.detect(modulePath.toString()); + m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString()); + m.setAnalyzedContent(detectionResult.getAnalyzedContent()); JsonNode dependencies = body.get("dependencies"); if(dependencies!=null){ resolveTransitiveDependencies(m,dependencies); @@ -235,10 +234,9 @@ private void getRootModuleWithDependencies(Module root,JsonNode node) { if(registryUrl == null || registryUrl.isBlank()) continue; Path modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString()); //} - Pair detectionResult = licenseDetector.detect(modulePath.toString()); - String license = detectionResult.first; - m.setLicense(license); - m.setAnalyzedContent(detectionResult.second); + var detectionResult = licenseDetector.detect(modulePath.toString()); + m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString()); + m.setAnalyzedContent(detectionResult.getAnalyzedContent()); resolveTransitiveDependencies(m,modulePath); scannedDependencies.add(m); // if(body.get("dependencies") != null) { @@ -270,10 +268,9 @@ private void resolveTransitiveDependencies(Module root,JsonNode node) { if(registryUrl == null || registryUrl.isBlank()) continue; modulePath = FileUtil.getFilePathFromURL(registryUrl, this.cloneLocation.toString()); } - Pair detectionResult = licenseDetector.detect(modulePath.toString()); - String license = detectionResult.first; - m.setLicense(license); - m.setAnalyzedContent(detectionResult.second); + var detectionResult = licenseDetector.detect(modulePath.toString()); + m.setLicense(detectionResult.getResultWithMostProbableLicenses().getLicensesAsString()); + m.setAnalyzedContent(detectionResult.getAnalyzedContent()); JsonNode dependencies = body.get("dependencies"); if(dependencies != null) { resolveTransitiveDependencies(m,dependencies);