More excludes. Multi-threaded slicing (#63)

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

build.sbt

@@ -3,7 +3,7 @@ ThisBuild / organization := "io.appthreat"
 ThisBuild / version      := "1.0.0"
 ThisBuild / scalaVersion := "3.3.0"
 
-val joernVersion      = "2.0.52"
+val joernVersion      = "2.0.56"
 
 lazy val atom = Projects.atom
 

src/main/scala/io/appthreat/atom/Atom.scala

@@ -35,14 +35,14 @@ import scala.util.{Failure, Properties, Success, Using}
 
 object Atom {
 
-  val DEFAULT_ATOM_OUT_FILE: String = if (Properties.isWin) "app.atom" else "app.⚛"
-  val DEFAULT_SLICE_OUT_FILE        = "slices.json"
-  val DEFAULT_SLICE_DEPTH           = 7
-  val DEFAULT_MAX_DEFS: Int         = 2000
-  val TYPE_PROPAGATION_ITERATIONS   = 1
-  private val MAVEN_JAR_PATH: File  = File.home / ".m2" / "repository"
-  private val GRADLE_JAR_PATH: File = File.home / ".gradle" / "caches" / "modules-2" / "files-2.1"
-  private val SBT_JAR_PATH: File    = File.home / ".ivy2" / "cache"
+  val DEFAULT_ATOM_OUT_FILE: String       = if (Properties.isWin) "app.atom" else "app.⚛"
+  val DEFAULT_SLICE_OUT_FILE              = "slices.json"
+  val DEFAULT_SLICE_DEPTH                 = 7
+  val DEFAULT_MAX_DEFS: Int               = 2000
+  private val TYPE_PROPAGATION_ITERATIONS = 1
+  private val MAVEN_JAR_PATH: File        = File.home / ".m2" / "repository"
+  private val GRADLE_JAR_PATH: File       = File.home / ".gradle" / "caches" / "modules-2" / "files-2.1"
+  private val SBT_JAR_PATH: File          = File.home / ".ivy2" / "cache"
   private val JAR_INFERENCE_PATHS: Set[String] =
     Set(MAVEN_JAR_PATH.pathAsString, GRADLE_JAR_PATH.pathAsString, SBT_JAR_PATH.pathAsString)
   private val ANDROID_JAR_PATH: Option[String] = Option(System.getenv("ANDROID_HOME")).flatMap { androidHome =>
@@ -298,7 +298,9 @@ object Atom {
           .createCpgWithOverlays(
             JavaConfig(fetchDependencies = true, inferenceJarPaths = JAR_INFERENCE_PATHS, enableTypeRecovery = true)
               .withInputPath(config.inputPath.pathAsString)
-              .withDefaultIgnoredFilesRegex(List("\\..*".r))
+              .withDefaultIgnoredFilesRegex(
+                List("\\..*".r, ".*build/(generated|intermediates|outputs|tmp).*" r, ".*src/test.*" r)
+              )
               .withOutputPath(outputAtomFile)
           )
       case Languages.JSSRC | Languages.JAVASCRIPT | "JS" | "TS" | "TYPESCRIPT" =>

src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala

@@ -12,12 +12,15 @@ import scala.annotation.unused
 import scala.collection.concurrent.TrieMap
 import scala.util.Try
 
+import java.util.concurrent.*
+
 /** A utility for slicing based off of usage references for identifiers and parameters. This is mainly tested around
   * JavaScript CPGs.
   */
 object UsageSlicing {
 
   private val resolver               = NoResolve
+  val exec: ExecutorService          = Executors.newWorkStealingPool(Runtime.getRuntime.availableProcessors() / 2)
   private val constructorTypeMatcher = Pattern.compile(".*new (\\w+)\\(.*")
   private val excludeOperatorCalls   = new AtomicBoolean(true)
 
@@ -37,35 +40,25 @@ object UsageSlicing {
       case None           => cpg.method
     }).withMethodNameFilter.withMethodParameterFilter.withMethodAnnotationFilter.declaration
 
-    def typeMap = TrieMap.from(cpg.typeDecl.map(f => (f.name, f.fullName)).toMap)
-
-    val fjp = ForkJoinPool.commonPool()
-
-    try {
-      val slices       = usageSlices(fjp, cpg, () => getDeclarations, typeMap)
-      val userDefTypes = userDefinedTypes(cpg)
-      ProgramUsageSlice(slices, userDefTypes)
-    } finally {
-      fjp.shutdown()
-    }
+    def typeMap      = TrieMap.from(cpg.typeDecl.map(f => (f.name, f.fullName)).toMap)
+    val slices       = usageSlices(cpg, () => getDeclarations, typeMap)
+    val userDefTypes = userDefinedTypes(cpg)
+    ProgramUsageSlice(slices, userDefTypes)
   }
 
   import io.shiftleft.semanticcpg.codedumper.CodeDumper.dump
 
-  private def usageSlices(
-    fjp: ForkJoinPool,
-    cpg: Cpg,
-    getDeclIdentifiers: () => Traversal[Declaration],
-    typeMap: TrieMap[String, String]
-  )(implicit config: UsagesConfig): List[MethodUsageSlice] = {
+  private def usageSlices(cpg: Cpg, getDeclIdentifiers: () => Traversal[Declaration], typeMap: TrieMap[String, String])(
+    implicit config: UsagesConfig
+  ): List[MethodUsageSlice] = {
     val language = cpg.metaData.language.headOption
     val root     = cpg.metaData.root.headOption
     getDeclIdentifiers()
       .to(LazyList)
       .filterNot(a => a.name.equals("*"))
       .filter(a => !a.name.startsWith("_tmp_") && atLeastNCalls(a, config.minNumCalls))
-      .map(a => fjp.submit(new TrackUsageTask(cpg, a, typeMap)))
-      .flatMap(_.get())
+      .map(a => exec.submit(new TrackUsageTask(cpg, a, typeMap)))
+      .flatMap(TimedGet)
       .groupBy { case (scope, _) => scope }
       .view
       .filterNot((m, _) => (m.fullName.startsWith("<operator") || m.fullName.startsWith("__builtin")))
@@ -86,6 +79,14 @@ object UsageSlicing {
       .toList
   }
 
+  private def TimedGet(dsf: Future[Option[(Method, ObjectUsageSlice)]]) = {
+    try {
+      dsf.get(5, TimeUnit.SECONDS)
+    } catch {
+      case _: Throwable => None
+    }
+  }
+
   /** Returns true if the given declaration is found to have at least n non-operator calls within its referenced
     * identifiers' scope.
     *
@@ -159,9 +160,9 @@ object UsageSlicing {
 
   private class TrackUsageTask(cpg: Cpg, tgt: Declaration, typeMap: TrieMap[String, String])(implicit
     config: UsagesConfig
-  ) extends RecursiveTask[Option[(Method, ObjectUsageSlice)]] {
+  ) extends Callable[Option[(Method, ObjectUsageSlice)]] {
 
-    override def compute(): Option[(Method, ObjectUsageSlice)] = {
+    override def call(): Option[(Method, ObjectUsageSlice)] = {
       val defNode = tgt match {
         case local: Local =>
           local.referencingIdentifiers.inCall.astParent.assignment

wrapper/nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",