Enable assembly plugin (#2)

* Move publish settings to individual frontends

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

* Skip function bodies by default for c/c++

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

* Rearrange test scripts. Bump version

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

---------

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

.github/workflows/containers.yml

@@ -35,7 +35,7 @@ jobs:
         run: rm -rf /home/runner/.cargo
         if: runner.os == 'Linux'
       - run: |
-          sbt createDistribution
+          sbt assembly createDistribution
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up QEMU

.github/workflows/master.yml

@@ -6,7 +6,6 @@ on:
     tags: ["*"]
 jobs:
   test:
-    concurrency: main
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/release.yml

@@ -27,8 +27,7 @@ jobs:
         run: rm -rf /home/runner/.cargo
         if: runner.os == 'Linux'
       - run: |
-          sbt scalafmtCheck +test
-          sbt createDistribution publish
+          sbt assembly createDistribution publish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: sha512sum target/chen.zip > target/chen.zip.sha512

build.sbt

@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.1"
+ThisBuild / version      := "0.0.2"
 ThisBuild / scalaVersion := "3.3.0"
 
 val cpgVersion = "1.4.22"
@@ -55,6 +55,7 @@ ThisBuild / scalacOptions ++= Seq(
   "17",
 )
 
+
 enablePlugins(JavaAppPackaging)
 
 lazy val createDistribution = taskKey[File]("Create a complete chen distribution")
@@ -73,18 +74,19 @@ ThisBuild / resolvers ++= Seq(
   "Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases/"
 )
 
+ThisBuild / assemblyMergeStrategy := {
+  case PathList("javax", "servlet", xs @ _*)         => MergeStrategy.first
+  case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
+  case "application.conf"                            => MergeStrategy.concat
+  case "unwanted.txt"                                => MergeStrategy.discard
+  case x => MergeStrategy.preferProject
+}
+
+ThisBuild / versionScheme := Some("early-semver")
+
 ThisBuild / Test / fork := true
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
-githubOwner := "appthreat"
-githubRepository := "chen"
-credentials +=
-  Credentials(
-    "GitHub Package Registry",
-    "maven.pkg.github.com",
-    "appthreat",
-    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
-  )
 publish / skip := true // don't publish the root project
 
 // Avoids running root tasks on the benchmarks project

console/build.sbt

@@ -31,3 +31,13 @@ libraryDependencies ++= Seq(
 )
 
 Test / compile := (Test / compile).dependsOn((Projects.c2cpg / stage)).value
+
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

console/src/main/scala/io/appthreat/console/BridgeBase.scala

@@ -43,10 +43,10 @@ case class Config(
   */
 trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandling with ServerHandling {
 
-  def slProduct: JProduct
+  def jProduct: JProduct
 
   protected def parseConfig(args: Array[String]): Config = {
-    val parser = new scopt.OptionParser[Config](slProduct.name) {
+    val parser = new scopt.OptionParser[Config](jProduct.name) {
       override def errorOnUnknownArgument = false
 
       note("Script execution")
@@ -155,7 +155,7 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
       arg[java.io.File]("<cpg.bin>")
         .optional()
         .action((x, c) => c.copy(cpgToLoad = Some(x.toScala)))
-        .text("CPG to load")
+        .text("Atom to load")
 
       opt[String]("for-input-path")
         .action((x, c) => c.copy(forInputPath = Some(x)))
@@ -199,7 +199,7 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
       GlobalReporting.enable()
       startHttpServer(config)
     } else if (config.pluginToRun.isDefined) {
-      runPlugin(config, slProduct.name)
+      runPlugin(config, jProduct.name)
     } else {
       startInteractiveShell(config)
     }
@@ -299,7 +299,7 @@ trait PluginHandling { this: BridgeBase =>
     new PluginManager(InstallConfig().rootPath).listPlugins().foreach(println)
     println("Available layer creators")
     println()
-    withTemporaryScript(codeToListPlugins(), slProduct.name) { file =>
+    withTemporaryScript(codeToListPlugins(), jProduct.name) { file =>
       runScript(config.copy(scriptFile = Some(file.path))).get
     }
   }

console/src/main/scala/io/appthreat/console/Console.scala

@@ -1,13 +1,10 @@
 package io.appthreat.console
 
 import better.files.File
-import dotty.tools.repl.State
 import io.appthreat.console.cpgcreation.{CpgGeneratorFactory, ImportCode}
 import io.appthreat.console.workspacehandling.{Project, WorkspaceLoader, WorkspaceManager}
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.appthreat.x2cpg.X2Cpg.defaultOverlayCreators
-import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.Overlays

console/src/main/scala/io/appthreat/console/workspacehandling/WorkspaceManager.scala

@@ -51,7 +51,6 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
       val pathToProject = projectNameToDir(projectName)
 
       if (project(projectName).isDefined) {
-        System.err.println(s"Project with name $projectName already exists - overwriting")
         removeProject(projectName)
       }
 
@@ -268,13 +267,11 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
       setActiveProject(name)
       project(name)
     } else {
-      val cpgFilename = baseCpgFilename(name)
-      report("Creating working copy of CPG to be safe")
+      val cpgFilename     = baseCpgFilename(name)
       val cpgFile         = File(cpgFilename)
       val workingCopyPath = projectDir(name).resolve(Project.workCpgFileName)
       val workingCopyName = workingCopyPath.toAbsolutePath.toString
       cp(cpgFile, workingCopyPath)
-      report(s"Loading base CPG from: $workingCopyName")
 
       val result = {
         val newCpg      = loader(workingCopyName)

dataflowengineoss/build.sbt

@@ -17,3 +17,12 @@ Antlr4 / antlr4PackageName := Some("io.appthreat.dataflowengineoss")
 Antlr4 / antlr4Version     := Versions.antlr
 Antlr4 / javaSource        := (Compile / sourceManaged).value
 Compile / doc / sources ~= (_ filter (_ => false))
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

macros/build.sbt

@@ -8,3 +8,12 @@ libraryDependencies ++= Seq(
 )
 
 enablePlugins(JavaAppPackaging)
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/build.sbt

@@ -118,3 +118,12 @@ generateScaladocs := {
 Universal / packageBin / mappings ++= sbt.Path.directory(new File("platform/src/main/resources/scripts"))
 
 maintainer := "cloud@appthreat.com"
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/c2cpg/build.sbt

@@ -23,3 +23,12 @@ enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 
 Universal / packageName       := name.value
 Universal / topLevelDirectory := None
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/Main.scala

@@ -15,9 +15,9 @@ final case class Config(
   logPreprocessor: Boolean = false,
   printIfDefsOnly: Boolean = false,
   includePathsAutoDiscovery: Boolean = false,
-  skipFunctionBodies: Boolean = false,
-  noImageLocations: Boolean = false,
-  useProjectIndex: Boolean = false
+  includeFunctionBodies: Boolean = false,
+  includeImageLocations: Boolean = false,
+  useProjectIndex: Boolean = true
 ) extends X2CpgConfig[Config] {
   def withIncludePaths(includePaths: Set[String]): Config = {
     this.copy(includePaths = includePaths).withInheritedFields(this)
@@ -47,12 +47,12 @@ final case class Config(
     this.copy(includePathsAutoDiscovery = value).withInheritedFields(this)
   }
 
-  def withSkipFunctionBodies(value: Boolean): Config = {
-    this.copy(skipFunctionBodies = value).withInheritedFields(this)
+  def withFunctionBodies(value: Boolean): Config = {
+    this.copy(includeFunctionBodies = value).withInheritedFields(this)
   }
 
-  def withNoImageLocations(value: Boolean): Config = {
-    this.copy(noImageLocations = value).withInheritedFields(this)
+  def withImageLocations(value: Boolean): Config = {
+    this.copy(includeImageLocations = value).withInheritedFields(this)
   }
 
   def withProjectIndexes(value: Boolean): Config = {
@@ -90,14 +90,14 @@ private object Frontend {
       opt[Unit]("with-include-auto-discovery")
         .text("enables auto discovery of system header include paths")
         .action((_, c) => c.withIncludePathsAutoDiscovery(true)),
-      opt[Unit]("skip-function-bodies")
-        .text("instructs the parser to skip function and method bodies.")
-        .action((_, c) => c.withSkipFunctionBodies(true)),
-      opt[Unit]("no-image-locations")
+      opt[Unit]("with-function-bodies")
+        .text("instructs the parser to parse function and method bodies.")
+        .action((_, c) => c.withFunctionBodies(true)),
+      opt[Unit]("with-image-locations")
         .text(
-          "performance optimization, allows the parser not to create image-locations. An image location explains how a name made it into the translation unit. Eg: via macro expansion or preprocessor."
+          "allows the parser to create image-locations. An image location explains how a name made it into the translation unit. Eg: via macro expansion or preprocessor."
         )
-        .action((_, c) => c.withNoImageLocations(true)),
+        .action((_, c) => c.withImageLocations(true)),
       opt[Unit]("with-project-index")
         .text("performance optimization, allows the parser to use an existing eclipse project(s) index(es).")
         .action((_, c) => c.withProjectIndexes(true)),

platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/CdtParser.scala

@@ -60,9 +60,9 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta
   // enables parsing of code behind disabled preprocessor defines:
   private var opts: Int = ILanguage.OPTION_PARSE_INACTIVE_CODE
   // instructs the parser to skip function and method bodies
-  if (config.skipFunctionBodies) opts |= ILanguage.OPTION_SKIP_FUNCTION_BODIES
+  if (!config.includeFunctionBodies) opts |= ILanguage.OPTION_SKIP_FUNCTION_BODIES
   // performance optimization, allows the parser not to create image-locations
-  if (config.noImageLocations) opts |= ILanguage.OPTION_NO_IMAGE_LOCATIONS
+  if (!config.includeImageLocations) opts |= ILanguage.OPTION_NO_IMAGE_LOCATIONS
 
   private def createParseLanguage(file: Path): ILanguage = {
     if (FileDefaults.isCPPFile(file.toString)) {

platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/io/ExcludeTest.scala

@@ -52,6 +52,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
       .withOutputPath(cpgOutFile.toString)
       .withIgnoredFiles(exclude)
       .withIgnoredFilesRegex(excludeRegex)
+      .withFunctionBodies(true)
     val c2cpg = new C2Cpg()
     val cpg   = c2cpg.createCpg(config).get
 

platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/PreprocessorPassTests.scala

@@ -29,7 +29,7 @@ class PreprocessorPassTests extends AnyWordSpec with Matchers {
         val filename = "test.c"
         val file     = dir / filename
         file.write(code)
-        val config = Config().withInputPath(dir.path.toString)
+        val config = Config().withInputPath(dir.path.toString).withFunctionBodies(true)
         val stmts  = new PreprocessorPass(config).run().toList
         stmts shouldBe List("SYMBOL", "FOO=true")
       }
@@ -55,7 +55,7 @@ class PreprocessorPassTests extends AnyWordSpec with Matchers {
         val filename = "test.c"
         val file     = dir / filename
         file.write(code)
-        val config = Config(defines = Set("SYMBOL")).withInputPath(dir.path.toString)
+        val config = Config(defines = Set("SYMBOL")).withInputPath(dir.path.toString).withFunctionBodies(true)
         val stmts  = new PreprocessorPass(config).run().toList
         stmts shouldBe List("SYMBOL=true", "FOO=true")
       }

platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/testfixtures/AbstractPassTest.scala

@@ -19,7 +19,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside {
         val cpg  = newEmptyCpg()
         val file = dir / fileName
         file.write(code)
-        val config = Config().withInputPath(dir.toString()).withOutputPath(dir.toString())
+        val config = Config().withInputPath(dir.toString()).withOutputPath(dir.toString()).withFunctionBodies(true)
         new AstCreationPass(cpg, config).createAndApply()
         f(cpg)
         file.delete()
@@ -31,7 +31,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside {
       File.usingTemporaryDirectory("c2cpgtest") { dir =>
         val file = dir / "file.c"
         file.write(code)
-        val config = Config().withInputPath(dir.toString()).withOutputPath(dir.toString())
+        val config = Config().withInputPath(dir.toString()).withOutputPath(dir.toString()).withFunctionBodies(true)
         new AstCreationPass(cpg, config).createAndApply()
       }
       cpg

platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/testfixtures/CCodeToCpgSuite.scala

@@ -18,6 +18,7 @@ trait C2CpgFrontend extends LanguageFrontend {
       .getOrElse(Config(includeComments = true))
       .withInputPath(sourceCodePath.getAbsolutePath)
       .withOutputPath(cpgOutFile.pathAsString)
+      .withFunctionBodies(true)
 
     c2cpg.createCpg(config).get
   }

platform/frontends/javasrc2cpg/build.sbt

@@ -49,3 +49,12 @@ packTestCode := {
   }
 }
 packTestCode := packTestCode.triggeredBy(Test / compile).value
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/jimple2cpg/build.sbt

@@ -11,3 +11,12 @@ libraryDependencies ++= Seq(
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 trapExit    := false
 Test / fork := true
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/jssrc2cpg/build.sbt

@@ -115,3 +115,12 @@ cleanFiles ++= Seq(
 
 Universal / packageName       := name.value
 Universal / topLevelDirectory := None
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/pysrc2cpg/build.sbt

@@ -27,3 +27,12 @@ javaCCTask := {
 }
 
 Compile / sourceGenerators += javaCCTask
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )

platform/frontends/x2cpg/build.sbt

@@ -16,3 +16,12 @@ enablePlugins(JavaAppPackaging)
 
 Universal / packageName       := name.value
 Universal / topLevelDirectory := None
+githubOwner := "appthreat"
+githubRepository := "chen"
+credentials +=
+  Credentials(
+    "GitHub Package Registry",
+    "maven.pkg.github.com",
+    "appthreat",
+    sys.env.getOrElse("GITHUB_TOKEN", "N/A")
+  )