Working scalapy integration (#3)

* Working scalapy integration Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Added polynote docker Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Added calltree and graphml to predef Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Sep 20, 2023 · db36aba · db36aba
1 parent 7276116
commit db36aba
Show file tree

Hide file tree

Showing 26 changed files with 387 additions and 85 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -67,3 +67,21 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,scope=chen
           cache-to: type=gha,mode=max,scope=chen
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta2
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/appthreat/chen-note
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ci/Dockerfile-note
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta2.outputs.tags }}
+          labels: ${{ steps.meta2.outputs.labels }}
+          cache-from: type=gha,scope=chen-note
+          cache-to: type=gha,mode=max,scope=chen-note
diff --git a/README.md b/README.md
@@ -1 +1,3 @@
 Code Hierarchy Exploration Net
+
+
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.3"
+ThisBuild / version      := "0.0.4"
 ThisBuild / scalaVersion := "3.3.0"
 
 val cpgVersion = "1.4.22"

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -25,9 +25,15 @@ ENV JAVA_VERSION=$JAVA_VERSION \
     PYTHON_CMD=python3 \
     PYTHONUNBUFFERED=1 \
     PYTHONIOENCODING="utf-8" \
+    JAVA_OPTS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1" \
+    JOERN_DATAFLOW_TRACKED_WIDTH=128 \
+    SCALAPY_PYTHON_LIBRARY=python3.11 \
     ANDROID_HOME=/opt/android-sdk-linux
 ENV PATH=${PATH}:/opt/platform:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
 
+WORKDIR /opt
+COPY ./ci/requirements.txt /opt/
+COPY ./ci/kernel.json /opt/
 RUN set -e; \
     ARCH_NAME="$(rpm --eval '%{_arch}')"; \
     url=; \
@@ -44,10 +50,17 @@ RUN set -e; \
     esac; \
     echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf install -y gcc git-core \
-        python3.11 python3.11-devel python3.11-pip \
+        python3.11 python3.11-devel python3.11-pip graphviz graphviz-gd graphviz-python3 \
         pcre2 findutils which tar gzip zip unzip sudo nodejs ncurses sqlite-devel \
     && alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1 \
     && python3 --version \
+    && curl -LO https://repo.almalinux.org/almalinux/9/CRB/${ARCH_NAME}/os/Packages/graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && rpm -ivh graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && rm graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && python3.11 -m pip install -r /opt/requirements.txt notebook owasp-depscan \
+    && python3.11 -m pip install virtualenv numpy GitPython pandas graphviz pydotplus networkx[default,extra] \
+    && python3.11 -m pip install torch --index-url https://download.pytorch.org/whl/cpu \
+    && python3.11 -m pip install torch_geometric pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cpu.html \
     && curl -s "https://get.sdkman.io" | bash \
     && source "$HOME/.sdkman/bin/sdkman-init.sh" \
     && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true" >> $HOME/.sdkman/etc/config \
@@ -67,12 +80,16 @@ RUN set -e; \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platform-tools' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platforms;android-33' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'build-tools;33.0.0' --sdk_root=/opt/android-sdk-linux \
-    && mkdir -p /opt
-WORKDIR /opt
+    && curl -Lo coursier https://git.io/coursier-cli \
+    && chmod +x coursier \
+    && ./coursier launch almond:0.14.0-RC13 --scala 3.3.0 -- --install \
+    && cp /opt/kernel.json /root/.local/share/jupyter/kernels/scala/ \
+    && sudo npm install -g @cyclonedx/cdxgen --omit=optional \
+    && rm -f coursier
 COPY ./target/chen.zip .
-RUN cd /opt/ \
-    && unzip -q chen.zip \
-    && rm chen.zip \
+COPY ./notebooks /opt/notebooks
+RUN unzip -q chen.zip \
+    && rm chen.zip kernel.json requirements.txt \
     && microdnf clean all
-
+ENV CLASSPATH=$CLASSPATH:/opt/platform/lib/*.jar:
 CMD ["chennai"]
diff --git a/ci/Dockerfile-note b/ci/Dockerfile-note
@@ -0,0 +1,98 @@
+FROM almalinux:9.2-minimal
+
+LABEL maintainer="appthreat" \
+      org.opencontainers.image.authors="Team AppThreat <cloud@appthreat.com>" \
+      org.opencontainers.image.source="https://github.com/appthreat/chen" \
+      org.opencontainers.image.url="https://github.com/appthreat/chen" \
+      org.opencontainers.image.version="1.0.0" \
+      org.opencontainers.image.vendor="appthreat" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.title="chen" \
+      org.opencontainers.image.description="Container image for AppThreat chen code analysis platform" \
+      org.opencontainers.docker.cmd="docker run --rm -it -v /tmp:/tmp -p 8192:8192 -v $(pwd):/opt/notebooks:rw -t ghcr.io/appthreat/chen-note --config /opt/config.yml"
+
+ARG JAVA_VERSION=20.0.2-graalce
+ARG MAVEN_VERSION=3.9.3
+ARG GRADLE_VERSION=8.2.1
+ARG POLYNOTE_VERSION="0.5.1"
+ARG SCALA_VERSION="2.12"
+ARG DIST_TAR="polynote-dist.tar.gz"
+
+ENV JAVA_VERSION=$JAVA_VERSION \
+    MAVEN_VERSION=$MAVEN_VERSION \
+    GRADLE_VERSION=$GRADLE_VERSION \
+    GRADLE_OPTS="-Dorg.gradle.daemon=false" \
+    JAVA_HOME="/opt/java/${JAVA_VERSION}" \
+    MAVEN_HOME="/opt/maven/${MAVEN_VERSION}" \
+    GRADLE_HOME="/opt/gradle/${GRADLE_VERSION}" \
+    POLYNOTE_SCALA_VERSION=${SCALA_VERSION} \
+    POLYNOTE_VERSION=${POLYNOTE_VERSION} \
+    PYTHON_CMD=python3 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING="utf-8" \
+    JAVA_OPTS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1" \
+    JOERN_DATAFLOW_TRACKED_WIDTH=128 \
+    SCALAPY_PYTHON_LIBRARY=python3.11 \
+    ANDROID_HOME=/opt/android-sdk-linux
+ENV PATH=${PATH}:/opt/platform:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
+
+WORKDIR /opt
+RUN set -e; \
+    ARCH_NAME="$(rpm --eval '%{_arch}')"; \
+    url=; \
+    case "${ARCH_NAME##*-}" in \
+        'x86_64') \
+            OS_ARCH_SUFFIX=''; \
+            GOBIN_VERSION='amd64'; \
+            ;; \
+        'aarch64') \
+            OS_ARCH_SUFFIX='-aarch64'; \
+            GOBIN_VERSION='arm64'; \
+            ;; \
+        *) echo >&2 "error: unsupported architecture: '$ARCH_NAME'"; exit 1 ;; \
+    esac; \
+    echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
+    && microdnf install -y gcc git-core wget \
+        python3.11 python3.11-devel python3.11-pip graphviz graphviz-gd graphviz-python3 \
+        pcre2 findutils which tar gzip zip unzip sudo nodejs ncurses sqlite-devel \
+    && alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1 \
+    && python3 --version \
+    && curl -s "https://get.sdkman.io" | bash \
+    && source "$HOME/.sdkman/bin/sdkman-init.sh" \
+    && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true" >> $HOME/.sdkman/etc/config \
+    && sdk install java $JAVA_VERSION \
+    && sdk install maven $MAVEN_VERSION \
+    && sdk install gradle $GRADLE_VERSION \
+    && sdk offline enable \
+    && mv /root/.sdkman/candidates/* /opt/ \
+    && rm -rf /root/.sdkman \
+    && curl -LO https://repo.almalinux.org/almalinux/9/CRB/${ARCH_NAME}/os/Packages/graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && rpm -ivh graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && rm graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
+    && python3.11 -m pip install owasp-depscan virtualenv ipython nbconvert numpy jedi jep GitPython pandas matplotlib graphviz pydotplus networkx[default,extra] \
+    && python3.11 -m pip install torch --index-url https://download.pytorch.org/whl/cpu \
+    && python3.11 -m pip install torch_geometric pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cpu.html \
+    && wget -q https://github.com/polynote/polynote/releases/download/$POLYNOTE_VERSION/$DIST_TAR \
+    && tar xfzp $DIST_TAR \
+    && echo "DIST_TAR=$DIST_TAR" \
+    && rm $DIST_TAR \
+    && useradd -ms /bin/bash polly \
+    && microdnf install -y epel-release \
+    && mkdir -p ${ANDROID_HOME}/cmdline-tools \
+    && curl -L https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip -o ${ANDROID_HOME}/cmdline-tools/android_tools.zip \
+    && unzip ${ANDROID_HOME}/cmdline-tools/android_tools.zip -d ${ANDROID_HOME}/cmdline-tools/ \
+    && rm ${ANDROID_HOME}/cmdline-tools/android_tools.zip \
+    && mv ${ANDROID_HOME}/cmdline-tools/cmdline-tools ${ANDROID_HOME}/cmdline-tools/latest \
+    && yes | /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager --licenses --sdk_root=/opt/android-sdk-linux \
+    && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platform-tools' --sdk_root=/opt/android-sdk-linux \
+    && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platforms;android-33' --sdk_root=/opt/android-sdk-linux \
+    && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'build-tools;33.0.0' --sdk_root=/opt/android-sdk-linux \
+    && sudo npm install -g @cyclonedx/cdxgen --omit=optional
+COPY ./target/chen.zip .
+COPY ./notebooks /opt/notebooks
+COPY ./ci/config.yml /opt/config.yml
+RUN unzip -q chen.zip \
+    && chown -R polly:polly /opt \
+    && microdnf clean all
+EXPOSE 8192
+ENTRYPOINT ["/opt/polynote/polynote.py"]
diff --git a/ci/config.yml b/ci/config.yml
@@ -0,0 +1,14 @@
+listen:
+  host: 0.0.0.0
+
+storage:
+  dir: /opt/notebooks
+
+dependencies:
+  python:
+    - GitPython
+    - pandas
+    - matplotlib
+    - graphviz
+    - pydotplus
+    - networkx[default,extra]
diff --git a/ci/kernel.json b/ci/kernel.json
@@ -0,0 +1,13 @@
+{
+  "argv": [
+    "java",
+    "-Djna.library.path=/usr/lib64",
+    "-cp",
+    "/root/.local/share/jupyter/kernels/scala/launcher.jar:/opt/platform/lib/*.jar",
+    "coursier.bootstrap.launcher.Launcher",
+    "--connection-file",
+    "{connection_file}"
+  ],
+  "display_name": "Scala",
+  "language": "scala"
+}
diff --git a/ci/requirements.txt b/ci/requirements.txt
@@ -0,0 +1,51 @@
+asttokens==2.4.0
+attrs==23.1.0
+backcall==0.2.0
+beautifulsoup4==4.12.2
+bleach==6.0.0
+comm==0.1.4
+debugpy==1.8.0
+decorator==5.1.1
+defusedxml==0.7.1
+executing==1.2.0
+fastjsonschema==2.18.0
+ipykernel==6.25.2
+ipython==8.15.0
+jedi==0.19.0
+Jinja2==3.1.2
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
+jupyter-console==6.6.3
+jupyter_client==8.3.1
+jupyter_core==5.3.1
+jupyterlab-pygments==0.2.2
+MarkupSafe==2.1.3
+matplotlib-inline==0.1.6
+mistune==3.0.1
+nbclient==0.8.0
+nbconvert==7.8.0
+nbformat==5.9.2
+nest-asyncio==1.5.8
+packaging==23.1
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+platformdirs==3.10.0
+prompt-toolkit==3.0.39
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.16.1
+python-dateutil==2.8.2
+pyzmq==25.1.1
+referencing==0.30.2
+rpds-py==0.10.3
+six==1.16.0
+soupsieve==2.5
+stack-data==0.6.2
+tinycss2==1.2.1
+tornado==6.3.3
+traitlets==5.10.0
+wcwidth==0.2.6
+webencodings==0.5.1
diff --git a/console/build.sbt b/console/build.sbt
@@ -27,11 +27,23 @@ libraryDependencies ++= Seq(
   "com.lihaoyi"          %% "os-lib"            % "0.9.1",
   "com.lihaoyi"          %% "pprint"            % "0.7.3",
   "com.lihaoyi"          %% "cask"              % CaskVersion,
+  "me.shadaj"            %% "scalapy-core"      % "0.5.2",
   "org.scalatest"        %% "scalatest"         % Versions.scalatest % Test
 )
 
+
 Test / compile := (Test / compile).dependsOn((Projects.c2cpg / stage)).value
 
+import ai.kien.python.Python
+
+lazy val python = Python()
+
+lazy val javaOpts = python.scalapyProperties.get.map {
+  case (k, v) => s"""-D$k=$v"""
+}.toSeq
+
+javaOptions ++= javaOpts
+
 githubOwner := "appthreat"
 githubRepository := "chen"
 credentials +=

diff --git a/console/src/main/scala/io/appthreat/console/BridgeBase.scala b/console/src/main/scala/io/appthreat/console/BridgeBase.scala
@@ -136,19 +136,19 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
 
       opt[String]("server-host")
         .action((x, c) => c.copy(serverHost = x))
-        .text("Hostname on which to expose the CPGQL server")
+        .text("Hostname on which to expose the Chen server")
 
       opt[Int]("server-port")
         .action((x, c) => c.copy(serverPort = x))
-        .text("Port on which to expose the CPGQL server")
+        .text("Port on which to expose the Chen server")
 
       opt[String]("server-auth-username")
         .action((x, c) => c.copy(serverAuthUsername = Option(x)))
-        .text("Basic auth username for the CPGQL server")
+        .text("Basic auth username for the Chen server")
 
       opt[String]("server-auth-password")
         .action((x, c) => c.copy(serverAuthPassword = Option(x)))
-        .text("Basic auth password for the CPGQL server")
+        .text("Basic auth password for the Chen server")
 
       note("Misc")
 
@@ -181,7 +181,7 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
     parser.parse(args, Config()).get
   }
 
-  /** Entry point for Joern's integrated REPL and plugin manager */
+  /** Entry point for Chen's integrated REPL and plugin manager */
   protected def run(config: Config): Unit = {
     if (config.listPlugins) {
       printPluginsAndLayerCreators(config)
@@ -206,7 +206,7 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
   }
 
   protected def createPredefFile(additionalLines: Seq[String] = Nil): Path = {
-    val tmpFile = Files.createTempFile("joern-predef", "sc")
+    val tmpFile = Files.createTempFile("chen-predef", "sc")
     Files.write(tmpFile, (predefLines ++ additionalLines).asJava)
     tmpFile.toAbsolutePath
   }

diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala
@@ -15,6 +15,7 @@ import overflowdb.traversal.help.Doc
 import scala.sys.process.Process
 import scala.util.control.NoStackTrace
 import scala.util.{Failure, Success, Try}
+import scala.collection.mutable.ListBuffer
 
 class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory)
     extends Reporting {
@@ -132,8 +133,8 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur
                  |""",
     example = "cpg.method.l"
   )
-  implicit def cpg: Cpg  = workspace.cpg
-  implicit def atom: Cpg = workspace.cpg
+  implicit def cpg: Cpg = workspace.cpg
+  def atom: Cpg         = workspace.cpg
 
   /** All cpgs loaded in the workspace
     */

diff --git a/console/src/main/scala/io/appthreat/console/ConsoleConfig.scala b/console/src/main/scala/io/appthreat/console/ConsoleConfig.scala
@@ -21,8 +21,8 @@ class InstallConfig(environment: Map[String, String] = sys.env) {
     *   - running a unit/integration test (note: the jars would be in the local cache, e.g. in ~/.coursier/cache)
     */
   lazy val rootPath: File = {
-    if (environment.contains("SHIFTLEFT_OCULAR_INSTALL_DIR")) {
-      environment("SHIFTLEFT_OCULAR_INSTALL_DIR").toFile
+    if (environment.contains("CHEN_INSTALL_DIR")) {
+      environment("CHEN_INSTALL_DIR").toFile
     } else {
       val uriToLibDir  = classOf[InstallConfig].getProtectionDomain.getCodeSource.getLocation.toURI
       val pathToLibDir = File(uriToLibDir).parent

diff --git a/console/src/test/scala/io/appthreat/console/ConsoleConfigTest.scala b/console/src/test/scala/io/appthreat/console/ConsoleConfigTest.scala
@@ -12,8 +12,8 @@ class ConsoleConfigTest extends AnyWordSpec with Matchers {
       config.rootPath shouldBe ProjectRoot.find
     }
 
-    "set the rootPath to SHIFTLEFT_OCULAR_INSTALL_DIR if it is defined" in {
-      val config = new InstallConfig(environment = Map("SHIFTLEFT_OCULAR_INSTALL_DIR" -> "/tmp"))
+    "set the rootPath to CHEN_INSTALL_DIR if it is defined" in {
+      val config = new InstallConfig(environment = Map("CHEN_INSTALL_DIR" -> "/tmp"))
       config.rootPath shouldBe File("/tmp")
     }
 

diff --git a/console/src/test/scala/io/appthreat/console/testing/ConsoleFixture.scala b/console/src/test/scala/io/appthreat/console/testing/ConsoleFixture.scala
@@ -40,7 +40,7 @@ object TestWorkspaceLoader extends WorkspaceLoader[Project] {
 
 class TestConsole(workspaceDir: String) extends Console[Project](TestWorkspaceLoader, File(workspaceDir)) {
   override def config =
-    new ConsoleConfig(install = new InstallConfig(Map("SHIFTLEFT_OCULAR_INSTALL_DIR" -> workspaceDir)))
+    new ConsoleConfig(install = new InstallConfig(Map("CHEN_INSTALL_DIR" -> workspaceDir)))
 
   override def importCode: ImportCode[Project] = new ImportCode(this) {
     override val generatorFactory = new TestCpgGeneratorFactory(config)

diff --git a/notebooks/README.md b/notebooks/README.md