Improvements to line number resolution

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

build.sbt

@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.15"
+ThisBuild / version      := "0.0.16"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/jartypereader/JarTypeReader.scala

@@ -67,7 +67,7 @@ object JarTypeReader {
   private def classTypeSignatureFromString(signature: String): ClassTypeSignature = {
     signature.split('.').toList match {
       case Nil =>
-        logger.warn(s"$signature is not a valid class signature")
+        logger.debug(s"$signature is not a valid class signature")
         ClassTypeSignature(None, NameWithTypeArgs("", Nil), Nil)
 
       case name :: Nil => ClassTypeSignature(None, typedName = NameWithTypeArgs(name, Nil), Nil)

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/AstCreationPass.scala

@@ -58,7 +58,7 @@ class AstCreationPass(config: Config, cpg: Cpg, sourcesOverride: Option[List[Str
           ).createAst()
         )
 
-      case None => logger.warn(s"Skipping AST creation for $filename")
+      case None => logger.debug(s"Skipping AST creation for $filename")
     }
   }
 
@@ -77,7 +77,7 @@ class AstCreationPass(config: Config, cpg: Cpg, sourcesOverride: Option[List[Str
       DependencyResolver.getDependencies(Paths.get(inputPath)) match {
         case Some(deps) => deps.toList
         case None =>
-          logger.warn(s"Could not fetch dependencies for project at path $inputPath")
+          logger.debug(s"Could not fetch dependencies for project at path $inputPath")
           List()
       }
     } else {

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/AstCreator.scala

@@ -402,7 +402,7 @@ class AstCreator(
   def getBindingTable(typeDecl: ResolvedReferenceTypeDeclaration): BindingTable = {
     val fullName = typeInfoCalc.fullName(typeDecl).getOrElse {
       val qualifiedName = typeDecl.getQualifiedName
-      logger.warn(s"Could not get full name for resolved type decl $qualifiedName. THIS SHOULD NOT HAPPEN!")
+      logger.debug(s"Could not get full name for resolved type decl $qualifiedName. THIS SHOULD NOT HAPPEN!")
       qualifiedName
     }
     bindingTableCache.getOrElseUpdate(
@@ -1217,7 +1217,7 @@ class AstCreator(
       case x: WhileStmt        => Seq(astForWhile(x))
       // case x: LocalClassDeclarationStmt => Seq(astForLocalClassDeclarationStmt(x))
       case x =>
-        logger.warn(s"Attempting to generate AST for unknown statement of type ${x.getClass}")
+        logger.debug(s"Attempting to generate AST for unknown statement of type ${x.getClass}")
         Seq(unknownAst(x))
     }
   }
@@ -1356,7 +1356,7 @@ class AstCreator(
         Ast()
       case iterableAstHead :: Nil => iterableAstHead
       case iterableAsts =>
-        logger.warn(
+        logger.debug(
           s"Found multiple ASTS for iterable expr $iterableExpression: $filename:l$lineNo\nDropping all but the first!"
         )
         iterableAsts.head
@@ -1469,7 +1469,7 @@ class AstCreator(
         None
       case variable :: Nil => Some(variable)
       case variable :: _ =>
-        logger.warn(s"ForEach statement defines multiple variables. Dropping all but the first: $filename$lineNo")
+        logger.debug(s"ForEach statement defines multiple variables. Dropping all but the first: $filename$lineNo")
         Some(variable)
     }
 
@@ -1614,13 +1614,13 @@ class AstCreator(
 
     val actualIteratorAst = astsForExpression(iterExpr, expectedType = ExpectedType.empty).toList match {
       case Nil =>
-        logger.warn(s"Could not create receiver ast for iterator $iterExpr")
+        logger.debug(s"Could not create receiver ast for iterator $iterExpr")
         None
 
       case ast :: Nil => Some(ast)
 
       case ast :: _ =>
-        logger.warn(s"Created multiple receiver asts for $iterExpr. Dropping all but the first.")
+        logger.debug(s"Created multiple receiver asts for $iterExpr. Dropping all but the first.")
         Some(ast)
     }
 
@@ -2062,7 +2062,7 @@ class AstCreator(
       Seq(assignAst)
     } else {
       if (partialConstructorQueue.size > 1) {
-        logger.warn("BUG: Received multiple partial constructors from assignment. Dropping all but the first.")
+        logger.debug("BUG: Received multiple partial constructors from assignment. Dropping all but the first.")
       }
       val partialConstructor = partialConstructorQueue.head
       partialConstructorQueue.clear()
@@ -2115,12 +2115,12 @@ class AstCreator(
             callAst(fieldAccess.copy, args)
 
           case _ =>
-            logger.warn(s"Attempting to copy field access without required children: ${fieldAccess.code}")
+            logger.debug(s"Attempting to copy field access without required children: ${fieldAccess.code}")
             Ast()
         }
 
       case Some(root) =>
-        logger.warn(s"Attempting to copy unhandled root type for var decl init: $root")
+        logger.debug(s"Attempting to copy unhandled root type for var decl init: $root")
         Ast()
 
       case None =>
@@ -3192,7 +3192,6 @@ class AstCreator(
     val evalStrat =
       if (parameter.getType.isPrimitiveType) EvaluationStrategies.BY_VALUE else EvaluationStrategies.BY_SHARING
     typeInfoCalc.registerType(typeFullName)
-
     val parameterNode = NewMethodParameterIn()
       .name(parameter.getName.toString)
       .code(parameter.toString)

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/typesolvers/EagerSourceTypeSolver.scala

@@ -55,11 +55,11 @@ class EagerSourceTypeSolver(
 
   override def setParent(parent: TypeSolver): Unit = {
     if (parent == null) {
-      logger.warn(s"Cannot set parent of type solver to null. setParent will be ignored.")
+      logger.debug(s"Cannot set parent of type solver to null. setParent will be ignored.")
     } else if (this.parent != null) {
-      logger.warn(s"Attempting to re-set type solver parent. setParent will be ignored.")
+      logger.debug(s"Attempting to re-set type solver parent. setParent will be ignored.")
     } else if (parent == this) {
-      logger.warn(s"Parent of TypeSolver cannot be itself. setParent will be ignored.")
+      logger.debug(s"Parent of TypeSolver cannot be itself. setParent will be ignored.")
     } else {
       this.parent = parent
     }

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/typesolvers/SimpleCombinedTypeSolver.scala

@@ -65,7 +65,7 @@ class SimpleCombinedTypeSolver extends TypeSolver {
             // RecordDeclarations aren't handled by JavaParser yet
             None
           case unhandled: Throwable =>
-            logger.warn("Caught unhandled exception", unhandled)
+            logger.debug("Caught unhandled exception", unhandled)
             None
         }
       }
@@ -86,11 +86,11 @@ class SimpleCombinedTypeSolver extends TypeSolver {
 
   override def setParent(parent: TypeSolver): Unit = {
     if (parent == null) {
-      logger.warn(s"Cannot set parent of type solver to null. setParent will be ignored.")
+      logger.debug(s"Cannot set parent of type solver to null. setParent will be ignored.")
     } else if (this.parent != null) {
-      logger.warn(s"Attempting to re-set type solver parent. setParent will be ignored.")
+      logger.debug(s"Attempting to re-set type solver parent. setParent will be ignored.")
     } else if (parent == this) {
-      logger.warn(s"Parent of TypeSolver cannot be itself. setParent will be ignored.")
+      logger.debug(s"Parent of TypeSolver cannot be itself. setParent will be ignored.")
     } else {
       this.parent = parent
     }

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/typesolvers/noncaching/JdkJarTypeSolver.scala

@@ -102,7 +102,7 @@ class JdkJarTypeSolver extends TypeSolver {
         case Success(_) => registerPackagesForJar(archivePath)
 
         case Failure(e) =>
-          logger.warn(s"Could not load jar at path $archivePath", e.getMessage)
+          logger.debug(s"Could not load jar at path $archivePath", e.getMessage)
       }
     }
   }
@@ -124,7 +124,7 @@ class JdkJarTypeSolver extends TypeSolver {
       }
     } catch {
       case ioException: IOException =>
-        logger.warn(s"Could register classes for archive at $archivePath", ioException.getMessage)
+        logger.debug(s"Could register classes for archive at $archivePath", ioException.getMessage)
     }
   }
 }

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/util/Delombok.scala

@@ -42,7 +42,7 @@ object Delombok {
         s"@${file.canonicalPath}"
 
       case Failure(t) =>
-        logger.warn(
+        logger.debug(
           s"Failed to create classpath file for delombok execution. Results may be missing on Windows systems",
           t
         )
@@ -59,13 +59,15 @@ object Delombok {
             tempDir.path.toAbsolutePath.toString
 
           case Failure(t) =>
-            logger.warn(s"Executing delombok failed", t)
-            logger.warn("Creating AST with original source instead. Some methods and type information will be missing.")
+            logger.debug(s"Executing delombok failed", t)
+            logger.debug(
+              "Creating AST with original source instead. Some methods and type information will be missing."
+            )
             projectDir
         }
 
       case Failure(e) =>
-        logger.warn(s"Failed to create temporary directory for delomboked source. Methods and types may be missing", e)
+        logger.debug(s"Failed to create temporary directory for delomboked source. Methods and types may be missing", e)
         projectDir
     }
   }
@@ -78,7 +80,7 @@ object Delombok {
       case Some("types-only")   => TypesOnly
       case Some("run-delombok") => RunDelombok
       case Some(value) =>
-        logger.warn(s"Found unrecognised delombok mode `$value`. Using default instead.")
+        logger.debug(s"Found unrecognised delombok mode `$value`. Using default instead.")
         Default
     }
   }

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/util/SourceParser.scala

@@ -69,22 +69,24 @@ class SourceParser private (originalInputPath: Path, analysisRoot: Path, typesRo
     val javaParserConfig =
       new ParserConfiguration()
         .setLanguageLevel(LanguageLevel.BLEEDING_EDGE)
+        .setAttributeComments(false)
+        .setLexicalPreservationEnabled(true)
         .setStoreTokens(storeTokens)
     val parseResult = new JavaParser(javaParserConfig).parse(file.toJava)
 
     parseResult.getProblems.asScala.toList match {
       case Nil => // Just carry on as usual
       case problems =>
-        logger.warn(s"Encountered problems while parsing file ${file.name}:")
+        logger.debug(s"Encountered problems while parsing file ${file.name}:")
         problems.foreach { problem =>
-          logger.warn(s"- ${problem.getMessage}")
+          logger.debug(s"- ${problem.getMessage}")
         }
     }
 
     parseResult.getResult.toScala match {
       case Some(result) if result.getParsed == Parsedness.PARSED => Some(result)
       case _ =>
-        logger.warn(s"Failed to parse file ${file.name}")
+        logger.debug(s"Failed to parse file ${file.name}")
         None
     }
   }

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "0.0.15"
+version = "0.0.16"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <cloud@appthreat.com>"]
 license = "Apache-2.0"