From 516358a9788248ff4aaabd67adeb1c6bc09053b7 Mon Sep 17 00:00:00 2001 From: danicheg Date: Sun, 12 Nov 2023 13:09:26 +0300 Subject: [PATCH 1/5] Add ScalaSettingsOps for 2.12 / 2.13 --- .../ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala | 6 ++++++ .../ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 plugin/src/main/scala-2.12/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala create mode 100644 plugin/src/main/scala-2.13/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala diff --git a/plugin/src/main/scala-2.12/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala b/plugin/src/main/scala-2.12/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala new file mode 100644 index 0000000..194204d --- /dev/null +++ b/plugin/src/main/scala-2.12/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala @@ -0,0 +1,6 @@ +package ch.epfl.scala.profilers.tools + +object ScalaSettingsOps { + def isScala212: Boolean = true + def isScala213: Boolean = false +} diff --git a/plugin/src/main/scala-2.13/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala b/plugin/src/main/scala-2.13/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala new file mode 100644 index 0000000..4f1a648 --- /dev/null +++ b/plugin/src/main/scala-2.13/ch/epfl/scala/profilers/tools/ScalaSettingsOps.scala @@ -0,0 +1,6 @@ +package ch.epfl.scala.profilers.tools + +object ScalaSettingsOps { + def isScala212: Boolean = false + def isScala213: Boolean = true +} From e1c8013b53553384222dd653a910d425ea3884d6 Mon Sep 17 00:00:00 2001 From: danicheg Date: Sun, 12 Nov 2023 13:11:11 +0300 Subject: [PATCH 2/5] Add the ProfileDbPath#toGraphsProfilePath method --- .../src/main/scala/ch.epfl.scala.profiledb/ProfileDbPath.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/profiledb/src/main/scala/ch.epfl.scala.profiledb/ProfileDbPath.scala b/profiledb/src/main/scala/ch.epfl.scala.profiledb/ProfileDbPath.scala index 6cac9d9..ffa0204 100644 --- a/profiledb/src/main/scala/ch.epfl.scala.profiledb/ProfileDbPath.scala +++ b/profiledb/src/main/scala/ch.epfl.scala.profiledb/ProfileDbPath.scala @@ -35,6 +35,9 @@ object ProfileDbPath { def toProfileDbPath(relativeSourceFile: RelativePath): RelativePath = Prefix.resolveRelative(addDbExtension(relativeSourceFile)) + def toGraphsProfilePath(path: AbsolutePath): AbsolutePath = + path.resolve(GraphsProfileDbRelativePath) + private[profiledb] def addDbExtension(path: RelativePath): RelativePath = { val realPath = path.underlying val extendedName = realPath.getFileName.toString + ProfileDbExtension From 2d6b21c061bde9274b3f77d2b950230421bd527c Mon Sep 17 00:00:00 2001 From: danicheg Date: Sun, 12 Nov 2023 13:26:34 +0300 Subject: [PATCH 3/5] Enhance the ProfilingImpl to write a global implicit searches flamegraph --- .../epfl/scala/profilers/ProfilingImpl.scala | 119 ++++++++++++------ 1 file changed, 81 insertions(+), 38 deletions(-) diff --git a/plugin/src/main/scala/ch/epfl/scala/profilers/ProfilingImpl.scala b/plugin/src/main/scala/ch/epfl/scala/profilers/ProfilingImpl.scala index c31cb53..6c6a03d 100644 --- a/plugin/src/main/scala/ch/epfl/scala/profilers/ProfilingImpl.scala +++ b/plugin/src/main/scala/ch/epfl/scala/profilers/ProfilingImpl.scala @@ -149,20 +149,45 @@ final class ProfilingImpl[G <: Global]( } } - def generateGraphData(outputDir: AbsolutePath): List[AbsolutePath] = { + def generateGraphData(outputDir: AbsolutePath, globalDirMaybe: Option[AbsolutePath]): List[AbsolutePath] = { Files.createDirectories(outputDir.underlying) + val randomId = java.lang.Long.toString(System.currentTimeMillis()) - val implicitGraphName = s"implicit-searches-$randomId" - val macroGraphName = s"macros-$randomId" - /* val dotFile = outputDir.resolve(s"$graphName.dot") + + /*val dotFile = outputDir.resolve(s"$graphName.dot") ProfilingAnalyzerPlugin.dottify(graphName, dotFile.underlying)*/ - val implicitFlamegraphFile = outputDir.resolve(s"$implicitGraphName.flamegraph") - ProfilingAnalyzerPlugin.foldImplicitStacks(implicitFlamegraphFile.underlying) - if (config.generateMacroFlamegraph) { - val macroFlamegraphFile = outputDir.resolve(s"$macroGraphName.flamegraph") - ProfilingMacroPlugin.foldMacroStacks(macroFlamegraphFile.underlying) - List(implicitFlamegraphFile, macroFlamegraphFile) - } else List(implicitFlamegraphFile) + + val implicitFlamegraphFiles = { + val mkImplicitGraphName: String => String = + postfix => s"implicit-searches-$postfix.flamegraph" + val compileUnitFlamegraphFile = outputDir.resolve(mkImplicitGraphName(randomId)) + + globalDirMaybe match { + case Some(globalDir) => + Files.createDirectories(globalDir.underlying) + + val globalFile = + globalDir + .resolve(mkImplicitGraphName("global")) + + List(compileUnitFlamegraphFile, globalFile) + + case None => + List(compileUnitFlamegraphFile) + } + } + + val macroFlamegraphFiles = + if (config.generateMacroFlamegraph) { + val macroGraphName = s"macros-$randomId" + val file = outputDir.resolve(s"$macroGraphName.flamegraph") + List(file) + } else Nil + + ProfilingAnalyzerPlugin.foldImplicitStacks(implicitFlamegraphFiles) + ProfilingMacroPlugin.foldMacroStacks(macroFlamegraphFiles) + + implicitFlamegraphFiles ::: macroFlamegraphFiles } private val registeredQuantities = QuantitiesHijacker.getRegisteredQuantities(global) @@ -190,20 +215,29 @@ final class ProfilingImpl[G <: Global]( private val implicitsDependants = new mutable.AnyRefMap[Type, mutable.HashSet[Type]]() private val searchIdChildren = perRunCaches.newMap[Int, List[analyzer.ImplicitSearch]]() - def foldImplicitStacks(outputPath: Path): Unit = { - // This part is memory intensive and hence the use of java collections - val stacksJavaList = new java.util.ArrayList[String]() - stackedNanos.foreach { - case (id, (nanos, tpe)) => - val names = - stackedNames.getOrElse(id, sys.error(s"Stack name for search id ${id} doesn't exist!")) - val stackName = names.mkString(";") - //val count = implicitSearchesByType.getOrElse(tpe, sys.error(s"No counter for ${tpe}")) - stacksJavaList.add(s"$stackName ${nanos / 1000}") - } - java.util.Collections.sort(stacksJavaList) - Files.write(outputPath, stacksJavaList, StandardOpenOption.WRITE, StandardOpenOption.CREATE) - } + def foldImplicitStacks(outputPaths: Seq[AbsolutePath]): Unit = + if (outputPaths.nonEmpty) { + // This part is memory intensive and hence the use of java collections + val stacksJavaList = new java.util.ArrayList[String]() + stackedNanos.foreach { + case (id, (nanos, _)) => + val names = + stackedNames.getOrElse(id, sys.error(s"Stack name for search id ${id} doesn't exist!")) + val stackName = names.mkString(";") + //val count = implicitSearchesByType.getOrElse(tpe, sys.error(s"No counter for ${tpe}")) + stacksJavaList.add(s"$stackName ${nanos / 1000}") + } + java.util.Collections.sort(stacksJavaList) + + outputPaths.foreach(path => + Files.write( + path.underlying, + stacksJavaList, + StandardOpenOption.APPEND, + StandardOpenOption.CREATE + ) + ) + } else () def dottify(graphName: String, outputPath: Path): Unit = { def clean(`type`: Type) = typeToString(`type`).replace("\"", "\'") @@ -454,19 +488,28 @@ final class ProfilingImpl[G <: Global]( private val stackedNanos = perRunCaches.newMap[Int, Long]() private val stackedNames = perRunCaches.newMap[Int, List[String]]() - def foldMacroStacks(outputPath: Path): Unit = { - // This part is memory intensive and hence the use of java collections - val stacksJavaList = new java.util.ArrayList[String]() - stackedNanos.foreach { - case (id, nanos) => - val names = - stackedNames.getOrElse(id, sys.error(s"Stack name for macro id ${id} doesn't exist!")) - val stackName = names.mkString(";") - stacksJavaList.add(s"$stackName ${nanos / 1000}") - } - java.util.Collections.sort(stacksJavaList) - Files.write(outputPath, stacksJavaList, StandardOpenOption.WRITE, StandardOpenOption.CREATE) - } + def foldMacroStacks(outputPaths: Seq[AbsolutePath]): Unit = + if (outputPaths.nonEmpty) { + // This part is memory intensive and hence the use of java collections + val stacksJavaList = new java.util.ArrayList[String]() + stackedNanos.foreach { + case (id, nanos) => + val names = + stackedNames.getOrElse(id, sys.error(s"Stack name for macro id ${id} doesn't exist!")) + val stackName = names.mkString(";") + stacksJavaList.add(s"$stackName ${nanos / 1000}") + } + java.util.Collections.sort(stacksJavaList) + + outputPaths.foreach(path => + Files.write( + path.underlying, + stacksJavaList, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE + ) + ) + } else () import scala.tools.nsc.Mode override def pluginsMacroExpand(t: Typer, expandee: Tree, md: Mode, pt: Type): Option[Tree] = { From 4081b757e264ba3a81b832126aebe5e2254069fe Mon Sep 17 00:00:00 2001 From: danicheg Date: Sun, 12 Nov 2023 13:28:25 +0300 Subject: [PATCH 4/5] Add the generate-global-flamegraph plugin option --- .../scala/ch/epfl/scala/PluginConfig.scala | 5 +- .../scala/ch/epfl/scala/ProfilingPlugin.scala | 52 ++++++++++++++----- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/plugin/src/main/scala/ch/epfl/scala/PluginConfig.scala b/plugin/src/main/scala/ch/epfl/scala/PluginConfig.scala index 3ad75a5..16886ca 100644 --- a/plugin/src/main/scala/ch/epfl/scala/PluginConfig.scala +++ b/plugin/src/main/scala/ch/epfl/scala/PluginConfig.scala @@ -2,12 +2,13 @@ package ch.epfl.scala import ch.epfl.scala.profiledb.utils.AbsolutePath -case class PluginConfig( +final case class PluginConfig( showProfiles: Boolean, noDb: Boolean, - sourceRoot: Option[AbsolutePath], + sourceRoot: AbsolutePath, printSearchIds: Set[Int], generateMacroFlamegraph: Boolean, + generateGlobalFlamegraph: Boolean, printFailedMacroImplicits: Boolean, concreteTypeParamsInImplicits: Boolean ) diff --git a/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala b/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala index daa1f2e..b37b82a 100644 --- a/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala +++ b/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala @@ -10,11 +10,10 @@ package ch.epfl.scala import java.nio.file.Files - import ch.epfl.scala.profiledb.{ProfileDb, ProfileDbPath} -import ch.epfl.scala.profiledb.utils.AbsolutePath +import ch.epfl.scala.profiledb.utils.{AbsolutePath, RelativePath} import ch.epfl.scala.profilers.ProfilingImpl -import ch.epfl.scala.profilers.tools.{Logger, SettingsOps} +import ch.epfl.scala.profilers.tools.{Logger, ScalaSettingsOps, SettingsOps} import scala.reflect.internal.util.{SourceFile, Statistics} import scala.reflect.io.Path @@ -34,6 +33,7 @@ class ProfilingPlugin(val global: Global) extends Plugin { private final lazy val SourceRoot = "sourceroot" private final lazy val PrintSearchResult = "print-search-result" private final lazy val GenerateMacroFlamegraph = "generate-macro-flamegraph" + private final lazy val GenerateGlobalFlamegraph = "generate-global-flamegraph" private final lazy val PrintFailedMacroImplicits = "print-failed-implicit-macro-candidates" private final lazy val NoProfileDb = "no-profiledb" private final lazy val ShowConcreteImplicitTparams = "show-concrete-implicit-tparams" @@ -55,13 +55,17 @@ class ProfilingPlugin(val global: Global) extends Plugin { } private final lazy val config = PluginConfig( - super.options.contains(ShowProfiles), - super.options.contains(NoProfileDb), - findOption(SourceRoot, SourceRootRegex).map(AbsolutePath.apply), - findSearchIds(findOption(PrintSearchResult, PrintSearchRegex)), - super.options.contains(GenerateMacroFlamegraph), - super.options.contains(PrintFailedMacroImplicits), - super.options.contains(ShowConcreteImplicitTparams) + showProfiles = super.options.contains(ShowProfiles), + noDb = super.options.contains(NoProfileDb), + sourceRoot = + findOption(SourceRoot, SourceRootRegex) + .map(AbsolutePath.apply) + .getOrElse(AbsolutePath.workingDirectory), + printSearchIds = findSearchIds(findOption(PrintSearchResult, PrintSearchRegex)), + generateMacroFlamegraph = super.options.contains(GenerateMacroFlamegraph), + generateGlobalFlamegraph = super.options.contains(GenerateGlobalFlamegraph), + printFailedMacroImplicits = super.options.contains(PrintFailedMacroImplicits), + concreteTypeParamsInImplicits = super.options.contains(ShowConcreteImplicitTparams) ) private lazy val logger = new Logger(global) @@ -96,10 +100,31 @@ class ProfilingPlugin(val global: Global) extends Plugin { } private def reportStatistics(graphsPath: AbsolutePath): Unit = { - val macroProfiler = implementation.macroProfiler - val persistedGraphData = implementation.generateGraphData(graphsPath) + val globalDir = + if (config.generateGlobalFlamegraph) { + val scalaDir = + if (ScalaSettingsOps.isScala212) + "scala-2.12" + else if (ScalaSettingsOps.isScala213) + "scala-2.13" + else + sys.error(s"Currently, only Scala 2.12 and 2.13 are supported, " + + s"but [${global.settings.source.value}] has been spotted") + + val globalDir = + ProfileDbPath.toGraphsProfilePath( + config.sourceRoot.resolve(RelativePath(s"target/$scalaDir/classes")) + ) + + Some(globalDir) + } else None + + val persistedGraphData = implementation.generateGraphData(graphsPath, globalDir) persistedGraphData.foreach(p => logger.info(s"Writing graph to ${p.underlying}")) + if (config.showProfiles) { + val macroProfiler = implementation.macroProfiler + logger.info("Macro data per call-site", macroProfiler.perCallSite) logger.info("Macro data per file", macroProfiler.perFile) logger.info("Macro data in total", macroProfiler.inTotal) @@ -181,10 +206,9 @@ class ProfilingPlugin(val global: Global) extends Plugin { if (outputPath.isEmpty) "." else outputPath } - private final def sourceRoot = config.sourceRoot.getOrElse(AbsolutePath.workingDirectory) private def dbPathFor(sourceFile: SourceFile): Option[ProfileDbPath] = { val absoluteSourceFile = AbsolutePath(sourceFile.file.path) - val targetPath = absoluteSourceFile.toRelative(sourceRoot) + val targetPath = absoluteSourceFile.toRelative(config.sourceRoot) if (targetPath.syntax.endsWith(".scala")) { val outputDir = getOutputDirFor(sourceFile.file) val absoluteOutput = AbsolutePath(outputDir.jfile) From 241c7a33c2d8f08f4d260967a11705b9a344e566 Mon Sep 17 00:00:00 2001 From: danicheg Date: Sun, 12 Nov 2023 14:20:55 +0300 Subject: [PATCH 5/5] Add the 'generate-global-flamegraph' option to optionsHelp --- plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala b/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala index b37b82a..4d6aede 100644 --- a/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala +++ b/plugin/src/main/scala/ch/epfl/scala/ProfilingPlugin.scala @@ -71,8 +71,11 @@ class ProfilingPlugin(val global: Global) extends Plugin { private lazy val logger = new Logger(global) private def pad20(option: String): String = option + (" " * (20 - option.length)) + override def init(ops: List[String], e: (String) => Unit): Boolean = true + override val optionsHelp: Option[String] = Some(s""" + |-P:$name:${pad20(GenerateGlobalFlamegraph)}: Creates a global flamegraph of implicit searches for all compilation units. Use the `-P:$name:$SourceRoot` option to manage the root directory, otherwise, a working directory (defined by the `user.dir` property) will be picked. |-P:$name:${pad20(SourceRoot)}:_ Sets the source root for this project. |-P:$name:${pad20(ShowProfiles)} Logs profile information for every call-site. |-P:$name:${pad20(ShowConcreteImplicitTparams)} Shows types in flamegraphs of implicits with concrete type params.