implement graphmlimporter, add tests in domain language (#180)

* WIP GraphML reimpl * WIP * implement graphmlimporter, add tests in domain language
joernio · Apr 26, 2024 · e8f5050 · e8f5050
1 parent a956e5b
commit e8f5050
Show file tree

Hide file tree

Showing 33 changed files with 691 additions and 153 deletions.
diff --git a/core/src/main/scala/flatgraph/DiffGraphApplier.scala b/core/src/main/scala/flatgraph/DiffGraphApplier.scala
@@ -7,9 +7,9 @@ import flatgraph.Edge.Direction.{Incoming, Outgoing}
 import scala.collection.{Iterator, mutable}
 
 object DiffGraphApplier {
-  def applyDiff(g: Graph, diff: DiffGraphBuilder): Unit = {
-    if (g.isClosed) throw new GraphClosedException(s"graph cannot be modified any longer since it's closed")
-    new DiffGraphApplier(g, diff).applyUpdate()
+  def applyDiff(graph: Graph, diff: DiffGraphBuilder): Unit = {
+    if (graph.isClosed) throw new GraphClosedException(s"graph cannot be modified any longer since it's closed")
+    new DiffGraphApplier(graph, diff).applyUpdate()
     diff.buffer = null
   }
 }

diff --git a/core/src/main/scala/flatgraph/Graph.scala b/core/src/main/scala/flatgraph/Graph.scala
@@ -97,6 +97,9 @@ class Graph(val schema: Schema, val storagePathMaybe: Option[Path] = None) exten
   def allEdges: Iterator[Edge] =
     allNodes.flatMap(Accessors.getEdgesOut)
 
+  def edgeCount(): Int =
+    allEdges.size
+
   /** Lookup nodes with a given label and property value via index. N.b. currently only supported for String properties. Context:
     * MultiDictIndex requires the key to be a String and this is using reverse indices, i.e. the lookup is from String -> GNode.
     */

diff --git a/core/src/main/scala/flatgraph/Schema.scala b/core/src/main/scala/flatgraph/Schema.scala
@@ -1,6 +1,7 @@
 package flatgraph
 
 import flatgraph.Edge.Direction
+import flatgraph.Schema.UndefinedKind
 
 object DefaultValue
 object NoProperty
@@ -96,7 +97,8 @@ object FormalQtyType {
 
 abstract class Schema {
   def getNumberOfNodeKinds: Int
-  def nodeKinds: Range = Range(0, getNumberOfNodeKinds)
+  def nodeKinds: Range        = Range(0, getNumberOfNodeKinds)
+  def nodeLabels: Seq[String] = nodeKinds.map(getNodeLabel)
 
   def getNumberOfEdgeKinds: Int
   def edgeKinds: Range = Range(0, getNumberOfEdgeKinds)
@@ -106,6 +108,9 @@ abstract class Schema {
 
   def getNodeLabel(nodeKind: Int): String
   def getNodeKindByLabel(label: String): Int
+  def getNodeKindByLabelMaybe(label: String): Option[Int] = {
+    Option(getNodeKindByLabel(label)).filterNot(_ == UndefinedKind)
+  }
 
   // So, the issue here is: We have a couple of pseudo-properties that can only exist at a single node kind
   // (theoretically same for edges). We want to allow our data-layout to alias these properties. This means that multiple
@@ -143,10 +148,10 @@ abstract class Schema {
 }
 
 class FreeSchema(
-  val nodeLabels: Array[String],
-  val propertyLabels: Array[String],
+  nodeLabels: Array[String],
+  propertyLabels: Array[String],
   nodePropertyPrototypes: Array[AnyRef],
-  val edgeLabels: Array[String],
+  edgeLabels: Array[String],
   edgePropertyPrototypes: Array[AnyRef],
   formalQuantities: Array[FormalQtyType.FormalQuantity] = null
 ) extends Schema {

diff --git a/domain-classes-generator/src/main/scala/flatgraph/codegen/DomainClassesGenerator.scala b/domain-classes-generator/src/main/scala/flatgraph/codegen/DomainClassesGenerator.scala
@@ -607,7 +607,7 @@ class DomainClassesGenerator(schema: Schema) {
          |import flatgraph.FormalQtyType
          |
          |object GraphSchema extends flatgraph.Schema {
-         |  val nodeLabels = Array($nodeLabelsSrc)
+         |  private val nodeLabels = IndexedSeq($nodeLabelsSrc)
          |  val nodeKindByLabel = nodeLabels.zipWithIndex.toMap
          |  val edgeLabels = Array(${edgeTypes.map { e => s""""${e.name}"""" }.mkString(",  ")})
          |  val edgeIdByLabel = edgeLabels.zipWithIndex.toMap

diff --git a/formats-tests/src/test/resources/graphml-small.xml b/formats-tests/src/test/resources/graphml-small.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" ?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-element"
+         xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.1/graphml.xsd">
+    <key id="labelV" for="node" attr.name="labelV" attr.type="string"></key>
+    <key id="name" for="node" attr.name="name" attr.type="string"></key>
+    <key id="songType" for="node" attr.name="songType" attr.type="string"></key>
+    <key id="labelE" for="edge" attr.name="labelE" attr.type="string"></key>
+    <graph id="G" edgedefault="directed">
+        <node id="1">
+            <data key="labelV">song</data>
+            <data key="name">HEY BO DIDDLEY</data>
+            <data key="songType">cover</data>
+        </node>
+        <node id="340">
+            <data key="labelV">artist</data>
+            <data key="name">Garcia</data>
+        </node>
+        <node id="527">
+            <data key="labelV">artist</data>
+            <data key="name">Bo_Diddley</data>
+        </node>
+        <edge id="7611" source="1" target="527">
+            <data key="labelE">writtenBy</data>
+        </edge>
+        <edge id="7612" source="1" target="340">
+            <data key="labelE">sungBy</data>
+        </edge>
+    </graph>
+</graphml>
diff --git a/formats-tests/src/test/resources/graphson-small.json b/formats-tests/src/test/resources/graphson-small.json
@@ -0,0 +1,106 @@
+{
+  "@type": "tinker:graph",
+  "@value": {
+    "edges": [{
+      "@type": "g:Edge",
+      "id": {
+        "@type": "g:Int64",
+        "@value": 0
+      },
+      "inV": {
+        "@type": "g:Int64",
+        "@value": 340
+      },
+      "inVLabel": "artist",
+      "label": "sungBy",
+      "outV": {
+        "@type": "g:Int64",
+        "@value": 1
+      },
+      "outVLabel": "song",
+      "properties": {
+
+      }
+    }, {
+      "@type": "g:Edge",
+      "id": {
+        "@type": "g:Int64",
+        "@value": 1
+      },
+      "inV": {
+        "@type": "g:Int64",
+        "@value": 527
+      },
+      "inVLabel": "artist",
+      "label": "writtenBy",
+      "outV": {
+        "@type": "g:Int64",
+        "@value": 1
+      },
+      "outVLabel": "song",
+      "properties": {
+
+      }
+    }],
+    "vertices": [{
+      "@type": "g:Vertex",
+      "id": {
+        "@type": "g:Int64",
+        "@value": 1
+      },
+      "label": "song",
+      "properties": {
+        "name": {
+          "@type": "g:VertexProperty",
+          "@value": "HEY BO DIDDLEY",
+          "id": {
+            "@type": "g:Int64",
+            "@value": 0
+          }
+        },
+        "songType": {
+          "@type": "g:VertexProperty",
+          "@value": "cover",
+          "id": {
+            "@type": "g:Int64",
+            "@value": 1
+          }
+        }
+      }
+    }, {
+      "@type": "g:Vertex",
+      "id": {
+        "@type": "g:Int64",
+        "@value": 340
+      },
+      "label": "artist",
+      "properties": {
+        "name": {
+          "@type": "g:VertexProperty",
+          "@value": "Garcia",
+          "id": {
+            "@type": "g:Int64",
+            "@value": 2
+          }
+        }
+      }
+    }, {
+      "@type": "g:Vertex",
+      "id": {
+        "@type": "g:Int64",
+        "@value": 527
+      },
+      "label": "artist",
+      "properties": {
+        "name": {
+          "@type": "g:VertexProperty",
+          "@value": "Bo_Diddley",
+          "id": {
+            "@type": "g:Int64",
+            "@value": 3
+          }
+        }
+      }
+    }]
+  }
+}
diff --git a/formats-tests/src/test/resources/neo4jcsv/invalid_column_content_data.csv b/formats-tests/src/test/resources/neo4jcsv/invalid_column_content_data.csv
@@ -0,0 +1,3 @@
+1,stringProp1,11,testNode,stringListProp1a;stringListProp1b,21;31;41
+2,stringProp2,,testNode,,
+3,,NOT_AN_INT,testNode,,
diff --git a/formats-tests/src/test/resources/neo4jcsv/invalid_column_content_header.csv b/formats-tests/src/test/resources/neo4jcsv/invalid_column_content_header.csv
@@ -0,0 +1 @@
+id:ID,StringProperty,IntProperty:int,:LABEL,StringListProperty:string[],IntListProperty:int[]
diff --git a/formats-tests/src/test/resources/neo4jcsv/testedges_data.csv b/formats-tests/src/test/resources/neo4jcsv/testedges_data.csv
@@ -0,0 +1,2 @@
+testEdge,1,2,9223372036854775807
+testEdge,2,3,
diff --git a/formats-tests/src/test/resources/neo4jcsv/testedges_header.csv b/formats-tests/src/test/resources/neo4jcsv/testedges_header.csv
@@ -0,0 +1 @@
+:TYPE,:START_ID,:END_ID,longProperty:long
diff --git a/formats-tests/src/test/resources/neo4jcsv/testnodes_data.csv b/formats-tests/src/test/resources/neo4jcsv/testnodes_data.csv
@@ -0,0 +1,3 @@
+1,stringProp1,11,testNode,stringListProp1a;stringListProp1b,21;31;41
+2,stringProp2,,testNode,,
+3,,13,testNode,,
diff --git a/formats-tests/src/test/resources/neo4jcsv/testnodes_header.csv b/formats-tests/src/test/resources/neo4jcsv/testnodes_header.csv
@@ -0,0 +1 @@
+id:ID,StringProperty,IntProperty:int,:LABEL,StringListProperty:string[],IntListProperty:int[]
diff --git a/formats-tests/src/test/resources/neo4jcsv/unsupported_multiple_labels_data.csv b/formats-tests/src/test/resources/neo4jcsv/unsupported_multiple_labels_data.csv
@@ -0,0 +1 @@
+1,LABEL1,11,LABEL2
diff --git a/formats-tests/src/test/resources/neo4jcsv/unsupported_multiple_labels_header.csv b/formats-tests/src/test/resources/neo4jcsv/unsupported_multiple_labels_header.csv
@@ -0,0 +1 @@
+id:ID,:LABEL,IntProperty:int,:LABEL
diff --git a/formats-tests/src/test/scala/flatgraph/formats/graphml/GraphMLTests.scala b/formats-tests/src/test/scala/flatgraph/formats/graphml/GraphMLTests.scala
@@ -1,7 +1,9 @@
 package flatgraph.formats.graphml
 
 import better.files.File
-import org.scalatest.matchers.should.Matchers._
+import flatgraph.testdomains.gratefuldead.GratefulDead
+import flatgraph.testdomains.gratefuldead.Language.*
+import org.scalatest.matchers.should.Matchers.*
 import org.scalatest.wordspec.AnyWordSpec
 import flatgraph.util.DiffTool
 
@@ -12,31 +14,28 @@ import scala.jdk.CollectionConverters.{CollectionHasAsScala, IterableHasAsJava}
 class GraphMLTests extends AnyWordSpec {
 
   "import minified gratefuldead graph" in {
-    pending
-//    ??? // TODO
-//    val graph = GratefulDead.newGraph()
-//    graph.nodeCount() shouldBe 0
-//
-//    GraphMLImporter.runImport(graph, Paths.get(getClass.getResource("/graphml-small.xml").toURI))
-//    graph.nodeCount() shouldBe 3
-//    graph.edgeCount() shouldBe 2
-//
-//    val node1 = graph.node(1)
-//    node1.label() shouldBe "song"
-//    val node340 = node1.out("sungBy").next()
-//    val node527 = node1.out("writtenBy").next()
-//
-//    node340.label shouldBe "artist"
-//    node340.property("name") shouldBe "Garcia"
-//    node340.out().hasNext shouldBe false
-//    node340.in().hasNext shouldBe true
-//
-//    node527.label shouldBe "artist"
-//    node527.property("name") shouldBe "Bo_Diddley"
-//    node527.out().hasNext shouldBe false
-//    node527.in().hasNext shouldBe true
-//
-//    graph.close()
+    val gratefulDead = GratefulDead.empty
+    val graph        = gratefulDead.graph
+    graph.nodeCount() shouldBe 0
+
+    GraphMLImporter.runImport(graph, Paths.get(getClass.getResource("/graphml-small.xml").toURI))
+    graph.nodeCount() shouldBe 3
+    graph.edgeCount() shouldBe 2
+
+    gratefulDead.song.size shouldBe 1
+    gratefulDead.song.name.l shouldBe List("HEY BO DIDDLEY")
+    gratefulDead.artist.size shouldBe 2
+    gratefulDead.artist.name.l shouldBe List("Garcia", "Bo_Diddley")
+
+    val Seq(boDiddley, garcia) = gratefulDead.artist.sortBy(_.name).l
+    val Seq(heyBoDiddley)      = gratefulDead.song.l
+
+    heyBoDiddley.sungBy shouldBe garcia
+    heyBoDiddley.writtenBy shouldBe boDiddley
+    garcia.sang.l shouldBe List(heyBoDiddley)
+    boDiddley.wrote.l shouldBe List(heyBoDiddley)
+
+    graph.close()
   }
 
 //  "Exporter should export valid xml" when {