From 4e885e0b74a96667e78fd69c9720be5fbccb3fbe Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Mon, 12 Feb 2024 16:09:07 +0100 Subject: [PATCH 1/3] add default only graph for cbd option --- lib/CBDShapeExtractor.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/CBDShapeExtractor.ts b/lib/CBDShapeExtractor.ts index 45be098..10eed21 100644 --- a/lib/CBDShapeExtractor.ts +++ b/lib/CBDShapeExtractor.ts @@ -1,7 +1,7 @@ import rdfDereference, { RdfDereferencer } from "rdf-dereference"; import { NodeLink, RDFMap, ShapesGraph, ShapeTemplate } from "./Shape"; import { Path, PathResult } from "./Path"; -import { BlankNode, Store } from "n3"; +import { BlankNode, DefaultGraph, Store } from "n3"; import { Quad, Term } from "@rdfjs/types"; class DereferenceNeeded { @@ -13,6 +13,10 @@ class DereferenceNeeded { } } +type CBDShapeExtractorOptions = { + cbdDefaultGraph: boolean; +}; + /** * Usage: * import {ShapeExtractor} from "extract-cbd-shape"; @@ -24,7 +28,16 @@ export class CBDShapeExtractor { dereferencer: RdfDereferencer; shapesGraph?: ShapesGraph; - constructor(shapesGraphStore?: Store, dereferencer?: RdfDereferencer) { + options: CBDShapeExtractorOptions; + + constructor( + shapesGraphStore?: Store, + dereferencer?: RdfDereferencer, + options: Partial = {}, + ) { + this.options = { + cbdDefaultGraph: options.cbdDefaultGraph || false, + }; if (!dereferencer) { this.dereferencer = rdfDereference; } else { @@ -306,7 +319,8 @@ export class CBDShapeExtractor { graphsToIgnore: Array, ) { extractedStar.addCBDTerm(id); - const quads = store.getQuads(id, null, null, null); + const graph = this.options.cbdDefaultGraph ? new DefaultGraph() : null; + const quads = store.getQuads(id, null, null, graph); //Iterate over the quads, add them to the result and check whether we should further get other quads based on blank nodes or the SHACL shape for (const q of quads) { From 780fca4089bbde835ef966b43244757540816321 Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Mon, 12 Feb 2024 16:46:14 +0100 Subject: [PATCH 2/3] add bulk ingest option --- lib/CBDShapeExtractor.ts | 45 +++++++++++++++ tests/06 - shapes and named graphs/data.ttl | 2 +- .../extraction-example.test.ts | 57 +++++++++++++++---- .../shape-example.ttl | 3 +- 4 files changed, 95 insertions(+), 12 deletions(-) diff --git a/lib/CBDShapeExtractor.ts b/lib/CBDShapeExtractor.ts index 10eed21..b95af29 100644 --- a/lib/CBDShapeExtractor.ts +++ b/lib/CBDShapeExtractor.ts @@ -56,6 +56,51 @@ export class CBDShapeExtractor { }); } + public async bulkExtract( + store: Store, + ids: Array, + shapeId?: Term, + graphsToIgnore?: Array, + itemExtracted?: (member: { subject: Term; quads: Quad[] }) => void, + ): Promise> { + const out: Array<{ subject: Term; quads: Quad[] }> = []; + const idSet = new Set(ids.map((x) => x.value)); + + const memberSpecificQuads: { [id: string]: Array } = {}; + const newStore = new Store(); + for (let quad of store.readQuads(null, null, null, null)) { + if (quad.graph.termType == "NamedNode" && idSet.has(quad.graph.value)) { + if (!(quad.graph.value in memberSpecificQuads)) { + memberSpecificQuads[quad.graph.value] = []; + } + memberSpecificQuads[quad.graph.value].push(quad); + } else { + newStore.add(quad); + } + } + + const promises = []; + for (let id of ids) { + const promise = this.extract( + newStore, + id, + shapeId, + (graphsToIgnore || []).slice(), + ).then((quads) => { + if (itemExtracted) { + itemExtracted({ subject: id, quads }); + } + + out.push({ subject: id, quads }); + }); + promises.push(promise); + } + + await Promise.all(promises); + + return out; + } + /** * Extracts: * * first level quads, diff --git a/tests/06 - shapes and named graphs/data.ttl b/tests/06 - shapes and named graphs/data.ttl index 9e99e9f..a3f91d1 100644 --- a/tests/06 - shapes and named graphs/data.ttl +++ b/tests/06 - shapes and named graphs/data.ttl @@ -20,4 +20,4 @@ ex:M1v1 { ex:M1v2 { ex:M1 rdfs:label "M1v2" . } - \ No newline at end of file + diff --git a/tests/06 - shapes and named graphs/extraction-example.test.ts b/tests/06 - shapes and named graphs/extraction-example.test.ts index 2faa396..fe26813 100644 --- a/tests/06 - shapes and named graphs/extraction-example.test.ts +++ b/tests/06 - shapes and named graphs/extraction-example.test.ts @@ -2,6 +2,7 @@ import { assert } from "chai"; import { NamedNode, Parser, Store, StreamParser, Term, Writer } from "n3"; import { CBDShapeExtractor } from "../../lib/CBDShapeExtractor"; import rdfDereference from "rdf-dereference"; +import { Quad, Term as RTerm } from "@rdfjs/types"; describe("Check weather all selected quads can be extracted", function () { let shapeStore = new Store(); @@ -9,18 +10,24 @@ describe("Check weather all selected quads can be extracted", function () { let dataStore = new Store(); before(async () => { let readStream = ( - await rdfDereference.dereference("./tests/06 - shapes and named graphs/shape-example.ttl", { - localFiles: true, - }) + await rdfDereference.dereference( + "./tests/06 - shapes and named graphs/shape-example.ttl", + { + localFiles: true, + }, + ) ).data; await new Promise((resolve, reject) => { shapeStore.import(readStream).on("end", resolve).on("error", reject); }); extractor = new CBDShapeExtractor(shapeStore); let readStream2 = ( - await rdfDereference.dereference("./tests/06 - shapes and named graphs/data-example.ttl", { - localFiles: true, - }) + await rdfDereference.dereference( + "./tests/06 - shapes and named graphs/data-example.ttl", + { + localFiles: true, + }, + ) ).data; await new Promise((resolve, reject) => { dataStore.import(readStream2).on("end", resolve).on("error", reject); @@ -29,11 +36,41 @@ describe("Check weather all selected quads can be extracted", function () { it("All quads from example should be extracted", async () => { let result = await extractor.extract( dataStore, - new NamedNode("http://example.org/important_point"), - new NamedNode("http://example.org/Shape") + new NamedNode("http://example.org/line"), + new NamedNode("http://example.org/shape"), ); // It should only have 6 quads - assert.equal(result.length, 3); + assert.equal(result.length, 6); + }); + + it("bulk - All quads from example should be extracted", async () => { + let called = 0; + const cb = (member: { subject: RTerm; quads: Quad[] }) => { + called += 1; + if (member.subject.value == "http://example.org/line") { + assert.equal(member.quads.length, 6); + return; + } + + if (member.subject.value == "http://example.org/important_point") { + assert.equal(member.quads.length, 2); + return; + } + assert.fail(); + }; + + let result = await extractor.bulkExtract( + dataStore, + [ + new NamedNode("http://example.org/line"), + new NamedNode("http://example.org/important_point"), + ], + new NamedNode("http://example.org/shape"), + undefined, + cb, + ); + // It should only have 6 quads + assert.equal(result.length, 2); + assert.equal(called, 2); }); - }); diff --git a/tests/06 - shapes and named graphs/shape-example.ttl b/tests/06 - shapes and named graphs/shape-example.ttl index dab5e4d..c1b42f9 100644 --- a/tests/06 - shapes and named graphs/shape-example.ttl +++ b/tests/06 - shapes and named graphs/shape-example.ttl @@ -7,6 +7,7 @@ ex:shape a sh:NodeShape; sh:path ex:point; sh:node ex:PointShape; ]. + ex:PointShape a sh:NodeShape ; sh:property [ @@ -18,4 +19,4 @@ ex:PointShape sh:path ex:y ; sh:datatype xsd:integer ; sh:minCount 1; - ]. \ No newline at end of file + ]. From e436c8d2102fd30a5306f760c752c0209f4b25e3 Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Tue, 13 Feb 2024 10:37:31 +0100 Subject: [PATCH 3/3] add member specific quads to member --- lib/CBDShapeExtractor.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/CBDShapeExtractor.ts b/lib/CBDShapeExtractor.ts index b95af29..8e067fc 100644 --- a/lib/CBDShapeExtractor.ts +++ b/lib/CBDShapeExtractor.ts @@ -67,12 +67,12 @@ export class CBDShapeExtractor { const idSet = new Set(ids.map((x) => x.value)); const memberSpecificQuads: { [id: string]: Array } = {}; + for (let id of ids) { + memberSpecificQuads[id.value] = []; + } const newStore = new Store(); for (let quad of store.readQuads(null, null, null, null)) { if (quad.graph.termType == "NamedNode" && idSet.has(quad.graph.value)) { - if (!(quad.graph.value in memberSpecificQuads)) { - memberSpecificQuads[quad.graph.value] = []; - } memberSpecificQuads[quad.graph.value].push(quad); } else { newStore.add(quad); @@ -87,6 +87,7 @@ export class CBDShapeExtractor { shapeId, (graphsToIgnore || []).slice(), ).then((quads) => { + quads.push(...memberSpecificQuads[id.value]); if (itemExtracted) { itemExtracted({ subject: id, quads }); }