From 0e687f74e541e72fa7a8f71038bbe500d8336012 Mon Sep 17 00:00:00 2001 From: Andy Date: Thu, 31 Oct 2024 23:28:51 -0700 Subject: [PATCH] fix(builder): add embedding model (#385) --- .../process/LLMNlExtractProcessor.java | 1 - .../physical/process/VectorizerProcessor.java | 5 +- .../physical/sink/impl/Neo4jSinkWriter.java | 28 ++++-- .../common/util/neo4j/Neo4jDriverManager.java | 11 ++- dev/release/docker-compose.yml | 1 + dev/release/mysql/buildx-release-mysql.sh | 4 +- dev/release/mysql/sql/initdb.sql | 1 - dev/release/python/Dockerfile | 7 +- .../python/build-release-python-aliyun.sh | 18 +++- dev/release/python/build-release-python.sh | 4 +- dev/release/server/buildx-release-server.sh | 4 +- dev/test/docker-compose.yml | 1 + .../reasoner/udf/builtin/udf/SplitPart.java | 32 ------- .../openspg/reasoner/udf/test/UdfTest.java | 17 ---- .../server/openapi/ProjectController.java | 87 +------------------ .../http/server/openapi/SchemaController.java | 81 +++++++++++++++++ server/biz/common/pom.xml | 4 + .../biz/common/impl/ProjectManagerImpl.java | 31 ++++++- .../service/alter/sync/Neo4jSyncer.java | 3 - 19 files changed, 177 insertions(+), 163 deletions(-) delete mode 100644 reasoner/udf/src/main/java/com/antgroup/openspg/reasoner/udf/builtin/udf/SplitPart.java diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/LLMNlExtractProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/LLMNlExtractProcessor.java index cc5a727cf..29f58a0e1 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/LLMNlExtractProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/LLMNlExtractProcessor.java @@ -117,7 +117,6 @@ public List call() throws Exception { log.info("LLMNlExtractProcessor invoke Chunks: {}", names); List result = (List) operatorFactory.invoke(config.getOperatorConfig(), record); - log.info("LLMNlExtractProcessor invoke result: {}", JSON.toJSONString(result)); List records = JSON.parseObject(JSON.toJSONString(result), new TypeReference>() {}); node.addTraceLog( diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/VectorizerProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/VectorizerProcessor.java index bb6c76367..e2bcbbdf9 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/VectorizerProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/VectorizerProcessor.java @@ -47,9 +47,12 @@ public List process(List inputs) { node.addTraceLog("Start vectorizer processor..."); List results = new ArrayList<>(); SubGraphRecord subGraph = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList()); + SubGraphRecord outputs = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList()); for (BaseRecord record : inputs) { SubGraphRecord spgRecord = (SubGraphRecord) record; + outputs.getResultNodes().addAll(spgRecord.getResultNodes()); + outputs.getResultEdges().addAll(spgRecord.getResultEdges()); Map map = mapper.convertValue(spgRecord, Map.class); node.addTraceLog( "invoke vectorizer processor operator:%s", config.getOperatorConfig().getClassName()); @@ -69,7 +72,7 @@ public List process(List inputs) { } results.add(subGraph); node.addTraceLog("post vectorizer complete..."); - node.setOutputs(subGraph); + node.setOutputs(outputs); node.setStatus(StatusEnum.FINISH); return results; } diff --git a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java index e4e52d53b..7da69893a 100644 --- a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java +++ b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java @@ -30,7 +30,6 @@ import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGPropertyRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName; -import com.antgroup.openspg.reasoner.runner.local.impl.LocalRunnerThreadPool; import com.antgroup.openspg.server.common.model.project.Project; import com.google.common.collect.Lists; import java.util.ArrayList; @@ -39,16 +38,23 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; @Slf4j public class Neo4jSinkWriter extends BaseSinkWriter { + private static final int NUM_THREADS = 10; + private ExecuteNode node; private Neo4jStoreClient client; private Project project; private static final String DOT = "."; + ExecutorService nodeExecutor; + ExecutorService edgeExecutor; public Neo4jSinkWriter(String id, String name, Neo4jSinkNodeConfig config) { super(id, name, config); @@ -63,6 +69,22 @@ public void doInit(BuilderContext context) throws BuilderException { } client = new Neo4jStoreClient(context.getGraphStoreUrl()); project = JSON.parseObject(context.getProject(), Project.class); + nodeExecutor = + new ThreadPoolExecutor( + NUM_THREADS, + NUM_THREADS, + 2 * 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + new ThreadPoolExecutor.CallerRunsPolicy()); + edgeExecutor = + new ThreadPoolExecutor( + NUM_THREADS, + NUM_THREADS, + 2 * 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + new ThreadPoolExecutor.CallerRunsPolicy()); } @Override @@ -90,8 +112,6 @@ private void batchWriteToNeo4j(List records) { public void writeToNeo4j(SubGraphRecord subGraphRecord) { subGraphRecord.getResultNodes().forEach(node -> convertProperties(node.getProperties())); subGraphRecord.getResultEdges().forEach(edge -> convertProperties(edge.getProperties())); - - ExecutorService nodeExecutor = LocalRunnerThreadPool.getThreadPoolExecutor(null); try { node.addTraceLog("Start Writer Nodes processor..."); List> nodeFutures = @@ -102,8 +122,6 @@ public void writeToNeo4j(SubGraphRecord subGraphRecord) { Thread.currentThread().interrupt(); throw new RuntimeException("Error during node upsert", e); } - - ExecutorService edgeExecutor = LocalRunnerThreadPool.getThreadPoolExecutor(null); try { node.addTraceLog("Start Writer Edges processor..."); List> edgeFutures = diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jDriverManager.java b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jDriverManager.java index 78c930d36..069f04deb 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jDriverManager.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jDriverManager.java @@ -14,9 +14,11 @@ import com.google.common.collect.Maps; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Config; import org.neo4j.driver.Driver; import org.neo4j.driver.GraphDatabase; @@ -32,7 +34,14 @@ public static Driver getNeo4jDriver(String uri, String user, String password) { if (instanceMap.get(uniqueKey) == null) { Driver driver; try { - driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password)); + Config config = + Config.builder() + .withMaxConnectionPoolSize(200) + .withMaxConnectionLifetime(2, TimeUnit.HOURS) + .withMaxTransactionRetryTime(300, TimeUnit.SECONDS) + .withConnectionAcquisitionTimeout(300, TimeUnit.SECONDS) + .build(); + driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password), config); driver.verifyConnectivity(); } catch (Exception e) { throw new RuntimeException("init Neo4j Client failed :" + uri, e); diff --git a/dev/release/docker-compose.yml b/dev/release/docker-compose.yml index 8983dd632..be22326b0 100644 --- a/dev/release/docker-compose.yml +++ b/dev/release/docker-compose.yml @@ -1,3 +1,4 @@ +version: "3.7" services: server: restart: always diff --git a/dev/release/mysql/buildx-release-mysql.sh b/dev/release/mysql/buildx-release-mysql.sh index 5a7088e43..bd2faf9bb 100644 --- a/dev/release/mysql/buildx-release-mysql.sh +++ b/dev/release/mysql/buildx-release-mysql.sh @@ -10,8 +10,6 @@ # or implied. docker buildx build -f Dockerfile --platform linux/arm64/v8,linux/amd64 --push \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-mysql:0.0.3 \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-mysql:latest \ - -t openspg/openspg-mysql:0.0.3 \ + -t openspg/openspg-mysql:0.5 \ -t openspg/openspg-mysql:latest \ . diff --git a/dev/release/mysql/sql/initdb.sql b/dev/release/mysql/sql/initdb.sql index 56d7d772f..29bb59ed6 100644 --- a/dev/release/mysql/sql/initdb.sql +++ b/dev/release/mysql/sql/initdb.sql @@ -234,7 +234,6 @@ CREATE TABLE `kg_ontology_ext` ( ) AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 COMMENT = 'schema的扩展属性'; INSERT INTO kg_biz_domain (`id`,`gmt_create`,`gmt_modified`,`name`,`status`,`description`,`global_config`) VALUES(1,'2023-09-01 00:00:00','2023-09-01 00:00:00','defaultTenant','VALID','',null); -INSERT INTO kg_project_info (`id`,`name`,`description`,`status`,`gmt_create`,`gmt_modified`,`namespace`,`biz_domain_id`) VALUES(1,'defaultProject','defaultProject','VALID','2023-09-01 00:00:00','2023-09-01 00:00:00','DEFAULT',1); INSERT INTO kg_ontology_entity (`id`,`original_id`,`name`,`name_zh`,`entity_category`,`layer`,`description`,`description_zh`,`status`,`with_index`,`scope`,`version`,`version_status`,`gmt_create`,`gmt_modified`,`transformer_id`,`operator_config`,`config`,`unique_name`) VALUES(1,1,'Thing','事物','ADVANCED','EXTENSION','Base class for all schema types, all of which inherit the type either directly or indirectly','所有schema类型的基类，它们都直接或者间接继承该类型','1','TRUE','PUBLIC',44,'ONLINE','2023-09-01 00:00:00','2023-09-01 00:00:00',0,null,null,'Thing'); INSERT INTO kg_ontology_entity (`id`,`original_id`,`name`,`name_zh`,`entity_category`,`layer`,`description`,`description_zh`,`status`,`with_index`,`scope`,`version`,`version_status`,`gmt_create`,`gmt_modified`,`transformer_id`,`operator_config`,`config`,`unique_name`) VALUES(2,2,'Text','文本','BASIC','CORE','文本','基本数据类型-文本','1','TRUE','PUBLIC',0,'ONLINE','2023-09-01 00:00:00','2023-09-01 00:00:00',0,null,'{"constrains":[{"id":"REQUIRE","name":"Required","nameZh":"值非空","value":null},{"id":"UNIQUE","name":"Unique","nameZh":"值唯一","value":null},{"id":"ENUM","name":"Enum","nameZh":"枚举","value":null},{"id":"MULTIVALUE","name":"Multi value","nameZh":"多值","value":null},{"id":"REGULAR","name":"Regular match","nameZh":"正则匹配","value":null}]}','Text'); diff --git a/dev/release/python/Dockerfile b/dev/release/python/Dockerfile index a96bd7319..fe00a566d 100644 --- a/dev/release/python/Dockerfile +++ b/dev/release/python/Dockerfile @@ -43,13 +43,10 @@ RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then \ RUN python3 -m venv /openspg_venv && \ . /openspg_venv/bin/activate && \ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture) && \ - pip3 install openspg-kag==0.0.3.20241022.2 -i https://artifacts.antgroup-inc.cn/artifact/repositories/simple-dev/ && \ + pip3 install openspg-kag==0.5.1 && \ pip3 install pemja==0.4.0 && \ pip3 install -U "http://alps-common.oss-cn-hangzhou-zmf.aliyuncs.com/nscommon/shiji/nscommon-0.0.1.tar.gz" &&\ echo "if (tty -s); then \n . /openspg_venv/bin/activate \nfi" >> ~/.bashrc -ADD openspg/dev/release/python/lib/builder*.jar /openspg_venv/lib/python3.8/site-packages/knext/builder/lib -ADD openspg/dev/release/python/lib/reasoner*.jar /openspg_venv/lib/python3.8/site-packages/knext/reasoner/lib - COPY openspg/ /openspg -#RUN git clone --depth=1 https://github.com/OpenSPG/openspg.git \ No newline at end of file +RUN git clone --depth=1 https://github.com/OpenSPG/KAG.git \ No newline at end of file diff --git a/dev/release/python/build-release-python-aliyun.sh b/dev/release/python/build-release-python-aliyun.sh index 7d94c3597..482fdebc2 100644 --- a/dev/release/python/build-release-python-aliyun.sh +++ b/dev/release/python/build-release-python-aliyun.sh @@ -10,7 +10,7 @@ # or implied. IMAGE="spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python" -VERSION="kag1" +VERSION="0.5" cd ../../../../ docker build -f openspg/dev/release/python/Dockerfile --platform linux/arm64/v8 --push \ -t ${IMAGE}:${VERSION}-arm64 \ @@ -31,4 +31,18 @@ docker manifest create \ ${IMAGE}:${VERSION}-amd64 \ ${IMAGE}:${VERSION}-arm64 -docker manifest push ${IMAGE}:${VERSION} \ No newline at end of file +docker manifest push ${IMAGE}:${VERSION} + +if docker manifest inspect ${IMAGE}:${LATEST} &> /dev/null; then + echo "Manifest already exists, removing it..." + docker manifest rm ${IMAGE}:${LATEST} +else + echo "Manifest does not exist, proceeding with creation and push." +fi + +docker manifest create \ + ${IMAGE}:${LATEST} \ + ${IMAGE}:${VERSION}-amd64 \ + ${IMAGE}:${VERSION}-arm64 + +docker manifest push ${IMAGE}:${LATEST} \ No newline at end of file diff --git a/dev/release/python/build-release-python.sh b/dev/release/python/build-release-python.sh index 138eeadee..e922996f9 100644 --- a/dev/release/python/build-release-python.sh +++ b/dev/release/python/build-release-python.sh @@ -11,8 +11,6 @@ # for amd64 docker build -f Dockerfile --platform linux/amd64 --push \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python:0.0.3 \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python:latest \ - -t openspg/openspg-python:0.0.3 \ + -t openspg/openspg-python:0.5 \ -t openspg/openspg-python:latest \ . diff --git a/dev/release/server/buildx-release-server.sh b/dev/release/server/buildx-release-server.sh index e4cc22009..ecfb53935 100644 --- a/dev/release/server/buildx-release-server.sh +++ b/dev/release/server/buildx-release-server.sh @@ -10,8 +10,6 @@ # or implied. docker buildx build -f Dockerfile --platform linux/arm64/v8,linux/amd64 --push \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-server:0.0.3 \ - -t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-server:latest \ - -t openspg/openspg-server:0.0.3 \ + -t openspg/openspg-server:0.5 \ -t openspg/openspg-server:latest \ . diff --git a/dev/test/docker-compose.yml b/dev/test/docker-compose.yml index d2c102f1e..9062bfdca 100644 --- a/dev/test/docker-compose.yml +++ b/dev/test/docker-compose.yml @@ -1,3 +1,4 @@ +version: "3.7" services: mysql: restart: always diff --git a/reasoner/udf/src/main/java/com/antgroup/openspg/reasoner/udf/builtin/udf/SplitPart.java b/reasoner/udf/src/main/java/com/antgroup/openspg/reasoner/udf/builtin/udf/SplitPart.java deleted file mode 100644 index dc1f5dc27..000000000 --- a/reasoner/udf/src/main/java/com/antgroup/openspg/reasoner/udf/builtin/udf/SplitPart.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OpenSPG Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.reasoner.udf.builtin.udf; - -import com.antgroup.openspg.reasoner.udf.model.UdfDefine; -import org.apache.commons.lang3.StringUtils; - -public class SplitPart { - @UdfDefine(name = "split_part", compatibleName = "SplitPart") - public String split(String str, String separator, Integer partNum) throws Exception { - String[] strings = StringUtils.split(str, separator); - int length = strings.length; - if (partNum < 0) { - partNum = length - 1; - } - if (partNum >= length) { - return ""; - } - return strings[partNum]; - } -} diff --git a/reasoner/udf/src/test/java/com/antgroup/openspg/reasoner/udf/test/UdfTest.java b/reasoner/udf/src/test/java/com/antgroup/openspg/reasoner/udf/test/UdfTest.java index 201076b12..82d0cb248 100644 --- a/reasoner/udf/src/test/java/com/antgroup/openspg/reasoner/udf/test/UdfTest.java +++ b/reasoner/udf/src/test/java/com/antgroup/openspg/reasoner/udf/test/UdfTest.java @@ -169,23 +169,6 @@ public void testSubStr() { Assert.assertEquals(rst2, "岁"); } - @Test - public void testSplitPart() { - UdfMng mng = UdfMngFactory.getUdfMng(); - IUdfMeta udfMeta = - mng.getUdfMeta( - "split_part", - Lists.newArrayList(KTString$.MODULE$, KTString$.MODULE$, KTInteger$.MODULE$)); - Object rst1 = udfMeta.invoke("Hello,World!", ",", 0); - Assert.assertEquals("Hello", rst1); - Object rst2 = udfMeta.invoke("Hello,Ni,Hao", ",", -1); - Assert.assertEquals("Hao", rst2); - Object rst3 = udfMeta.invoke("A省B市C村XXX", "村", 0); - Assert.assertEquals("A省B市C", rst3); - Object rst4 = udfMeta.invoke("A省B市C村XXX", "村", 5); - Assert.assertEquals("", rst4); - } - @Test public void testCast() { UdfMng mng = UdfMngFactory.getUdfMng(); diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java index 0fc078519..251e31292 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java @@ -13,20 +13,6 @@ package com.antgroup.openspg.server.api.http.server.openapi; -import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.core.schema.model.BasicInfo; -import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum; -import com.antgroup.openspg.core.schema.model.alter.SchemaDraft; -import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; -import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; -import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum; -import com.antgroup.openspg.core.schema.model.predicate.Property; -import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig; -import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; -import com.antgroup.openspg.core.schema.model.type.EntityType; -import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; -import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; -import com.antgroup.openspg.core.schema.model.type.SPGTypeRef; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectCreateRequest; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; import com.antgroup.openspg.server.api.facade.dto.schema.request.SchemaAlterRequest; @@ -35,10 +21,7 @@ import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; import com.antgroup.openspg.server.biz.common.ProjectManager; import com.antgroup.openspg.server.common.model.project.Project; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import java.util.List; -import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; @@ -68,7 +51,8 @@ public Project action() { if (request.getAutoSchema() == null || Boolean.TRUE.equals(request.getAutoSchema())) { SchemaAlterRequest request = new SchemaAlterRequest(); request.setProjectId(project.getId()); - request.setSchemaDraft(getDefaultSchemaDraft(project.getNamespace())); + request.setSchemaDraft( + SchemaController.getDefaultSchemaDraft(project.getNamespace())); schemaController.alterSchema(request); } return project; @@ -76,73 +60,6 @@ public Project action() { }); } - private SchemaDraft getDefaultSchemaDraft(String namespace) { - SchemaDraft sd = new SchemaDraft(); - List alterSpgTypes = getDefaultSchema(namespace); - sd.setAlterSpgTypes(alterSpgTypes); - return sd; - } - - private List getDefaultSchema(String namespace) { - List schemaTypes = Lists.newArrayList(); - - Map chunkProperties = Maps.newHashMap(); - chunkProperties.put("content", IndexTypeEnum.TEXT_AND_VECTOR); - - Map properties = Maps.newHashMap(); - properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR); - properties.put("semanticType", IndexTypeEnum.TEXT); - - schemaTypes.add(getBaseSPGType(namespace, "Chunk", "文本块", chunkProperties)); - schemaTypes.add(getBaseSPGType(namespace, "ArtificialObject", "人造物体", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Astronomy", "天文学", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Building", "建筑", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Creature", "生物", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Concept", "概念", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Date", "日期", properties)); - schemaTypes.add(getBaseSPGType(namespace, "GeographicLocation", "地理位置", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Keyword", "关键词", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Medicine", "药物", properties)); - schemaTypes.add(getBaseSPGType(namespace, "NaturalScience", "自然科学", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Organization", "组织机构", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Person", "人物", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Transport", "运输", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Works", "作品", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Event", "事件", properties)); - schemaTypes.add(getBaseSPGType(namespace, "Others", "其他", properties)); - schemaTypes.add(getBaseSPGType(namespace, "SemanticConcept", "语义概念", properties)); - return schemaTypes; - } - - private EntityType getBaseSPGType( - String namespace, String label, String nameZh, Map properties) { - List propertyList = Lists.newArrayList(); - for (String pro : properties.keySet()) { - - Property property = - new Property( - new BasicInfo<>(new PredicateIdentifier(pro), pro, StringUtils.EMPTY), - null, - new SPGTypeRef( - new BasicInfo<>(new SPGTypeIdentifier(null, "Text"), "文本", "文本"), - SPGTypeEnum.BASIC_TYPE), - false, - new PropertyAdvancedConfig().setIndexType(properties.get(pro))); - property.setAlterOperation(AlterOperationEnum.CREATE); - propertyList.add(property); - } - - EntityType entityType = - new EntityType( - new BasicInfo(new SPGTypeIdentifier(namespace, label), nameZh, StringUtils.EMPTY), - ParentTypeInfo.THING, - propertyList, - null, - null); - entityType.setAlterOperation(AlterOperationEnum.CREATE); - return entityType; - } - @RequestMapping(method = RequestMethod.GET) public ResponseEntity query( @RequestParam(required = false) Long tenantId, diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java index b380287e0..9cdbf7133 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java @@ -13,23 +13,37 @@ package com.antgroup.openspg.server.api.http.server.openapi; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.core.schema.model.BasicInfo; import com.antgroup.openspg.core.schema.model.SchemaExtInfo; +import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum; +import com.antgroup.openspg.core.schema.model.alter.SchemaDraft; +import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum; import com.antgroup.openspg.core.schema.model.predicate.Property; +import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig; import com.antgroup.openspg.core.schema.model.predicate.Relation; import com.antgroup.openspg.core.schema.model.predicate.SubProperty; import com.antgroup.openspg.core.schema.model.semantic.PredicateSemantic; import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; import com.antgroup.openspg.core.schema.model.type.BaseSPGType; +import com.antgroup.openspg.core.schema.model.type.EntityType; import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; import com.antgroup.openspg.core.schema.model.type.ProjectSchema; import com.antgroup.openspg.core.schema.model.type.SPGTypeAdvancedConfig; +import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; +import com.antgroup.openspg.core.schema.model.type.SPGTypeRef; import com.antgroup.openspg.server.api.facade.dto.schema.request.*; import com.antgroup.openspg.server.api.http.server.BaseController; import com.antgroup.openspg.server.api.http.server.HttpBizCallback; import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; import com.antgroup.openspg.server.biz.common.util.AssertUtils; import com.antgroup.openspg.server.biz.schema.SchemaManager; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.util.List; +import java.util.Map; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; @@ -45,6 +59,73 @@ public class SchemaController extends BaseController { @Autowired private SchemaManager schemaManager; + public static SchemaDraft getDefaultSchemaDraft(String namespace) { + SchemaDraft sd = new SchemaDraft(); + List alterSpgTypes = getDefaultSchema(namespace); + sd.setAlterSpgTypes(alterSpgTypes); + return sd; + } + + public static List getDefaultSchema(String namespace) { + List schemaTypes = Lists.newArrayList(); + + Map chunkProperties = Maps.newHashMap(); + chunkProperties.put("content", IndexTypeEnum.TEXT_AND_VECTOR); + + Map properties = Maps.newHashMap(); + properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR); + properties.put("semanticType", IndexTypeEnum.TEXT); + + schemaTypes.add(getBaseSPGType(namespace, "Chunk", "文本块", chunkProperties)); + schemaTypes.add(getBaseSPGType(namespace, "ArtificialObject", "人造物体", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Astronomy", "天文学", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Building", "建筑", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Creature", "生物", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Concept", "概念", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Date", "日期", properties)); + schemaTypes.add(getBaseSPGType(namespace, "GeographicLocation", "地理位置", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Keyword", "关键词", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Medicine", "药物", properties)); + schemaTypes.add(getBaseSPGType(namespace, "NaturalScience", "自然科学", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Organization", "组织机构", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Person", "人物", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Transport", "运输", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Works", "作品", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Event", "事件", properties)); + schemaTypes.add(getBaseSPGType(namespace, "Others", "其他", properties)); + schemaTypes.add(getBaseSPGType(namespace, "SemanticConcept", "语义概念", properties)); + return schemaTypes; + } + + public static EntityType getBaseSPGType( + String namespace, String label, String nameZh, Map properties) { + List propertyList = Lists.newArrayList(); + for (String pro : properties.keySet()) { + + Property property = + new Property( + new BasicInfo<>(new PredicateIdentifier(pro), pro, StringUtils.EMPTY), + null, + new SPGTypeRef( + new BasicInfo<>(new SPGTypeIdentifier(null, "Text"), "文本", "文本"), + SPGTypeEnum.BASIC_TYPE), + false, + new PropertyAdvancedConfig().setIndexType(properties.get(pro))); + property.setAlterOperation(AlterOperationEnum.CREATE); + propertyList.add(property); + } + + EntityType entityType = + new EntityType( + new BasicInfo(new SPGTypeIdentifier(namespace, label), nameZh, StringUtils.EMPTY), + ParentTypeInfo.THING, + propertyList, + null, + null); + entityType.setAlterOperation(AlterOperationEnum.CREATE); + return entityType; + } + @RequestMapping(value = "/alterSchema", method = RequestMethod.POST) @ResponseBody public ResponseEntity alterSchema(@RequestBody SchemaAlterRequest request) { diff --git a/server/biz/common/pom.xml b/server/biz/common/pom.xml index 90f8390c3..6148a9747 100644 --- a/server/biz/common/pom.xml +++ b/server/biz/common/pom.xml @@ -35,5 +35,9 @@ com.antgroup.openspg.cloudext cloudext-impl-graph-store-neo4j + + com.alibaba + pemja + diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java index 8dc205c2c..331396966 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java @@ -44,6 +44,7 @@ public class ProjectManagerImpl implements ProjectManager { @Override public Project create(ProjectCreateRequest request) { JSONObject config = setDatabase(request.getConfig(), request.getNamespace()); + createNeo4jDatabase(request.getNamespace(), config); Project project = new Project( null, @@ -52,7 +53,6 @@ public Project create(ProjectCreateRequest request) { request.getNamespace(), request.getTenantId(), config.toJSONString()); - createNeo4jDatabase(project.getNamespace(), config); Long projectId = projectRepository.save(project); project.setId(projectId); return project; @@ -62,6 +62,7 @@ public Project create(ProjectCreateRequest request) { public Project update(ProjectCreateRequest request) { Project project = projectRepository.queryById(request.getId()); JSONObject config = setDatabase(request.getConfig(), project.getNamespace()); + config = setVectorDimensions(config, project); Project update = new Project(request.getId(), null, null, null, null, config.toJSONString()); return projectRepository.update(update); } @@ -83,6 +84,30 @@ private JSONObject setDatabase(String configStr, String namespace) { return config; } + private JSONObject setVectorDimensions(JSONObject config, Project project) { + JSONObject oldConfig = JSONObject.parseObject(project.getConfig()); + String vectorDimensions = null; + if (oldConfig.containsKey(CommonConstants.VECTORIZER)) { + vectorDimensions = + oldConfig + .getJSONObject(CommonConstants.VECTORIZER) + .getString(CommonConstants.VECTOR_DIMENSIONS); + } + if (StringUtils.isBlank(vectorDimensions)) { + return config; + } + if (config.containsKey(CommonConstants.VECTORIZER)) { + config + .getJSONObject(CommonConstants.VECTORIZER) + .put(CommonConstants.VECTOR_DIMENSIONS, vectorDimensions); + } else { + JSONObject graphStore = new JSONObject(); + graphStore.put(CommonConstants.VECTOR_DIMENSIONS, vectorDimensions); + config.put(CommonConstants.VECTORIZER, graphStore); + } + return config; + } + @Override public Project queryById(Long projectId) { return projectRepository.queryById(projectId); @@ -104,9 +129,13 @@ public void createNeo4jDatabase(String namespace, JSONObject config) { } if (graphStore.containsKey(Neo4jConstants.USER)) { user = graphStore.getString(Neo4jConstants.USER); + } else { + graphStore.put(Neo4jConstants.USER, user); } if (graphStore.containsKey(Neo4jConstants.PASSWORD)) { password = graphStore.getString(Neo4jConstants.PASSWORD); + } else { + graphStore.put(Neo4jConstants.PASSWORD, password); } Neo4jAdminUtils driver = new Neo4jAdminUtils(host, user, password, database); diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/sync/Neo4jSyncer.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/sync/Neo4jSyncer.java index 5ad1c17a1..7ef616517 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/sync/Neo4jSyncer.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/sync/Neo4jSyncer.java @@ -56,9 +56,6 @@ private int getVectorDimensions(Project project) { if (StringUtils.isNotBlank(config)) { JSONObject vectorizerConfig = JSON.parseObject(config).getJSONObject(CommonConstants.VECTORIZER); - if (vectorizerConfig == null) { - return Neo4jCommonUtils.DEFAULT_VECTOR_DIMENSIONS; - } Integer vectorDimensions = vectorizerConfig.getInteger(CommonConstants.VECTOR_DIMENSIONS); if (vectorDimensions != null) return vectorDimensions; }