Merge pull request #48 from zhuzhongshu123/component_refactor

refactor(kag): refactor core components

.github/workflows/code-format-check.yml → .github/workflows/code-check.yml

@@ -1,4 +1,4 @@
-name: Code Format Check
+name: CI
 
 on:
   push:
@@ -22,7 +22,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pre-commit
-
+          pip install pre-commit pytest pytest-cov 
+          pip install -r requirements.txt
+          pip install -e .
+          pip install black==24.10.0
       - name: Run pre-commit
-        run: pre-commit run --all-files
+        run: pre-commit run --all-files
+
+      # - name: Run unit tests
+      #   run: pushd tests/unit && pytest && popd
+

.pre-commit-config.yaml

@@ -4,9 +4,14 @@ repos:
     hooks:
       - id: black
         files: ^kag/.*\.py$
+        exclude: |
+          (?x)^(
+            kag/solver/logic/core_modules/rule_runner/rule_runner.py |
+            kag/solver/logic/core_modules/parser/logic_node_parser.py
+          )$
   - repo: https://github.com/pycqa/flake8
     rev: 4.0.1
     hooks:
       - id: flake8
         files: ^kag/.*\.py$
-        
+

kag/__init__.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # Apache License
 # Version 2.0, January 2004
 # http://www.apache.org/licenses/
@@ -204,6 +205,18 @@
 __package_name__ = "openspg-kag"
 __version__ = "0.5.2-beta1"
 
-from kag.common.env import init_env
+# Register Built-in Components
+from kag.common.conf import init_env
 
 init_env()
+
+import kag.interface
+
+import kag.builder.component
+import kag.builder.default_chain
+import kag.builder.runner
+import kag.builder.prompt
+import kag.solver.prompt
+import kag.common.vectorize_model
+import kag.common.llm
+import kag.solver

kag/builder/__init__.py

@@ -1,10 +0,0 @@
-# Copyright 2023 OpenSPG Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-# in compliance with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied.

kag/builder/component/__init__.py

@@ -10,13 +10,71 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied.
 
+from kag.builder.component.external_graph.external_graph import (
+    DefaultExternalGraphLoader,
+)
+from kag.builder.component.extractor.kag_extractor import KAGExtractor
+from kag.builder.component.extractor.spg_extractor import SPGExtractor
+from kag.builder.component.aligner.kag_aligner import KAGAligner
+from kag.builder.component.aligner.spg_aligner import SPGAligner
+from kag.builder.component.postprocessor.kag_postprocessor import KAGPostProcessor
+
 from kag.builder.component.mapping.spg_type_mapping import SPGTypeMapping
 from kag.builder.component.mapping.relation_mapping import RelationMapping
+from kag.builder.component.mapping.spo_mapping import SPOMapping
+from kag.builder.component.reader.csv_reader import CSVReader
+from kag.builder.component.reader.json_reader import JSONReader
+from kag.builder.component.reader.yuque_reader import YuqueReader
+from kag.builder.component.reader.dataset_reader import (
+    MusiqueCorpusReader,
+    HotpotqaCorpusReader,
+)
+from kag.builder.component.reader.file_reader import FileReader
+from kag.builder.component.reader.directory_reader import DirectoryReader
+
+
+from kag.builder.component.record_parser.pdf_parser import PDFParser
+from kag.builder.component.record_parser.markdown_parser import MarkDownParser
+from kag.builder.component.record_parser.docx_parser import DocxParser
+from kag.builder.component.record_parser.txt_parser import TXTParser
+from kag.builder.component.record_parser.dict_parser import DictParser
+
+
+from kag.builder.component.splitter.length_splitter import LengthSplitter
+from kag.builder.component.splitter.pattern_splitter import PatternSplitter
+from kag.builder.component.splitter.outline_splitter import OutlineSplitter
+from kag.builder.component.splitter.semantic_splitter import SemanticSplitter
+from kag.builder.component.vectorizer.batch_vectorizer import BatchVectorizer
 from kag.builder.component.writer.kg_writer import KGWriter
 
 
 __all__ = [
+    "DefaultExternalGraphLoader",
+    "KAGExtractor",
+    "SPGExtractor",
+    "KAGAligner",
+    "SPGAligner",
+    "KAGPostProcessor",
+    "KGWriter",
     "SPGTypeMapping",
     "RelationMapping",
+    "SPOMapping",
+    "TXTParser",
+    "PDFParser",
+    "MarkDownParser",
+    "DocxParser",
+    "DictParser",
+    "JSONReader",
+    "HotpotqaCorpusReader",
+    "MusiqueCorpusReader",
+    "FileReader",
+    "DirectoryReader",
+    "YuqueReader",
+    "CSVReader",
+    "LengthSplitter",
+    "PatternSplitter",
+    "OutlineSplitter",
+    "SemanticSplitter",
+    "BatchVectorizer",
     "KGWriter",
 ]

kag/builder/component/aligner/__init__.py

@@ -1,11 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2023 OpenSPG Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-# in compliance with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied.

kag/builder/component/aligner/kag_post_processor.py → kag/builder/component/aligner/kag_aligner.py

@@ -13,11 +13,12 @@
 from typing import List, Sequence, Dict, Type
 
 from kag.builder.model.sub_graph import SubGraph
-from kag.interface.builder import AlignerABC
+from kag.interface import AlignerABC
 from knext.common.base.runnable import Input, Output
 
 
-class KAGPostProcessorAligner(AlignerABC):
+@AlignerABC.register("kag")
+class KAGAligner(AlignerABC):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 

kag/builder/component/aligner/spg_post_processor.py → kag/builder/component/aligner/spg_aligner.py

@@ -12,19 +12,21 @@
 
 from typing import List, Type, Dict
 
-from kag.interface.builder import AlignerABC
+from kag.interface import AlignerABC
 from knext.schema.client import BASIC_TYPES
+from kag.common.conf import KAG_PROJECT_CONF
 from kag.builder.model.spg_record import SPGRecord
 from kag.builder.model.sub_graph import SubGraph
 from knext.common.base.runnable import Input, Output
 from knext.schema.client import SchemaClient
 from knext.schema.model.base import ConstraintTypeEnum, BaseSpgType
 
 
-class SPGPostProcessorAligner(AlignerABC):
+@AlignerABC.register("spg")
+class SPGAligner(AlignerABC):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.spg_types = SchemaClient(project_id=self.project_id).load()
+        self.spg_types = SchemaClient(project_id=KAG_PROJECT_CONF.project_id).load()
 
     @property
     def input_types(self) -> Type[Input]: