diff --git a/.gitignore b/.gitignore index 2ca8c583..fbd40bf4 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,6 @@ docker/benchmark/database/data dataset/TPC-H V3.0.1 dataset/SynData dataset/databases +dataset/data-importer/all-data !dataset/newyork-taxi.zip !dataset/newyork-taxi-sample.txt -`` \ No newline at end of file diff --git a/adapter/adapter-postgis/README.md b/adapter/adapter-postgis/README.md new file mode 100644 index 00000000..4cab02d5 --- /dev/null +++ b/adapter/adapter-postgis/README.md @@ -0,0 +1,6 @@ +# Postgis适配说明 +Point类在com.hufudb.openhufu.data.storage.Point中定义 + +在proto传输过程中，Point类型以字符串类型Point(x y)形式传递 + +在生成SQL语句时，需要将Point类型翻译为ST_GeomFromText('Point(x y)', 4326)，从而表示为geometry类型对象 \ No newline at end of file diff --git a/adapter/adapter-postgis/pom.xml b/adapter/adapter-postgis/pom.xml new file mode 100644 index 00000000..7c45daa5 --- /dev/null +++ b/adapter/adapter-postgis/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + + openhufu-adapter + com.hufudb.openhufu + 1.0.0-SNAPSHOT + + + openhufu-adapter-postgis + + + + + + + com.hufudb.openhufu + openhufu-owner + ${project.version} + provided + + + + org.postgresql + postgresql + + + net.postgis + postgis-jdbc + + + com.google.guava + guava + + + + + + + kr.motd.maven + os-maven-plugin + + + org.jacoco + jacoco-maven-plugin + + + report-aggregate + verify + + report-aggregate + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + + package + + single + + + + + + + diff --git a/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapter.java b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapter.java new file mode 100644 index 00000000..3014a6c3 --- /dev/null +++ b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapter.java @@ -0,0 +1,32 @@ +package com.hufudb.openhufu.owner.adapter.postgis; + +import java.sql.Connection; +import java.sql.Statement; +import java.sql.SQLException; +import java.sql.ResultSet; +import com.hufudb.openhufu.owner.adapter.AdapterTypeConverter; +import com.hufudb.openhufu.owner.adapter.jdbc.JDBCAdapter; +import com.hufudb.openhufu.data.storage.DataSet; +import com.hufudb.openhufu.data.storage.EmptyDataSet; +import com.hufudb.openhufu.expression.Translator; +import com.hufudb.openhufu.data.schema.Schema; + + +public class PostgisAdapter extends JDBCAdapter { + PostgisAdapter(String catalog, Connection connection, Statement statement, + AdapterTypeConverter converter, Translator translator) { + super(catalog, connection, statement, converter, translator); + } + + @Override + protected DataSet executeSQL(String sql, Schema schema) { + try { + ResultSet rs = statement.executeQuery(sql); + LOG.info("Execute {}", sql); + return new PostgisResultDataSet(schema, rs); + } catch (SQLException e) { + LOG.error("Fail to execute SQL [{}]: {}", sql, e.getMessage()); + return EmptyDataSet.INSTANCE; + } + } +} diff --git a/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapterFactory.java b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapterFactory.java new file mode 100644 index 00000000..56818a9e --- /dev/null +++ b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapterFactory.java @@ -0,0 +1,43 @@ +package com.hufudb.openhufu.owner.adapter.postgis; + +import com.hufudb.openhufu.common.enums.DataSourceType; +import com.hufudb.openhufu.owner.adapter.AdapterConfig; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import com.hufudb.openhufu.owner.adapter.AdapterFactory; +import com.hufudb.openhufu.expression.BasicTranslator; +import com.hufudb.openhufu.owner.adapter.Adapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgisAdapterFactory implements AdapterFactory { + + static final Logger LOG = LoggerFactory.getLogger(PostgisAdapterFactory.class); + + public PostgisAdapterFactory() { + try { + Class.forName("org.postgresql.Driver"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public Adapter create(AdapterConfig config) { + try { + Connection connection = DriverManager.getConnection(config.url, config.user, config.passwd); + Statement statement = connection.createStatement(); + return new PostgisAdapter(config.catalog, connection, statement, new PostgisTypeConverter(), + new BasicTranslator(getType().getType())); + } catch (Exception e) { + LOG.error("Fail to create csv adapter: {}", config.url, e); + return null; + } + } + + @Override + public DataSourceType getType() { + return DataSourceType.POSTGIS; + } +} diff --git a/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisResultDataSet.java b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisResultDataSet.java new file mode 100644 index 00000000..8e0aa8bf --- /dev/null +++ b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisResultDataSet.java @@ -0,0 +1,91 @@ +package com.hufudb.openhufu.owner.adapter.postgis; + +import java.sql.ResultSet; +import java.util.List; +import com.google.common.collect.ImmutableList; +import com.hufudb.openhufu.data.schema.Schema; +import com.hufudb.openhufu.data.storage.ResultDataSet; +import com.hufudb.openhufu.proto.OpenHuFuData.ColumnDesc; +import org.postgresql.util.PGobject; + +/** + * PostGIS extension of ResultDataSet + */ +public class PostgisResultDataSet extends ResultDataSet { + + public PostgisResultDataSet(Schema schema, ResultSet result) { + super(schema, result); + } + + @Override + protected List generateGetters() { + ImmutableList.Builder builder = ImmutableList.builder(); + int i = 1; + for (ColumnDesc col : schema.getColumnDescs()) { + final int idx = i; + switch (col.getType()) { + case BLOB: + builder.add(() -> { + return result.getBytes(idx); + }); + break; + case BOOLEAN: + builder.add(() -> { + return result.getBoolean(idx); + }); + break; + case BYTE: + case SHORT: + case INT: + builder.add(() -> { + return result.getInt(idx); + }); + break; + case DATE: + builder.add(() -> { + return result.getDate(idx); + }); + break; + case TIME: + builder.add(() -> { + return result.getTime(idx); + }); + break; + case TIMESTAMP: + builder.add(() -> { + return result.getTimestamp(idx); + }); + break; + case LONG: + builder.add(() -> { + return result.getLong(idx); + }); + break; + case STRING: + builder.add(() -> { + return result.getString(idx); + }); + break; + case DOUBLE: + builder.add(() -> { + return result.getDouble(idx); + }); + break; + case FLOAT: + builder.add(() -> { + return result.getFloat(idx); + }); + break; + case GEOMETRY: + builder.add(() -> { + return PostgisUtils.fromPGPoint(((PGobject) (result.getObject(idx)))); + }); + break; + default: + break; + } + ++i; + } + return builder.build(); + } +} diff --git a/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisTypeConverter.java b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisTypeConverter.java new file mode 100644 index 00000000..155fb9b3 --- /dev/null +++ b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisTypeConverter.java @@ -0,0 +1,48 @@ +package com.hufudb.openhufu.owner.adapter.postgis; + +import com.hufudb.openhufu.owner.adapter.AdapterTypeConverter; +import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType; + +public class PostgisTypeConverter extends AdapterTypeConverter { + @Override + public ColumnType convert(int type, String typeName) { + switch (typeName) { + case "real": + case "float4": + return ColumnType.FLOAT; + case "float8": + case "double precision": + case "numeric": + return ColumnType.DOUBLE; + case "TINYINT": + return ColumnType.BYTE; + case "SMALLINT": + return ColumnType.SHORT; + case "int2": + case "int4": + return ColumnType.INT; + case "oid": + case "int8": + return ColumnType.LONG; + case "varchar": + case "char": + case "bpchar": + case "text": + case "name": + return ColumnType.STRING; + case "bit": + case "bool": + return ColumnType.BOOLEAN; + case "date": + return ColumnType.DATE; + case "time": + return ColumnType.TIME; + case "timestamp": + return ColumnType.TIMESTAMP; + case "geometry": + return ColumnType.GEOMETRY; + default: + return ColumnType.STRING; + } + } +} diff --git a/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisUtils.java b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisUtils.java new file mode 100644 index 00000000..f1d34296 --- /dev/null +++ b/adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisUtils.java @@ -0,0 +1,23 @@ +package com.hufudb.openhufu.owner.adapter.postgis; + +import java.sql.SQLException; +import org.postgresql.util.PGobject; +import org.postgis.GeometryBuilder; +import com.hufudb.openhufu.data.storage.Point; + +public class PostgisUtils { + private PostgisUtils() {} + + /** + * convert PostGIS Point to Pair in java + */ + public static Point fromPGPoint(PGobject pgpoint) { + try { + org.postgis.Point p = GeometryBuilder.geomFromString(pgpoint.getValue()).getPoint(0); + return new Point(p.x, p.y); + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + } +} diff --git a/adapter/adapter-postgis/src/main/resources/META-INF/services/com.hufudb.openhufu.owner.adapter.AdapterFactory b/adapter/adapter-postgis/src/main/resources/META-INF/services/com.hufudb.openhufu.owner.adapter.AdapterFactory new file mode 100644 index 00000000..3b3502da --- /dev/null +++ b/adapter/adapter-postgis/src/main/resources/META-INF/services/com.hufudb.openhufu.owner.adapter.AdapterFactory @@ -0,0 +1 @@ +com.hufudb.openhufu.owner.adapter.postgis.PostgisAdapterFactory \ No newline at end of file diff --git a/adapter/pom.xml b/adapter/pom.xml index a5b74c02..df23fc79 100755 --- a/adapter/pom.xml +++ b/adapter/pom.xml @@ -15,6 +15,7 @@ adapter-csv + adapter-postgis diff --git a/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialBenchmarkTest.java b/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialCSVTest.java similarity index 61% rename from benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialBenchmarkTest.java rename to benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialCSVTest.java index 143ffa6a..130faf27 100644 --- a/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialBenchmarkTest.java +++ b/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialCSVTest.java @@ -31,20 +31,20 @@ import static org.junit.Assert.assertEquals; -public class OpenHuFuSpatialBenchmarkTest { +public class OpenHuFuSpatialCSVTest { private static final Logger LOG = LoggerFactory.getLogger(OpenHuFuBenchmark.class); private static final OpenHuFuUser user = new OpenHuFuUser(); @BeforeClass public static void setUp() throws IOException { LinkedTreeMap userConfigs = new Gson().fromJson(Files.newBufferedReader( - Path.of(OpenHuFuBenchmark.class.getClassLoader().getResource("spatial-user-configs.json") + Path.of(OpenHuFuBenchmark.class.getClassLoader().getResource("spatial-csv-configs.json") .getPath())), LinkedTreeMap.class); List endpoints = (List) userConfigs.get("owners"); List globalTableConfigs = new Gson().fromJson(new Gson().toJson(userConfigs.get("tables")), new TypeToken>() {}.getType()); - LOG.info("Init benchmark of OpenHuFuSpatial..."); + LOG.info("Init benchmark of OpenHuFuSpatialCSV..."); for (String endpoint : endpoints) { user.addOwner(endpoint, null); } @@ -63,7 +63,7 @@ public void printLine(ResultSet it) throws SQLException { } @Test - public void testSqlSelect() throws SQLException { + public void testSelect() throws SQLException { String sql = "select * from spatial"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); @@ -78,7 +78,7 @@ public void testSqlSelect() throws SQLException { } @Test - public void testSqlSpatialDistance() throws SQLException { + public void testSpatialDistance() throws SQLException { String sql = "select Distance(S_POINT, POINT(1404050, -4762163)) from spatial"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); @@ -93,7 +93,7 @@ public void testSqlSpatialDistance() throws SQLException { } @Test - public void testSqlRangeQuery() throws SQLException { + public void testRangeQuery() throws SQLException { String sql = "select * from spatial where DWithin(POINT(1404050, -4762163), S_POINT, 5)"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); @@ -108,7 +108,7 @@ public void testSqlRangeQuery() throws SQLException { } @Test - public void testSqlRangeCount() throws SQLException { + public void testRangeCount() throws SQLException { String sql = "select count(*) from spatial where DWithin(POINT(1404050, -4762163), S_POINT, 5)"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); @@ -123,8 +123,8 @@ public void testSqlRangeCount() throws SQLException { } @Test - public void testSqlRangeJoin() throws SQLException { - String sql = "select * from join_left s1 join spatial s2 on DWithin(s1.JL_POINT, s2.S_POINT, 500000)"; + public void testKNNQuery1() throws SQLException { + String sql = "select S_ID from spatial order by Distance(POINT(1404050, -4762163), S_POINT) asc limit 10"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); long count = 0; @@ -132,14 +132,14 @@ public void testSqlRangeJoin() throws SQLException { printLine(dataset); ++count; } - assertEquals(78, count); + assertEquals(10, count); dataset.close(); } } @Test - public void testSqlKNNQuery1() throws SQLException { - String sql = "select S_ID from spatial order by Distance(POINT(1404050, -4762163), S_POINT) asc limit 10"; + public void testKNNQuery2() throws SQLException { + String sql = "select S_ID from spatial where KNN(POINT(1404050, -4762163), S_POINT, 10)"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); long count = 0; @@ -153,8 +153,8 @@ public void testSqlKNNQuery1() throws SQLException { } @Test - public void testSqlKNNQuery2() throws SQLException { - String sql = "select S_ID from spatial where KNN(POINT(1404050, -4762163), S_POINT, 10)"; + public void testRangeJoin() throws SQLException { + String sql = "select * from join_left s1 join spatial s2 on DWithin(s1.JL_POINT, s2.S_POINT, 500000)"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); long count = 0; @@ -162,13 +162,13 @@ public void testSqlKNNQuery2() throws SQLException { printLine(dataset); ++count; } - assertEquals(10, count); + assertEquals(78, count); dataset.close(); } } @Test - public void testSqlKNNJOIN() throws SQLException { + public void testKNNJOIN() throws SQLException { String sql = "select s1.JL_ID, s2.S_ID from join_left s1 join spatial s2 on KNN(s1.JL_POINT, s2.S_POINT, 5)"; try (Statement stmt = user.createStatement()) { ResultSet dataset = stmt.executeQuery(sql); @@ -181,76 +181,4 @@ public void testSqlKNNJOIN() throws SQLException { dataset.close(); } } - - @Test - public void testSelect() { - String tableName = SpatialTableName.SPATIAL.getName(); - LeafPlan plan = new LeafPlan(); - plan.setTableName(tableName); - plan.setSelectExps(ExpressionFactory - .createInputRef(user.getOpenHuFuTableSchema(tableName).getSchema())); - DataSet dataset = user.executeQuery(plan); - DataSetIterator it = dataset.getIterator(); - long count = 0; - while (it.next()) { - for (int i = 0; i < it.size(); i++) { - System.out.print(it.get(i) + "|"); - } - System.out.println(); - ++count; - } - assertEquals(3000, count); - dataset.close(); - } - - @Test - public void testSpatialDistance() { - String tableName = SpatialTableName.SPATIAL.getName(); - LeafPlan plan = new LeafPlan(); - plan.setTableName(tableName); - - // select Distance(S_POINT, POINT((1404050.076199729, -4762163.267865509)) from spatial; - Expression pointFunc = - ExpressionFactory.createLiteral(ColumnType.GEOMETRY, GeometryUtils.fromString("POINT(1404050.076199729 -4762163.267865509)")); - Expression distanceFunc = - ExpressionFactory.createScalarFunc(ColumnType.DOUBLE, "Distance", - ImmutableList.of(pointFunc, pointFunc)); - plan.setSelectExps(ImmutableList.of(distanceFunc)); - DataSet dataset = user.executeQuery(plan); - DataSetIterator it = dataset.getIterator(); - int count = 0; - assertEquals(1, it.size()); - while (it.next()) { - assertEquals(0.0, it.get(0)); - count++; - } - assertEquals(3000, count); - } - - @Test - public void testSpatialDWithin() { - String tableName = SpatialTableName.SPATIAL.getName(); - LeafPlan plan = new LeafPlan(); - plan.setTableName(tableName); - plan.setSelectExps( - ExpressionFactory.createInputRef(user.getOpenHuFuTableSchema(tableName).getSchema())); - // select * from spatial where DWithin(S_POINT, POINT((1404050.076199729, -4762163.267865509), 0.1); - Expression pointFunc = - ExpressionFactory.createLiteral(ColumnType.GEOMETRY, GeometryUtils.fromString("POINT(1404050.076199729 -4762163.267865509)")); - Expression dwithinFunc = - ExpressionFactory.createScalarFunc(ColumnType.BOOLEAN, "DWithin", - ImmutableList.of( - ExpressionFactory.createInputRef(1, ColumnType.GEOMETRY, Modifier.PUBLIC), - pointFunc, ExpressionFactory.createLiteral(ColumnType.DOUBLE, 0.1))); - plan.setWhereExps(ImmutableList.of(dwithinFunc)); - DataSet dataset = user.executeQuery(plan); - DataSetIterator it = dataset.getIterator(); - int count = 0; - assertEquals(2, it.size()); - while (it.next()) { - assertEquals(0L, it.get(0)); - count++; - } - assertEquals(1, count); - } } diff --git a/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialPostgisTest.java b/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialPostgisTest.java new file mode 100644 index 00000000..24e35692 --- /dev/null +++ b/benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialPostgisTest.java @@ -0,0 +1,218 @@ +package com.hufudb.openhufu.benchmark; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.hufudb.openhufu.benchmark.enums.SpatialTableName; +import com.hufudb.openhufu.core.table.GlobalTableConfig; +import com.hufudb.openhufu.data.storage.DataSet; +import com.hufudb.openhufu.data.storage.DataSetIterator; +import com.hufudb.openhufu.data.storage.utils.GeometryUtils; +import com.hufudb.openhufu.expression.ExpressionFactory; +import com.hufudb.openhufu.plan.LeafPlan; +import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType; +import com.hufudb.openhufu.proto.OpenHuFuData.Modifier; +import com.hufudb.openhufu.proto.OpenHuFuPlan.Expression; +import com.hufudb.openhufu.user.OpenHuFuUser; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OpenHuFuSpatialPostgisTest { + + private static final Logger LOG = LoggerFactory.getLogger(OpenHuFuBenchmark.class); + private static final OpenHuFuUser user = new OpenHuFuUser(); + + @BeforeClass + public static void setUp() throws IOException { + LinkedTreeMap userConfigs = new Gson().fromJson(Files.newBufferedReader( + Path.of(OpenHuFuBenchmark.class.getClassLoader().getResource("spatial-postgis-configs.json") + .getPath())), + LinkedTreeMap.class); + List endpoints = (List) userConfigs.get("owners"); + List globalTableConfigs = + new Gson().fromJson(new Gson().toJson(userConfigs.get("tables")), + new TypeToken>() { + }.getType()); + LOG.info("Init benchmark of OpenHuFuSpatialPOSTGIS..."); + for (String endpoint : endpoints) { + user.addOwner(endpoint, null); + } + + for (GlobalTableConfig config : globalTableConfigs) { + user.createOpenHuFuTable(config); + } + LOG.info("Init finish"); + } + + public void printLine(ResultSet it) throws SQLException { + for (int i = 1; i <= it.getMetaData().getColumnCount(); i++) { + System.out.print(it.getString(i) + "|"); + } + System.out.println(); + } + + @Test + public void testSelect() throws SQLException { + String sql = "select * from osm_a"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + int count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(400, count); + dataset.close(); + } + } + + @Test + public void testSpatialDistance() throws SQLException { + String sql = "select id, Distance(location, POINT(0, 0)) from osm_a"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(400, count); + dataset.close(); + } + } + + @Test + public void testRangeQuery() throws SQLException { + String sql = "select * from osm_a where DWithin(POINT(0, 0), location, 50)"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + dataset.close(); + assertEquals(30, count); + } + } + + /* + Result: osm_a_1: 14, osm_a_2: 16, osm_a_3: 0, osm_a_4: 0 + Validation SQL: + SELECT COUNT(*) from osm_a_1 where ST_DWithin('SRID=4326;POINT (0 0)', location, 50.0) + SELECT COUNT(*) from osm_a_2 where ST_DWithin('SRID=4326;POINT (0 0)', location, 50.0) + SELECT COUNT(*) from osm_a_3 where ST_DWithin('SRID=4326;POINT (0 0)', location, 50.0) + SELECT COUNT(*) from osm_a_4 where ST_DWithin('SRID=4326;POINT (0 0)', location, 50.0) + */ + @Test + public void testRangeCount() throws SQLException { + String sql = "select count(*) from osm_a where DWithin(POINT(0, 0), location, 50)"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + dataset.next(); + assertEquals(30, dataset.getInt(1)); + dataset.close(); + } + } + + /* + Valication SQL: + SELECT id, location, distance + FROM ((SELECT id as id, + st_astext(location) as location, + 'SRID=4326;POINT (0 0)' <-> location as distance + FROM osm_a_1) + union + (SELECT id as id, + st_astext(location) as location, + 'SRID=4326;POINT (0 0)' <-> location as distance + FROM osm_a_2) + union + (SELECT id as id, + st_astext(location) as location, + 'SRID=4326;POINT (0 0)' <-> location as distance + FROM osm_a_3) + union + (SELECT id as id, + st_astext(location) as location, + 'SRID=4326;POINT (0 0)' <-> location as distance + FROM osm_a_4)) AS new_osm_a + ORDER BY distance + ASC + LIMIT 10 + */ + @Test + public void testKNNQuery1() throws SQLException { + String sql = + "select id, location from osm_a order by Distance(POINT(0, 0), location) asc limit 10"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(10, count); + dataset.close(); + } + } + + @Test + public void testKNNQuery2() throws SQLException { + String sql = "select id, location from osm_a where KNN(POINT(0, 0), location, 10)"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(10, count); + dataset.close(); + } + } + + @Test + public void testRangeJoin() throws SQLException { + String sql = + "select * from osm_b join osm_a on DWithin(osm_b.location, osm_a.location, 5)"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(220, count); + dataset.close(); + } + } + + @Test + public void testKNNJOIN() throws SQLException { + String sql = "select * from osm_b join osm_a on KNN(osm_b.location, osm_a.location, 5)"; + try (Statement stmt = user.createStatement()) { + ResultSet dataset = stmt.executeQuery(sql); + long count = 0; + while (dataset.next()) { + printLine(dataset); + ++count; + } + assertEquals(200, count); + dataset.close(); + } + } +} diff --git a/benchmark/src/test/resources/spatial-user-configs.json b/benchmark/src/test/resources/spatial-csv-configs.json similarity index 100% rename from benchmark/src/test/resources/spatial-user-configs.json rename to benchmark/src/test/resources/spatial-csv-configs.json diff --git a/benchmark/src/test/resources/spatial-postgis-configs.json b/benchmark/src/test/resources/spatial-postgis-configs.json new file mode 100644 index 00000000..82f3aa79 --- /dev/null +++ b/benchmark/src/test/resources/spatial-postgis-configs.json @@ -0,0 +1,52 @@ +{ + "owners": [ + "localhost:12345", + "localhost:12346", + "localhost:12347", + "localhost:12348" + ], + "tables": [ + { + "tableName": "osm_a", + "localTables": [ + { + "endpoint": "localhost:12345", + "localName": "osm_a" + }, + { + "endpoint": "localhost:12346", + "localName": "osm_a" + }, + { + "endpoint": "localhost:12347", + "localName": "osm_a" + }, + { + "endpoint": "localhost:12348", + "localName": "osm_a" + } + ] + }, + { + "tableName": "osm_b", + "localTables": [ + { + "endpoint": "localhost:12345", + "localName": "osm_b" + }, + { + "endpoint": "localhost:12346", + "localName": "osm_b" + }, + { + "endpoint": "localhost:12347", + "localName": "osm_b" + }, + { + "endpoint": "localhost:12348", + "localName": "osm_b" + } + ] + } + ] +} \ No newline at end of file diff --git a/common/src/main/java/com/hufudb/openhufu/common/enums/DataSourceType.java b/common/src/main/java/com/hufudb/openhufu/common/enums/DataSourceType.java index fbd5540a..768f1d5f 100644 --- a/common/src/main/java/com/hufudb/openhufu/common/enums/DataSourceType.java +++ b/common/src/main/java/com/hufudb/openhufu/common/enums/DataSourceType.java @@ -6,7 +6,8 @@ */ public enum DataSourceType { - CSV("CSV"); + CSV("CSV"), + POSTGIS("POSTGIS"); DataSourceType(String type) { this.type = type; diff --git a/common/src/main/java/com/hufudb/openhufu/common/exception/ErrorFormatter.java b/common/src/main/java/com/hufudb/openhufu/common/exception/ErrorFormatter.java index 0a07e61b..664c6d86 100644 --- a/common/src/main/java/com/hufudb/openhufu/common/exception/ErrorFormatter.java +++ b/common/src/main/java/com/hufudb/openhufu/common/exception/ErrorFormatter.java @@ -8,7 +8,7 @@ private ErrorFormatter() {} static String format(ErrorCode error, Object... args) { if (error == null) { - throw new IllegalArgumentException("BaymaxError cannot be null"); + throw new IllegalArgumentException("OpenHuFuError cannot be null"); } if (error.getDesc() == null) { diff --git a/core/pom.xml b/core/pom.xml index ec724b66..83e83311 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -65,7 +65,6 @@ com.google.guava guava - 30.0-jre junit diff --git a/core/src/main/java/com/hufudb/openhufu/core/implementor/UserSideImplementor.java b/core/src/main/java/com/hufudb/openhufu/core/implementor/UserSideImplementor.java index 0ba1f314..1320e29f 100644 --- a/core/src/main/java/com/hufudb/openhufu/core/implementor/UserSideImplementor.java +++ b/core/src/main/java/com/hufudb/openhufu/core/implementor/UserSideImplementor.java @@ -130,10 +130,6 @@ private boolean isPrivacyKNNJoin(BinaryPlan plan) { return plan.getJoinCond().getCondition().getStr().equals("knn"); } - private DataSet privacySpatialJoin(BinaryPlan plan, boolean isDistanceJoin) { - return privacySpatialJoin(plan, isDistanceJoin, false); - } - private DataSet privacySpatialJoin(BinaryPlan plan, boolean isDistanceJoin, boolean isUsingKNNFunc) { DataSet left = ownerSideQuery(plan.getChildren().get(0)); @@ -212,7 +208,7 @@ public DataSet implement(Plan plan) { } if (isMultiParty(plan)) { if (plan instanceof BinaryPlan && isPrivacyRangeJoin((BinaryPlan) plan)) { - return privacySpatialJoin((BinaryPlan) plan, true); + return privacySpatialJoin((BinaryPlan) plan, true, false); } if (plan instanceof BinaryPlan && isPrivacyKNNJoin((BinaryPlan) plan)) { return privacySpatialJoin((BinaryPlan) plan, false, isUsingKNNFuc); @@ -239,7 +235,7 @@ private DataSet privacyKNN(UnaryPlan plan, boolean isUsingKNNFunc) { // if (USE_DP) { right = kNNRadiusQuery(plan) * 2; // } - double deviation = 1e-6; + double deviation = 1e-10; int loop = 0; long count = 0L; if (USE_DP) { @@ -270,11 +266,13 @@ private DataSet privacyKNN(UnaryPlan plan, boolean isUsingKNNFunc) { } else if (sign > 0) { right = mid; } else { + LOG.info("kNN radius is {}", mid); DataSet dataSet = ArrayDataSet.materialize(kNNCircleRangeQuery(plan, mid, isUsingKNNFunc)); return dataSet; } loop++; } + LOG.info("kNN radius is {}", right); return kNNCircleRangeQuery(plan, right, isUsingKNNFunc); } diff --git a/core/src/main/java/com/hufudb/openhufu/core/implementor/spatial/knn/BinarySearchKNN.java b/core/src/main/java/com/hufudb/openhufu/core/implementor/spatial/knn/BinarySearchKNN.java index 73b390d2..72fcdad3 100644 --- a/core/src/main/java/com/hufudb/openhufu/core/implementor/spatial/knn/BinarySearchKNN.java +++ b/core/src/main/java/com/hufudb/openhufu/core/implementor/spatial/knn/BinarySearchKNN.java @@ -25,8 +25,8 @@ public static Plan generateKNNRadiusQueryPlan(UnaryPlan originalPlan) { OpenHuFuPlan.Expression distance = originalLeaf.getSelectExps() .get(originalLeaf.getOrders().get(0).getRef()); leafPlan.setSelectExps(ImmutableList.of(distance)); - leafPlan.setOrders(ImmutableList.of(OpenHuFuPlan.Collation.newBuilder().setRef(0) - .setDirection(OpenHuFuPlan.Direction.ASC).build())); +// leafPlan.setOrders(ImmutableList.of(OpenHuFuPlan.Collation.newBuilder().setRef(0) +// .setDirection(OpenHuFuPlan.Direction.ASC).build())); leafPlan.setOffset(originalLeaf.getFetch() - 1); leafPlan.setFetch(1); LOG.info(leafPlan.toString()); @@ -53,7 +53,7 @@ public static Plan generatePrivacyComparePlan(UnaryPlan originalPlan, double ran leafPlan.setWhereExps(whereExps); leafPlan.setAggExps(ImmutableList.of(ExpressionFactory.createAggFunc(OpenHuFuData.ColumnType.LONG, OpenHuFuData.Modifier.PROTECTED, AggFuncType.COUNT.getId(), ImmutableList.of()))); - leafPlan.setOrders(originalLeaf.getOrders()); +// leafPlan.setOrders(originalLeaf.getOrders()); UnaryPlan unaryPlan = new UnaryPlan(leafPlan); unaryPlan.setSelectExps(ImmutableList.of(ExpressionFactory diff --git a/data/src/main/java/com/hufudb/openhufu/data/storage/Point.java b/data/src/main/java/com/hufudb/openhufu/data/storage/Point.java new file mode 100644 index 00000000..b73f569a --- /dev/null +++ b/data/src/main/java/com/hufudb/openhufu/data/storage/Point.java @@ -0,0 +1,94 @@ +package com.hufudb.openhufu.data.storage; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Point implements Serializable { + private static PointParser parser = new PointParser(); + + private double x; + private double y; + + public Point(double x, double y) { + this.x = x; + this.y = y; + } + + public double getX() { + return this.x; + } + + public double getY() { + return this.y; + } + + @Override + public String toString() { + return String.format("POINT(%f %f)", getX(), getY()); + } + + public byte[] toBytes() { + byte[] tmp = ByteBuffer.allocate(Double.BYTES).putDouble(x).array(); + byte[] encoded = new byte[Double.BYTES * 2]; + for (int i = 0; i < Double.BYTES; ++i) { + encoded[i] = tmp[i]; + } + tmp = ByteBuffer.allocate(Double.BYTES).putDouble(y).array(); + for (int i = 0; i < Double.BYTES; ++i) { + encoded[i + Double.BYTES] = tmp[i]; + } + return encoded; + } + + public static Point fromBytes(byte[] encoded) { + byte[] tmp = new byte[Double.BYTES]; + for (int i = 0; i < Double.BYTES; ++i) { + tmp[i] = encoded[i]; + } + double x = ByteBuffer.wrap(tmp).getDouble(); + for (int i = 0; i < Double.BYTES; ++i) { + tmp[i] = encoded[i + Double.BYTES]; + } + double y = ByteBuffer.wrap(tmp).getDouble(); + return new Point(x, y); + } + + @Override + public boolean equals(Object obj) { + return obj == this || (obj instanceof Point && ((Point) obj).getX() == x && ((Point) obj).getY() == y); + } + + private static class PointParser { + private final static Logger LOG = LoggerFactory.getLogger(PointParser.class); + + private static String pointRex = "\\(\\s*([\\-]?[0-9]+[.]?[0-9]*)\\s+([\\-]?[0-9]+[.]?[0-9]*)\\s*\\)"; + private Pattern pointPattern; + + PointParser() { + pointPattern = Pattern.compile(pointRex); + } + + Point parse(String pointStr) { + if (pointStr == null) { + return null; + } + Matcher pointMatcher = pointPattern.matcher(pointStr); + if (pointMatcher.find()) { + String xStr = pointMatcher.group(1); + String yStr = pointMatcher.group(2); + return new Point(Double.parseDouble(xStr), Double.parseDouble(yStr)); + } else { + LOG.debug("can't parse {} to Point", pointStr); + return null; + } + } + } + + public static Point parse(String p) { + return parser.parse(p); + } +} diff --git a/data/src/main/java/com/hufudb/openhufu/data/storage/ProtoColumn.java b/data/src/main/java/com/hufudb/openhufu/data/storage/ProtoColumn.java index ed3422a7..53d781ac 100644 --- a/data/src/main/java/com/hufudb/openhufu/data/storage/ProtoColumn.java +++ b/data/src/main/java/com/hufudb/openhufu/data/storage/ProtoColumn.java @@ -72,7 +72,7 @@ public class ProtoColumn implements Column { break; case BYTESCOL: if (type == ColumnType.GEOMETRY) { - this.getter = (rowNum) -> GeometryUtils.fromBytes(column.getBytescol().getCell(rowNum).toByteArray()); + this.getter = (rowNum) -> Point.fromBytes(column.getBytescol().getCell(rowNum).toByteArray()); } else { this.getter = (rowNum) -> column.getBytescol().getCell(rowNum).toByteArray(); } @@ -140,7 +140,7 @@ public static class Builder { break; case GEOMETRY: bytesBuilder = BytesColumn.newBuilder(); - appender = (val) -> bytesBuilder.addCell(ByteString.copyFrom((GeometryUtils.toBytes((Geometry) val)))); + appender = (val) -> bytesBuilder.addCell(ByteString.copyFrom(((Point) val).toBytes())); break; case STRING: strBuilder = StringColumn.newBuilder(); diff --git a/data/src/main/java/com/hufudb/openhufu/data/storage/RandomDataSet.java b/data/src/main/java/com/hufudb/openhufu/data/storage/RandomDataSet.java index b28ab3f4..edae28c2 100644 --- a/data/src/main/java/com/hufudb/openhufu/data/storage/RandomDataSet.java +++ b/data/src/main/java/com/hufudb/openhufu/data/storage/RandomDataSet.java @@ -16,9 +16,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.math3.distribution.LaplaceDistribution; import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.Point; /* * Used for security union, insert fake record to dataset @@ -134,14 +132,9 @@ private Object getRandomValueFromData(int columnIndex) { case BOOLEAN: return lap.sample() > 0.0; case GEOMETRY: - Geometry geometry = (Geometry) originRows.get(r).get(columnIndex); - if (geometry instanceof Point) { - Point p = (Point) geometry; - return geoFactory.createPoint( - new Coordinate(p.getX() + lap.sample(), p.getX() + lap.sample())); - } else { - throw new OpenHuFuException(ErrorCode.DATA_TYPE_NOT_SUPPORT, type); - } + Point p = (Point) originRows.get(r).get(columnIndex); + return geoFactory.createPoint( + new Coordinate(p.getX() + lap.sample(), p.getX() + lap.sample())); case STRING: return originRows.get(r).get(columnIndex); default: diff --git a/dataset/data-importer/config/osm/schema-osm_10_4.json b/dataset/data-importer/config/osm/schema-osm_10_4.json new file mode 100644 index 00000000..6fd8ade0 --- /dev/null +++ b/dataset/data-importer/config/osm/schema-osm_10_4.json @@ -0,0 +1,181 @@ +{ + "databases": [ + { + "host": "localhost", + "port": 54321, + "username": "hufu", + "password": "hufu", + "database": "postgres", + "schemas": [ + { + "name": "osm_a_1", + "csv_path": "./data/osm_a_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_2", + "csv_path": "./data/osm_a_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_3", + "csv_path": "./data/osm_a_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_4", + "csv_path": "./data/osm_a_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_1", + "csv_path": "./data/osm_b_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_2", + "csv_path": "./data/osm_b_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_3", + "csv_path": "./data/osm_b_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_4", + "csv_path": "./data/osm_b_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/dataset/data-importer/config/traffic/schema-traffic.json b/dataset/data-importer/config/traffic/schema-traffic.json new file mode 100644 index 00000000..6fd8ade0 --- /dev/null +++ b/dataset/data-importer/config/traffic/schema-traffic.json @@ -0,0 +1,181 @@ +{ + "databases": [ + { + "host": "localhost", + "port": 54321, + "username": "hufu", + "password": "hufu", + "database": "postgres", + "schemas": [ + { + "name": "osm_a_1", + "csv_path": "./data/osm_a_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_2", + "csv_path": "./data/osm_a_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_3", + "csv_path": "./data/osm_a_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_4", + "csv_path": "./data/osm_a_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_1", + "csv_path": "./data/osm_b_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_2", + "csv_path": "./data/osm_b_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_3", + "csv_path": "./data/osm_b_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_4", + "csv_path": "./data/osm_b_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/dataset/data-importer/data/osm_a_1.csv b/dataset/data-importer/data/osm_a_1.csv new file mode 100644 index 00000000..aea5ec43 --- /dev/null +++ b/dataset/data-importer/data/osm_a_1.csv @@ -0,0 +1,100 @@ +14415434,120.3737,14.61726 +21568794,44.3091228,33.2916473 +21661571,44.341939,33.3069229 +24670219,44.524256,33.3599914 +25276726,44.3081183,33.38169 +25351252,44.3787444,33.2667707 +25389234,121.0010696,14.5377703 +25463304,121.6882437,11.6036244 +25526436,44.4684724,33.3054919 +25629553,121.0624496,14.6292214 +25812375,44.6282332,33.1601434 +25944349,69.2908213,34.6157949 +26028404,44.4856443,33.3043685 +26097147,123.0450146,13.1289588 +26171308,121.7318581,14.3084893 +26222396,124.8793468,11.6567481 +26259685,123.9202765,12.1508055 +26294960,120.3162682,12.0368285 +26327763,125.5014798,10.0036441 +26483920,120.1213634,30.2375373 +26596242,96.1682915,16.8665171 +26745559,44.3141309,33.2493962 +26926420,45.0766114,65.4257472 +26959161,42.4664162,61.634988 +27061892,123.5326333,10.8401531 +27176435,120.3991795,14.8459081 +27416888,44.3475783,33.2274505 +27542801,76.3077241,44.7942926 +27609746,29.1995646,41.2210391 +27640229,35.8285452,36.2419552 +27655171,27.3302267,36.9990946 +27672404,28.6154065,36.736068 +27685921,28.8312382,36.6106342 +27711930,29.3712898,36.2452784 +27729070,27.5674765,37.2766363 +27745701,27.9287956,40.5022925 +27769157,27.064115,38.876104 +27799542,26.653336,39.29679 +27838737,38.1869566,40.9402448 +27860282,38.582471,40.9437588 +27882759,40.7245766,43.0857391 +27898495,39.1510179,47.028367 +27915495,38.06904,45.966708 +27931803,35.0433739,45.6072251 +27948131,36.2964034,46.6406565 +27964321,38.1359709,47.0502501 +27981028,35.715931,46.577824 +27998271,34.4949831,45.87363 +28016798,39.8960281,47.2592847 +28034401,39.395543,47.1162419 +28052284,33.511702,44.630977 +28070346,34.448194,46.154603 +28106589,44.4725196,33.2537324 +28192645,120.9820061,31.1694976 +28389397,120.57514,31.50677 +28576100,120.219103,31.4017333 +28997564,121.416956,14.1181201 +29980265,35.2251452,31.7623533 +30876868,50.6069026,26.2478296 +31203707,55.2958218,25.2731495 +32043108,26.9395187,38.4459356 +32428912,121.0762708,14.8206508 +32928353,121.0358611,14.6230309 +33680046,121.0537179,14.5447191 +34053013,34.8835908,32.4653555 +35278399,120.1215609,30.2786694 +35454778,120.9962666,14.659686 +35936082,47.7577266,11.111727 +35949160,43.4854209,11.3680605 +36088067,53.0871779,12.1588107 +36104452,53.878457,12.3239228 +36117556,42.9434216,12.910389 +36358313,102.581882,2.0862926 +44531152,80.2085349,6.0406226 +56356015,121.0169994,14.3932009 +57397172,139.6686558,36.1683097 +57435617,139.5684016,35.8815831 +58644819,74.222925,35.9376364 +59603146,100.3182971,5.368205 +59831242,120.9744771,22.5528834 +59955160,121.4931931,29.4805768 +60902136,73.085825,32.0304265 +63158015,119.7117966,26.2547665 +63330741,117.2330801,23.6368 +63488207,119.6077873,23.7514345 +63663462,119.5999101,26.4540601 +63820077,117.4848962,23.7175173 +64024071,119.555705,26.0803565 +64245043,118.53847,25.78973 +64440341,119.9046799,26.3516 +64611834,119.8656599,26.6487301 +64652315,117.55044,24.1112301 +64692455,117.37746,23.94567 +64765670,117.8899001,24.0278699 +68200512,122.0195568,29.1772412 +69704845,120.2713338,22.7009236 +69970622,120.1290071,26.6900366 +70929315,121.4722421,23.3671993 +71160087,120.1126751,26.7078023 +71389144,121.5657733,22.0128067 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_a_2.csv b/dataset/data-importer/data/osm_a_2.csv new file mode 100644 index 00000000..86896ecb --- /dev/null +++ b/dataset/data-importer/data/osm_a_2.csv @@ -0,0 +1,100 @@ +71612327,120.3523788,26.7093933 +72545536,122.15969,29.6622999 +72736923,116.9570509,23.4481839 +72907020,114.3384948,22.4571889 +73076673,114.287281,22.2381828 +73235601,122.020915,29.8915718 +73444204,114.7453801,22.63546 +73598878,114.94357,22.71135 +73786965,122.1426594,29.983726 +73939956,31.6096749,31.2027421 +74432006,113.1752166,22.6847998 +74595368,112.5410593,21.6344634 +74763101,121.6418801,29.57197 +74937706,113.1453501,22.7339801 +75142806,121.8168472,29.2943362 +75298007,121.94913,29.5493 +75452874,113.2682571,22.256029 +75651878,38.4061972,44.469752 +75796382,112.38253,21.95561 +76009911,113.1678299,22.6467001 +76211681,113.7216754,21.9576254 +76444609,112.5895335,21.6943802 +76977163,121.9669134,30.1747084 +77176538,111.6402499,21.5156299 +77379511,122.145985,30.124742 +77771242,122.37195,30.2828 +78634431,122.3580193,30.4682892 +79395024,122.47156,30.71529 +81320424,121.4075856,31.2152037 +82526619,105.7994801,21.230662 +84910564,119.8694484,32.2802301 +85354363,121.5132238,28.4982891 +85837054,121.3069498,28.1718167 +86152312,120.30219,27.29207 +88081145,121.5905949,31.2487378 +90439589,46.9765988,50.9533349 +92574907,55.3012683,25.2640728 +94596348,55.1780065,25.0672711 +96706735,47.5882555,52.0844486 +100364881,103.3974007,18.4502995 +105749082,103.03339,18.1291919 +110668532,104.8275353,16.9899132 +115079736,101.617865,17.8899532 +116236209,48.0781459,29.8152903 +117647268,47.7539777,30.5719398 +118447382,49.1939972,27.4222023 +118481499,50.0005189,26.8167411 +119354434,50.9938815,25.934558 +121842046,51.5218267,25.4799541 +122118067,50.6928884,29.1196762 +122564954,48.8145592,30.3199216 +123830600,49.1637212,30.4996187 +124486203,51.4085473,24.6225639 +125661650,53.320781,24.5848806 +125864401,54.0369922,24.1991719 +128488777,60.0163874,56.8441014 +130004204,58.7310575,23.5106636 +130057562,56.4008509,26.1960622 +130085263,58.937399,20.5458775 +130177617,38.4572394,37.12898 +130511290,108.4612501,21.89342 +130737723,109.5775801,21.7420999 +130934992,109.2054346,21.7244872 +130965702,53.7828773,16.8804596 +131024496,108.42707,21.6693 +131112627,108.5665667,21.6716309 +131353521,109.90894,21.46695 +131594247,108.4905099,21.9543399 +131865452,110.4184635,21.2150639 +132428666,42.1979883,13.6520604 +132532333,110.1887894,20.2469475 +132763052,108.6596628,18.8087419 +132991220,108.197206,15.8632987 +133213496,77.5458072,55.3042518 +133304941,40.0486019,15.938802 +133337788,40.0824702,15.6507053 +133368889,110.3364701,20.8347 +133408634,41.3795502,14.3796667 +133439681,110.2006495,20.9596563 +133725378,110.3020931,19.7004739 +133990179,110.5120499,18.91958 +134251022,110.16987,20.87119 +134496508,109.54084,19.9174301 +134914546,40.1421091,16.5316621 +135056706,42.1577605,17.5690555 +135310582,41.75969,16.5927222 +135349037,75.6005461,55.1256434 +135925252,38.1046629,18.4006968 +136012109,39.6477324,20.5136523 +136043546,40.9529451,19.439267 +137183463,38.9600281,22.879556 +137215411,38.9452269,21.955209 +137274423,106.2997621,52.6124507 +137817103,40.9153238,37.8427615 +140495914,37.2314247,24.8371063 +140700608,36.7938094,25.6915466 +140782859,34.8318995,28.9089783 +141937242,111.0848261,30.8604912 +150112832,120.1951842,30.2779014 +150579953,121.2822326,38.7911708 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_a_3.csv b/dataset/data-importer/data/osm_a_3.csv new file mode 100644 index 00000000..dc5ff7a6 --- /dev/null +++ b/dataset/data-importer/data/osm_a_3.csv @@ -0,0 +1,100 @@ +150671265,43.6978412,40.6419104 +150885995,121.2402344,38.7901914 +150928609,121.1829376,36.7004839 +150975728,120.4645201,33.37543 +151204991,119.7422,33.9067701 +151835973,119.59533,33.78823 +152659783,50.3170924,53.1958565 +152901303,125.81776,36.96366 +153090148,124.98204,38.58684 +153266226,124.87888,38.50199 +153469736,125.3568601,37.6890199 +153634452,124.8197601,38.1973001 +153769577,124.84775,38.10698 +153927414,125.996452,37.0659797 +158053346,103.7745384,1.3912416 +161489071,43.1626923,41.0424744 +164849636,117.61817,39.2281201 +166007740,48.5613477,53.122222 +166337749,48.543651,53.0849532 +166606335,121.318524,39.0017423 +166906328,121.84084,41.0922101 +167218305,121.66017,39.30513 +167452530,125.8784149,39.0630275 +167679794,125.4780292,40.6136267 +167904044,125.7008276,39.0009241 +168114184,125.194214,40.57737 +168277423,123.66906,40.1623 +168502708,124.344137,39.979682 +168725201,124.7386471,39.5529347 +174592046,121.0815206,14.6034146 +176169915,60.6054333,56.7894899 +179371093,49.9656084,53.0035005 +180142835,73.0177047,33.6904133 +182660813,41.3479921,42.796745 +186689838,60.6235345,56.8189332 +188255433,50.1236951,53.2311441 +191756980,76.5689993,8.9623859 +192580294,39.3734917,33.155406 +193685057,76.2335889,10.2036141 +197167084,84.9124469,56.5265612 +198532106,120.4583283,16.5189301 +199278001,62.8576755,25.2682457 +199572159,66.2881592,25.5234086 +201816320,75.0721258,12.4957702 +202209319,67.9489106,24.1591521 +203040166,68.9386323,23.7935969 +203386322,70.6774031,24.4766152 +203475268,70.2164478,23.8010417 +203533992,70.4199937,22.8078415 +203694633,69.112182,22.4056005 +204029254,69.9701946,23.8842534 +204201534,71.1208624,23.4382269 +204333202,71.3818699,20.909468 +205065547,103.8716616,13.4629291 +205333826,72.8281163,21.0351912 +206101419,103.8511927,1.3694845 +206259429,72.7629613,19.9809049 +206283372,73.0536879,18.8168981 +206303450,73.1850386,17.3847724 +206400585,73.3466392,16.5532058 +207096233,48.0168223,25.4343633 +207688739,74.1210973,14.8025173 +208409150,73.1248479,6.8954545 +208442723,55.3112249,24.9955498 +208544822,73.1302094,2.1956815 +209288413,73.1084081,0.2486094 +210374976,73.3573479,54.9560403 +211100543,84.5020729,56.4799527 +213953798,80.0310552,15.2127313 +215527926,84.9807063,56.4492483 +218546670,58.5484471,23.609777 +220771394,60.581616,56.790753 +221243487,58.2448129,22.9035 +221608779,44.8384291,41.810192 +223380920,101.6746486,27.4759554 +223594214,44.8933495,41.6855592 +223965863,44.6912118,41.7164316 +224247358,99.4457,28.13063 +226050753,98.3884279,24.9908486 +226139188,80.5791593,15.8631455 +226257548,81.3021849,16.3025483 +226879758,53.0982277,18.4596994 +229092795,49.5977081,40.5580915 +230376542,85.5679443,19.76445 +231711652,60.4926529,56.8097086 +234786196,50.0921602,53.2520103 +237388894,86.9548452,20.7161291 +237577592,39.7154442,59.5274345 +238209281,98.8999801,26.4938429 +239684793,88.0491384,22.3818041 +239755637,88.5156896,21.8634504 +240067373,87.8621395,22.5384304 +240231003,156.4632782,52.4872658 +240298905,158.3435233,53.0557491 +240353981,160.7329289,56.3657271 +240369955,156.7324483,57.1099294 +240409722,89.4307548,22.4868894 +240473411,99.194419,25.278361 +240704179,70.5441076,43.137254 +240839256,162.8129303,58.8625635 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_a_4.csv b/dataset/data-importer/data/osm_a_4.csv new file mode 100644 index 00000000..737a1526 --- /dev/null +++ b/dataset/data-importer/data/osm_a_4.csv @@ -0,0 +1,100 @@ +241034400,90.396671,22.7125749 +241120974,90.0643257,22.598012 +241142449,90.0688147,22.6857414 +241282923,129.1871777,37.4525529 +241297243,129.260493,35.2919694 +241529223,65.5385154,44.7628592 +241587021,90.5556089,22.0230836 +241672062,131.8088215,43.4300611 +241686432,131.893931,43.0060895 +241700723,131.12278,42.6637101 +241719286,130.4671043,42.3246059 +241736069,129.85779,41.7947699 +241795531,136.0461143,34.6843764 +241832760,177.2537927,62.5726059 +241842246,177.6437475,52.0435478 +241849483,178.9899129,51.5814075 +241880872,60.6323183,56.7549653 +241909212,91.9611694,22.4223397 +241921105,92.0370412,21.6842614 +242534346,77.098403,43.8233194 +242827600,101.667215,54.2356262 +243188235,78.8549785,46.2188837 +243348002,108.4845913,51.7238391 +243526498,81.7668078,46.4349912 +243739398,79.2420272,42.8913351 +243842096,77.2130258,43.3852886 +243867028,75.8851092,30.8211267 +243902031,92.8400349,20.7345866 +243916003,93.3106398,20.4158211 +243924451,93.3279639,20.404973 +243932930,93.2357311,20.084314 +243942749,93.8224713,20.0106243 +244071742,93.6393138,19.9271893 +244080349,93.7844467,19.8054044 +244122424,94.0366219,19.7401381 +244150448,83.2822038,50.2744386 +244158445,93.8365808,19.4440634 +244171773,93.9614832,19.3441836 +244182121,93.8013473,19.2092319 +244197625,93.7114799,19.0145368 +244232589,94.2091235,18.7440159 +244297361,94.5654657,17.3386718 +244322604,19.2487234,74.4698867 +244500851,74.5697444,42.8737688 +244535467,126.1991754,38.9882288 +244546614,126.8713792,38.2549986 +244565227,126.7356446,37.892149 +244578362,126.192139,37.846454 +244589079,126.2870159,37.7127921 +244604153,126.0812999,37.2821099 +244617347,126.6271037,37.167507 +244629102,126.8575299,36.865333 +244646061,126.5363424,36.2515016 +244660012,126.4538008,36.3806785 +244674316,128.4077579,35.4181336 +244686319,126.2538599,35.5972101 +244696318,128.76216,35.0158899 +244707288,128.4938917,35.0793639 +244718072,128.21885,34.94333 +244729809,126.0477104,35.0810725 +244740610,128.4737831,34.9135432 +244751496,127.61135,34.87019 +244762136,128.0867644,34.9186627 +244772493,126.3315439,34.724201 +244782356,126.1753401,34.7717 +244792289,128.42851,34.7268201 +244803329,127.4542929,34.5011423 +244813166,126.0591339,34.3221852 +244820745,126.282236,34.3778354 +244829747,126.33082,34.56454 +244837476,126.3578184,34.476545 +244845477,126.7582557,34.2084262 +244852626,126.0904364,34.2790243 +244860580,126.0330433,34.4835889 +244868835,126.3243296,33.9149447 +244901626,40.3239321,48.3000949 +245301016,127.4635248,39.7882582 +245320456,127.5158676,39.7725005 +245338839,127.6264851,39.268585 +245557758,67.3595703,37.2229132 +245589676,77.4175451,10.9942894 +245597199,92.75651,11.66468 +245605249,78.5351379,12.2643464 +245614509,92.6802671,12.9481368 +245627219,75.0598477,14.3767003 +245647578,66.6510457,38.3074186 +245664639,72.8430982,19.0515338 +245695555,88.1820416,21.5691489 +245721262,84.5138522,57.063833 +245755202,81.6389537,26.9746137 +245914775,58.6984819,37.8621572 +245999540,146.8870631,43.8116697 +246500044,103.8230262,1.304687 +246563921,141.5575942,43.1176894 +246776735,56.2112432,26.2354671 +246799101,154.7804278,49.2830605 +247131357,87.2801235,59.7098963 +247240865,61.397599,41.2124398 +247439828,89.75979,27.5216494 +247686150,78.6964994,10.8352842 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_b_1.csv b/dataset/data-importer/data/osm_b_1.csv new file mode 100644 index 00000000..4e3deefe --- /dev/null +++ b/dataset/data-importer/data/osm_b_1.csv @@ -0,0 +1,10 @@ +247732141,38.5136449,37.4615308 +247841906,78.3039674,42.7494033 +247867962,73.563631,45.7466262 +247917194,73.0922993,33.7014519 +248047312,80.1584233,6.166971 +248133552,38.2166545,35.9219827 +248171621,39.0702286,35.9115961 +248223416,161.3518238,69.4031367 +248263747,42.8638318,36.6872651 +248355015,43.66525,32.57725 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_b_2.csv b/dataset/data-importer/data/osm_b_2.csv new file mode 100644 index 00000000..f8bfeb27 --- /dev/null +++ b/dataset/data-importer/data/osm_b_2.csv @@ -0,0 +1,10 @@ +248477840,100.5532274,13.6985287 +248669614,73.0118094,33.6395785 +248831354,100.4997393,13.6884696 +248865093,75.0554287,46.4276157 +249001471,160.6179578,69.6500175 +249049930,75.942792,46.5456018 +249158991,73.285,39.042 +249344841,48.4567122,32.579337 +249358129,69.76,50.65025 +249595142,100.5021068,13.7757738 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_b_3.csv b/dataset/data-importer/data/osm_b_3.csv new file mode 100644 index 00000000..9b39ac6e --- /dev/null +++ b/dataset/data-importer/data/osm_b_3.csv @@ -0,0 +1,10 @@ +249737084,72.6598436,33.6069083 +249870255,136.7755151,54.1200365 +249899753,142.5428975,47.7253096 +249922157,141.9771981,51.5383145 +249942994,143.0555,53.231 +250037485,141.835556,46.6123485 +250138258,66.6031122,41.0425917 +250190746,100.13435,27.40576 +250296314,99.303768,26.459198 +250541319,100.1857667,26.4853855 \ No newline at end of file diff --git a/dataset/data-importer/data/osm_b_4.csv b/dataset/data-importer/data/osm_b_4.csv new file mode 100644 index 00000000..862e4341 --- /dev/null +++ b/dataset/data-importer/data/osm_b_4.csv @@ -0,0 +1,10 @@ +250677566,66.4625946,41.0518493 +250753139,40.4735684,35.023372 +250876123,100.4381758,50.6428239 +250955352,81.24725,30.722 +251021131,79.69125,33.648 +251082994,121.0525115,14.6923415 +251171805,118.1324731,31.4902273 +251186151,101.8119276,23.7371377 +251346117,77.4795734,12.8897647 +251638518,72.7451536,41.8204911 \ No newline at end of file diff --git a/dataset/data-importer/importer.py b/dataset/data-importer/importer.py new file mode 100644 index 00000000..186bc926 --- /dev/null +++ b/dataset/data-importer/importer.py @@ -0,0 +1,102 @@ +import psycopg2 +import csv +import json +import ast + + +def load_schema(schema_path): + with open(schema_path, 'r', encoding='utf-8') as f: + schema = json.load(f) + return schema['databases'] + +def connect(host, port, user, password, dbname): + con = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return con + +def build_create_table_sql(schemas): + sqls = [] + for schema in schemas: + sql = "DROP TABLE IF EXISTS " + schema['name'] + sqls.append(sql) + sql = "CREATE TABLE "; + sql += schema['name'] + "(\n" + for column in schema['columns']: + sql += " {} {},\n".format(column['name'], column['type']) + sql = sql[:-2] + sql += "\n);" + sqls.append(sql) + return sqls + +def create_table(con, schemas): + cur = con.cursor() + sqls = build_create_table_sql(schemas) + for sql in sqls: + print(sql) + cur.execute(sql) + con.commit() + cur.close() + +def load_data(con, schemas): + cur = con.cursor() + for schema in schemas: + sql = "INSERT INTO " + schema['name'] + values = [] + ids = [] + columns = schema['columns'] + for column in columns: + if column['type'] == "geometry": + values.append("ST_GeomFromText('point(%s %s)', 4326)") + ids.extend(column['index'][0:2]) + else: + values.append("%s") + ids.append(column['index'][0]) + sql += " VALUES(" + ",".join(values) + ")" + print(sql) + with open(schema['csv_path'], 'r', encoding='utf-8') as f: + rows = csv.reader(f) + for i, row in enumerate(rows): + fields = [] + for j in ids: + try: + fields.append(ast.literal_eval(row[j])) + except: + fields.append(str(row[j])) + cur.execute(sql, tuple(fields)) + if i != 0 and i % 1000 == 0: + print ("insert " + str(i+1) + " rows") + con.commit() + print ("insert " + str(i+1) + " rows") + con.commit() + +def create_index(con, schemas): + cur = con.cursor() + for schema in schemas: + sql = "CREATE INDEX " + schema['name'] + "_geom_idx" + columns = schema['columns'] + for column in columns: + if column['type'] == "geometry": + sql = sql + " ON " + schema['name'] + " USING GIST(" + column['name'] + ")" + print ("start create index on table " + schema['name']) + cur.execute(sql) + print ("create finish") + con.commit() + +def import_data(databases): + for database in databases: + dbname = database['database'] + host = database['host'] + port = database['port'] + username = database['username'] + password = database['password'] + con = connect(host, port, username, password, dbname) + create_table(con, database['schemas']) + load_data(con, database['schemas']) + # create_index(con, database['schemas']) + con.close() + +# osm traffic +# directory +# simulate or +if __name__ == "__main__": + databases = load_schema("schema.json") + import_data(databases) diff --git a/dataset/data-importer/schema-osm.json b/dataset/data-importer/schema-osm.json new file mode 100644 index 00000000..dbadb57d --- /dev/null +++ b/dataset/data-importer/schema-osm.json @@ -0,0 +1,181 @@ +{ + "databases": [ + { + "host": "localhost", + "port": 54321, + "username": "hufu", + "password": "hufu", + "database": "postgres", + "schemas": [ + { + "name": "osm_a_1", + "csv_path": "./all-data/OSM/osm_a_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_2", + "csv_path": "./data/osm_a_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_3", + "csv_path": "./data/osm_a_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_4", + "csv_path": "./data/osm_a_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_1", + "csv_path": "./data/osm_b_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_2", + "csv_path": "./data/osm_b_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_3", + "csv_path": "./data/osm_b_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_4", + "csv_path": "./data/osm_b_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/dataset/data-importer/schema.json b/dataset/data-importer/schema.json new file mode 100644 index 00000000..b114401a --- /dev/null +++ b/dataset/data-importer/schema.json @@ -0,0 +1,181 @@ +{ + "databases": [ + { + "host": "localhost", + "port": 54321, + "username": "hufu", + "password": "hufu", + "database": "osm_db", + "schemas": [ + { + "name": "osm_a_1", + "csv_path": "./data/osm_a_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_2", + "csv_path": "./data/osm_a_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_3", + "csv_path": "./data/osm_a_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_a_4", + "csv_path": "./data/osm_a_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_1", + "csv_path": "./data/osm_b_1.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_2", + "csv_path": "./data/osm_b_2.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_3", + "csv_path": "./data/osm_b_3.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + }, + { + "name": "osm_b_4", + "csv_path": "./data/osm_b_4.csv", + "columns": [ + { + "name": "id", + "type": "int", + "index": [ + 0 + ] + }, + { + "name": "location", + "type": "geometry", + "index": [ + 1, + 2 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/OwnerService.java b/owner/src/main/java/com/hufudb/openhufu/owner/OwnerService.java index 70e32f86..f7d709db 100644 --- a/owner/src/main/java/com/hufudb/openhufu/owner/OwnerService.java +++ b/owner/src/main/java/com/hufudb/openhufu/owner/OwnerService.java @@ -73,7 +73,7 @@ public void query(QueryPlanProto request, StreamObserver responseO output.stream(); output.close(); } catch (Exception e) { - throw new OpenHuFuException(ErrorCode.QUERY_ERROR, e); + throw new OpenHuFuException(e, ErrorCode.QUERY_ERROR); } } diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/adapter/Adapter.java b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/Adapter.java index ad1f81cd..f1fec652 100644 --- a/owner/src/main/java/com/hufudb/openhufu/owner/adapter/Adapter.java +++ b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/Adapter.java @@ -3,6 +3,7 @@ import com.hufudb.openhufu.data.schema.SchemaManager; import com.hufudb.openhufu.data.storage.DataSet; import com.hufudb.openhufu.plan.Plan; +import java.util.List; public interface Adapter { SchemaManager getSchemaManager(); diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/adapter/AdapterTypeConverter.java b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/AdapterTypeConverter.java new file mode 100644 index 00000000..0da3cfeb --- /dev/null +++ b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/AdapterTypeConverter.java @@ -0,0 +1,28 @@ +package com.hufudb.openhufu.owner.adapter; + +import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType; +import java.sql.Types; + +public abstract class AdapterTypeConverter { + public ColumnType convert(int type, String typeName) { + switch (type) { + case Types.BOOLEAN: + return ColumnType.BOOLEAN; + case Types.INTEGER: + return ColumnType.INT; + case Types.BIGINT: + return ColumnType.LONG; + case Types.FLOAT: + return ColumnType.FLOAT; + case Types.DOUBLE: + case Types.NUMERIC: + return ColumnType.DOUBLE; + case Types.BLOB: + return ColumnType.BLOB; + case Types.CHAR: + case Types.VARCHAR: + default: + return ColumnType.STRING; + } + } +} diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCAdapter.java b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCAdapter.java new file mode 100644 index 00000000..58c52906 --- /dev/null +++ b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCAdapter.java @@ -0,0 +1,164 @@ +package com.hufudb.openhufu.owner.adapter.jdbc; + +import com.hufudb.openhufu.data.schema.Schema; +import com.hufudb.openhufu.data.schema.SchemaManager; +import com.hufudb.openhufu.data.schema.TableSchema; +import com.hufudb.openhufu.data.storage.DataSet; +import com.hufudb.openhufu.data.storage.EmptyDataSet; +import com.hufudb.openhufu.data.storage.ResultDataSet; +import com.hufudb.openhufu.expression.Translator; +import com.hufudb.openhufu.owner.adapter.Adapter; +import com.hufudb.openhufu.owner.adapter.AdapterTypeConverter; +import com.hufudb.openhufu.plan.Plan; +import com.hufudb.openhufu.proto.OpenHuFuPlan.PlanType; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base adapter for datasource with jdbc support + */ +public abstract class JDBCAdapter implements Adapter { + protected final static Logger LOG = LoggerFactory.getLogger(JDBCAdapter.class); + + protected String catalog; + protected Connection connection; + protected Statement statement; + protected final AdapterTypeConverter converter; + protected final SchemaManager schemaManager; + protected final JDBCTranslator translator; + + protected JDBCAdapter(String catalog, Connection connection, Statement statement, + AdapterTypeConverter converter, Translator translator) { + this.catalog = catalog; + this.connection = connection; + this.statement = statement; + this.converter = converter; + this.schemaManager = new SchemaManager(); + this.translator = new JDBCTranslator(translator); + loadAllTableSchema(); + } + + public void loadAllTableSchema() { + try { + DatabaseMetaData meta = connection.getMetaData(); + ResultSet rs = meta.getTables(catalog, null, "%", new String[] {"TABLE"}); + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + schemaManager.addLocalTable(getTableSchema(tableName, meta)); + } + rs.close(); + } catch (Exception e) { + LOG.error("Fail to load all table info: {}", e.getMessage()); + e.printStackTrace(); + } + } + + @Override + public DataSet query(Plan queryPlan) { + String sql = generateSQL(queryPlan); + Schema schema = queryPlan.getOutSchema(); + if (!sql.isEmpty()) { + return executeSQL(sql, schema); + } else { + return EmptyDataSet.INSTANCE; + } + } + + public ResultSet query(String sql) throws SQLException { + return statement.executeQuery(sql); + } + + public void execute(String sql) throws SQLException { + statement.execute(sql); + } + @Override + public void init() { + // do nothing + } + + @Override + public void shutdown() { + try { + statement.close(); + connection.close(); + } catch (Exception e) { + LOG.error("Fail to close statement/connection: {}", e.getMessage()); + } + } + + @Override + public SchemaManager getSchemaManager() { + return schemaManager; + } + + protected TableSchema getTableSchema(String tableName, DatabaseMetaData meta) { + try { + ResultSet rc = meta.getColumns(catalog, null, tableName, null); + TableSchema.Builder TableSchemaBuilder = TableSchema.newBuilder(); + TableSchemaBuilder.setTableName(tableName); + while (rc.next()) { + String columnName = rc.getString("COLUMN_NAME"); + TableSchemaBuilder.add(columnName, converter.convert(rc.getType(), rc.getString("TYPE_NAME"))); + } + rc.close(); + return TableSchemaBuilder.build(); + } catch (Exception e) { + LOG.error("Error when load TableSchema of {}: ", tableName, e.getMessage()); + return null; + } + } + + protected String generateSQL(Plan plan) { + assert plan.getPlanType().equals(PlanType.LEAF); + String actualTableName = schemaManager.getActualTableName(plan.getTableName()); + Schema tableSchema = schemaManager.getActualSchema(plan.getTableName()); + LOG.info("Query {}: {}", actualTableName, tableSchema); + final List filters = translator.translateExps(tableSchema, plan.getWhereExps()); + final List selects = translator.translateExps(tableSchema, plan.getSelectExps()); + final List groups = + plan.getGroups().stream().map(ref -> selects.get(ref)).collect(Collectors.toList()); + // order by + List order = translator.translateOrders(selects, plan.getOrders()); + StringBuilder sql = new StringBuilder(); + // select from clause + if (!plan.getAggExps().isEmpty()) { + final List aggs = translator.translateAgg(selects, plan.getAggExps()); + sql.append(String.format("SELECT %s from %s", String.join(",", aggs), actualTableName)); + } else { + sql.append(String.format("SELECT %s from %s", String.join(",", selects), actualTableName)); + } + // where clause + if (!filters.isEmpty()) { + sql.append(String.format(" where %s", String.join(" AND ", filters))); + } + if (!groups.isEmpty()) { + sql.append(String.format(" group by %s", String.join(",", groups))); + } + if (!order.isEmpty()) { + sql.append(String.format(" order by %s", String.join(",", order))); + } + if (plan.getFetch() != 0) { + sql.append(" LIMIT ").append(plan.getFetch() + plan.getOffset()); + } + return sql.toString(); + } + + protected DataSet executeSQL(String sql, Schema schema) { + try { + ResultSet rs = statement.executeQuery(sql); + LOG.info("Execute {}", sql); + return new ResultDataSet(schema, rs); + } catch (SQLException e) { + LOG.error("Fail to execute SQL [{}]: {}", sql, e.getMessage()); + return EmptyDataSet.INSTANCE; + } + } +} diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCTranslator.java b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCTranslator.java new file mode 100644 index 00000000..2a9ccfb3 --- /dev/null +++ b/owner/src/main/java/com/hufudb/openhufu/owner/adapter/jdbc/JDBCTranslator.java @@ -0,0 +1,31 @@ +package com.hufudb.openhufu.owner.adapter.jdbc; + +import com.hufudb.openhufu.data.schema.Schema; +import com.hufudb.openhufu.expression.Translator; +import com.hufudb.openhufu.proto.OpenHuFuPlan.Collation; +import com.hufudb.openhufu.proto.OpenHuFuPlan.Expression; +import java.util.List; +import java.util.stream.Collectors; + +public class JDBCTranslator { + Translator translator; + + JDBCTranslator(Translator translator) { + this.translator = translator; + } + + public List translateExps(Schema schema, List exps) { + translator.setInput(schema.getColumnDescs().stream().map(col -> col.getName()).collect(Collectors.toList())); + return exps.stream().map(exp -> translator.translate(exp)).collect(Collectors.toList()); + } + + public List translateAgg(List selectExpStrs, List aggs) { + translator.setInput(selectExpStrs); + return aggs.stream().map(exp -> translator.translate(exp)).collect(Collectors.toList()); + } + + public List translateOrders(List outExpStrs, List orders) { + return orders.stream().map(order -> String.format("%s %s", outExpStrs.get(order.getRef()), + order.getDirection().toString())).collect(Collectors.toList()); + } +} diff --git a/owner/src/main/java/com/hufudb/openhufu/owner/implementor/OwnerSideImplementor.java b/owner/src/main/java/com/hufudb/openhufu/owner/implementor/OwnerSideImplementor.java index 3e7dc1f2..71f64c4b 100644 --- a/owner/src/main/java/com/hufudb/openhufu/owner/implementor/OwnerSideImplementor.java +++ b/owner/src/main/java/com/hufudb/openhufu/owner/implementor/OwnerSideImplementor.java @@ -1,5 +1,8 @@ package com.hufudb.openhufu.owner.implementor; +import com.hufudb.openhufu.proto.OpenHuFuData.Modifier; +import com.hufudb.openhufu.proto.OpenHuFuPlan.Expression; +import com.hufudb.openhufu.proto.OpenHuFuPlan.OperatorType; import java.util.List; import java.util.concurrent.ExecutorService; import com.hufudb.openhufu.data.storage.DataSet; @@ -91,6 +94,13 @@ public DataSet unaryQuery(UnaryPlan unary) { public DataSet leafQuery(LeafPlan leaf) { try { DataSet localDataSet = dataSourceAdapter.query(leaf); + if (leaf.hasAgg()) { + for(Expression expression : leaf.getAggExps()) { + if (expression.getOpType()== OperatorType.AGG_FUNC && expression.getModifier() == Modifier.PROTECTED) { + return localDataSet; + } + } + } OwnerUnion union = getUnion(); return union.union(localDataSet, rpc, leaf.getTaskInfo()); } catch (Exception e) { diff --git a/plan/src/main/java/com/hufudb/openhufu/expression/BasicTranslator.java b/plan/src/main/java/com/hufudb/openhufu/expression/BasicTranslator.java new file mode 100644 index 00000000..ddd7fefd --- /dev/null +++ b/plan/src/main/java/com/hufudb/openhufu/expression/BasicTranslator.java @@ -0,0 +1,235 @@ +package com.hufudb.openhufu.expression; + +import com.google.common.collect.ImmutableList; +import com.hufudb.openhufu.data.storage.utils.DateUtils; +import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType; +import com.hufudb.openhufu.proto.OpenHuFuPlan.Expression; +import com.hufudb.openhufu.proto.OpenHuFuPlan.OperatorType; +import com.hufudb.openhufu.udf.UDFLoader; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class BasicTranslator implements Translator { + protected String dataSource; + protected List inputStrs; + + public BasicTranslator(String dataSource) { + this.dataSource = dataSource; + this.inputStrs = ImmutableList.of(); + } + + public void setInput(List inputs) { + this.inputStrs = inputs; + } + + public String translate(Expression exp) { + switch (exp.getOpType()) { + case REF: + return inputRef(exp); + case LITERAL: + return literal(exp); + case PLUS: + case MINUS: + // binary + case GT: + case GE: + case LT: + case LE: + case EQ: + case NE: + case TIMES: + case DIVIDE: + case MOD: + case AND: + case OR: + case LIKE: + return binary(exp); + // unary + case AS: + case NOT: + case PLUS_PRE: + case MINUS_PRE: + case IS_NULL: + case IS_NOT_NULL: + return unary(exp); + case CASE: + return caseCall(exp); + case SCALAR_FUNC: + return scalarFunc(exp); + case AGG_FUNC: + return aggregateFunc(exp); + default: + throw new RuntimeException("can't translate " + exp); + } + } + + protected String inputRef(Expression ref) { + int idx = ref.getI32(); + return inputStrs.get(idx); + } + + protected String literal(Expression literal) { + ColumnType type = literal.getOutType(); + switch (type) { + case BOOLEAN: + return String.valueOf(literal.getB()); + case BYTE: + case SHORT: + case INT: + return String.valueOf(literal.getI32()); + case DATE: + return String.format("date '%s'", DateUtils.longToDate(literal.getI64()).toString()); + case TIME: + return String.format("time '%s'", DateUtils.intToTime(literal.getI32()).toString()); + case TIMESTAMP: + return String.format("timestamp '%s'", DateUtils.longToTimestamp(literal.getI64()).toString()); + case INTERVAL: + return String.format("interval '%d' second", literal.getI64() / 1000); + case LONG: + return String.valueOf(literal.getI64()); + case FLOAT: + return String.valueOf(literal.getF32()); + case DOUBLE: + return String.valueOf(literal.getF64()); + case STRING: + return String.format("'%s'", literal.getStr()); + case GEOMETRY: + return String.format("'SRID=4326;%s'", literal.getStr()); + default: + throw new RuntimeException("can't translate literal " + literal); + } + } + + protected String unary(Expression exp) { + OperatorType type = exp.getOpType(); + String in = translate(exp.getIn(0)); + switch (type) { + case AS: + return String.format("(%s)", in); + case PLUS_PRE: + return String.format("(+%s)", in); + case MINUS_PRE: + return String.format("(-%s)", in); + case IS_NULL: + return String.format("(%s IS NULL)", in); + case IS_NOT_NULL: + return String.format("(%s IS NOT NULL)", in); + case NOT: + return String.format("(NOT %s)", in); + default: + throw new RuntimeException("can't translate unary " + exp); + } + } + + protected String binary(Expression exp) { + String left = translate(exp.getIn(0)); + String right = translate(exp.getIn(1)); + String op; + switch (exp.getOpType()) { + case GT: + op = ">"; + break; + case GE: + op = ">="; + break; + case LT: + op = "<"; + break; + case LE: + op = "<="; + break; + case EQ: + op = "="; + break; + case NE: + op = "<>"; + break; + case PLUS: + op = "+"; + break; + case MINUS: + op = "-"; + break; + case TIMES: + op = "*"; + break; + case DIVIDE: + op = "/"; + break; + case MOD: + op = "%"; + break; + case AND: + op = "AND"; + break; + case OR: + op = "OR"; + break; + case LIKE: + op = "LIKE"; + break; + default: + throw new RuntimeException("can't translate binary " + exp); + } + return String.format("(%s %s %s)", left, op, right); + } + + protected String caseCall(Expression exp) { + List inputs = + exp.getInList().stream().map(e -> translate(e)).collect(Collectors.toList()); + List caseList = new ArrayList<>(); + for (int i = 1; i < inputs.size(); i += 2) { + caseList.add(String.format("WHEN %s THEN %s", inputs.get(i - 1), inputs.get(i))); + } + String elseCase = String.format("ELSE %s", inputs.get(inputs.size() - 1)); + return String.format("CASE %s %s END", String.join(" ", caseList), elseCase); + } + + protected String scalarFunc(Expression exp) { + List inputs = + exp.getInList().stream().map(e -> translate(e)).collect(Collectors.toList()); + if (ScalarFuncType.support(exp.getStr())) { + ScalarFuncType func = ScalarFuncType.of(exp.getStr()); + switch (func) { + case ABS: + if (inputs.size() != 1) { + throw new RuntimeException("ABS need 1 arguements, but give " + inputs.size()); + } + return String.format("ABS(%s)", inputs.get(0)); + default: + throw new RuntimeException("Unsupported scalar function"); + } + } + return UDFLoader.translateScalar(exp.getStr(), dataSource, inputs); + } + + protected String aggregateFunc(Expression exp) { + AggFuncType type = AggFuncType.of(exp.getI32()); + List inputRefs = ExpressionUtils.getAggInputs(exp); + switch (type) { + case GROUPKEY: + return inputStrs.get(inputRefs.get(0)); + case SUM: + return String.format("SUM(%s)", inputStrs.get(inputRefs.get(0))); + case COUNT: + if (inputRefs.isEmpty()) { + return String.format("COUNT(*)"); + } else if (inputRefs.size() == 1) { + return String.format("COUNT(%s)", inputStrs.get(inputRefs.get(0))); + } else { + List inputs = + inputRefs.stream().map(ref -> inputStrs.get(ref)).collect(Collectors.toList()); + return String.format("COUNT((%s))", String.join(",", inputs)); + } + case AVG: + return String.format("AVG(%s)", inputStrs.get(inputRefs.get(0))); + case MAX: + return String.format("MAX(%s)", inputStrs.get(inputRefs.get(0))); + case MIN: + return String.format("MIN(%s)", inputStrs.get(inputRefs.get(0))); + default: + throw new RuntimeException("can't translate aggFunc " + exp); + } + } +} diff --git a/plan/src/main/java/com/hufudb/openhufu/expression/Translator.java b/plan/src/main/java/com/hufudb/openhufu/expression/Translator.java new file mode 100644 index 00000000..4fd44d17 --- /dev/null +++ b/plan/src/main/java/com/hufudb/openhufu/expression/Translator.java @@ -0,0 +1,12 @@ +package com.hufudb.openhufu.expression; + +import com.hufudb.openhufu.proto.OpenHuFuPlan.Expression; +import java.util.List; + +/** + * Convert an Expression into String + */ +public interface Translator { + void setInput(List inputs); + String translate(Expression exp); +} diff --git a/plan/src/main/java/com/hufudb/openhufu/udf/ScalarUDF.java b/plan/src/main/java/com/hufudb/openhufu/udf/ScalarUDF.java index d6a2c089..46747ad9 100644 --- a/plan/src/main/java/com/hufudb/openhufu/udf/ScalarUDF.java +++ b/plan/src/main/java/com/hufudb/openhufu/udf/ScalarUDF.java @@ -11,4 +11,5 @@ public interface ScalarUDF { String getName(); ColumnType getOutType(List inTypes); Object implement(List inputs); + String translate(String dataSource, List inputs); } diff --git a/plan/src/main/java/com/hufudb/openhufu/udf/UDFLoader.java b/plan/src/main/java/com/hufudb/openhufu/udf/UDFLoader.java index 52067c22..6541b17c 100644 --- a/plan/src/main/java/com/hufudb/openhufu/udf/UDFLoader.java +++ b/plan/src/main/java/com/hufudb/openhufu/udf/UDFLoader.java @@ -38,6 +38,14 @@ public static Object implementScalar(String funcName, List inputs) { return UDFLoader.scalarUDFs.get(funcName).implement(inputs); } + public static String translateScalar(String funcName, String dataSource, List inputs) { + if (!UDFLoader.scalarUDFs.containsKey(funcName)) { + LOG.error("Unsupported scalar UDF {}", funcName); + throw new RuntimeException("Unsupported scalar UDF"); + } + return UDFLoader.scalarUDFs.get(funcName).translate(dataSource, inputs); + } + public static ColumnType getScalarOutType(String funcName, List inputs) { return UDFLoader.scalarUDFs.get(funcName) .getOutType(inputs.stream().map(in -> in.getOutType()).collect(Collectors.toList())); diff --git a/pom.xml b/pom.xml index 6f68b37e..e6de8192 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,10 @@ 1.43.2 3.12.0 + + 42.6.0 + 2.5.0 + 4.13.1 @@ -83,6 +87,12 @@ 3.8 + + com.google.guava + guava + 30.0-jre + + commons-cli commons-cli @@ -102,6 +112,18 @@ compile + + + org.postgresql + postgresql + ${postgresql.version} + + + net.postgis + postgis-jdbc + ${postgis.version} + + junit diff --git a/release/config/spatialOwner1.json b/release/config/spatial-csv/spatial-csv-owner1.json similarity index 100% rename from release/config/spatialOwner1.json rename to release/config/spatial-csv/spatial-csv-owner1.json diff --git a/release/config/spatialOwner2.json b/release/config/spatial-csv/spatial-csv-owner2.json similarity index 100% rename from release/config/spatialOwner2.json rename to release/config/spatial-csv/spatial-csv-owner2.json diff --git a/release/config/spatialOwner3.json b/release/config/spatial-csv/spatial-csv-owner3.json similarity index 100% rename from release/config/spatialOwner3.json rename to release/config/spatial-csv/spatial-csv-owner3.json diff --git a/release/config/spatial-postgis/spatial-postgis-owner1.json b/release/config/spatial-postgis/spatial-postgis-owner1.json new file mode 100644 index 00000000..64e9fff9 --- /dev/null +++ b/release/config/spatial-postgis/spatial-postgis-owner1.json @@ -0,0 +1,51 @@ +{ + "id": 1, + "port": 12345, + "hostname": "localhost", + "implementorconfigpath": "./config/owner.yml", + "adapterconfig": { + "datasource": "POSTGIS", + "url": "jdbc:postgresql://localhost:54321/osm_db", + "catalog": "osm_db", + "user": "hufu", + "passwd": "hufu" + }, + "tables": [ + { + "actualName": "osm_a_1", + "publishedName": "osm_a", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "protected", + "columnId": 1 + } + ] + }, + { + "actualName": "osm_b_1", + "publishedName": "osm_b", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "public", + "columnId": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/release/config/spatial-postgis/spatial-postgis-owner2.json b/release/config/spatial-postgis/spatial-postgis-owner2.json new file mode 100644 index 00000000..ba8a6d19 --- /dev/null +++ b/release/config/spatial-postgis/spatial-postgis-owner2.json @@ -0,0 +1,51 @@ +{ + "id": 2, + "port": 12346, + "hostname": "localhost", + "implementorconfigpath": "./config/owner.yml", + "adapterconfig": { + "datasource": "POSTGIS", + "url": "jdbc:postgresql://localhost:54321/osm_db", + "catalog": "osm_db", + "user": "hufu", + "passwd": "hufu" + }, + "tables": [ + { + "actualName": "osm_a_2", + "publishedName": "osm_a", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "protected", + "columnId": 1 + } + ] + }, + { + "actualName": "osm_b_2", + "publishedName": "osm_b", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "public", + "columnId": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/release/config/spatial-postgis/spatial-postgis-owner3.json b/release/config/spatial-postgis/spatial-postgis-owner3.json new file mode 100644 index 00000000..18fe5789 --- /dev/null +++ b/release/config/spatial-postgis/spatial-postgis-owner3.json @@ -0,0 +1,51 @@ +{ + "id": 3, + "port": 12347, + "hostname": "localhost", + "implementorconfigpath": "./config/owner.yml", + "adapterconfig": { + "datasource": "POSTGIS", + "url": "jdbc:postgresql://localhost:54321/osm_db", + "catalog": "osm_db", + "user": "hufu", + "passwd": "hufu" + }, + "tables": [ + { + "actualName": "osm_a_3", + "publishedName": "osm_a", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "protected", + "columnId": 1 + } + ] + }, + { + "actualName": "osm_b_3", + "publishedName": "osm_b", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "public", + "columnId": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/release/config/spatial-postgis/spatial-postgis-owner4.json b/release/config/spatial-postgis/spatial-postgis-owner4.json new file mode 100644 index 00000000..de5eeb8f --- /dev/null +++ b/release/config/spatial-postgis/spatial-postgis-owner4.json @@ -0,0 +1,51 @@ +{ + "id": 4, + "port": 12348, + "hostname": "localhost", + "implementorconfigpath": "./config/owner.yml", + "adapterconfig": { + "datasource": "POSTGIS", + "url": "jdbc:postgresql://localhost:54321/osm_db", + "catalog": "osm_db", + "user": "hufu", + "passwd": "hufu" + }, + "tables": [ + { + "actualName": "osm_a_4", + "publishedName": "osm_a", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "protected", + "columnId": 1 + } + ] + }, + { + "actualName": "osm_b_4", + "publishedName": "osm_b", + "publishedColumns": [ + { + "name": "id", + "type": "INT", + "modifier": "public", + "columnId": 0 + }, + { + "name": "location", + "type": "GEOMETRY", + "modifier": "public", + "columnId": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/release/spatialOwner_all.sh b/release/spatialOwner_all.sh deleted file mode 100755 index 28370504..00000000 --- a/release/spatialOwner_all.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -bash ./owner.sh start ./config/spatialOwner1.json -bash ./owner.sh start ./config/spatialOwner2.json -bash ./owner.sh start ./config/spatialOwner3.json \ No newline at end of file diff --git a/release/owner_all.sh b/release/start_all_owner.sh similarity index 100% rename from release/owner_all.sh rename to release/start_all_owner.sh diff --git a/release/start_all_spatial_csv_owner.sh b/release/start_all_spatial_csv_owner.sh new file mode 100755 index 00000000..e3bac510 --- /dev/null +++ b/release/start_all_spatial_csv_owner.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +bash ./owner.sh start ./config/spatial-csv/spatial-csv-owner1.json +bash ./owner.sh start ./config/spatial-csv/spatial-csv-owner2.json +bash ./owner.sh start ./config/spatial-csv/spatial-csv-owner3.json \ No newline at end of file diff --git a/release/start_all_spatial_postgis_owner.sh b/release/start_all_spatial_postgis_owner.sh new file mode 100755 index 00000000..f10916e0 --- /dev/null +++ b/release/start_all_spatial_postgis_owner.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#simulate 4 owner in 1 postgis + +set -e + +bash ./owner.sh start ./config/spatial-postgis/spatial-postgis-owner1.json +bash ./owner.sh start ./config/spatial-postgis/spatial-postgis-owner2.json +bash ./owner.sh start ./config/spatial-postgis/spatial-postgis-owner3.json +bash ./owner.sh start ./config/spatial-postgis/spatial-postgis-owner4.json \ No newline at end of file diff --git a/scripts/build/package.sh b/scripts/build/package.sh index 31b43039..c7c81c6b 100755 --- a/scripts/build/package.sh +++ b/scripts/build/package.sh @@ -11,14 +11,18 @@ if [ $# -eq 0 ]; then mkdir -p ./release/lib cp owner/target/*-with-dependencies.jar ./release/bin/owner_server.jar cp adapter/adapter-csv/target/*-with-dependencies.jar ./release/adapter/adapter_csv.jar - cp adapter/adapter-csv/target/*-with-dependencies.jar ./release/adapter/adapter_csv.jar + cp adapter/adapter-postgis/target/*-with-dependencies.jar ./release/adapter/adapter_postgis.jar cp benchmark/target/benchmark.jar ./release/bin/benchmark.jar elif [ $1 == "owner" ]; then mvn install -T ${thread} -DskipTests -pl $1 cp owner/target/*-with-dependencies.jar ./release/bin/owner_server.jar elif [ $1 == "adapter" ]; then - mvn install -T ${thread} -DskipTests -pl $1 + mvn install -T ${thread} -DskipTests -amd -pl $1 cp adapter/adapter-csv/target/*-with-dependencies.jar ./release/adapter/adapter_csv.jar + cp adapter/adapter-postgis/target/*-with-dependencies.jar ./release/adapter/adapter_postgis.jar +elif [ $1 == "udf" ]; then + mvn install -T ${thread} -DskipTests -amd -pl $1 + cp udf/spatial-udf/target/*-with-dependencies.jar ./release/udf/scalar/spatial_udf.jar elif [ $1 == "benchmark" ]; then mvn install -T ${thread} -DskipTests -pl $1 cp benchmark/target/benchmark.jar ./release/bin/benchmark.jar diff --git a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/DWithin.java b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/DWithin.java index 25bce951..a78fb1dd 100644 --- a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/DWithin.java +++ b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/DWithin.java @@ -44,4 +44,14 @@ public Object implement(List inputs) { return left.distance(right) <= ((Number) inputs.get(2)) .doubleValue(); } + + @Override + public String translate(String dataSource, List inputs) { + switch(dataSource) { + case "POSTGIS": + return String.format("ST_DWithin(%s, %s, %s)", inputs.get(0), inputs.get(1), inputs.get(2)); + default: + throw new RuntimeException("Unsupported datasource for Distance UDF"); + } + } } diff --git a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Distance.java b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Distance.java index 44ebf212..362f8622 100644 --- a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Distance.java +++ b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Distance.java @@ -18,6 +18,7 @@ public String getName() { public ColumnType getOutType(List inTypes) { return ColumnType.DOUBLE; } + public Double distance(Geometry left, Geometry right) { return (Double) implement(ImmutableList.of(left, right)); @@ -40,4 +41,14 @@ public Object implement(List inputs) { Geometry right = (Geometry) inputs.get(1); return left.distance(right); } + + @Override + public String translate(String dataSource, List inputs) { + switch(dataSource) { + case "POSTGIS": + return String.format("%s <-> %s", inputs.get(0), inputs.get(1)); + default: + throw new RuntimeException("Unsupported datasource for Distance UDF"); + } + } } diff --git a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/KNN.java b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/KNN.java index a5ad50fa..9baca833 100644 --- a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/KNN.java +++ b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/KNN.java @@ -18,27 +18,17 @@ public ColumnType getOutType(List inTypes) { return ColumnType.BOOLEAN; } - public Boolean knn(Geometry left, Geometry right, Integer distance) { - return (Boolean) implement(ImmutableList.of(left, right, distance)); + public Boolean knn(Geometry left, Geometry right, Integer count) { + return (Boolean) implement(ImmutableList.of(left, right, count)); } @Override public Object implement(List inputs) { throw new RuntimeException(); -// if (inputs.size() != 3) { -// LOG.error("KNN UDF expect 3 parameters, but give {}", inputs.size()); -// throw new RuntimeException("KNN UDF expect 3 parameters"); -// } -// if (inputs.get(0) == null || inputs.get(1) == null || inputs.get(2) == null) { -// return null; -// } -// if (!(inputs.get(0) instanceof Geometry) || !(inputs.get(1) instanceof Geometry) -// || !(inputs.get(2) instanceof Integer)) { -// LOG.error("KNN UDF requires (Point, Point, Integer)"); -// throw new RuntimeException("KNN UDF requires (Point, Point)"); -// } -// Geometry left = (Geometry) inputs.get(0); -// Geometry right = (Geometry) inputs.get(1); -// return true; + } + + @Override + public String translate(String dataSource, List inputs) { + return ""; } } diff --git a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Point.java b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Point.java index a62a42e4..4335a378 100644 --- a/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Point.java +++ b/udf/spatial-udf/src/main/java/com/hufudb/openhufu/udf/Point.java @@ -40,4 +40,14 @@ public Object implement(List inputs) { ((Number) inputs.get(1)).doubleValue()); return geoFactory.createPoint(coordinate); } + + @Override + public String translate(String dataSource, List inputs) { + switch (dataSource) { + case "POSTGIS": + return String.format("'SRID=4326;POINT(%s %s)'", inputs.get(0), inputs.get(1)); + default: + throw new RuntimeException("Unsupported datasource for Point UDF"); + } + } }