Add introspection support for PostgreSQL and MSSQL data stores

changelog.md

@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+- Fixes an issue where client API is still generated even if all entities contain unsupported field(s)
+- Fixes an issue where unique indexes are declared twice in script.sql in one-to-one associations
+- Fixes an issue where the cardinality of the first association is taken as the cardinality of all the other associations between same entities
+
+### Added
+- [Add introspection support for PostgreSQL databases](https://github.com/ballerina-platform/ballerina-library/issues/6333)
+- [Add introspection support for MSSQL databases](https://github.com/ballerina-platform/ballerina-library/issues/6460)
+
+## [1.3.0] - 2024-05-03
+
 ### Changed
 - [Refactor migrate command and test cases to support more scenarios](https://github.com/ballerina-platform/ballerina-library/issues/6189)
 - [Fix a bug in the `migrate` command where tables with `@sql:Name` annotations are recreated](https://github.com/ballerina-platform/ballerina-library/issues/6374)

docs/spec/spec.md

@@ -3,7 +3,7 @@
 _Owners_: @daneshk @sahanHe  
 _Reviewers_: @daneshk  
 _Created_: 2022/07/26   
-_Updated_: 2024/04/10  
+_Updated_: 2024/05/06  
 _Edition_: Swan Lake  
 
 ## Introduction
@@ -214,13 +214,13 @@ Behaviour of the `push` command,
 ```bash
 bal persist pull --datastore mysql --host localhost --port 3306 --user root --database persist
 ```
-| Command Parameter |                                   Description                                    | Mandatory | Default Value |
-|:-----------------:|:--------------------------------------------------------------------------------:|:---------:|:-------------:|
-|    --datastore    | used to indicate the preferred data store. Currently, only 'mysql' is supported. |    No     |     mysql     |
-|      --host       |                        used to indicate the database host                        |    Yes    |     None      |
-|      --port       |                        used to indicate the database port                        |    No     |     3306      |
-|      --user       |                        used to indicate the database user                        |    Yes    |     None      |
-|    --database     |                        used to indicate the database name                        |    Yes    |     None      |
+| Command Parameter |                                       Description                                       | Mandatory |   Default Value    |
+|:-----------------:|:---------------------------------------------------------------------------------------:|:---------:|:------------------:|
+|    --datastore    | used to indicate the preferred data store. supports `mysql`, `postgresql`, and `mssql`. |    No     |       mysql        |
+|      --host       |                           used to indicate the database host                            |    Yes    |        None        |
+|      --port       |                           used to indicate the database port                            |    No     | datastore specific |
+|      --user       |                           used to indicate the database user                            |    Yes    |        None        |
+|    --database     |                           used to indicate the database name                            |    Yes    |        None        |
 
 This command will introspect the schema of the database and create a `model.bal` file with the entities and relations based on the schema of the database. The database configuration should be provided as command-line arguments.
 

examples/hospital_unsupported/Ballerina.toml

@@ -15,8 +15,3 @@ url = "https://maven.wso2.org/nexus/content/groups/public/"
 id="wso2-nexus-snapshot"
 url = "https://maven.wso2.org/nexus/content/repositories/snapshots/"
 
-[[platform.java17.dependency]]
-groupId = "io.ballerina.stdlib"
-artifactId = "persist.sql-native"
-version = "1.3.0-SNAPSHOT"
-

persist-cli-tests/src/test/java/io/ballerina/persist/tools/ToolingGenerateTest.java

@@ -939,6 +939,25 @@ public void testGenerateIgnoreSQLAnnotationsForRedis() {
         assertGeneratedSources("tool_test_generate_104_redis");
     }
 
+    @Test(enabled = true)
+    @Description("There are entities with 'sql' annotations")
+    public void testGenerateWithAllUnsupportedEntities() {
+        updateOutputBallerinaToml("tool_test_generate_105");
+        executeGenerateCommand("tool_test_generate_105", "mysql", "entities");
+        assertGeneratedSources("tool_test_generate_105");
+    }
+
+    @Test(enabled = true)
+    @Description("The model has a one-to-one relation where the unique index name is managed")
+    public void testGenerateEntitiesWithUniqueIndexAnnotatedForeignKeys() {
+        String subDir = "tool_test_generate_106";
+        updateOutputBallerinaToml(subDir);
+        executeGenerateCommand(subDir, "mysql", "entities");
+        executeGenerateCommand(subDir, "mssql", "mssql_entities");
+        executeGenerateCommand(subDir, "postgresql", "postgresql_entities");
+        assertGeneratedSources(subDir);
+    }
+
     private void updateOutputBallerinaToml(String fileName) {
         String tomlFileName = "Ballerina.toml";
         Path filePath = Paths.get("src", "test", "resources", "test-src", "output", fileName, tomlFileName);

persist-cli-tests/src/test/java/io/ballerina/persist/tools/utils/DatabaseTestUtils.java

@@ -40,6 +40,11 @@
 import java.util.ArrayList;
 import java.util.Locale;
 
+import static io.ballerina.persist.nodegenerator.syntax.constants.BalSyntaxConstants.CREATE_DATABASE_SQL_FORMAT;
+import static io.ballerina.persist.nodegenerator.syntax.constants.BalSyntaxConstants.DROP_DATABASE_SQL_FORMAT;
+import static io.ballerina.persist.nodegenerator.syntax.constants.BalSyntaxConstants.JDBC_URL_WITH_DATABASE_MSSQL;
+import static io.ballerina.persist.nodegenerator.syntax.constants.BalSyntaxConstants.JDBC_URL_WITH_DATABASE_MYSQL;
+import static io.ballerina.persist.nodegenerator.syntax.constants.BalSyntaxConstants.JDBC_URL_WITH_DATABASE_POSTGRESQL;
 import static io.ballerina.persist.tools.utils.GeneratedSourcesTestUtils.GENERATED_SOURCES_DIRECTORY;
 import static io.ballerina.persist.tools.utils.GeneratedSourcesTestUtils.INPUT_RESOURCES_DIRECTORY;
 import static io.ballerina.projects.util.ProjectConstants.BALLERINA_TOML;
@@ -201,17 +206,50 @@ private static void validateTable(Connection connection, PersistTable persistTab
         }
     }
 
-    public static void createFromDatabaseScript(String packageName, String datasource,
+    public static void resetPostgreSqlDatabase(DatabaseConfiguration dbConfig, boolean recreate) {
+
+        String url = String.format(JDBC_URL_WITH_DATABASE_POSTGRESQL, "postgresql",  dbConfig.getHost(),
+                dbConfig.getPort(), "postgres");
+        resetDatabase(dbConfig, recreate, url);
+    }
+
+    public static void resetMsSqlDatabase(DatabaseConfiguration dbConfig, boolean recreate) {
+
+        String url = String.format(JDBC_URL_WITH_DATABASE_MSSQL,
+                PersistToolsConstants.SupportedDataSources.MSSQL_DB_ALT,  dbConfig.getHost(), dbConfig.getPort(),
+                "master");
+        resetDatabase(dbConfig, recreate, url);
+    }
+
+    private static void resetDatabase(DatabaseConfiguration dbConfig, boolean recreate, String url) {
+        try (Connection connection = DriverManager.getConnection(url, dbConfig.getUsername(), dbConfig.getPassword())) {
+            try (Statement statement = connection.createStatement()) {
+                statement.execute(String.format(DROP_DATABASE_SQL_FORMAT, dbConfig.getDatabase()));
+                if (recreate) {
+                    statement.execute(String.format(CREATE_DATABASE_SQL_FORMAT, dbConfig.getDatabase()));
+                }
+                statement.executeBatch();
+                PrintStream outStream = System.out;
+                outStream.println("Database reset successfully");
+            }
+        } catch (SQLException e) {
+            errStream.println("Failed to reset database: " + e.getMessage());
+        }
+    }
+
+    public static void createFromDatabaseScript(String packageName, String datastore,
                                                 DatabaseConfiguration dbConfig) {
         Path sourcePath = Paths.get(INPUT_RESOURCES_DIRECTORY, packageName);
 
         String url;
-        if (datasource.equals(PersistToolsConstants.SupportedDataSources.MSSQL_DB)) {
-            url = String.format("jdbc:sqlserver://%s:%s", dbConfig.getHost(), dbConfig.getPort());
-        } else if (datasource.equals(PersistToolsConstants.SupportedDataSources.POSTGRESQL_DB)) {
-            url = String.format("jdbc:postgresql://%s:%s/", dbConfig.getHost(), dbConfig.getPort());
+        if (datastore.equals(PersistToolsConstants.SupportedDataSources.MSSQL_DB)) {
+            url = String.format(JDBC_URL_WITH_DATABASE_MSSQL, PersistToolsConstants.SupportedDataSources.MSSQL_DB_ALT,
+                    dbConfig.getHost(), dbConfig.getPort(), dbConfig.getDatabase());
+        } else if (datastore.equals(PersistToolsConstants.SupportedDataSources.POSTGRESQL_DB)) {
+            url = String.format(JDBC_URL_WITH_DATABASE_POSTGRESQL, "postgresql",  dbConfig.getHost(),
+                    dbConfig.getPort(), dbConfig.getDatabase());
         } else {
-            url = String.format("jdbc:mysql://%s:%s", dbConfig.getHost(), dbConfig.getPort());
+            url = String.format(JDBC_URL_WITH_DATABASE_MYSQL, "mysql", dbConfig.getHost(), dbConfig.getPort(), "mysql");
         }
 
         try (Connection connection = DriverManager.getConnection(url, dbConfig.getUsername(), dbConfig.getPassword())) {

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_105/Ballerina.toml

@@ -0,0 +1,9 @@
+[package]
+distribution = "2201.8.4"
+name = "tool_test_generate_105"
+org = "wso2"
+version = "0.1.0"
+
+[build-options]
+observabilityIncluded = true
+

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_105/main.bal

@@ -0,0 +1,21 @@
+// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/io;
+
+public function main() {
+    io:println("Hello, World!");
+}

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_105/persist/model.bal

@@ -0,0 +1,49 @@
+// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/persist as _;
+import ballerinax/persist.sql;
+
+public enum UserGender {
+    MALE,
+    FEMALE
+}
+
+public type User record {|
+    readonly int id;
+    string name;
+    UserGender gender;
+    string nic;
+    decimal? salary;
+    Car[] cars;
+    Car? drives;
+    //Unsupported[set('a','b','c')] setType;
+|};
+
+public type Car record {|
+    readonly int id;
+    string name;
+    string model;
+    @sql:Index {name: ["ownerId"]}
+    int ownerId;
+    @sql:Relation {keys: ["ownerId"]}
+    User owner;
+    @sql:Name {value: "DRIVER_ID"}
+    int driverId;
+    @sql:Relation {keys: ["driverId"]}
+    User driver;
+    //Unsupported[set('a','b','c')] setType;
+|};

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_106/Ballerina.toml

@@ -0,0 +1,10 @@
+[package]
+distribution = "2201.8.4"
+name = "tool_test_generate_106"
+org = "wso2"
+version = "0.1.0"
+
+[build-options]
+observabilityIncluded = true
+
+

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_106/main.bal

@@ -0,0 +1,21 @@
+// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/io;
+
+public function main() {
+    io:println("Hello, World!");
+}

persist-cli-tests/src/test/resources/test-src/input/tool_test_generate_106/persist/model.bal

@@ -0,0 +1,57 @@
+// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/persist as _;
+import ballerinax/persist.sql;
+
+
+public type User record {|
+    readonly int id;
+    string name;
+    string nic;
+    decimal? salary;
+    Car? drives;
+|};
+
+public type Car record {|
+    readonly int id;
+    string name;
+    string model;
+    @sql:UniqueIndex
+    int driverId;
+    @sql:Relation {keys: ["driverId"]}
+    User driver;
+|};
+
+public type User2 record {|
+    readonly int id;
+    readonly string nic;
+    string name;
+    decimal? salary;
+    Car2? drives;
+|};
+
+public type Car2 record {|
+    readonly int id;
+    string name;
+    string model;
+    @sql:UniqueIndex {name: "driver_idx"}
+    int driverId;
+    @sql:UniqueIndex {name: "driver_idx"}
+    string driverNic;
+    @sql:Relation {keys: ["driverId", "driverNic"]}
+    User2 driver;
+|};

persist-cli-tests/src/test/resources/test-src/input/tool_test_pull_28_postgresql/Ballerina.toml

@@ -0,0 +1,13 @@
+[package]
+distribution = "2201.8.4"
+name = "tool_test_pull_28_postgresql"
+org = "wso2"
+version = "0.1.0"
+
+[build-options]
+observabilityIncluded = true
+
+[[platform.java17.dependency]]
+artifactId = "persist.sql-native"
+groupId = "io.ballerina.stdlib"
+version = "1.2.0"

persist-cli-tests/src/test/resources/test-src/input/tool_test_pull_28_postgresql/main.bal

@@ -0,0 +1,21 @@
+// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/io;
+
+public function main() {
+    io:println("Hello, World!");
+}

persist-cli-tests/src/test/resources/test-src/input/tool_test_pull_28_postgresql/script.sql

@@ -0,0 +1,24 @@
+-- Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com).
+--
+-- WSO2 LLC. licenses this file to you under the Apache License,
+-- Version 2.0 (the "License"); you may not use this file except
+-- in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+CREATE TABLE "User" (
+	"id" INT NOT NULL,
+	"name" VARCHAR(191) NOT NULL,
+	"gender" VARCHAR(6) CHECK ("gender" IN ('MALE', 'FEMALE')) NOT NULL,
+	"nic" VARCHAR(191) NOT NULL,
+	"salary" DECIMAL(65,30) NOT NULL,
+	PRIMARY KEY("id")
+);

persist-cli-tests/src/test/resources/test-src/input/tool_test_pull_29_postgresql/Ballerina.toml

@@ -0,0 +1,13 @@
+[package]
+distribution = "2201.8.4"
+name = "tool_test_pull_29_postgresql"
+org = "wso2"
+version = "0.1.0"
+
+[build-options]
+observabilityIncluded = true
+
+[[platform.java17.dependency]]
+artifactId = "persist.sql-native"
+groupId = "io.ballerina.stdlib"
+version = "1.2.0"