Introspection Support for Bal Persist

.gitignore

@@ -24,7 +24,9 @@ hs_err_pid*
 
 build
 .gradle/
-target
+
+/target/
+!persist-cli-tests/src/resources/test-src/**/target/
 
 # IDEA Files
 .idea/
@@ -44,3 +46,6 @@ bin/
 # Ballerina
 velocity.log*
 *Ballerina.lock
+
+# Jar Dependency Versions
+version.properties

changelog.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - [Added support for PostgreSQL as a datasource](https://github.com/ballerina-platform/ballerina-standard-library/issues/5829)
 - [Integrate the persist code generation to the bal build command](https://github.com/ballerina-platform/ballerina-library/issues/5784)
+- [Added introspection support for MySQL databases](https://github.com/ballerina-platform/ballerina-library/issues/6014)
+- [Added advanced annotation support for SQL databases](https://github.com/ballerina-platform/ballerina-library/issues/6013)
 
 ## [1.2.1] - 2021-11-21
 

docs/spec/spec.md

@@ -22,6 +22,7 @@ The conforming implementation of the specification is released and included in t
 2. [Initializing Persistence Layer in Bal Project](#2-initializing-the-bal-project-with-persistence-layer)
 3. [Generating Persistence Derived Types, Clients, and Database Schema](#3-generating-persistence-derived-types-and-clients)
 4. [Push Persistence Schema to the Data Provider](#4-push-persistence-schema-to-the-data-provider)
+5. [Pull Persistence Schema from the Data Provider](#5-pull-persistence-schema-from-the-data-provider)
 
 ## 1. Overview
 This specification elaborates on the `Persist CLI Tool` commands.
@@ -206,3 +207,33 @@ Behaviour of the `push` command,
 - User should add the relevant configuration to the Ballerina.toml file.
 - The user should have initiated the persistence layer in the project and executed the `generate` command to generate the SQL script.
 - If the user invokes the command twice, it will not fail. It will rerun the SQL script against the database.
+
+## 5. Pull Persistence Schema from the Data Provider
+
+```bash
+bal persist pull --datastore mysql --host localhost --port 3306 --user root --database persist
+```
+| Command Parameter |                                      Description                                      | Mandatory | Default Value |
+|:-----------------:|:-------------------------------------------------------------------------------------:|:---------:|:-------------:|
+|    --datastore    | used to indicate the preferred database client. Currently, only 'mysql' is supported. |    No     |     mysql     |
+|      --host       |                          used to indicate the database host                           |    Yes    |     None      |
+|      --port       |                          used to indicate the database port                           |    No     |     3306      |
+|      --user       |                          used to indicate the database user                           |    Yes    |     None      |
+|    --database     |                          used to indicate the database name                           |    Yes    |     None      |
+
+This command will introspect the schema of the database and create a `model.bal` file with the entities and relations based on the schema of the database. The database configuration should be provided as command-line arguments.
+
+This command should execute within a Ballerina project.  
+
+The `persist` directory is created if it is not already present. If a `model.bal` file is already present in the `persist` directory, it will prompt the user to confirm overwriting the existing `model.bal` file.
+
+Running the `pull` command will,
+1. Create a `model.bal` file with the entities and relations based on the introspected schema of the database.
+2. Not change the schema of the database in any way.
+
+Behaviour of the `pull` command,
+- User should invoke the command within a Ballerina project.
+- User should provide the relevant database configuration as command-line arguments.
+- The database password is not provided as a command-line argument. The user will be prompted to enter the password.
+- If the user invokes the command while a `model.bal` file exists in the `persist` directory, it will prompt the user to confirm overwriting the existing `model.bal` file.
+- The user must execute the `generate` command to generate the derived types and client API after running the `pull` command in order to use the client API in the project.

examples/build.gradle

@@ -24,6 +24,7 @@ description = 'Ballerina - Persist Examples'
 
 def ballerinaDist = "${project.buildDir}/ballerina-distribution"
 def examples = ["medical-center", "rainier"]
+def introspectionExamples = ["hospital"]
 
 configurations {
     balTools
@@ -124,6 +125,14 @@ task copyTestResources() {
                 }
             }
         }
+        copy {
+            introspectionExamples.each { example ->
+                into buildDir
+                into("generated-examples/${example}/") {
+                    from "${example}"
+                }
+            }
+        }
     }
 
 }
@@ -248,6 +257,26 @@ task generateInMemoryExamples {
     }
 }
 
+task generateIntrospectionExamples {
+    doLast {
+        introspectionExamples.each { example ->
+            try {
+                exec {
+                    workingDir "${project.projectDir}/build/generated-examples/${example}"
+                    if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
+                        commandLine 'sh', '-c', "${ballerinaDist}/bin/bal persist generate --datastore=mysql --module=entities"
+                    } else {
+                        commandLine 'cmd', '/c', "${ballerinaDist}/bin/bal.bat persist generate --datastore=mysql --module=entities"
+                    }
+                }
+            } catch (Exception e) {
+                println("Example '${example}' Generation failed: " + e.message)
+                throw e
+            }
+        }
+    }
+}
+
 // task generateGSheetExamples {
 //     doLast {
 //         examples.each { example ->
@@ -306,13 +335,13 @@ def checkExecResult(executionResult, failText, standardOutput) {
 task startTestDockerContainer(type: Exec) {
     if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
         def standardOutput = new ByteArrayOutputStream()
-        commandLine 'sh', '-c', 'docker run --platform linux/amd64 --rm -d --name ballerina-persist-tools-example -e MYSQL_ROOT_PASSWORD="Test123#" -p 3308:3306 -t mysql:8.0.21'
+        commandLine 'sh', '-c', 'docker run --platform linux/amd64 --rm -d --name ballerina-persist-tools-example -e MYSQL_ROOT_PASSWORD="Test123#" -e MYSQL_DATABASE="hospital" -p 3308:3306 -t mysql:8.0.21'
         def healthCheck = 1
         def counter = 0
         doLast {
             checkExecResult(executionResult, 'Error', standardOutput)
             while (healthCheck != 0 && counter < 12) {
-                sleep(60 * 1000)
+                sleep(30 * 1000)
                 healthCheck = checkTestDockerContainerStatus('ballerina-persist-tools-example')
                 counter = counter + 1
             }
@@ -406,6 +435,47 @@ task testInMemoryExamples {
     }
 }
 
+task testIntrospectionExamples {
+    dependsOn(startTestDockerContainer)
+    doLast {
+        introspectionExamples.each { example ->
+            try {
+                if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
+                    //this part is not necessary. this is added so that the build will not fail due to the issue with
+                    //mysql connector for ballerina requiring a 'first login' on a different mysql client. This is a
+                    //temporary fix until the issue is resolved.
+                    try {
+                        exec {
+                            workingDir "${project.projectDir}/build/generated-examples/${example}"
+                            commandLine 'sh', '-c', "${ballerinaDist}/bin/bal persist pull --host localhost --database nonexistentdb --user root --port 3308"
+                            standardInput = new ByteArrayInputStream("Test123#\ny\n".getBytes())
+                        }
+                    } catch (Exception ignored) {}
+                    exec {
+                        workingDir "${project.projectDir}/build/generated-examples/${example}"
+                        commandLine 'sh', '-c', "${ballerinaDist}/bin/bal test"
+                    }
+                } else {
+                    try {
+                        exec {
+                            workingDir "${project.projectDir}/build/generated-examples/${example}"
+                            commandLine 'cmd', '/c', "${ballerinaDist}/bin/bal.bat persist pull --host localhost --database hris --user root --port 3308"
+                            standardInput = new ByteArrayInputStream("Test123#\ny\n".getBytes())
+                        }
+                    } catch (Exception ignored) {}
+                    exec {
+                        workingDir "${project.projectDir}/build/generated-examples/${example}"
+                        commandLine 'cmd', '/c', "${ballerinaDist}/bin/bal.bat test"
+                    }
+                }
+            } catch (Exception e) {
+                println("Example '${example}' Tests failed: " + e.message)
+                throw e
+            }
+        }
+    }
+}
+
 task automateGSheetTable {
     doLast {
         try {
@@ -450,13 +520,15 @@ copyPersistTool.dependsOn ":persist-cli:build"
 copyTestResources.dependsOn copyPersistTool
 
 initInMemoryExamples.dependsOn copyTestResources
+generateIntrospectionExamples.dependsOn copyTestResources
 generateInMemoryExamples.dependsOn initInMemoryExamples
 testInMemoryExamples.dependsOn generateInMemoryExamples
 
 initDbExamples.dependsOn testInMemoryExamples
 generateDbExamples.dependsOn initDbExamples
 pushExamples.dependsOn generateDbExamples
 testDbExamples.dependsOn pushExamples
+testIntrospectionExamples.dependsOn generateIntrospectionExamples
 
 // initGSheetExamples.dependsOn testDbExamples
 // generateGSheetExamples.dependsOn initGSheetExamples
@@ -465,5 +537,6 @@ testDbExamples.dependsOn pushExamples
 
 build.dependsOn testInMemoryExamples
 build.dependsOn testDbExamples
+build.dependsOn testIntrospectionExamples
 //build.dependsOn testGSheetExamples
 build.finalizedBy stopTestDockerContainer

examples/hospital/Ballerina.toml

@@ -0,0 +1,16 @@
+[package]
+org = "wso2"
+name = "Demo"
+version = "0.1.0"
+distribution = "2201.8.4"
+
+[build-options]
+observabilityIncluded = true
+
+[[platform.java17.repository]]
+id="wso2-nexus"
+url = "https://maven.wso2.org/nexus/content/groups/public/"
+
+[[platform.java17.repository]]
+id="wso2-nexus-snapshot"
+url = "https://maven.wso2.org/nexus/content/repositories/snapshots/"