wip

ScorpioBroker · Nov 27, 2023 · e9ad9cf · e9ad9cf
1 parent aa7ab18
commit e9ad9cf
Show file tree

Hide file tree

Showing 20 changed files with 292 additions and 102 deletions.
diff --git a/fullSetup/NGSILDHL7Flow.json b/fullSetup/NGSILDHL7Flow.json
diff --git a/fullSetup/NiFi_Flow_full.json b/fullSetup/NiFi_Flow_full.json
diff --git a/fullSetup/docker-compose.yml → fullSetup/docker-compose-full.yml b/fullSetup/docker-compose.yml → fullSetup/docker-compose-full.yml
@@ -15,8 +15,9 @@ services:
       - "8443:8443"
       - "8080:8080"
     environment:
-      ENTITY_TYPES: Medication,Patient
-      AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      ENTITY_TYPES: Medication,https://fiware.github.io/data-models/context.jsonld,Patient,https://smartdatamodels.org/dataModel.Hl7/Patient,MedicationAdministration,https://smartdatamodels.org/dataModel.Hl7/MedicationAdministration,MedicationStatement,MedicationDispense,Observation,Media
+      #AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      FORCE_ARRAY: performer
       NGSILD_HOST: http://scorpio:9090
       NOTIFICATION_PATH: notify
       NOTIFICATION_PORT: 4444
@@ -50,33 +51,16 @@ services:
       #   HTTP Port
       NIFI_WEB_HTTP_PORT: 8080
     volumes:
-      - ./NGSILDHL7Flow.json:/home/nifi/NGSILDHL7Flow.json
+      - ./NiFi_Flow_full.json:/home/nifi/NGSILDHL7Flow.json
       - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
       - ./download:/home/nifi/download
+      - ./mapping.csv:/home/nifi/mapping.csv
+      #- ./load:/home/nifi/load
+      #- ./original:/home/nifi/original
     entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
-  zookeeper:
-    image: zookeeper
-    ports:
-      - "2181"
-    logging:
-      driver: none
-  kafka:
-    image: bitnami/kafka
-    ports:
-      - "9092"
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka 
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_LOG_RETENTION_MS: 10000
-      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 5000
-      ALLOW_PLAINTEXT_LISTENER: "yes"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - zookeeper
-#    logging:
-#      driver: none
+      - nifireg
+      - scorpio
   postgres:
     image: postgis/postgis
     ports:
@@ -88,16 +72,10 @@ services:
     logging:
       driver: none
 #    volumes:
-#      - /var/lib/docker/volumes/b089a32ad3162e1f7152fe9a87bdb10f4340f02372ae802c65ea870ccefb863f/_data:/var/lib/postgresql/data
+#      - ./mylocaldbfolder:/var/lib/postgresql/data
   scorpio:
-#    image: scorpiobroker/scorpio:scorpio-aaio-no-eureka_1.1.0-SNAPSHOT
-    image: scorpiobroker/all-in-one-runner:java-kafka-3.0.11
+    image: scorpiobroker/all-in-one-runner:java-4.1.10
     ports:
       - "9090:9090"
-#    environment:
-#      SPRING_SECURITY_MODE: deactivated
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWT-SET-URI: http://localhost:1234/realms/scorpio/protocol/openid-connect/certs
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER-URI: http://localhost:1234/realms/scorpio
     depends_on:
       - postgres
-      - kafka
diff --git a/fullSetup/mapping.csv b/fullSetup/mapping.csv
@@ -0,0 +1,3 @@
+id;urn:fdbd3b62-57c1-11ee-a711-02a0527110a8:20231024:heart_rate_variability1;my_internal_id
+subject/reference;Patient/fdbd3b62-57c1-11ee-a711-02a0527110a8;
+id;;
diff --git a/onlymapping/NiFi_Flowmappingonlyfile2file.json b/onlymapping/NiFi_Flowmappingonlyfile2file.json
diff --git a/onlymapping/docker-compose-extra-mapping.yml b/onlymapping/docker-compose-extra-mapping.yml
@@ -0,0 +1,62 @@
+version: '3'
+
+services:
+#  testserver:
+#    image: testserver
+#    ports:
+#      - "55551:55551"
+  nifireg:
+    image: apache/nifi-registry
+    ports:
+      - "18080:18080"
+  nifi:
+    image: apache/nifi
+    ports: 
+      - "8443:8443"
+      - "8080:8080"
+    environment:
+      ENTITY_TYPES: Medication,https://fiware.github.io/data-models/context.jsonld,Patient,https://smartdatamodels.org/dataModel.Hl7/Patient,MedicationAdministration,https://smartdatamodels.org/dataModel.Hl7/MedicationAdministration,MedicationStatement,MedicationDispense,Observation,Media
+      #AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      FORCE_ARRAY: performer
+      NGSILD_HOST: http://scorpio:9090
+      NOTIFICATION_PATH: notify
+      NOTIFICATION_PORT: 4444
+      NOTIFICATION_HOST: http://nifi
+      HL7_HOST: http://testserver:55551
+      TEMP_INTERVAL: 20
+      NIFI_REG: http://nifireg:18080
+      MY_HOST: http://nifi:8080
+      ########## JVM ##########
+
+      # -Xms
+      #   The initial JVM heap size.
+      #NIFI_JVM_HEAP_INIT: 512m
+
+      # -Xmx
+      #   The maximum JVM heap size.
+      #NIFI_JVM_HEAP_MAX: 512m
+
+      # -Xdebug
+      #   The JVM Debugger can be enabled by setting this environment variable to any value.
+      #NIFI_JVM_DEBUGGER: ''
+
+
+
+      ########## Web ##########
+
+      # nifi.web.http.host
+      NIFI_WEB_HTTP_HOST: '0.0.0.0'
+
+      # nifi.web.http.port
+      #   HTTP Port
+      NIFI_WEB_HTTP_PORT: 8080
+    volumes:
+      - ./NiFi_Flowmappingonlyfile2file.json:/home/nifi/NGSILDHL7Flow.json
+      - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
+      - ./download:/home/nifi/download
+      - ./mapping.csv:/home/nifi/mapping.csv
+      - ./load:/home/nifi/load
+      - ./original:/home/nifi/original
+    entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
+    depends_on:
+      - nifireg
diff --git a/onlymapping/loadtemplate.sh b/onlymapping/loadtemplate.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+touch bla.temp
+/opt/nifi/scripts/start.sh &
+pid=$!
+if [[ ! -f "created" ]]; then
+while true; do
+    curl -s --head http://localhost:8080/nifi | head -n 1 | grep "HTTP/1.[01] [23].." > /dev/null
+    if [ $? -eq 0 ]; then
+        echo "NIFI Website is up."
+        break
+    else
+        echo "NIFI Website is down."
+        sleep 5
+    fi
+done
+# Set the nifi cli path
+NIFI_CLI=/opt/nifi/nifi-toolkit-current/bin/cli.sh
+
+# Set the nifi url
+NIFI_URL=http://localhost:8080/nifi
+
+REG_CLIENT_EXISTS=$($NIFI_CLI nifi get-reg-client-id -rcn regclient1|grep -c regclient1)
+if [ $REG_CLIENT_EXISTS -gt 0 ]; then
+echo "Creating reg client"
+$NIFI_CLI nifi create-reg-client -u ${MY_HOST} --registryClientUrl ${NIFI_REG} --registryClientName regclient1
+fi
+echo "checking bucket"
+BUCKETS_EXISTS=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}| grep -c "MyBucket")
+
+
+
+
+if [ $BUCKETS_EXISTS -lt 1 ]; then
+echo "bucket does not exist"
+$NIFI_CLI registry create-bucket -u ${NIFI_REG} --bucketName MyBucket
+BUCKET_ID=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}|grep MyBucket|awk '{print $3}')
+$NIFI_CLI registry create-flow -u ${NIFI_REG} --bucketIdentifier $BUCKET_ID --flowName MyFlow
+FLOWID=$($NIFI_CLI registry list-flows -u ${NIFI_REG} --bucketIdentifier=$BUCKET_ID|grep MyFlow|awk '{print $3}')
+$NIFI_CLI registry import-flow-version -u ${NIFI_REG} -f $FLOWID -i /home/nifi/NGSILDHL7Flow.json
+$NIFI_CLI nifi pg-import --bucketIdentifier $BUCKET_ID --flowIdentifier $FLOWID --flowVersion 1
+fi
+PG_ID=$($NIFI_CLI nifi pg-list|grep 1|awk '{print $4}')
+$NIFI_CLI nifi pg-enable-services --processGroupId $PG_ID
+$NIFI_CLI nifi pg-start --processGroupId $PG_ID
+touch created
+fi
+wait $pid
diff --git a/onlymapping/mapping.csv b/onlymapping/mapping.csv
@@ -0,0 +1,3 @@
+id;urn:fdbd3b62-57c1-11ee-a711-02a0527110a8:20231024:heart_rate_variability1;my_internal_id
+subject/reference;Patient/fdbd3b62-57c1-11ee-a711-02a0527110a8;
+id;;
diff --git a/sub2LiveSystem/NGSILDHL7Flow.json b/sub2LiveSystem/NGSILDHL7Flow.json
diff --git a/sub2LiveSystem/NiFi_Flowliveupdate.json b/sub2LiveSystem/NiFi_Flowliveupdate.json
diff --git a/tempInterval2File/docker-compose.yml → sub2LiveSystem/docker-compose-liveupdate.yml b/tempInterval2File/docker-compose.yml → sub2LiveSystem/docker-compose-liveupdate.yml
@@ -15,8 +15,9 @@ services:
       - "8443:8443"
       - "8080:8080"
     environment:
-      ENTITY_TYPES: Medication,Patient
-      AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      ENTITY_TYPES: Medication,https://fiware.github.io/data-models/context.jsonld,Patient,https://smartdatamodels.org/dataModel.Hl7/Patient,MedicationAdministration,https://smartdatamodels.org/dataModel.Hl7/MedicationAdministration,MedicationStatement,MedicationDispense,Observation,Media
+      #AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      FORCE_ARRAY: performer
       NGSILD_HOST: http://scorpio:9090
       NOTIFICATION_PATH: notify
       NOTIFICATION_PORT: 4444
@@ -50,33 +51,16 @@ services:
       #   HTTP Port
       NIFI_WEB_HTTP_PORT: 8080
     volumes:
-      - ./NGSILDHL7Flow.json:/home/nifi/NGSILDHL7Flow.json
+      - ./NiFi_Flowliveupdate.json:/home/nifi/NGSILDHL7Flow.json
       - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
       - ./download:/home/nifi/download
+      - ./mapping.csv:/home/nifi/mapping.csv
+      #- ./load:/home/nifi/load
+      #- ./original:/home/nifi/original
     entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
-  zookeeper:
-    image: zookeeper
-    ports:
-      - "2181"
-    logging:
-      driver: none
-  kafka:
-    image: bitnami/kafka
-    ports:
-      - "9092"
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka 
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_LOG_RETENTION_MS: 10000
-      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 5000
-      ALLOW_PLAINTEXT_LISTENER: "yes"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - zookeeper
-#    logging:
-#      driver: none
+      - nifireg
+      - scorpio
   postgres:
     image: postgis/postgis
     ports:
@@ -88,16 +72,10 @@ services:
     logging:
       driver: none
 #    volumes:
-#      - /var/lib/docker/volumes/b089a32ad3162e1f7152fe9a87bdb10f4340f02372ae802c65ea870ccefb863f/_data:/var/lib/postgresql/data
+#      - ./mylocaldbfolder:/var/lib/postgresql/data
   scorpio:
-#    image: scorpiobroker/scorpio:scorpio-aaio-no-eureka_1.1.0-SNAPSHOT
-    image: scorpiobroker/all-in-one-runner:java-kafka-3.0.11
+    image: scorpiobroker/all-in-one-runner:java-4.1.10
     ports:
       - "9090:9090"
-#    environment:
-#      SPRING_SECURITY_MODE: deactivated
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWT-SET-URI: http://localhost:1234/realms/scorpio/protocol/openid-connect/certs
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER-URI: http://localhost:1234/realms/scorpio
     depends_on:
       - postgres
-      - kafka
diff --git a/sub2LiveSystem/mapping.csv b/sub2LiveSystem/mapping.csv
@@ -0,0 +1,3 @@
+id;urn:fdbd3b62-57c1-11ee-a711-02a0527110a8:20231024:heart_rate_variability1;my_internal_id
+subject/reference;Patient/fdbd3b62-57c1-11ee-a711-02a0527110a8;
+id;;
diff --git a/tempInterval2File/NGSILDHL7Flow.json b/tempInterval2File/NGSILDHL7Flow.json
diff --git a/tempInterval2File/NiFi_Flowtemporal2nd.json b/tempInterval2File/NiFi_Flowtemporal2nd.json
diff --git a/sub2LiveSystem/docker-compose.yml → ...nterval2File/docker-compose-tempquery.yml b/sub2LiveSystem/docker-compose.yml → ...nterval2File/docker-compose-tempquery.yml
@@ -15,8 +15,9 @@ services:
       - "8443:8443"
       - "8080:8080"
     environment:
-      ENTITY_TYPES: Medication,Patient
-      AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      ENTITY_TYPES: Medication,https://fiware.github.io/data-models/context.jsonld,Patient,https://smartdatamodels.org/dataModel.Hl7/Patient,MedicationAdministration,https://smartdatamodels.org/dataModel.Hl7/MedicationAdministration,MedicationStatement,MedicationDispense,Observation,Media
+      #AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      FORCE_ARRAY: performer
       NGSILD_HOST: http://scorpio:9090
       NOTIFICATION_PATH: notify
       NOTIFICATION_PORT: 4444
@@ -50,33 +51,16 @@ services:
       #   HTTP Port
       NIFI_WEB_HTTP_PORT: 8080
     volumes:
-      - ./NGSILDHL7Flow.json:/home/nifi/NGSILDHL7Flow.json
+      - ./NiFi_Flowtemporal2nd.json:/home/nifi/NGSILDHL7Flow.json
       - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
       - ./download:/home/nifi/download
+      - ./mapping.csv:/home/nifi/mapping.csv
+      #- ./load:/home/nifi/load
+      #- ./original:/home/nifi/original
     entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
-  zookeeper:
-    image: zookeeper
-    ports:
-      - "2181"
-    logging:
-      driver: none
-  kafka:
-    image: bitnami/kafka
-    ports:
-      - "9092"
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka 
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_LOG_RETENTION_MS: 10000
-      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 5000
-      ALLOW_PLAINTEXT_LISTENER: "yes"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - zookeeper
-#    logging:
-#      driver: none
+      - nifireg
+      - scorpio
   postgres:
     image: postgis/postgis
     ports:
@@ -88,16 +72,10 @@ services:
     logging:
       driver: none
 #    volumes:
-#      - /var/lib/docker/volumes/b089a32ad3162e1f7152fe9a87bdb10f4340f02372ae802c65ea870ccefb863f/_data:/var/lib/postgresql/data
+#      - ./mylocaldbfolder:/var/lib/postgresql/data
   scorpio:
-#    image: scorpiobroker/scorpio:scorpio-aaio-no-eureka_1.1.0-SNAPSHOT
-    image: scorpiobroker/all-in-one-runner:java-kafka-3.0.11
+    image: scorpiobroker/all-in-one-runner:java-4.1.10
     ports:
       - "9090:9090"
-#    environment:
-#      SPRING_SECURITY_MODE: deactivated
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWT-SET-URI: http://localhost:1234/realms/scorpio/protocol/openid-connect/certs
-#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER-URI: http://localhost:1234/realms/scorpio
     depends_on:
       - postgres
-      - kafka
diff --git a/tempInterval2File/mapping.csv b/tempInterval2File/mapping.csv
@@ -0,0 +1,3 @@
+id;urn:fdbd3b62-57c1-11ee-a711-02a0527110a8:20231024:heart_rate_variability1;my_internal_id
+subject/reference;Patient/fdbd3b62-57c1-11ee-a711-02a0527110a8;
+id;;
diff --git a/tempInterval2FileNoMapping/NiFi_Flowtemp2ndnomapping.json b/tempInterval2FileNoMapping/NiFi_Flowtemp2ndnomapping.json