initial

ScorpioBroker · Jun 2, 2023 · aa7ab18 · aa7ab18
1 parent eba89c3
commit aa7ab18
Show file tree

Hide file tree

Showing 13 changed files with 456 additions and 70 deletions.
diff --git a/README.md b/README.md
diff --git a/Untitled Diagram.drawio b/Untitled Diagram.drawio
diff --git a/docker-compose-postgis.yml b/docker-compose-postgis.yml
diff --git a/fixedfiwaredatamodel.jsonld b/fixedfiwaredatamodel.jsonld
diff --git a/fullSetup/NGSILDHL7Flow.json b/fullSetup/NGSILDHL7Flow.json
diff --git a/fullSetup/docker-compose.yml b/fullSetup/docker-compose.yml
@@ -0,0 +1,103 @@
+version: '3'
+
+services:
+#  testserver:
+#    image: testserver
+#    ports:
+#      - "55551:55551"
+  nifireg:
+    image: apache/nifi-registry
+    ports:
+      - "18080:18080"
+  nifi:
+    image: apache/nifi
+    ports: 
+      - "8443:8443"
+      - "8080:8080"
+    environment:
+      ENTITY_TYPES: Medication,Patient
+      AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      NGSILD_HOST: http://scorpio:9090
+      NOTIFICATION_PATH: notify
+      NOTIFICATION_PORT: 4444
+      NOTIFICATION_HOST: http://nifi
+      HL7_HOST: http://testserver:55551
+      TEMP_INTERVAL: 20
+      NIFI_REG: http://nifireg:18080
+      MY_HOST: http://nifi:8080
+      ########## JVM ##########
+
+      # -Xms
+      #   The initial JVM heap size.
+      #NIFI_JVM_HEAP_INIT: 512m
+
+      # -Xmx
+      #   The maximum JVM heap size.
+      #NIFI_JVM_HEAP_MAX: 512m
+
+      # -Xdebug
+      #   The JVM Debugger can be enabled by setting this environment variable to any value.
+      #NIFI_JVM_DEBUGGER: ''
+
+
+
+      ########## Web ##########
+
+      # nifi.web.http.host
+      NIFI_WEB_HTTP_HOST: '0.0.0.0'
+
+      # nifi.web.http.port
+      #   HTTP Port
+      NIFI_WEB_HTTP_PORT: 8080
+    volumes:
+      - ./NGSILDHL7Flow.json:/home/nifi/NGSILDHL7Flow.json
+      - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
+      - ./download:/home/nifi/download
+    entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
+  zookeeper:
+    image: zookeeper
+    ports:
+      - "2181"
+    logging:
+      driver: none
+  kafka:
+    image: bitnami/kafka
+    ports:
+      - "9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka 
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_LOG_RETENTION_MS: 10000
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 5000
+      ALLOW_PLAINTEXT_LISTENER: "yes"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - zookeeper
+#    logging:
+#      driver: none
+  postgres:
+    image: postgis/postgis
+    ports:
+      - "5432"
+    environment:
+      POSTGRES_USER: ngb
+      POSTGRES_PASSWORD: ngb
+      POSTGRES_DB: ngb
+    logging:
+      driver: none
+#    volumes:
+#      - /var/lib/docker/volumes/b089a32ad3162e1f7152fe9a87bdb10f4340f02372ae802c65ea870ccefb863f/_data:/var/lib/postgresql/data
+  scorpio:
+#    image: scorpiobroker/scorpio:scorpio-aaio-no-eureka_1.1.0-SNAPSHOT
+    image: scorpiobroker/all-in-one-runner:java-kafka-3.0.11
+    ports:
+      - "9090:9090"
+#    environment:
+#      SPRING_SECURITY_MODE: deactivated
+#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWT-SET-URI: http://localhost:1234/realms/scorpio/protocol/openid-connect/certs
+#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER-URI: http://localhost:1234/realms/scorpio
+    depends_on:
+      - postgres
+      - kafka
diff --git a/fullSetup/loadtemplate.sh b/fullSetup/loadtemplate.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+touch bla.temp
+/opt/nifi/scripts/start.sh &
+pid=$!
+if [[ ! -f "created" ]]; then
+while true; do
+    curl -s --head http://localhost:8080/nifi | head -n 1 | grep "HTTP/1.[01] [23].." > /dev/null
+    if [ $? -eq 0 ]; then
+        echo "NIFI Website is up."
+        break
+    else
+        echo "NIFI Website is down."
+        sleep 5
+    fi
+done
+# Set the nifi cli path
+NIFI_CLI=/opt/nifi/nifi-toolkit-current/bin/cli.sh
+
+# Set the nifi url
+NIFI_URL=http://localhost:8080/nifi
+
+REG_CLIENT_EXISTS=$($NIFI_CLI nifi get-reg-client-id -rcn regclient1|grep -c regclient1)
+if [ $REG_CLIENT_EXISTS -gt 0 ]; then
+echo "Creating reg client"
+$NIFI_CLI nifi create-reg-client -u ${MY_HOST} --registryClientUrl ${NIFI_REG} --registryClientName regclient1
+fi
+echo "checking bucket"
+BUCKETS_EXISTS=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}| grep -c "MyBucket")
+
+
+
+
+if [ $BUCKETS_EXISTS -lt 1 ]; then
+echo "bucket does not exist"
+$NIFI_CLI registry create-bucket -u ${NIFI_REG} --bucketName MyBucket
+BUCKET_ID=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}|grep MyBucket|awk '{print $3}')
+$NIFI_CLI registry create-flow -u ${NIFI_REG} --bucketIdentifier $BUCKET_ID --flowName MyFlow
+FLOWID=$($NIFI_CLI registry list-flows -u ${NIFI_REG} --bucketIdentifier=$BUCKET_ID|grep MyFlow|awk '{print $3}')
+$NIFI_CLI registry import-flow-version -u ${NIFI_REG} -f $FLOWID -i /home/nifi/NGSILDHL7Flow.json
+$NIFI_CLI nifi pg-import --bucketIdentifier $BUCKET_ID --flowIdentifier $FLOWID --flowVersion 1
+fi
+PG_ID=$($NIFI_CLI nifi pg-list|grep 1|awk '{print $4}')
+$NIFI_CLI nifi pg-enable-services --processGroupId $PG_ID
+$NIFI_CLI nifi pg-start --processGroupId $PG_ID
+touch created
+fi
+wait $pid
diff --git a/sub2LiveSystem/NGSILDHL7Flow.json b/sub2LiveSystem/NGSILDHL7Flow.json
diff --git a/sub2LiveSystem/docker-compose.yml b/sub2LiveSystem/docker-compose.yml
@@ -0,0 +1,103 @@
+version: '3'
+
+services:
+#  testserver:
+#    image: testserver
+#    ports:
+#      - "55551:55551"
+  nifireg:
+    image: apache/nifi-registry
+    ports:
+      - "18080:18080"
+  nifi:
+    image: apache/nifi
+    ports: 
+      - "8443:8443"
+      - "8080:8080"
+    environment:
+      ENTITY_TYPES: Medication,Patient
+      AT_CONTEXT: https://fiware.github.io/data-models/context.jsonld
+      NGSILD_HOST: http://scorpio:9090
+      NOTIFICATION_PATH: notify
+      NOTIFICATION_PORT: 4444
+      NOTIFICATION_HOST: http://nifi
+      HL7_HOST: http://testserver:55551
+      TEMP_INTERVAL: 20
+      NIFI_REG: http://nifireg:18080
+      MY_HOST: http://nifi:8080
+      ########## JVM ##########
+
+      # -Xms
+      #   The initial JVM heap size.
+      #NIFI_JVM_HEAP_INIT: 512m
+
+      # -Xmx
+      #   The maximum JVM heap size.
+      #NIFI_JVM_HEAP_MAX: 512m
+
+      # -Xdebug
+      #   The JVM Debugger can be enabled by setting this environment variable to any value.
+      #NIFI_JVM_DEBUGGER: ''
+
+
+
+      ########## Web ##########
+
+      # nifi.web.http.host
+      NIFI_WEB_HTTP_HOST: '0.0.0.0'
+
+      # nifi.web.http.port
+      #   HTTP Port
+      NIFI_WEB_HTTP_PORT: 8080
+    volumes:
+      - ./NGSILDHL7Flow.json:/home/nifi/NGSILDHL7Flow.json
+      - ./loadtemplate.sh:/home/nifi/loadtemplate.sh
+      - ./download:/home/nifi/download
+    entrypoint: ["bash", "/home/nifi/loadtemplate.sh"]
+  zookeeper:
+    image: zookeeper
+    ports:
+      - "2181"
+    logging:
+      driver: none
+  kafka:
+    image: bitnami/kafka
+    ports:
+      - "9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka 
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_LOG_RETENTION_MS: 10000
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 5000
+      ALLOW_PLAINTEXT_LISTENER: "yes"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - zookeeper
+#    logging:
+#      driver: none
+  postgres:
+    image: postgis/postgis
+    ports:
+      - "5432"
+    environment:
+      POSTGRES_USER: ngb
+      POSTGRES_PASSWORD: ngb
+      POSTGRES_DB: ngb
+    logging:
+      driver: none
+#    volumes:
+#      - /var/lib/docker/volumes/b089a32ad3162e1f7152fe9a87bdb10f4340f02372ae802c65ea870ccefb863f/_data:/var/lib/postgresql/data
+  scorpio:
+#    image: scorpiobroker/scorpio:scorpio-aaio-no-eureka_1.1.0-SNAPSHOT
+    image: scorpiobroker/all-in-one-runner:java-kafka-3.0.11
+    ports:
+      - "9090:9090"
+#    environment:
+#      SPRING_SECURITY_MODE: deactivated
+#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWT-SET-URI: http://localhost:1234/realms/scorpio/protocol/openid-connect/certs
+#      SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER-URI: http://localhost:1234/realms/scorpio
+    depends_on:
+      - postgres
+      - kafka
diff --git a/sub2LiveSystem/loadtemplate.sh b/sub2LiveSystem/loadtemplate.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+touch bla.temp
+/opt/nifi/scripts/start.sh &
+pid=$!
+if [[ ! -f "created" ]]; then
+while true; do
+    curl -s --head http://localhost:8080/nifi | head -n 1 | grep "HTTP/1.[01] [23].." > /dev/null
+    if [ $? -eq 0 ]; then
+        echo "NIFI Website is up."
+        break
+    else
+        echo "NIFI Website is down."
+        sleep 5
+    fi
+done
+# Set the nifi cli path
+NIFI_CLI=/opt/nifi/nifi-toolkit-current/bin/cli.sh
+
+# Set the nifi url
+NIFI_URL=http://localhost:8080/nifi
+
+REG_CLIENT_EXISTS=$($NIFI_CLI nifi get-reg-client-id -rcn regclient1|grep -c regclient1)
+if [ $REG_CLIENT_EXISTS -gt 0 ]; then
+echo "Creating reg client"
+$NIFI_CLI nifi create-reg-client -u ${MY_HOST} --registryClientUrl ${NIFI_REG} --registryClientName regclient1
+fi
+echo "checking bucket"
+BUCKETS_EXISTS=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}| grep -c "MyBucket")
+
+
+
+
+if [ $BUCKETS_EXISTS -lt 1 ]; then
+echo "bucket does not exist"
+$NIFI_CLI registry create-bucket -u ${NIFI_REG} --bucketName MyBucket
+BUCKET_ID=$($NIFI_CLI registry list-buckets -u ${NIFI_REG}|grep MyBucket|awk '{print $3}')
+$NIFI_CLI registry create-flow -u ${NIFI_REG} --bucketIdentifier $BUCKET_ID --flowName MyFlow
+FLOWID=$($NIFI_CLI registry list-flows -u ${NIFI_REG} --bucketIdentifier=$BUCKET_ID|grep MyFlow|awk '{print $3}')
+$NIFI_CLI registry import-flow-version -u ${NIFI_REG} -f $FLOWID -i /home/nifi/NGSILDHL7Flow.json
+$NIFI_CLI nifi pg-import --bucketIdentifier $BUCKET_ID --flowIdentifier $FLOWID --flowVersion 1
+fi
+PG_ID=$($NIFI_CLI nifi pg-list|grep 1|awk '{print $4}')
+$NIFI_CLI nifi pg-enable-services --processGroupId $PG_ID
+$NIFI_CLI nifi pg-start --processGroupId $PG_ID
+touch created
+fi
+wait $pid
diff --git a/tempInterval2File/NGSILDHL7Flow.json b/tempInterval2File/NGSILDHL7Flow.json