Merge pull request #914 from samply/release-v0.19.3

Release v0.19.3

.github/distributed-test/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   zookeeper:
     image: "docker.io/bitnami/zookeeper:3"
@@ -92,6 +91,7 @@ services:
     - kafka-topic-creator
     - cassandra-1
     - cassandra-2
+    restart: unless-stopped
 
   blaze-2:
     image: "blaze:latest"
@@ -121,6 +121,7 @@ services:
     - kafka-topic-creator
     - cassandra-1
     - cassandra-2
+    restart: unless-stopped
 
   ingress:
     image: "haproxy:2.3"

.github/doc-copy-data-test/docker-compose.override.yml

@@ -0,0 +1,6 @@
+services:
+  src:
+    image: "blaze:latest"
+
+  dst:
+    image: "blaze:latest"

.github/openid-auth-test/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   keycloak:
     image: "jboss/keycloak:15.0.2"

.github/scripts/download-all-resources.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+EXPECTED_SIZE=$(curl -s "$BASE?_summary=count" | jq -r .total)
+
+FILE_NAME=$(uuidgen)
+blazectl --no-progress --server "$BASE" download -o "$FILE_NAME.ndjson"
+
+test "download size" "$(wc -l "$FILE_NAME.ndjson" | xargs | cut -d ' ' -f1)" "$EXPECTED_SIZE"

.github/scripts/download-resources.sh

@@ -8,6 +8,6 @@ TYPE=$1
 EXPECTED_SIZE=$(curl -s "$BASE/${TYPE}?_summary=count" | jq -r .total)
 
 FILE_NAME=$(uuidgen)
-blazectl --no-progress --server $BASE download "$TYPE" -o "$FILE_NAME".ndjson
+blazectl --no-progress --server "$BASE" download "$TYPE" -o "$FILE_NAME.ndjson"
 
-test "download size" "$(wc -l "$FILE_NAME".ndjson | xargs | cut -d ' ' -f1)" "$EXPECTED_SIZE"
+test "download size" "$(wc -l "$FILE_NAME.ndjson" | xargs | cut -d ' ' -f1)" "$EXPECTED_SIZE"

.github/scripts/install-blazectl.sh

@@ -1,8 +1,8 @@
 #!/bin/bash -e
 
-VERSION=0.12.0
+VERSION="0.13.0"
 
-curl -sLO https://github.com/samply/blazectl/releases/download/v${VERSION}/blazectl-${VERSION}-linux-amd64.tar.gz
-tar xzf blazectl-${VERSION}-linux-amd64.tar.gz
-rm blazectl-${VERSION}-linux-amd64.tar.gz
+curl -sLO "https://github.com/samply/blazectl/releases/download/v$VERSION/blazectl-$VERSION-linux-amd64.tar.gz"
+tar xzf "blazectl-$VERSION-linux-amd64.tar.gz"
+rm "blazectl-$VERSION-linux-amd64.tar.gz"
 sudo mv ./blazectl /usr/local/bin/blazectl

.github/workflows/build.yml

@@ -297,6 +297,9 @@ jobs:
     - name: Count Resources
       run: blazectl count-resources --server http://localhost:8080/fhir
 
+    - name: Download All Resources
+      run: .github/scripts/download-all-resources.sh
+
     - name: Download Patient Resources
       run: .github/scripts/download-resources.sh Patient
 
@@ -857,6 +860,50 @@ jobs:
     - name: Authenticated Request
       run: .github/scripts/authenticated-request.sh
 
+  doc-copy-data-test:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@v3
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@v3
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Test Setup
+      run: docker-compose -f docs/data-sync/copy/docker-compose.yml -f .github/doc-copy-data-test/docker-compose.override.yml up -d
+
+    - name: Wait for Source Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Load Data Into Source Blaze
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Wait for Destination Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8082/health
+
+    - name: Copy All Resources from Source to Destination
+      run: scripts/copy-data.sh http://localhost:8080/fhir http://localhost:8082/fhir
+
+    - name: Download Source Patients
+      run: blazectl download --server http://localhost:8080/fhir Patient | jq -c 'del(.meta.versionId) | del(.meta.lastUpdated)' > src-patients.ndjson
+
+    - name: Download Destination Patients
+      run: blazectl download --server http://localhost:8082/fhir Patient | jq -c 'del(.meta.versionId) | del(.meta.lastUpdated)' > dst-patients.ndjson
+
+    - name: Compare Source and Destination Patients
+      run: diff src-patients.ndjson dst-patients.ndjson
+
   distributed-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -922,6 +969,9 @@ jobs:
     - name: Count Resources
       run: blazectl count-resources --server http://localhost:8080/fhir
 
+    - name: Download All Resources
+      run: .github/scripts/download-all-resources.sh
+
     - name: Download Patient Resources
       run: .github/scripts/download-resources.sh Patient
 
@@ -1248,6 +1298,7 @@ jobs:
     - bundle-with-references-test
     - jepsen-test
     - openid-auth-test
+    - doc-copy-data-test
     - distributed-test
     - jepsen-distributed-test
     runs-on: ubuntu-22.04

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v0.19.3
+
+### Bugfixes
+
+* Fix System Search Paging ([#910](https://github.com/samply/blaze/pull/910))
+
+### Documentation
+
+* Extend Documentation of Data Sync ([#911](https://github.com/samply/blaze/pull/911))
+
+The full changelog can be found [here](https://github.com/samply/blaze/milestone/61?closed=1).
+
 ## v0.19.2
 
 ### Bugfixes

Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get update && apt-get upgrade -y && \
     rm -rf /var/lib/apt/lists/
 
 RUN mkdir -p /app/data && chown 1001:1001 /app/data
-COPY target/blaze-0.19.2-standalone.jar /app/
+COPY target/blaze-0.19.3-standalone.jar /app/
 
 WORKDIR /app
 USER 1001
@@ -16,4 +16,4 @@ ENV INDEX_DB_DIR="/app/data/index"
 ENV TRANSACTION_DB_DIR="/app/data/transaction"
 ENV RESOURCE_DB_DIR="/app/data/resource"
 
-CMD ["java", "-jar", "blaze-0.19.2-standalone.jar"]
+CMD ["java", "-jar", "blaze-0.19.3-standalone.jar"]

README.md

@@ -15,7 +15,7 @@ The goal of this project is to provide a FHIR® Store with an internal CQL Evalu
 
 Blaze passes all [Touchstone FHIR 4.0.1 Basic Tests][12] and almost all [CQL Tests][3]. Please refer to the [Conformance](docs/conformance.md) section and report any issues you encounter during evaluation.
 
-Latest release: [v0.19.2][5]
+Latest release: [v0.19.3][5]
 
 ## Quick Start
 
@@ -73,7 +73,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 
 [3]: <https://cql.hl7.org/tests.html>
 [4]: <https://alexanderkiel.gitbook.io/blaze/deployment>
-[5]: <https://github.com/samply/blaze/releases/tag/v0.19.2>
+[5]: <https://github.com/samply/blaze/releases/tag/v0.19.3>
 [6]: <https://www.yourkit.com/java/profiler/>
 [7]: <https://www.yourkit.com/.net/profiler/>
 [8]: <https://www.yourkit.com/youmonitor/>

build.clj

@@ -2,7 +2,7 @@
   (:require [clojure.tools.build.api :as b]))
 
 (def lib 'samply/blaze)
-(def version "0.19.2")
+(def version "0.19.3")
 (def class-dir "target/classes")
 (def basis (b/create-basis {:project "deps.edn"}))
 (def uber-file (format "target/%s-%s-standalone.jar" (name lib) version))

deps.edn

@@ -61,7 +61,7 @@
     {:mvn/version "0.4.6"}
 
     org.clojure/tools.namespace
-    {:mvn/version "1.4.0"}}}
+    {:mvn/version "1.4.1"}}}
 
   :kaocha
   {:extra-deps
@@ -75,7 +75,7 @@
 
    :extra-deps
    {org.clojure/tools.namespace
-    {:mvn/version "1.4.0"}}}
+    {:mvn/version "1.4.1"}}}
 
   :outdated
   {:replace-deps

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.2'
 services:
   blaze:
     image: "samply/blaze:0.19"

docs/authentication/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   keycloak:
     image: "jboss/keycloak:15.0.2"

docs/consistency/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   zookeeper-1:
     image: "docker.io/bitnami/zookeeper:3"
@@ -149,7 +148,7 @@ services:
       HEAP_NEWSIZE: "200M"
 
   blaze-1:
-    image: "samply/blaze:jepsen"
+    image: "samply/blaze:0.19"
     hostname: "blaze-1"
     environment:
       JAVA_TOOL_OPTIONS: "-Xmx4g -Dclojure.server.repl='{:address,\"0.0.0.0\",:port,5555,:accept,clojure.core.server/repl}'"
@@ -176,9 +175,10 @@ services:
     - cassandra-1
     - cassandra-2
     - cassandra-3
+    restart: unless-stopped
 
   blaze-2:
-    image: "samply/blaze:jepsen"
+    image: "samply/blaze:0.19"
     hostname: "blaze-2"
     environment:
       JAVA_TOOL_OPTIONS: "-Xmx4g"
@@ -205,6 +205,7 @@ services:
     - cassandra-1
     - cassandra-2
     - cassandra-3
+    restart: unless-stopped
 
   ingress:
     image: "haproxy:2.3"

docs/data-sync.md

@@ -1,23 +1,93 @@
 # Sync Resources from Another FHIR Server to Blaze
 
+## Using Subscriptions
+
 If you want to facilitate the CQL engine or other features of Blaze, but you can't or don't like to use Blaze as your primary FHIR server, you can configure your primary FHIR server to automatically sync every change to Blaze by using the [subscription][1] mechanism.
 
 In this example we use [HAPI][2] as our primary FHIR server. In the `docs/data-sync` directory, you can find a Docker Compose file with a setup of a HAPI and a Blaze server. Please start the containers by running:
 
 ```sh
-docker-compose up
+docker-compose -f docs/data-sync/subscription/docker-compose.yml up
 ```
 
-after both servers are up and running, you can create two subscriptions, one for Patient resources and one for Observations. Please run:
+After both servers are up and running, you can create two subscriptions, one for Patient resources and one for Observations. Please run:
 
 ```sh
 curl -H 'Content-Type: application/fhir+json' -d @subscription-bundle.json http://localhost:8090/fhir
 ```
 
 After you created the subscriptions, you can import or change data on the HAPI server, and it will be synced automatically to the Blaze server.
 
-One problem, you may encounter is that if you issue transactions against HAPI with resources referencing each other, HAPI will not send them in the right order to Blaze, so that Blaze is complaining about violated referential integrity.
+Because the subscription mechanism doesn't send the resources in the right order to satisfy referential integrity, Blaze is started with `ENFORCE_REFERENTIAL_INTEGRITY` set to `false`.
+
+
+## Create a Full Clone of a Blaze Server
+
+Another use-case would be to copy all data from one Blaze server to another. That can be useful to either:
+
+* remove the history from a Blaze server,
+* create a snapshot of all resources,
+* migrate from Blaze to another FHIR server or the other way around.
+
+### Setup Test Environment
+
+In order to test copying all data from one Blaze server to another, start the following Docker Compose project:
+
+```sh
+docker-compose -f docs/data-sync/copy/docker-compose.yml up
+```
+
+You should see a `src` server started at port 8080 and a `dst` server started at port 8082.
+
+### Load Data into the Source Server
+
+Next, load some data into the source server:
+
+```sh
+blazectl upload --server http://localhost:8080/fhir .github/test-data/synthea
+```
+
+After that finishes, you can use `blazectl count-resources` to ensure that the source server has data and the destination server hasn't:
+
+```sh
+blazectl count-resources --server http://localhost:8080/fhir
+blazectl count-resources --server http://localhost:8082/fhir
+```
 
+### Copy All Resources from Source to Destination
+
+The `copy-data.sh` script uses [GNU Parallel][3]. You may have to install that first.
+
+```sh
+scripts/copy-data.sh http://localhost:8080/fhir http://localhost:8082/fhir
+```
+
+The script outputs `Successfully send transaction bundle` for each transaction bundle send to the destination server.
+
+You can use `blazectl count-resources` to see whether the resource counts of the destination server equals the resource counts of the source server:
+
+```sh
+blazectl count-resources --server http://localhost:8080/fhir
+blazectl count-resources --server http://localhost:8082/fhir
+```
+
+You can also compare the resource contents between the source and the destination server by downloading all resources (of a type), removing the `Meta.versionId` and `Meta.lastUpdated` values that will be different on the destination server:
+
+```sh
+blazectl download --server http://localhost:8080/fhir Patient | jq -c 'del(.meta.versionId) | del(.meta.lastUpdated)' > src-patients.ndjson 
+blazectl download --server http://localhost:8082/fhir Patient | jq -c 'del(.meta.versionId) | del(.meta.lastUpdated)' > dst-patients.ndjson 
+diff src-patients.ndjson dst-patients.ndjson
+```
+
+### Save All Resources from the Source Server
+
+If you don't like to copy the data into the destination server immediately, you can also save the transaction bundles on disk and use `blazectl upload` later to upload them to the destination server.
+
+```sh
+mkdir dst
+scripts/save-data.sh http://localhost:8080/fhir dst
+```
 
 [1]: <https://www.hl7.org/fhir/subscription.html>
 [2]: <https://hapifhir.io>
+[3]: <https://www.gnu.org/software/parallel/>

docs/data-sync/copy/docker-compose.yml

@@ -0,0 +1,28 @@
+services:
+  src:
+    image: "samply/blaze:0.19"
+    environment:
+      BASE_URL: "http://localhost:8080"
+      JAVA_TOOL_OPTIONS: "-Xmx2g"
+      LOG_LEVEL: "debug"
+    ports:
+    - "8080:8080"
+    volumes:
+    - "blaze-data-src:/app/data"
+
+  dst:
+    image: "samply/blaze:0.19"
+    environment:
+      BASE_URL: "http://localhost:8082"
+      SERVER_PORT: "8082"
+      JAVA_TOOL_OPTIONS: "-Xmx2g"
+      ENFORCE_REFERENTIAL_INTEGRITY: "false"
+      LOG_LEVEL: "debug"
+    ports:
+    - "8082:8082"
+    volumes:
+    - "blaze-data-dst:/app/data"
+
+volumes:
+  blaze-data-src:
+  blaze-data-dst: