diff --git a/.github/scripts/check-bloom-filter.sh b/.github/scripts/check-bloom-filter.sh new file mode 100755 index 000000000..1c2426358 --- /dev/null +++ b/.github/scripts/check-bloom-filter.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +HASH="$1" +PATIENT_COUNT="$2" + +test "patient count" "$(curl -s "$BASE/__admin/cql/bloom-filters" | jq -r ".[] | select(.hash == \"$HASH\") | .patientCount")" "$PATIENT_COUNT" diff --git a/.github/scripts/check-patient-last-change-index-missing.sh b/.github/scripts/check-patient-last-change-index-missing.sh new file mode 100755 index 000000000..7963cd461 --- /dev/null +++ b/.github/scripts/check-patient-last-change-index-missing.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +curl -s "$BASE/__admin/dbs/index/column-families" | jq -r '.[].name' | grep -q "patient-last-change-index" + +test "exit code" "$?" "1" diff --git a/.github/scripts/check-patient-last-change-index-state.sh b/.github/scripts/check-patient-last-change-index-state.sh new file mode 100755 index 000000000..0bd79ab16 --- /dev/null +++ b/.github/scripts/check-patient-last-change-index-state.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +STATE="$(curl -s "$BASE/__admin/dbs/index/column-families/patient-last-change-index/state" | jq -r .type)" + +test "state" "$STATE" "$1" diff --git a/.github/scripts/test-cql-expr-cache-metrics.sh b/.github/scripts/test-cql-expr-cache-metrics.sh new file mode 100755 index 000000000..f3f723891 --- /dev/null +++ b/.github/scripts/test-cql-expr-cache-metrics.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +URL="http://localhost:8081/metrics" + +num-metrics() { + NAME="$1" + FILTER="$2" + curl -s "$URL" | grep "$NAME" | grep -c "$FILTER" +} + +# CQL expression cache is available +test "blaze_cache_estimated_size cql-expr-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"cql-expr-cache\"")" "1" + +# other caches are still available +test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1" +test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1" diff --git a/.github/scripts/test-metrics.sh b/.github/scripts/test-metrics.sh index a739a5c2e..b57741b8a 100755 --- a/.github/scripts/test-metrics.sh +++ b/.github/scripts/test-metrics.sh @@ -15,6 +15,5 @@ test "blaze_rocksdb_block_cache_data_miss index" "$(num-metrics "blaze_rocksdb_b test "blaze_rocksdb_block_cache_data_miss transaction" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"transaction\"")" "1" test "blaze_rocksdb_block_cache_data_miss resource" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"resource\"")" "1" -test "blaze_rocksdb_table_reader_usage_bytes index" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"index\"")" "14" -test "blaze_rocksdb_table_reader_usage_bytes transaction" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"transaction\"")" "1" -test "blaze_rocksdb_table_reader_usage_bytes resource" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"resource\"")" "1" +test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1" +test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30cbf0b02..38b6a6d50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ on: - master - develop tags: - - 'v*.*.*' + - 'v*.*.*' pull_request: branches: - master @@ -25,7 +25,7 @@ jobs: cljfmt: 0.12.0 - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Check Formatting run: make fmt @@ -40,7 +40,7 @@ jobs: clj-kondo: '2024.03.13' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Lint run: make lint @@ -58,7 +58,7 @@ jobs: run: npm install -g fsh-sushi - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Build working-directory: job-ig @@ -83,6 +83,7 @@ jobs: - async - byte-buffer - byte-string + - cache-collector - cassandra - coll - cql @@ -148,7 +149,7 @@ jobs: cli: '1.11.2.1446' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -189,7 +190,7 @@ jobs: cli: '1.11.2.1446' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -223,7 +224,7 @@ jobs: cli: '1.11.2.1446' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -250,7 +251,7 @@ jobs: uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3 - name: Build and Export to Docker - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6 with: context: . tags: blaze:latest @@ -268,7 +269,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Setup Node uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 @@ -282,7 +283,7 @@ jobs: uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3 - name: Build and Export to Docker - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6 with: context: modules/frontend tags: blaze-frontend:latest @@ -316,7 +317,7 @@ jobs: run: docker load --input /tmp/${{ matrix.image }}.tar - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Run Trivy Vulnerability Scanner uses: aquasecurity/trivy-action@master @@ -333,6 +334,158 @@ jobs: with: sarif_file: trivy-results.sarif + cql-expr-cache-test: + needs: build + runs-on: ubuntu-22.04 + + steps: + - name: Check out Git repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Install Blazectl + run: .github/scripts/install-blazectl.sh + + - name: Download Blaze Image + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 + with: + name: blaze-image + path: /tmp + + - name: Load Blaze Image + run: docker load --input /tmp/blaze.tar + + - name: Run Blaze + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -e CQL_EXPR_CACHE_SIZE=1000 -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Docker Logs + run: docker logs blaze + + - name: Check Capability Statement + run: .github/scripts/check-capability-statement.sh + + - name: Ensure that the State of PatientLastChange Index is Current + run: .github/scripts/check-patient-last-change-index-state.sh current + + - name: Load Data + run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea + + - name: Prometheus Metrics + run: .github/scripts/test-cql-expr-cache-metrics.sh + + - name: Check Total-Number of Resources are 92114 + run: .github/scripts/check-total-number-of-resources.sh 92114 + + - name: Evaluate CQL Query 1 + run: .github/scripts/evaluate-measure.sh q1 56 + + - name: Evaluate CQL Query 1 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q1 56 + + - name: Evaluate CQL Query 1 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q1 56 + + - name: Evaluate CQL Query 1 on Individual Patients + run: .github/scripts/evaluate-patient-q1-measure.sh + + - name: Evaluate CQL Query 2 + run: .github/scripts/evaluate-measure.sh q2 42 + + - name: Evaluate CQL Query 2 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q2 42 + + - name: Evaluate CQL Query 2 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q2 42 + + - name: Evaluate CQL Query 4 + run: .github/scripts/evaluate-measure.sh q4 0 + + - name: Check Bloom Filter + run: .github/scripts/check-bloom-filter.sh 484cc96511a406d40ee8eeb4de6a49b05b3766ddef8eb05ac57c6139d774eddc 0 + + - name: Evaluate CQL Query 4 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q4 0 + + - name: Evaluate CQL Query 4 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q4 0 + + - name: Evaluate CQL Query 7 + run: .github/scripts/evaluate-measure.sh q7 81 + + - name: Evaluate CQL Query 7 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q7 81 + + - name: Evaluate CQL Query 7 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q7 81 + + - name: Evaluate CQL Query 14 + run: .github/scripts/evaluate-measure.sh q14 96 + + - name: Check Bloom Filter + run: .github/scripts/check-bloom-filter.sh fab7fc40f75f294787d8b2090788a291cdaf71640d6dcfe5441c00403225a256 96 + + - name: Evaluate CQL Query 14 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q14 96 + + - name: Evaluate CQL Query 14 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q14 96 + + - name: Evaluate CQL Query 17 + run: .github/scripts/evaluate-measure.sh q17 120 + + - name: Evaluate CQL Query 17 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q17 120 + + - name: Evaluate CQL Query 17 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q17 120 + + - name: Evaluate CQL Query 20 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q20-stratifier-city 120 + + - name: Evaluate CQL Query 21 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q21-stratifier-city-of-only-women 64 + + - name: Evaluate CQL Query 26 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q26-stratifier-bmi 120 + + - name: Evaluate CQL Query 27 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q27-stratifier-calculated-bmi 120 + + - name: Evaluate CQL Query 32 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q32-stratifier-underweight 120 + + - name: Evaluate CQL Query 36 + run: .github/scripts/evaluate-measure.sh q36-parameter 86 + + - name: Check Bloom Filter + run: .github/scripts/check-bloom-filter.sh d4fc6cde1636852f9e362a68ca7be027a66bf7cb38ebff9c256c3eb2179c2639 86 + + - name: Evaluate CQL Query 36 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q36-parameter 86 + + - name: Evaluate CQL Query 37 + run: .github/scripts/evaluate-measure.sh q37-overlaps 24 + + - name: Check Bloom Filter + run: .github/scripts/check-bloom-filter.sh 8a572962e0540de1bb4f4bf5c0101ff06be3ae69f4cd489509a6cf8d1646d1e1 24 + + - name: Evaluate CQL Query 37 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q37-overlaps 24 + + - name: Evaluate CQL Query 37 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q37-overlaps 24 + + - name: Evaluate CQL Query 46 + run: .github/scripts/evaluate-measure.sh q46-between-date 19 + + - name: Check Bloom Filter + run: .github/scripts/check-bloom-filter.sh 5a1fe6d1b996aed4783f0ee04e6e74927d7ddccc407b0818a8a800769458020b 19 + + - name: Evaluate CQL Query 46 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q46-between-date 19 + integration-test: needs: build runs-on: ubuntu-22.04 @@ -345,7 +498,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -736,7 +889,7 @@ jobs: - name: Delete run: .github/scripts/delete.sh - - name: Delete Violating Referential Integrity + - name: Delete Violating Referential Integrity run: .github/scripts/check-referential-integrity-for-delete.sh 409 - name: Batch @@ -751,7 +904,7 @@ jobs: - name: Transaction Read/Write run: .github/scripts/transaction-rw.sh - - name: Transactional Delete Preserving Referential Integrity + - name: Transactional Delete Preserving Referential Integrity run: .github/scripts/transactional-delete.sh - name: Transaction with Invalid (Null) Resource @@ -856,7 +1009,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -924,7 +1077,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -968,7 +1121,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1007,7 +1160,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1048,7 +1201,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1083,7 +1236,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1121,7 +1274,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1150,7 +1303,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1181,7 +1334,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1224,7 +1377,7 @@ jobs: cli: '1.11.2.1446' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -1268,7 +1421,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1309,7 +1462,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1341,7 +1494,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Install Blazectl run: .github/scripts/install-blazectl.sh @@ -1385,7 +1538,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1423,7 +1576,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Setup Node uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 @@ -1507,7 +1660,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Image uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 @@ -1548,6 +1701,52 @@ jobs: - name: Fetch Patient Expecting an Error run: .github/scripts/fetch-resource-0-with-missing-resource-content.sh + # This test ensures that older versions of Blaze will migrate successfully to + # the new database schema especially building the PatientLastChange index. + build-patient-last-change-index-test: + needs: build + runs-on: ubuntu-22.04 + + steps: + - name: Check out Git repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Install Blazectl + run: .github/scripts/install-blazectl.sh + + - name: Download Blaze Image + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 + with: + name: blaze-image + path: /tmp + + - name: Load Blaze Image + run: docker load --input /tmp/blaze.tar + + - name: Run Blaze in the Version 0.27 before Introduction of the PatientLastChange Index + run: docker run --name blaze --rm -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -p 8080:8080 -v blaze-data:/app/data samply/blaze:0.27 + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Load Data + run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea + + - name: Ensure that the PatientLastChange Index does not exist + run: .github/scripts/check-patient-last-change-index-missing.sh + + - name: Shut down Blaze + run: docker stop blaze + + - name: Run Latest Blaze with the PatientLastChange Index + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -e LOG_LEVEL=debug -p 8080:8080 -v blaze-data:/app/data blaze:latest + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Ensure that the State of PatientLastChange Index is Current + run: .github/scripts/check-patient-last-change-index-state.sh current + jepsen-distributed-test: needs: build runs-on: ubuntu-22.04 @@ -1565,7 +1764,7 @@ jobs: cli: '1.11.2.1446' - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -1628,6 +1827,7 @@ jobs: if: github.event_name != 'pull_request' || (github.event.pull_request.base.repo.full_name == github.event.pull_request.head.repo.full_name) needs: - image-scan + - cql-expr-cache-test - integration-test - integration-test-synthea-1000 - not-enforcing-referential-integrity-test @@ -1646,6 +1846,7 @@ jobs: - frontend-test - missing-resource-content-test - custom-search-parameters-test + - build-patient-last-change-index-test runs-on: ubuntu-22.04 permissions: packages: write @@ -1662,7 +1863,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Download Blaze Uberjar if: ${{ matrix.image.context == '.' }} @@ -1717,7 +1918,7 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push to GHCR - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6 with: context: ${{ matrix.image.context }} platforms: linux/amd64,linux/arm64 @@ -1748,7 +1949,7 @@ jobs: - name: Build and push to DockerHub if: ${{ env.dockerhub_username != '' }} - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6 with: context: ${{ matrix.image.context }} platforms: linux/amd64,linux/arm64 @@ -1767,7 +1968,7 @@ jobs: steps: - name: Release - uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2 + uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 # v2 with: files: target/blaze-*-standalone.jar @@ -1775,7 +1976,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Setup Node uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ba14fd272..0095bd965 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -30,7 +30,7 @@ jobs: run: npm install -g fsh-sushi - name: Check out Git repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Cache Local Maven Repo uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -46,7 +46,7 @@ jobs: run: make test-coverage - name: Codecov Upload - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4 + uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4 with: name: codecov-umbrella token: ${{ secrets.CODECOV_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d620215f..4829bd81a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## v0.28.0 + +**!!! IMPORTANT !!!** + +This release contains database schema additions, so that the database can't be opened with older versions of Blaze. Downgrades are not possible. + +### Notes + +A new index called `PatientLastChange` which will be automatically created and filled at the startup of Blaze. The index is used for the new CQL cache feature ([#1051](https://github.com/samply/blaze/issues/1051)). Except for the caching, Blaze will work fully during index buildup. The index will be ready once the log message "Finished building PatientLastChange index of main node." appears. + +After the index buildup, the CQL Cache can be activated by setting `CQL_EXPR_CACHE_SIZE` to a size of heap memory in MiB and restarting Blaze. + +### New Features + +* Cache Results of the Exists CQL Expression ([#1051](https://github.com/samply/blaze/issues/1051)) + +### Bugfixes + +* Fix JSON Rendering in UI ([#1813](https://github.com/samply/blaze/issues/1813)) + +The full changelog can be found [here](https://github.com/samply/blaze/milestone/91?closed=1). + +## v0.28.0-rc.1 + +### New Features + +* Cache Results of the Exists CQL Expression ([#1051](https://github.com/samply/blaze/issues/1051)) + +### Bugfixes + +* Fix JSON Rendering in UI ([#1813](https://github.com/samply/blaze/issues/1813)) + +The full changelog can be found [here](https://github.com/samply/blaze/milestone/91?closed=1). + ## v0.27.1 ### Enhancements diff --git a/Dockerfile b/Dockerfile index 66e4462ff..122b0bfd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get upgrade -y && \ rm -rf /var/lib/apt/lists/ RUN mkdir -p /app/data && chown 1001:1001 /app/data -COPY target/blaze-0.27.1-standalone.jar /app/ +COPY target/blaze-0.28.0-standalone.jar /app/ WORKDIR /app USER 1001 @@ -20,4 +20,4 @@ ENV RESOURCE_DB_DIR="/app/data/resource" ENV ADMIN_INDEX_DB_DIR="/app/data/admin-index" ENV ADMIN_TRANSACTION_DB_DIR="/app/data/admin-transaction" -CMD ["java", "-jar", "blaze-0.27.1-standalone.jar"] +CMD ["java", "-jar", "blaze-0.28.0-standalone.jar"] diff --git a/Makefile b/Makefile index cb44a3c2c..0ca2d8caa 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ $(MODULES): $(MAKE) -C $@ $(MAKECMDGOALS) fmt-root: - cljfmt check + cljfmt check resources src test deps.edn fmt: $(MODULES) fmt-root diff --git a/README.md b/README.md index ec3b367a6..26f39fb69 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ A demo installation can be found [here](https://blaze.life.uni-leipzig.de/fhir) Blaze is widely used in the [Medical Informatics Initiative](https://www.medizininformatik-initiative.de) in Germany and in [Biobanks](https://www.bbmri-eric.eu) across Europe. A 1.0 version is expected in the next months. -Latest release: [v0.27.1][5] +Latest release: [v0.28.0][5] ## Quick Start @@ -75,7 +75,7 @@ Unless required by applicable law or agreed to in writing, software distributed [3]: [4]: -[5]: +[5]: [6]: [7]: [8]: diff --git a/build.clj b/build.clj index c2c1db77d..50c11f0c8 100644 --- a/build.clj +++ b/build.clj @@ -2,7 +2,7 @@ (:require [clojure.tools.build.api :as b])) (def lib 'samply/blaze) -(def version "0.27.1") +(def version "0.28.0") (def class-dir "target/classes") (def basis (b/create-basis {:project "deps.edn"})) (def uber-file (format "target/%s-%s-standalone.jar" (name lib) version)) diff --git a/cljfmt.edn b/cljfmt.edn index 5e57e0508..f65b7b03c 100644 --- a/cljfmt.edn +++ b/cljfmt.edn @@ -12,4 +12,5 @@ deftests [[:block 1]] satisfies-prop [[:block 1]] clojure.test.check.properties/for-all [[:block 1]] - blaze.metrics.core/collector [[:block 0]]}} + blaze.metrics.core/collector [[:block 0]] + reify-expr [[:inner 0] [:inner 1]]}} diff --git a/dev/blaze/dev.clj b/dev/blaze/dev.clj index b20dd3bf2..8faa1c770 100644 --- a/dev/blaze/dev.clj +++ b/dev/blaze/dev.clj @@ -1,20 +1,22 @@ (ns blaze.dev (:require - [blaze.byte-string :as bs] - [blaze.db.api :as d] - [blaze.db.api-spec] - [blaze.db.cache-collector.protocols :as ccp] - [blaze.db.resource-cache :as resource-cache] - [blaze.db.resource-store :as rs] - [blaze.db.tx-log :as tx-log] - [blaze.spec] - [blaze.system :as system] - [blaze.system-spec] - [clojure.repl :refer [pst]] - [clojure.spec.test.alpha :as st] - [clojure.tools.namespace.repl :refer [refresh]] - [java-time.api :as time] - [taoensso.timbre :as log])) + [blaze.byte-string :as bs] + [blaze.cache-collector.protocols :as ccp] + [blaze.cache-collector.protocols :as ccp] + [blaze.db.api :as d] + [blaze.db.api-spec] + [blaze.db.resource-cache :as resource-cache] + [blaze.db.resource-store :as rs] + [blaze.db.tx-log :as tx-log] + [blaze.elm.expression :as-alias expr] + [blaze.spec] + [blaze.system :as system] + [blaze.system-spec] + [clojure.repl :refer [pst]] + [clojure.spec.test.alpha :as st] + [clojure.tools.namespace.repl :refer [refresh]] + [java-time.api :as time] + [taoensso.timbre :as log])) (defonce system nil) @@ -58,6 +60,11 @@ (resource-cache/invalidate-all! (:blaze.db/resource-cache system)) ) +;; CQL Expression Cache +(comment + (str (ccp/-stats (::expr/cache system))) + ) + ;; RocksDB Stats (comment (.reset (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats])) diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 60b7f397c..cdd9f9f12 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -27,7 +27,7 @@ export default defineConfig({ nav: [ {text: 'Home', link: '/'}, { - text: "v0.27.1", + text: "v0.28.0", items: [ { text: 'Changelog', diff --git a/docs/api/admin.md b/docs/api/admin.md new file mode 100644 index 000000000..2a05ecbf7 --- /dev/null +++ b/docs/api/admin.md @@ -0,0 +1,7 @@ +# Admin API + +## OpenAPI Spec + +```sh +curl http://localhost:8080/fhir/__admin/openapi.json +``` diff --git a/docs/api/operation-measure-evaluate-measure.md b/docs/api/operation-measure-evaluate-measure.md index cb805190b..60c1fa595 100644 --- a/docs/api/operation-measure-evaluate-measure.md +++ b/docs/api/operation-measure-evaluate-measure.md @@ -6,6 +6,9 @@ The official documentation can be found [here][1]. * [blazectl][2] supports async requests since [v0.15.0][3] * cancellation of async requests is fully supported. the evaluation is stopped almost immediately * the `FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT` is only applied to synchronous requests +* [`CQL_EXPR_CACHE_SIZE`](../deployment/environment-variables.md) can be set to enable a cache of certain CQL expressions that will speed up evaluations +* a detailed documentation how to use the $evaluate-measure API can be found [here](../cql-queries/api.md) +* a documentation how to use $evaluate-measure via blazectl can be found [here](../cql-queries/blazectl.md) [1]: [2]: diff --git a/docs/cql-queries.md b/docs/cql-queries.md index 7f8bb1273..eb85f01b6 100644 --- a/docs/cql-queries.md +++ b/docs/cql-queries.md @@ -13,158 +13,8 @@ If you like to use the command line, please look into [this section](cql-queries If you'd like to use the CQL Evaluation API directly, please read the [CQL API Documentation](cql-queries/api.md). -## Install the Quality Reporting UI +## Resources -The most accessible way to create and execute CQL queries is to use the Quality Reporting UI. The Quality Reporting UI is a desktop application which you can download [here](https://github.com/samply/blaze-quality-reporting-ui). +You can learn more about CQL queries in the [Author's Guide][1] at HL7. -## Run Blaze - -If you don't already have Blaze running, you can read about how to do it in [Deployment](deployment/README.md). If you have Docker available just run: - -``` -docker run -p 8080:8080 -v blaze-data:/app/data samply/blaze:latest -``` - -Start the Quality Reporting UI. You should see an empty measure list. - -![](cql-queries/measures.png) - -In the upper right corner, you see **Localhost 8080**. If Blaze runs on Localhost port 8080, you can continue, otherwise you have to go into Settings and add your Blaze servers location. - -### Add Your Blaze Server - -Go into **Settings** and click on **Add**. - -![](cql-queries/settings-server.png) - -Enter a **Name** and a **URL**. Please be aware that URLs of Blaze FHIR endpoints have the path `/fhir` in it by default. You can find your FHIR endpoint URL of Blaze in the logs in a line like this: - -```text -Init FHIR RESTful API with base URL: http://localhost:8080/fhir -``` - -### Create Your First Library - -Blaze uses the FHIR [Quality Reporting](https://www.hl7.org/fhir/clinicalreasoning-quality-reporting.html) Module, to execute CQL queries. In Quality Reporting, CQL Query Expressions reside in [Library](https://www.hl7.org/fhir/library.html) resources and are referenced in [Measure](https://www.hl7.org/fhir/measure.html) resources. In order to create a Library resource, go to **Libraries** and click on **New Library**. - -![](cql-queries/libraries-new.png) - -After you created your Library, you can give it a name by clicking at **Edit:** - -![](cql-queries/library-title-edit.png) - -After naming your Library, you have to give it a canonical URL. We just use localhost for our URL here: - -![](cql-queries/library-url-edit.png) - -Next we add the CQL source code for our single **InInitialPopulation** query expression: - -![](cql-queries/library-cql.png) - -```text -library Covid19 -using FHIR version '4.0.0' -include FHIRHelpers version '4.0.0' - -codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' - -define InInitialPopulation: - exists([Condition: Code 'U07.1' from icd10]) -``` - -### Create Your First Measure - -You can create Measure resources under **Measures** by clicking on **New Measure**. After giving our Measure a name, we also have to give it a canonical URL: - -![](cql-queries/measure-url-edit.png) - -After that, we have to reference our previously created Library to our Measure by clicking on the **Edit** button in the right sidebar: - -![](cql-queries/measure-library-edit.png) - -Because the Measure comes with an initial population definition by default, we will only check it by clicking on **initial-population**: - -![](cql-queries/measure-initial-population.png) - -Here we see our CQL expression **InInitialPopulation** from our Library referenced: - -![](cql-queries/measure-initial-population-criteria.png) - -### Generating a First Report - -To generate a Report, we click on **Generate New Report**: - -![](cql-queries/measure-generate-report.png) - -After some time, a MeasureReport will appear in the list of reports of our Measure: - -![](cql-queries/measure-report-list.png) - -Please be patient, because currently there is no progress bar. If nothing appears for a long time, you can use the menu to go back to all measure, open our measure again and look if ou see a report with a fitting timestamp. - -All reports are persisted in Blaze and are shown in the UI with their creation timestamp. - -After you open the report, you will see that your **initial-population** has a count of zero. - -![](cql-queries/measure-report-1.png) - -### Import Patients with COVID-19 Diagnoses - -If you POST the following Bundle to the transaction endpoint of Blaze, you will have two patients, one with a COVID-19 condition and one without: - -```text -{ - "resourceType": "Bundle", - "type": "transaction", - "entry": [ - { - "request": { - "method": "PUT", - "url": "Patient/0" - }, - "resource": { - "resourceType": "Patient", - "id": "0" - } - }, - { - "request": { - "method": "PUT", - "url": "Condition/0-condition" - }, - "resource": { - "resourceType": "Condition", - "id": "0-condition", - "code": { - "coding": [ - { - "code": "U07.1", - "system": "http://hl7.org/fhir/sid/icd-10", - "version": "2016" - } - ] - }, - "subject": { - "reference": "Patient/0" - } - } - }, - { - "request": { - "method": "PUT", - "url": "Patient/1" - }, - "resource": { - "resourceType": "Patient", - "id": "1" - } - } - ] -} -``` - -After importing patients, the result of the initial population will be one: - -![](cql-queries/measure-report-2.png) - -You can learn more about CQL queries in the [Author's Guide](https://cql.hl7.org/02-authorsguide.html) at HL7. +[1]: diff --git a/docs/deployment/environment-variables.md b/docs/deployment/environment-variables.md index 00e5194bb..cb62d4621 100644 --- a/docs/deployment/environment-variables.md +++ b/docs/deployment/environment-variables.md @@ -41,7 +41,7 @@ The three database directories must not exist on the first start of Blaze and wi | TRANSACTION_DB_WAL_DIR | \ | v0.18 | | The directory were the transaction log write ahead log (WAL) files are stored. Empty means same dir as database files. | | RESOURCE_DB_DIR | resource ² | v0.8 | | The directory were the resource files are stored. This directory must not exist on the first start of Blaze and will be created by | | RESOURCE_DB_WAL_DIR | \ | v0.18 | | The directory were the resource write ahead log (WAL) files are stored. Empty means same dir as database files. | -| DB_BLOCK_CACHE_SIZE | 128 | v0.8 | | The size of the [block cache][2] of the DB in MB. | +| DB_BLOCK_CACHE_SIZE | 128 | v0.8 | | The size of the [block cache][2] of the DB in MiB. This cache is outside of the JVM heap. | | DB_RESOURCE_CACHE_SIZE | 100000 | v0.8 | | The size of the resource cache of the DB in number of resources. | | DB_MAX_BACKGROUND_JOBS | 4 | v0.8 | | The maximum number of the [background jobs][3] used for DB compactions. | | DB_RESOURCE_INDEXER_THREADS | 4 | v0.8 | | The number threads used for indexing resources. Try 8 or 16 depending on your hardware. | @@ -58,7 +58,7 @@ The distributed storage variant only uses the index database locally. |:-----------------------------------|:---------------|:------|:-------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| | INDEX_DB_DIR | index ² | v0.8 | | The directory were the index database files are stored. | | INDEX_DB_WAL_DIR | \ | v0.18 | | The directory were the index database write ahead log (WAL) files are stored. Empty means same dir as database files. | -| DB_BLOCK_CACHE_SIZE | 128 | v0.8 | | The size of the [block cache][2] of the DB in MB. | +| DB_BLOCK_CACHE_SIZE | 128 | v0.8 | | The size of the [block cache][2] of the DB in MiB. This cache is outside of the JVM heap. | | DB_RESOURCE_CACHE_SIZE | 100000 | v0.8 | | The size of the resource cache of the DB in number of resources. | | DB_MAX_BACKGROUND_JOBS | 4 | v0.8 | | The maximum number of the [background jobs][3] used for DB compactions. | | DB_RESOURCE_INDEXER_THREADS | 4 | v0.8 | | The number threads used for indexing resources. Try 8 or 16 depending on your hardware. | @@ -86,30 +86,33 @@ More information about distributed deployment are available [here](distributed-b ### Other Environment Variables -| Name | Default | Since | Depr ¹ | Description | -|:----------------------------------------|:---------------------------|:-------|---------|:-----------------------------------------------------------------------------------------------| -| PROXY_HOST | — | v0.6 | — | REMOVED: use -Dhttp.proxyHost | -| PROXY_PORT | — | v0.6 | — | REMOVED: use -Dhttp.proxyPort | -| PROXY_USER | — | v0.6.1 | — | REMOVED: try [SOCKS Options][1] | -| PROXY_PASSWORD | — | v0.6.1 | — | REMOVED: try [SOCKS Options][1] | -| CONNECTION_TIMEOUT | 5 s | v0.6.3 | — | connection timeout for outbound HTTP requests | -| REQUEST_TIMEOUT | 30 s | v0.6.3 | — | REMOVED | -| TERM_SERVICE_URI | [http://tx.fhir.org/r4][6] | v0.6 | v0.11 | Base URI of the terminology service | -| BASE_URL | `http://localhost:8080` | — | — | The URL under which Blaze is accessible by clients. | -| CONTEXT_PATH | /fhir | v0.11 | — | Context path under which the FHIR RESTful API will be accessible. | -| SERVER_PORT | 8080 | — | — | The port of the main HTTP server | -| METRICS_SERVER_PORT | 8081 | v0.6 | — | The port of the Prometheus metrics server | -| LOG_LEVEL | info | v0.6 | — | one of trace, debug, info, warn or error | -| JAVA_TOOL_OPTIONS | — | — | — | JVM options \(Docker only\) | -| FHIR_OPERATION_EVALUATE_MEASURE_THREADS | — | v0.8 | v0.23.3 | The number threads used for $evaluate-measure executions. | -| FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT | 3600000 (1h) | v0.19 | — | Timeout in milliseconds for synchronous $evaluate-measure executions. | -| OPENID_PROVIDER_URL | — | v0.11 | — | [OpenID Connect][4] provider URL to enable [authentication][5] | -| OPENID_CLIENT_TRUST_STORE | — | v0.26 | — | A PKCS #12 trust store containing CA certificates needed for the [OpenID Connect][4] provider. | -| OPENID_CLIENT_TRUST_STORE_PASS | — | v0.26 | — | The password for the PKCS #12 trust store. | -| ENFORCE_REFERENTIAL_INTEGRITY | true | v0.14 | — | Enforce referential integrity on resource create, update and delete. | -| DB_SYNC_TIMEOUT | 10000 | v0.15 | — | Timeout in milliseconds for all reading FHIR interactions acquiring the newest database state. | -| DB_SEARCH_PARAM_BUNDLE | — | v0.21 | — | Name of a custom search parameter bundle file. | -| ENABLE_ADMIN_API | — | v0.26 | — | Set to `true` if the optional Admin API should be enabled. Needed by the frontend. | +| Name | Default | Since | Depr ¹ | Description | +|:----------------------------------------|:---------------------------|:-------|---------|:------------------------------------------------------------------------------------------------------------| +| PROXY_HOST | — | v0.6 | — | REMOVED: use -Dhttp.proxyHost | +| PROXY_PORT | — | v0.6 | — | REMOVED: use -Dhttp.proxyPort | +| PROXY_USER | — | v0.6.1 | — | REMOVED: try [SOCKS Options][1] | +| PROXY_PASSWORD | — | v0.6.1 | — | REMOVED: try [SOCKS Options][1] | +| CONNECTION_TIMEOUT | 5 s | v0.6.3 | — | connection timeout for outbound HTTP requests | +| REQUEST_TIMEOUT | 30 s | v0.6.3 | — | REMOVED | +| TERM_SERVICE_URI | [http://tx.fhir.org/r4][6] | v0.6 | v0.11 | Base URI of the terminology service | +| BASE_URL | `http://localhost:8080` | — | — | The URL under which Blaze is accessible by clients. | +| CONTEXT_PATH | /fhir | v0.11 | — | Context path under which the FHIR RESTful API will be accessible. | +| SERVER_PORT | 8080 | — | — | The port of the main HTTP server | +| METRICS_SERVER_PORT | 8081 | v0.6 | — | The port of the Prometheus metrics server | +| LOG_LEVEL | info | v0.6 | — | one of trace, debug, info, warn or error | +| JAVA_TOOL_OPTIONS | — | — | — | JVM options \(Docker only\) | +| FHIR_OPERATION_EVALUATE_MEASURE_THREADS | — | v0.8 | v0.23.3 | The number threads used for $evaluate-measure executions. | +| FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT | 3600000 (1h) | v0.19 | — | Timeout in milliseconds for synchronous $evaluate-measure executions. | +| OPENID_PROVIDER_URL | — | v0.11 | — | [OpenID Connect][4] provider URL to enable [authentication][5] | +| OPENID_CLIENT_TRUST_STORE | — | v0.26 | — | A PKCS #12 trust store containing CA certificates needed for the [OpenID Connect][4] provider. | +| OPENID_CLIENT_TRUST_STORE_PASS | — | v0.26 | — | The password for the PKCS #12 trust store. | +| ENFORCE_REFERENTIAL_INTEGRITY | true | v0.14 | — | Enforce referential integrity on resource create, update and delete. | +| DB_SYNC_TIMEOUT | 10000 | v0.15 | — | Timeout in milliseconds for all reading FHIR interactions acquiring the newest database state. | +| DB_SEARCH_PARAM_BUNDLE | — | v0.21 | — | Name of a custom search parameter bundle file. | +| ENABLE_ADMIN_API | — | v0.26 | — | Set to `true` if the optional Admin API should be enabled. Needed by the frontend. | +| CQL_EXPR_CACHE_SIZE | — | v0.28 | — | Size of the CQL expression cache in MiB. This cache is part of the JVM heap. Will be disabled if not given. | +| CQL_EXPR_CACHE_REFRESH | PT24H | v0.28 | — | The duration after which a Bloom filter of the CQL expression cache will be refreshed. | +| CQL_EXPR_CACHE_THREADS | 4 | v0.28 | — | The maximum number of parallel Bloom filter calculations for the CQL expression cache. | ¹ Deprecated diff --git a/docs/deployment/manual-deployment.md b/docs/deployment/manual-deployment.md index c1a7edeb7..949aef0a3 100644 --- a/docs/deployment/manual-deployment.md +++ b/docs/deployment/manual-deployment.md @@ -2,12 +2,12 @@ The installation works under Windows, Linux and macOS. The only dependency is an installed OpenJDK 11 or 17 with 17 recommended. Blaze is tested with [Eclipse Temurin][1]. -Blaze runs on the JVM and comes as single JAR file. Download the most recent version [here](https://github.com/samply/blaze/releases/tag/v0.27.1). Look for `blaze-0.27.1-standalone.jar`. +Blaze runs on the JVM and comes as single JAR file. Download the most recent version [here](https://github.com/samply/blaze/releases/tag/v0.28.0). Look for `blaze-0.28.0-standalone.jar`. After the download, you can start blaze with the following command (Linux, macOS): ```sh -java -jar blaze-0.27.1-standalone.jar +java -jar blaze-0.28.0-standalone.jar ``` Blaze will run with an in-memory, volatile database for testing and demo purposes. @@ -17,14 +17,14 @@ Blaze can be run with durable storage by setting the environment variables `STOR Under Linux/macOS: ```sh -STORAGE=standalone java -jar blaze-0.27.1-standalone.jar +STORAGE=standalone java -jar blaze-0.28.0-standalone.jar ``` Under Windows, you need to set the Environment variables in the PowerShell before starting Blaze: ```powershell $Env:STORAGE="standalone" -java -jar blaze-0.27.1-standalone.jar +java -jar blaze-0.28.0-standalone.jar ``` This will create three directories called `index`, `transaction` and `resource` inside the current working directory, one for each database part used. @@ -42,7 +42,7 @@ The output should look like this: 2021-06-27T11:02:37.834Z ee086ef908c1 main INFO [blaze.core:64] - JVM version: 16.0.2 2021-06-27T11:02:37.834Z ee086ef908c1 main INFO [blaze.core:65] - Maximum available memory: 1738 MiB 2021-06-27T11:02:37.835Z ee086ef908c1 main INFO [blaze.core:66] - Number of available processors: 8 -2021-06-27T11:02:37.836Z ee086ef908c1 main INFO [blaze.core:67] - Successfully started Blaze version 0.27.1 in 8.2 seconds +2021-06-27T11:02:37.836Z ee086ef908c1 main INFO [blaze.core:67] - Successfully started Blaze version 0.28.0 in 8.2 seconds ``` In order to test connectivity, query the health endpoint: @@ -62,7 +62,7 @@ that should return: ```json { "name": "Blaze", - "version": "0.27.1" + "version": "0.28.0" } ``` diff --git a/docs/deployment/standalone-backend.md b/docs/deployment/standalone-backend.md index 6418eb93c..dca3d57aa 100644 --- a/docs/deployment/standalone-backend.md +++ b/docs/deployment/standalone-backend.md @@ -27,7 +27,7 @@ Blaze should log something like this: 2023-06-09T08:30:30.126Z b45689460ff3 main INFO [blaze.core:67] - JVM version: 17.0.7 2023-06-09T08:30:30.126Z b45689460ff3 main INFO [blaze.core:68] - Maximum available memory: 1738 MiB 2023-06-09T08:30:30.126Z b45689460ff3 main INFO [blaze.core:69] - Number of available processors: 2 -2023-06-09T08:30:30.126Z b45689460ff3 main INFO [blaze.core:70] - Successfully started 🔥 Blaze version 0.27.1 in 9.0 seconds +2023-06-09T08:30:30.126Z b45689460ff3 main INFO [blaze.core:70] - Successfully started 🔥 Blaze version 0.28.0 in 9.0 seconds ``` In order to test connectivity, query the health endpoint: @@ -47,7 +47,7 @@ that should return: ```json { "name": "Blaze", - "version": "0.27.1" + "version": "0.28.0" } ``` diff --git a/docs/implementation/cql.md b/docs/implementation/cql.md new file mode 100644 index 000000000..178f353cb --- /dev/null +++ b/docs/implementation/cql.md @@ -0,0 +1,88 @@ +# CQL + +## Expression Cache + +* bloom filter + * the set we like to build is the set of expressions returning true + * if the bloom filter returns false, we can be sure that the expression is not in the set of expressions returning true, so it will certainly return false + * the number of expressions returning true is far less then the number of expressions returning false + * we'll fill the Bloom filter with the expressions that returned true + * the problem is the following + * if we don't have filled the filter with all expression, the answer we get has no value + * we don't know whether the expression isn't in the set because we didn't put it there or because if returned false + * but we could use two filters + * one for all expressions returning true and one for all returning false + * if the expression isn't in both filters, it is new and so we have the evaluate it + * we could use a bloom filter for each expression + * then we would see whether we have a bloom filter or not + * we would insert Patients for which the expression returns true + * the first query evaluation is only used for insertion + * after that we mark the filter as ready for query + * now we can determine whether a expression is not true for a certain Patient + * + +* we'll use one Bloom filter per expression +* that Bloom filters will be stored in a Caffeine cache by expression hash +* each Bloom filter will be assigned the t of its creation +* the Bloom filters of all expressions of a query will be collected at the start of the query evaluation +* if a Bloom filter isn't found, its calculation will be queued and carried out asynchronously +* existing Bloom filters are immutable and will be used in query evaluation + * the Patient ID will be used to test whether this Patient isn't in the Bloom filter + * if the Patient ID wasn't found, the expression will return false + * if the Patient ID is found, the expression will be evaluated normally + +### Bloom Filter Calculation + +* the Bloom filter will be calculated for a particular exists expression +* it will be calculated based on a database with a particular t +* that t will be assigned to the Bloom filter +* the calculation will evaluate the expression for each patent of the database +* the ID's of Patients for which the expression returns true will be put into the Bloom filter + +## Or Expressions + +Expressions like: + +``` +define InInitialPopulation: + exists [Observation] or + exists [Condition] or + exists [Encounter] or + exists [Specimen] +``` + +are compiled to Blaze expressions: + +```edn +(or + (or + (or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (exists + (retrieve + "Encounter"))) + (exists + (retrieve + "Specimen"))) +``` + +which can be represented as the following tree + +``` + ^ + ^ | + ^ | | +O C E S +``` + +``` + ^ +| ^ +| | ^ +S E C O +``` diff --git a/docs/implementation/database.md b/docs/implementation/database.md index fff496494..87f2b15bf 100644 --- a/docs/implementation/database.md +++ b/docs/implementation/database.md @@ -38,16 +38,17 @@ There are two different sets of indices, ones which depend on the database value ### Indices depending on t -| Name | Key Parts | Value | -|--------------|-----------|-------------------------------| -| ResourceAsOf | type id t | content-hash, num-changes, op | -| TypeAsOf | type t id | content-hash, num-changes, op | -| SystemAsOf | t type id | content-hash, num-changes, op | -| TxSuccess | t | instant | -| TxError | t | anomaly | -| TByInstant | instant | t | -| TypeStats | type t | total, num-changes | -| SystemStats | t | total, num-changes | +| Name | Key Parts | Value | +|-------------------|-----------|-------------------------------| +| ResourceAsOf | type id t | content-hash, num-changes, op | +| TypeAsOf | type t id | content-hash, num-changes, op | +| SystemAsOf | t type id | content-hash, num-changes, op | +| PatientLastChange | pat-id t | - | +| TxSuccess | t | instant | +| TxError | t | anomaly | +| TByInstant | instant | t | +| TypeStats | type t | total, num-changes | +| SystemStats | t | total, num-changes | #### ResourceAsOf @@ -83,12 +84,16 @@ In addition to direct resource lookup, the `ResourceAsOf` index is used for list #### TypeAsOf -The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `id`, the entries will be first ordered by time, newest first, and second by resource identifier. +The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `id`, the entries will be first ordered by time, newest first, and second by resource identifier. #### SystemAsOf In the same way the `TypeAsOf` index uses a different key ordering in comparison to the `ResourceAsOf` index, the `SystemAsOf` index will use the key order `t`, `type` and `id` in order to provide a global time axis order by resource type and by identifier secondarily. +#### PatientLastChange + +The `PatientLastChange` index contains all changes to resources in the compartment of a particular Patient on reverse chronological order. Using the `PatientLastChange` index it's possible to detect the `t` of the last change in a Patient compartment. The CQL cache uses this index to invalidate cached results of expressions in the Patient context. + #### TxSuccess The `TxSuccess` index contains the real point in time, as `java.time.Instant`, successful transactions happened. In other words, this index maps each `t` which is just a monotonically increasing number to a real point in time. @@ -115,14 +120,14 @@ The `SystemStats` index keeps track of the total number of resources, and the nu The indices not depending on `t` directly point to the resource versions by their content hash. -| Name | Key Parts | Value | +| Name | Key Parts | Value | |-------------------------------------|----------------------------------------------------------------|-------| | SearchParamValueResource | search-param, type, value, id, hash-prefix | - | | ResourceSearchParamValue | type, id, hash-prefix, search-param, value | - | | CompartmentSearchParamValueResource | comp-code, comp-id, search-param, type, value, id, hash-prefix | - | | CompartmentResourceType | comp-code, comp-id, type, id | - | | SearchParam | code, type | id | -| ActiveSearchParams | id | - | +| ActiveSearchParams | id | - | #### SearchParamValueResource diff --git a/docs/monitoring/blaze.json b/docs/monitoring/blaze.json index 33d5b5886..f61b26da3 100644 --- a/docs/monitoring/blaze.json +++ b/docs/monitoring/blaze.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 74, + "id": 79, "links": [ { "asDropdown": false, @@ -447,7 +447,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "blaze_db_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}", + "expr": "blaze_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}", "hide": false, "interval": "", "legendFormat": "", @@ -546,7 +546,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_db_cache_hits_total[1m]) + rate(blaze_db_cache_misses_total[1m]))", + "expr": "rate(blaze_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_cache_hits_total[1m]) + rate(blaze_cache_misses_total[1m]))", "hide": false, "interval": "", "legendFormat": "", @@ -644,7 +644,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", + "expr": "rate(blaze_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", "hide": false, "interval": "", "legendFormat": "", @@ -742,7 +742,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", + "expr": "rate(blaze_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", "hide": false, "interval": "", "legendFormat": "", @@ -754,142 +754,143 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 14 }, "id": 160, - "panels": [], - "title": "RocksDB Block Cache", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 162, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "blaze_rocksdb_block_cache_usage_bytes{job=\"$job\",instance=\"$instance\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "normal", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": true, - "expr": "blaze_rocksdb_block_cache_pinned_usage_bytes{job=\"$job\",instance=\"$instance\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "pinned", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "B", - "step": 10 + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 162, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "blaze_rocksdb_block_cache_usage_bytes{job=\"$job\",instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "normal", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "blaze_rocksdb_block_cache_pinned_usage_bytes{job=\"$job\",instance=\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "pinned", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Usage Bytes", + "type": "timeseries" } ], - "title": "Usage Bytes", - "type": "timeseries" + "title": "RocksDB Block Cache", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -898,1347 +899,3018 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 15 }, "id": 68, - "panels": [], - "repeat": "database", - "repeatDirection": "h", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "refId": "A" - } - ], - "title": "RocksDB $database Database", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, - "thresholdsStyle": { - "mode": "off" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_block_cache_data_hit_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m]) / (rate(blaze_rocksdb_block_cache_data_hit_total[1m]) + rate(blaze_rocksdb_block_cache_data_miss_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Data Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Added blocks per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, + "unit": "ops" + }, + "overrides": [ { - "color": "red", - "value": 80 + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] }, - "unit": "percentunit" + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_block_cache_data_add_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Data Additions", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 23 - }, - "id": 63, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_block_cache_data_insert_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Data Insert Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_block_cache_index_hit_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m]) / (rate(blaze_rocksdb_block_cache_index_hit_total[1m]) + rate(blaze_rocksdb_block_cache_index_miss_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Index Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Added blocks per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 22 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_block_cache_index_add_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Index Additions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_block_cache_index_insert_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Index Insert Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_keys_read_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "read", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_keys_written_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "written", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_keys_updated_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "updated", + "range": true, + "refId": "C" + } + ], + "title": "Key Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_seek_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "seek", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_next_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "next", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_prev_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "prev", + "range": true, + "refId": "C" + } + ], + "title": "Iterator Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 28 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_iterators_created_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "created", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_iterators_deleted_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "deleted", + "range": true, + "refId": "B" + } + ], + "title": "Iterators Created/Deleted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Open Iterators. Should be zero. If not please file an issue.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 266, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "blaze_rocksdb_iterators_created_total{job=\"$job\",instance=\"$instance\", name=\"$database\"} - blaze_rocksdb_iterators_deleted_total", + "hide": false, + "interval": "", + "legendFormat": "created", + "range": true, + "refId": "A" + } + ], + "title": "Iterators Open", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 34 + }, + "id": 114, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_compression_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": " {{name}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Compression Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 115, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_decompression_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Decompression Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 113, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_compaction_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": " {{name}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Compaction Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_stall_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Stall Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_wal_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "WAL Bytes Written", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 155, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_wal_syncs_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "WAL Sync", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_block_cache_data_hit_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m]) / (rate(blaze_rocksdb_block_cache_data_hit_total[1m]) + rate(blaze_rocksdb_block_cache_data_miss_total[1m]))", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Data Hit Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Added blocks per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 112, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 119, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_flush_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": " {{name}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Memtable Flush Seconds", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_block_cache_data_add_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Data Additions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 121, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "editorMode": "code", + "expr": "rate(blaze_rocksdb_file_opens_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "File Opens", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_block_cache_data_insert_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Data Insert Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "description": "The total number of writes ending up with a timeout.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, - "thresholdsStyle": { - "mode": "off" - } + "overrides": [] }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 29 - }, - "id": 69, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_write_timeout_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Timeouts", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_block_cache_index_hit_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m]) / (rate(blaze_rocksdb_block_cache_index_hit_total[1m]) + rate(blaze_rocksdb_block_cache_index_miss_total[1m]))", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Index Hit Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Added blocks per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 29 - }, - "id": 79, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_block_cache_index_add_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Index Additions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 52 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "id": 157, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 29 - }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_blocks_decompressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks Decompressed", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_block_cache_index_insert_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Block Cache Index Insert Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 234, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 35 - }, - "id": 64, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_keys_read_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_keys_written_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "written", - "range": true, - "refId": "B" + "editorMode": "code", + "expr": "rate(blaze_rocksdb_bloom_filter_useful_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "useful", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_bloom_filter_full_positive_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "FullFilter not avoided", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(blaze_rocksdb_bloom_filter_full_true_positive_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "FullFilter not avoided but exists", + "range": true, + "refId": "C" + } + ], + "title": "Bloom Filter", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_keys_updated_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "updated", - "range": true, - "refId": "C" - } - ], - "title": "Key Operations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, + "unit": "ops" + }, + "overrides": [ { - "color": "red", - "value": 80 + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 35 - }, - "id": 65, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 58 }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_seek_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "seek", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "id": 156, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_next_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "next", - "range": true, - "refId": "B" + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_blocks_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks Compressed", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_prev_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "prev", - "range": true, - "refId": "C" - } - ], - "title": "Iterator Operations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 58 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 35 - }, - "id": 95, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": true, + "expr": "rate(blaze_rocksdb_blocks_not_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks Not Compressed", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_iterators_created_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "created", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "description": "Values are per Column Family and stacked.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_iterators_deleted_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "deleted", - "range": true, - "refId": "B" - } - ], - "title": "Iterators Created/Deleted", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 58 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 186, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 41 - }, - "id": 113, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "exemplar": false, + "expr": "blaze_rocksdb_table_reader_usage_bytes{job=\"$job\",instance=\"$instance\", name=\"$database\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{column_family}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "Table Reader Memory Usage", + "type": "timeseries" } - }, - "pluginVersion": "8.4.4", + ], + "repeat": "database", + "repeatDirection": "h", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_compaction_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": " {{name}}", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Compaction Seconds", - "type": "timeseries" + "title": "RocksDB $database Database", + "type": "row" }, { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 59, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(http_fhir_requests_total{job=\"$job\",instance=\"$instance\"}[1m])) by (interaction, code)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{interaction}} {{code}}", + "refId": "A", + "step": 10 + } + ], + "title": "Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "repeat": "quantile", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 41 - }, - "id": 114, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(${quantile:raw}, sum(rate(http_fhir_request_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (interaction, method, le))", + "interval": "", + "legendFormat": "{{interaction}} ({{method}})", + "range": true, + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "Durations $quantile Quantile", + "type": "timeseries" } - }, - "pluginVersion": "8.4.4", + ], "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_compression_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": " {{name}}", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Compression Seconds", - "type": "timeseries" + "title": "FHIR RESTful API Requests", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 261, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 259, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "repeat": "quantile", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 41 - }, - "id": 115, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(${quantile:raw}, sum(rate(fhir_generate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (format, le))", + "interval": "", + "legendFormat": "{{format}}", + "range": true, + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "Durations $quantile Quantile", + "type": "timeseries" } + ], + "title": "HTTP Response Generation", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "pluginVersion": "8.4.4", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 46, + "panels": [], "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_decompression_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 10 + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" } ], - "title": "Decompression Seconds", - "type": "timeseries" + "title": "Evaluate Measure", + "type": "row" }, { "datasource": { @@ -2296,7 +3968,7 @@ } ] }, - "unit": "ops" + "unit": "s" }, "overrides": [] }, @@ -2304,39 +3976,34 @@ "h": 6, "w": 8, "x": 0, - "y": 47 + "y": 19 }, - "id": 155, + "id": 48, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_wal_syncs_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, + "expr": "histogram_quantile(0.5, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", + "legendFormat": "{{subject_type}}", "refId": "A" } ], - "title": "WAL Sync", + "title": "Evaluation Durations 0.5 Quantile", "type": "timeseries" }, { @@ -2344,7 +4011,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -2374,7 +4040,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2396,7 +4062,7 @@ } ] }, - "unit": "percentunit" + "unit": "s" }, "overrides": [] }, @@ -2404,39 +4070,34 @@ "h": 6, "w": 8, "x": 8, - "y": 47 + "y": 19 }, - "id": 94, + "id": 49, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_stall_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, + "expr": "histogram_quantile(0.9, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", + "legendFormat": "{{subject_type}}", "refId": "A" } ], - "title": "Stall Seconds", + "title": "Evaluation Durations 0.9 Quantile", "type": "timeseries" }, { @@ -2444,7 +4105,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -2474,7 +4134,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2496,7 +4156,7 @@ } ] }, - "unit": "binBps" + "unit": "s" }, "overrides": [] }, @@ -2504,39 +4164,34 @@ "h": 6, "w": 8, "x": 16, - "y": 47 + "y": 19 }, - "id": 97, + "id": 50, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_wal_bytes_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "range": true, + "expr": "histogram_quantile(0.99, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", + "legendFormat": "{{subject_type}}", "refId": "A" } ], - "title": "WAL Bytes Written", + "title": "Evaluation Durations 0.99 Quantile", "type": "timeseries" }, { @@ -2544,7 +4199,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of writes ending up with a timeout.", + "description": "Estimated number of CQL expressions covered by a Bloom filter.", "fieldConfig": { "defaults": { "color": { @@ -2574,7 +4229,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2596,17 +4251,17 @@ } ] }, - "unit": "ops" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 8, + "w": 6, "x": 0, - "y": 53 + "y": 25 }, - "id": 98, + "id": 271, "options": { "legend": { "calcs": [], @@ -2627,16 +4282,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_write_timeout_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "blaze_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"cql-expr-cache\"}", "hide": false, "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "", "range": true, "refId": "A" } ], - "title": "Write Timeouts", + "title": "Estimated Number of Bloom Filters", "type": "timeseries" }, { @@ -2681,6 +4335,7 @@ }, "links": [], "mappings": [], + "max": 1, "min": 0, "thresholds": { "mode": "absolute", @@ -2701,17 +4356,14 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 8, - "y": 53 + "w": 6, + "x": 6, + "y": 25 }, - "id": 112, - "links": [], + "id": 272, "options": { "legend": { - "calcs": [ - "mean" - ], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2729,19 +4381,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_flush_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "format": "time_series", + "expr": "rate(blaze_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"cql-expr-cache\"}[1m]) / (rate(blaze_cache_hits_total[1m]) + rate(blaze_cache_misses_total[1m]))", + "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": " {{name}}", - "metric": "process_cpu_seconds_total", + "legendFormat": "", "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Memtable Flush Seconds", + "title": "Hit Ratio", "type": "timeseries" }, { @@ -2778,7 +4426,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2806,11 +4454,11 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 16, - "y": 53 + "w": 6, + "x": 12, + "y": 25 }, - "id": 154, + "id": 273, "options": { "legend": { "calcs": [], @@ -2831,15 +4479,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_rocksdb_file_opens_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"cql-expr-cache\"}[1m])", "hide": false, "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "", "range": true, "refId": "A" } ], - "title": "File Opens", + "title": "Loads", "type": "timeseries" }, { @@ -2847,7 +4495,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -2874,10 +4521,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2901,40 +4548,15 @@ }, "unit": "ops" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 59 + "w": 6, + "x": 18, + "y": 25 }, - "id": 156, + "id": 274, "options": { "legend": { "calcs": [], @@ -2955,16 +4577,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_blocks_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"cql-expr-cache\"}[1m])", "hide": false, "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "", "range": true, "refId": "A" } ], - "title": "Blocks Compressed", + "title": "Evictions", "type": "timeseries" }, { @@ -2972,7 +4593,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3002,7 +4622,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -3010,6 +4630,7 @@ }, "links": [], "mappings": [], + "max": 1, "min": 0, "thresholds": { "mode": "absolute", @@ -3024,42 +4645,17 @@ } ] }, - "unit": "ops" + "unit": "percentunit" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 8, - "y": 59 + "x": 0, + "y": 31 }, - "id": 157, + "id": 275, "options": { "legend": { "calcs": [], @@ -3080,16 +4676,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_blocks_decompressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_cql_expr_cache_bloom_filter_useful_total{job=\"$job\",instance=\"$instance\"}[1m]) / (rate(blaze_cql_expr_cache_bloom_filter_useful_total[1m]) + rate(blaze_cql_expr_cache_bloom_filter_not_useful_total[1m]))", "hide": false, "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "", "range": true, "refId": "A" } ], - "title": "Blocks Decompressed", + "title": "Bloom Filter Useful Ratio", "type": "timeseries" }, { @@ -3097,6 +4692,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Should be always 1%.", "fieldConfig": { "defaults": { "color": { @@ -3134,6 +4730,7 @@ }, "links": [], "mappings": [], + "max": 0.02, "min": 0, "thresholds": { "mode": "absolute", @@ -3148,23 +4745,23 @@ } ] }, - "unit": "ops" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 59 + "x": 9, + "y": 31 }, - "id": 234, + "id": 276, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -3179,41 +4776,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_rocksdb_bloom_filter_useful_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_cql_expr_cache_bloom_filter_false_positive_total{job=\"$job\",instance=\"$instance\"}[1m]) / (rate(blaze_cql_expr_cache_bloom_filter_useful_total[1m]))", "hide": false, "interval": "", - "legendFormat": "useful", + "legendFormat": "", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_bloom_filter_full_positive_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "FullFilter not avoided", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(blaze_rocksdb_bloom_filter_full_true_positive_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "FullFilter not avoided but exists", - "range": true, - "refId": "C" } ], - "title": "Bloom Filter", + "title": "Bloom Filter False Positive Rate", "type": "timeseries" }, { @@ -3221,7 +4792,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Values are per Column Family and stacked.", "fieldConfig": { "defaults": { "color": { @@ -3248,10 +4818,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -3273,23 +4843,22 @@ } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 16, + "w": 8, "x": 0, - "y": 65 + "y": 37 }, - "id": 186, - "links": [], + "id": 277, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -3297,7 +4866,7 @@ "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { @@ -3305,19 +4874,13 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": false, - "expr": "blaze_rocksdb_table_reader_usage_bytes{job=\"$job\",instance=\"$instance\", name=\"$database\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{column_family}}", - "metric": "process_cpu_seconds_total", + "expr": "histogram_quantile(0.5, sum(rate(blaze_cql_expr_cache_bloom_filter_creation_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "legendFormat": "{{subject_type}}", "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Table Reader Memory Usage", + "title": "Bloom Filter Creation Durations 0.5 Quantile", "type": "timeseries" }, { @@ -3325,7 +4888,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3355,7 +4917,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -3377,55 +4939,30 @@ } ] }, - "unit": "ops" + "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 65 + "x": 8, + "y": 37 }, - "id": 158, + "id": 280, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { @@ -3433,44 +4970,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(blaze_rocksdb_blocks_not_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", + "expr": "histogram_quantile(0.9, sum(rate(blaze_cql_expr_cache_bloom_filter_creation_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "legendFormat": "{{subject_type}}", "range": true, "refId": "A" } ], - "title": "Blocks Not Compressed", + "title": "Bloom Filter Creation Durations 0.9 Quantile", "type": "timeseries" }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 71 - }, - "id": 59, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "FHIR RESTful API Requests", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -3527,18 +5035,17 @@ } ] }, - "unit": "ops" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 24, - "x": 0, - "y": 72 + "w": 8, + "x": 16, + "y": 37 }, - "id": 61, - "links": [], + "id": 281, "options": { "legend": { "calcs": [], @@ -3551,24 +5058,21 @@ "sort": "none" } }, - "pluginVersion": "8.4.4", + "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(http_fhir_requests_total{job=\"$job\",instance=\"$instance\"}[1m])) by (interaction, code)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{interaction}} {{code}}", - "refId": "A", - "step": 10 + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(blaze_cql_expr_cache_bloom_filter_creation_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "legendFormat": "{{subject_type}}", + "range": true, + "refId": "A" } ], - "title": "Total", + "title": "Bloom Filter Creation Durations 0.99 Quantile", "type": "timeseries" }, { @@ -3627,7 +5131,7 @@ } ] }, - "unit": "s" + "unit": "bytes" }, "overrides": [] }, @@ -3635,9 +5139,9 @@ "h": 6, "w": 8, "x": 0, - "y": 78 + "y": 43 }, - "id": 62, + "id": 278, "options": { "legend": { "calcs": [], @@ -3651,8 +5155,6 @@ } }, "pluginVersion": "8.4.4", - "repeat": "quantile", - "repeatDirection": "h", "targets": [ { "datasource": { @@ -3661,35 +5163,21 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(${quantile:raw}, sum(rate(http_fhir_request_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (interaction, method, le))", + "expr": "histogram_quantile(0.5, sum(rate(blaze_cql_expr_cache_bloom_filter_bytes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", "interval": "", - "legendFormat": "{{interaction}} ({{method}})", + "legendFormat": "{{op}}", "range": true, "refId": "B" } ], - "title": "Durations $quantile Quantile", + "title": "Bloom Filter Bytes 0.5 Quantile", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 84 - }, - "id": 261, - "panels": [], - "title": "HTTP Response Generation", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3741,17 +5229,17 @@ } ] }, - "unit": "s" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 0, - "y": 85 + "x": 8, + "y": 43 }, - "id": 259, + "id": 279, "options": { "legend": { "calcs": [], @@ -3765,8 +5253,6 @@ } }, "pluginVersion": "8.4.4", - "repeat": "quantile", - "repeatDirection": "h", "targets": [ { "datasource": { @@ -3775,351 +5261,21 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(${quantile:raw}, sum(rate(fhir_generate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (format, le))", + "expr": "histogram_quantile(0.9, sum(rate(blaze_cql_expr_cache_bloom_filter_bytes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", "interval": "", - "legendFormat": "{{format}}", + "legendFormat": "{{op}}", "range": true, "refId": "B" } ], - "title": "Durations $quantile Quantile", + "title": "Bloom Filter Bytes 0.9 Quantile", "type": "timeseries" }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 91 - }, - "id": 46, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", - "legendFormat": "{{subject_type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Evaluation Durations 0.5 Quantile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 49 - }, - "hiddenSeries": false, - "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.9, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", - "legendFormat": "{{subject_type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Evaluation Durations 0.9 Quantile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 49 - }, - "hiddenSeries": false, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(fhir_evaluate_measure_evaluate_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (subject_type, le))", - "legendFormat": "{{subject_type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Evaluation Durations 0.99 Quantile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Evaluate Measure", - "type": "row" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 92 - }, - "id": 86, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Node", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "At 100 % the indexer is fully utilized.", "fieldConfig": { "defaults": { "color": { @@ -4157,7 +5313,6 @@ }, "links": [], "mappings": [], - "max": 1, "min": 0, "thresholds": { "mode": "absolute", @@ -4172,23 +5327,23 @@ } ] }, - "unit": "percentunit" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 0, - "y": 93 + "x": 16, + "y": 43 }, - "id": 22, + "id": 282, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", @@ -4202,628 +5357,748 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": true, - "expr": "1 - rate(blaze_db_node_duration_seconds_sum{job=\"$job\",instance=\"$instance\",op=\"poll-tx-queue\"}[1m])", + "expr": "histogram_quantile(0.99, sum(rate(blaze_cql_expr_cache_bloom_filter_bytes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", "interval": "", - "legendFormat": "", + "legendFormat": "{{op}}", + "range": true, "refId": "B" } ], - "title": "Indexer Utilization", + "title": "Bloom Filter Bytes 0.99 Quantile", "type": "timeseries" }, { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 86, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "At 100 % the indexer is fully utilized.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + "exemplar": true, + "expr": "1 - rate(blaze_db_node_duration_seconds_sum{job=\"$job\",instance=\"$instance\",op=\"poll-tx-queue\"}[1m])", + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Indexer Utilization", + "type": "timeseries" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index-resources" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index-resources" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 93 - }, - "id": 87, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 80 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate(blaze_db_node_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (op, le))", + "interval": "", + "legendFormat": "{{op}}", + "refId": "B" + } + ], + "title": "Durations 0.5 Quantile", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.5, sum(rate(blaze_db_node_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (op, le))", - "interval": "", - "legendFormat": "{{op}}", - "refId": "B" - } - ], - "title": "Durations 0.5 Quantile", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 80 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 93 - }, - "id": 88, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(blaze_db_node_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (op, le))", + "interval": "", + "legendFormat": "{{op}}", + "refId": "B" + } + ], + "title": "Durations 0.99 Quantile", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(blaze_db_node_duration_seconds_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (op, le))", - "interval": "", - "legendFormat": "{{op}}", - "refId": "B" - } - ], - "title": "Durations 0.99 Quantile", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Mean number of commands per transaction.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "Mean number of commands per transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 86 + }, + "id": 90, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 99 - }, - "id": 90, - "options": { - "legend": { - "calcs": [ - "mean" + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "interval": "", + "legendFormat": "", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "title": "Transaction Size 0.5 Quantile", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.5, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Transaction Size 0.5 Quantile", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "90% quantile of number of commands per transaction.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "90% quantile of number of commands per transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 86 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 99 - }, - "id": 91, - "options": { - "legend": { - "calcs": [ - "mean" + "exemplar": true, + "expr": "histogram_quantile(0.9, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "interval": "", + "legendFormat": "", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "title": "Transaction Size 0.9 Quantile", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.9, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Transaction Size 0.9 Quantile", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "99% quantile of number of commands per transaction.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "99% quantile of number of commands per transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 86 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 99 - }, - "id": 92, - "options": { - "legend": { - "calcs": [ - "mean" + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "interval": "", + "legendFormat": "", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "title": "Transaction Size 0.99 Quantile", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.4.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Transaction Size 0.99 Quantile", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "1% quantile of number of commands per transaction.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "1% quantile of number of commands per transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 92 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 105 - }, - "id": 93, - "options": { - "legend": { - "calcs": [ - "mean" + "exemplar": true, + "expr": "histogram_quantile(0.01, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", + "interval": "", + "legendFormat": "", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "Transaction Size 0.01 Quantile", + "type": "timeseries" } - }, - "pluginVersion": "8.4.4", + ], "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": true, - "expr": "histogram_quantile(0.01, sum(rate(blaze_db_node_transaction_sizes_bucket{job=\"$job\",instance=\"$instance\"}[1m])) by (le))", - "interval": "", - "legendFormat": "", - "refId": "B" + "refId": "A" } ], - "title": "Transaction Size 0.01 Quantile", - "type": "timeseries" + "title": "Node", + "type": "row" }, { "collapsed": true, @@ -4835,7 +6110,7 @@ "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 50 }, "id": 103, "panels": [ @@ -4886,7 +6161,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4902,7 +6178,7 @@ "h": 6, "w": 8, "x": 0, - "y": 132 + "y": 51 }, "id": 104, "options": { @@ -4981,7 +6257,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4997,7 +6274,7 @@ "h": 6, "w": 8, "x": 8, - "y": 132 + "y": 51 }, "id": 105, "options": { @@ -5076,7 +6353,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5092,7 +6370,7 @@ "h": 6, "w": 8, "x": 16, - "y": 132 + "y": 51 }, "id": 106, "options": { @@ -5171,7 +6449,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5187,7 +6466,7 @@ "h": 6, "w": 8, "x": 0, - "y": 138 + "y": 57 }, "id": 109, "options": { @@ -5268,7 +6547,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5284,7 +6564,7 @@ "h": 6, "w": 8, "x": 8, - "y": 138 + "y": 57 }, "id": 110, "options": { @@ -5365,7 +6645,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5381,7 +6662,7 @@ "h": 6, "w": 8, "x": 16, - "y": 138 + "y": 57 }, "id": 111, "options": { @@ -5438,7 +6719,7 @@ "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 51 }, "id": 31, "panels": [ @@ -6271,7 +7552,7 @@ "h": 1, "w": 24, "x": 0, - "y": 113 + "y": 52 }, "id": 24, "panels": [ @@ -6670,7 +7951,7 @@ "h": 1, "w": 24, "x": 0, - "y": 114 + "y": 53 }, "id": 37, "panels": [ @@ -7175,7 +8456,7 @@ "h": 1, "w": 24, "x": 0, - "y": 115 + "y": 54 }, "id": 34, "panels": [], @@ -7255,7 +8536,7 @@ "h": 7, "w": 8, "x": 0, - "y": 116 + "y": 55 }, "id": 4, "options": { @@ -7349,7 +8630,7 @@ "h": 7, "w": 8, "x": 8, - "y": 116 + "y": 55 }, "id": 42, "options": { @@ -7444,7 +8725,7 @@ "h": 7, "w": 8, "x": 16, - "y": 116 + "y": 55 }, "id": 83, "options": { @@ -7539,7 +8820,7 @@ "h": 7, "w": 8, "x": 0, - "y": 123 + "y": 62 }, "id": 2, "options": { @@ -7633,7 +8914,7 @@ "h": 7, "w": 8, "x": 8, - "y": 123 + "y": 62 }, "id": 43, "options": { @@ -7727,7 +9008,7 @@ "h": 7, "w": 8, "x": 16, - "y": 123 + "y": 62 }, "id": 82, "options": { @@ -7822,7 +9103,7 @@ "h": 7, "w": 8, "x": 0, - "y": 130 + "y": 69 }, "id": 122, "options": { @@ -7917,7 +9198,7 @@ "h": 7, "w": 8, "x": 8, - "y": 130 + "y": 69 }, "id": 210, "options": { @@ -7950,7 +9231,7 @@ "type": "timeseries" } ], - "refresh": "", + "refresh": "10s", "revision": 1, "schemaVersion": 38, "style": "dark", @@ -7985,7 +9266,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(blaze_db_cache_hits_total, job)", + "definition": "label_values(blaze_cache_hits_total, job)", "hide": 0, "includeAll": false, "label": "Job", @@ -7993,7 +9274,7 @@ "name": "job", "options": [], "query": { - "query": "label_values(blaze_db_cache_hits_total, job)", + "query": "label_values(blaze_cache_hits_total, job)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -8012,14 +9293,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)", + "definition": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], "query": { - "query": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)", + "query": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -8129,6 +9410,6 @@ "timezone": "", "title": "Blaze", "uid": "Q-h9isMWk", - "version": 1, + "version": 5, "weekStart": "" } diff --git a/docs/performance/cql.md b/docs/performance/cql.md index d36383746..909ae2841 100644 --- a/docs/performance/cql.md +++ b/docs/performance/cql.md @@ -87,42 +87,42 @@ The same can be said for the LEA58 system. | Dataset | System | Code | # Hits | Time (s) | StdDev | Pat./s | |---------|--------|---------|-------:|---------:|-------:|--------:| -| 100k | LEA25 | 17861-6 | 2 k | 0.16 | 0.011 | 630.6 k | -| 100k | LEA25 | 8310-5 | 60 k | 0.25 | 0.017 | 393.1 k | -| 100k | LEA25 | 72514-3 | 100 k | 0.29 | 0.009 | 341.5 k | -| 100k | LEA36 | 17861-6 | 2 k | 0.09 | 0.006 | 1.086 M | -| 100k | LEA36 | 8310-5 | 60 k | 0.12 | 0.003 | 807.1 k | -| 100k | LEA36 | 72514-3 | 100 k | 0.17 | 0.006 | 604.8 k | -| 100k | LEA47 | 17861-6 | 2 k | 0.06 | 0.002 | 1.629 M | -| 100k | LEA47 | 8310-5 | 60 k | 0.08 | 0.004 | 1.307 M | -| 100k | LEA47 | 72514-3 | 100 k | 0.09 | 0.003 | 1.068 M | -| 100k | LEA58 | 17861-6 | 2 k | 0.07 | 0.002 | 1.504 M | -| 100k | LEA58 | 8310-5 | 60 k | 0.08 | 0.001 | 1.298 M | -| 100k | LEA58 | 72514-3 | 100 k | 0.08 | 0.002 | 1.207 M | -| 100k-fh | LEA25 | 788-0 | 2 k | 0.21 | 0.009 | 475.3 k | -| 100k-fh | LEA25 | 44261-6 | 57 k | 0.30 | 0.012 | 331.4 k | -| 100k-fh | LEA25 | 72514-3 | 100 k | 0.38 | 0.021 | 265.8 k | -| 100k-fh | LEA36 | 788-0 | 2 k | 0.12 | 0.007 | 860.0 k | -| 100k-fh | LEA36 | 44261-6 | 57 k | 0.17 | 0.008 | 573.5 k | -| 100k-fh | LEA36 | 72514-3 | 100 k | 0.20 | 0.007 | 490.9 k | -| 100k-fh | LEA47 | 788-0 | 2 k | 0.07 | 0.002 | 1.415 M | -| 100k-fh | LEA47 | 44261-6 | 57 k | 0.10 | 0.002 | 995.8 k | -| 100k-fh | LEA47 | 72514-3 | 100 k | 0.12 | 0.004 | 809.9 k | -| 100k-fh | LEA58 | 788-0 | 2 k | 0.06 | 0.003 | 1.659 M | -| 100k-fh | LEA58 | 44261-6 | 57 k | 0.07 | 0.002 | 1.521 M | -| 100k-fh | LEA58 | 72514-3 | 100 k | 0.08 | 0.001 | 1.232 M | -| 1M | LEA25 | 17861-6 | 25 k | 8.04 | 0.059 | 124.3 k | -| 1M | LEA25 | 8310-5 | 603 k | 11.40 | 0.043 | 87.7 k | -| 1M | LEA25 | 72514-3 | 998 k | 13.16 | 0.049 | 76.0 k | -| 1M | LEA36 | 17861-6 | 25 k | 3.90 | 0.009 | 256.1 k | -| 1M | LEA36 | 8310-5 | 603 k | 5.74 | 0.023 | 174.2 k | -| 1M | LEA36 | 72514-3 | 998 k | 6.68 | 0.036 | 149.6 k | -| 1M | LEA47 | 17861-6 | 25 k | 0.59 | 0.003 | 1.705 M | -| 1M | LEA47 | 8310-5 | 603 k | 0.64 | 0.003 | 1.557 M | -| 1M | LEA47 | 72514-3 | 998 k | 0.76 | 0.006 | 1.324 M | -| 1M | LEA58 | 17861-6 | 25 k | 0.61 | 0.005 | 1.633 M | -| 1M | LEA58 | 8310-5 | 603 k | 0.67 | 0.005 | 1.495 M | -| 1M | LEA58 | 72514-3 | 998 k | 0.75 | 0.003 | 1.336 M | +| 100k | LEA25 | 17861-6 | 2 k | 0.08 | 0.005 | 1.203 M | +| 100k | LEA25 | 8310-5 | 60 k | 0.29 | 0.008 | 342.8 k | +| 100k | LEA25 | 72514-3 | 100 k | 0.42 | 0.008 | 237.7 k | +| 100k | LEA36 | 17861-6 | 2 k | 0.05 | 0.001 | 2.000 M | +| 100k | LEA36 | 8310-5 | 60 k | 0.15 | 0.003 | 667.8 k | +| 100k | LEA36 | 72514-3 | 100 k | 0.23 | 0.005 | 442.2 k | +| 100k | LEA47 | 17861-6 | 2 k | 0.05 | 0.002 | 1.873 M | +| 100k | LEA47 | 8310-5 | 60 k | 0.09 | 0.004 | 1.104 M | +| 100k | LEA47 | 72514-3 | 100 k | 0.14 | 0.004 | 731.9 k | +| 100k | LEA58 | 17861-6 | 2 k | 0.05 | 0.002 | 1.845 M | +| 100k | LEA58 | 8310-5 | 60 k | 0.08 | 0.001 | 1.315 M | +| 100k | LEA58 | 72514-3 | 100 k | 0.09 | 0.002 | 1.116 M | +| 100k-fh | LEA25 | 788-0 | 2 k | 0.08 | 0.005 | 1.299 M | +| 100k-fh | LEA25 | 44261-6 | 57 k | 0.26 | 0.017 | 379.7 k | +| 100k-fh | LEA25 | 72514-3 | 100 k | 0.40 | 0.025 | 250.9 k | +| 100k-fh | LEA36 | 788-0 | 2 k | 0.05 | 0.001 | 1.854 M | +| 100k-fh | LEA36 | 44261-6 | 57 k | 0.13 | 0.004 | 756.1 k | +| 100k-fh | LEA36 | 72514-3 | 100 k | 0.20 | 0.003 | 489.4 k | +| 100k-fh | LEA47 | 788-0 | 2 k | 0.05 | 0.001 | 1.938 M | +| 100k-fh | LEA47 | 44261-6 | 57 k | 0.08 | 0.003 | 1.332 M | +| 100k-fh | LEA47 | 72514-3 | 100 k | 0.09 | 0.002 | 1.100 M | +| 100k-fh | LEA58 | 788-0 | 2 k | 0.05 | 0.002 | 1.930 M | +| 100k-fh | LEA58 | 44261-6 | 57 k | 0.07 | 0.001 | 1.385 M | +| 100k-fh | LEA58 | 72514-3 | 100 k | 0.09 | 0.001 | 1.161 M | +| 1M | LEA25 | 17861-6 | 25 k | 0.47 | 0.011 | 2.148 M | +| 1M | LEA25 | 8310-5 | 603 k | 10.69 | 1.232 | 93.6 k | +| 1M | LEA25 | 72514-3 | 998 k | 16.74 | 1.959 | 59.8 k | +| 1M | LEA36 | 17861-6 | 25 k | 0.44 | 0.003 | 2.283 M | +| 1M | LEA36 | 8310-5 | 603 k | 4.61 | 0.031 | 217.0 k | +| 1M | LEA36 | 72514-3 | 998 k | 7.15 | 0.018 | 139.8 k | +| 1M | LEA47 | 17861-6 | 25 k | 0.47 | 0.004 | 2.138 M | +| 1M | LEA47 | 8310-5 | 603 k | 0.64 | 0.008 | 1.555 M | +| 1M | LEA47 | 72514-3 | 998 k | 0.97 | 0.009 | 1.032 M | +| 1M | LEA58 | 17861-6 | 25 k | 0.48 | 0.005 | 2.069 M | +| 1M | LEA58 | 8310-5 | 603 k | 0.63 | 0.004 | 1.587 M | +| 1M | LEA58 | 72514-3 | 998 k | 0.73 | 0.006 | 1.375 M | ### Example CQL Query @@ -195,42 +195,42 @@ For the LEA58 system, the relative performance is the same for all datasets. | Dataset | System | Code | Value | # Hits | Time (s) | StdDev | Pat./s | |---------|--------|---------|--------:|-------:|---------:|-------:|--------:| -| 100k | LEA25 | 29463-7 | 13.6 kg | 10 k | 2.64 | 0.033 | 37.9 k | -| 100k | LEA25 | 29463-7 | 75.3 kg | 50 k | 1.41 | 0.007 | 70.7 k | -| 100k | LEA25 | 29463-7 | 185 kg | 100 k | 0.35 | 0.018 | 285.1 k | -| 100k | LEA36 | 29463-7 | 13.6 kg | 10 k | 1.04 | 0.012 | 95.9 k | -| 100k | LEA36 | 29463-7 | 75.3 kg | 50 k | 0.71 | 0.006 | 141.3 k | -| 100k | LEA36 | 29463-7 | 185 kg | 100 k | 0.20 | 0.009 | 511.1 k | -| 100k | LEA47 | 29463-7 | 13.6 kg | 10 k | 0.68 | 0.004 | 147.2 k | -| 100k | LEA47 | 29463-7 | 75.3 kg | 50 k | 0.47 | 0.010 | 214.1 k | -| 100k | LEA47 | 29463-7 | 185 kg | 100 k | 0.12 | 0.004 | 855.7 k | -| 100k | LEA58 | 29463-7 | 13.6 kg | 10 k | 0.47 | 0.004 | 210.7 k | -| 100k | LEA58 | 29463-7 | 75.3 kg | 50 k | 0.31 | 0.002 | 320.2 k | -| 100k | LEA58 | 29463-7 | 185 kg | 100 k | 0.09 | 0.001 | 1.057 M | -| 100k-fh | LEA25 | 29463-7 | 13.6 kg | 100 k | 1.40 | 0.046 | 71.3 k | -| 100k-fh | LEA25 | 29463-7 | 75.3 kg | 100 k | 0.86 | 0.007 | 116.4 k | -| 100k-fh | LEA25 | 29463-7 | 185 kg | 100 k | 0.44 | 0.018 | 228.2 k | -| 100k-fh | LEA36 | 29463-7 | 13.6 kg | 100 k | 0.78 | 0.003 | 127.9 k | -| 100k-fh | LEA36 | 29463-7 | 75.3 kg | 100 k | 0.49 | 0.015 | 202.8 k | -| 100k-fh | LEA36 | 29463-7 | 185 kg | 100 k | 0.25 | 0.006 | 397.9 k | -| 100k-fh | LEA47 | 29463-7 | 13.6 kg | 100 k | 0.45 | 0.005 | 222.7 k | -| 100k-fh | LEA47 | 29463-7 | 75.3 kg | 100 k | 0.29 | 0.005 | 349.2 k | -| 100k-fh | LEA47 | 29463-7 | 185 kg | 100 k | 0.15 | 0.005 | 663.4 k | -| 100k-fh | LEA58 | 29463-7 | 13.6 kg | 100 k | 0.30 | 0.002 | 331.9 k | -| 100k-fh | LEA58 | 29463-7 | 75.3 kg | 100 k | 0.19 | 0.002 | 536.2 k | -| 100k-fh | LEA58 | 29463-7 | 185 kg | 100 k | 0.10 | 0.004 | 976.5 k | -| 1M | LEA25 | 29463-7 | 13.6 kg | 99 k | 719.84 | 3.734 | 1.4 k | -| 1M | LEA25 | 29463-7 | 75.3 kg | 500 k | 479.52 | 11.096 | 2.1 k | -| 1M | LEA25 | 29463-7 | 185 kg | 998 k | 103.51 | 40.442 | 9.7 k | -| 1M | LEA36 | 29463-7 | 13.6 kg | 99 k | 432.80 | 1.586 | 2.3 k | -| 1M | LEA36 | 29463-7 | 75.3 kg | 500 k | 265.29 | 1.618 | 3.8 k | -| 1M | LEA36 | 29463-7 | 185 kg | 998 k | 7.72 | 0.045 | 129.5 k | -| 1M | LEA47 | 29463-7 | 13.6 kg | 99 k | 138.82 | 1.378 | 7.2 k | -| 1M | LEA47 | 29463-7 | 75.3 kg | 500 k | 8.09 | 0.015 | 123.6 k | -| 1M | LEA47 | 29463-7 | 185 kg | 998 k | 1.03 | 0.004 | 973.7 k | -| 1M | LEA58 | 29463-7 | 13.6 kg | 99 k | 4.18 | 0.004 | 239.3 k | -| 1M | LEA58 | 29463-7 | 75.3 kg | 500 k | 2.67 | 0.008 | 374.2 k | -| 1M | LEA58 | 29463-7 | 185 kg | 998 k | 0.78 | 0.003 | 1.288 M | +| 100k | LEA25 | 29463-7 | 13.6 kg | 10 k | 0.35 | 0.019 | 286.1 k | +| 100k | LEA25 | 29463-7 | 75.3 kg | 50 k | 0.84 | 0.025 | 118.8 k | +| 100k | LEA25 | 29463-7 | 185 kg | 100 k | 1.23 | 0.022 | 81.1 k | +| 100k | LEA36 | 29463-7 | 13.6 kg | 10 k | 0.14 | 0.007 | 698.5 k | +| 100k | LEA36 | 29463-7 | 75.3 kg | 50 k | 0.36 | 0.011 | 275.7 k | +| 100k | LEA36 | 29463-7 | 185 kg | 100 k | 0.57 | 0.016 | 176.4 k | +| 100k | LEA47 | 29463-7 | 13.6 kg | 10 k | 0.08 | 0.007 | 1.252 M | +| 100k | LEA47 | 29463-7 | 75.3 kg | 50 k | 0.19 | 0.004 | 519.2 k | +| 100k | LEA47 | 29463-7 | 185 kg | 100 k | 0.35 | 0.016 | 286.2 k | +| 100k | LEA58 | 29463-7 | 13.6 kg | 10 k | 0.07 | 0.002 | 1.409 M | +| 100k | LEA58 | 29463-7 | 75.3 kg | 50 k | 0.13 | 0.004 | 780.9 k | +| 100k | LEA58 | 29463-7 | 185 kg | 100 k | 0.18 | 0.006 | 543.8 k | +| 100k-fh | LEA25 | 29463-7 | 13.6 kg | 100 k | 6.40 | 0.072 | 15.6 k | +| 100k-fh | LEA25 | 29463-7 | 75.3 kg | 100 k | 3.23 | 0.037 | 31.0 k | +| 100k-fh | LEA25 | 29463-7 | 185 kg | 100 k | 1.18 | 0.017 | 84.7 k | +| 100k-fh | LEA36 | 29463-7 | 13.6 kg | 100 k | 2.45 | 0.023 | 40.8 k | +| 100k-fh | LEA36 | 29463-7 | 75.3 kg | 100 k | 1.27 | 0.020 | 78.6 k | +| 100k-fh | LEA36 | 29463-7 | 185 kg | 100 k | 0.50 | 0.005 | 199.6 k | +| 100k-fh | LEA47 | 29463-7 | 13.6 kg | 100 k | 0.78 | 0.021 | 128.3 k | +| 100k-fh | LEA47 | 29463-7 | 75.3 kg | 100 k | 0.45 | 0.006 | 221.1 k | +| 100k-fh | LEA47 | 29463-7 | 185 kg | 100 k | 0.17 | 0.005 | 572.4 k | +| 100k-fh | LEA58 | 29463-7 | 13.6 kg | 100 k | 0.74 | 0.022 | 134.4 k | +| 100k-fh | LEA58 | 29463-7 | 75.3 kg | 100 k | 0.43 | 0.007 | 234.5 k | +| 100k-fh | LEA58 | 29463-7 | 185 kg | 100 k | 0.18 | 0.004 | 565.8 k | +| 1M | LEA25 | 29463-7 | 13.6 kg | 99 k | 2.91 | 0.169 | 344.2 k | +| 1M | LEA25 | 29463-7 | 75.3 kg | 500 k | 15.89 | 1.056 | 62.9 k | +| 1M | LEA25 | 29463-7 | 185 kg | 998 k | 27.61 | 0.948 | 36.2 k | +| 1M | LEA36 | 29463-7 | 13.6 kg | 99 k | 1.11 | 0.012 | 901.9 k | +| 1M | LEA36 | 29463-7 | 75.3 kg | 500 k | 3.19 | 0.042 | 313.1 k | +| 1M | LEA36 | 29463-7 | 185 kg | 998 k | 10.61 | 0.030 | 94.3 k | +| 1M | LEA47 | 29463-7 | 13.6 kg | 99 k | 0.60 | 0.018 | 1.664 M | +| 1M | LEA47 | 29463-7 | 75.3 kg | 500 k | 1.55 | 0.012 | 646.4 k | +| 1M | LEA47 | 29463-7 | 185 kg | 998 k | 2.17 | 0.022 | 460.9 k | +| 1M | LEA58 | 29463-7 | 13.6 kg | 99 k | 0.57 | 0.012 | 1.754 M | +| 1M | LEA58 | 29463-7 | 75.3 kg | 500 k | 0.93 | 0.011 | 1.078 M | +| 1M | LEA58 | 29463-7 | 185 kg | 998 k | 1.30 | 0.022 | 772.0 k | ### CQL Query @@ -302,30 +302,30 @@ The same can be said for the LEA58 system. | Dataset | System | # Hits | Time (s) | StdDev | Pat./s | |---------|--------|-------:|---------:|-------:|--------:| -| 100k | LEA25 | 395 | 0.70 | 0.011 | 142.0 k | -| 100k | LEA25 | 95 k | 0.44 | 0.022 | 227.6 k | -| 100k | LEA36 | 395 | 0.40 | 0.010 | 249.3 k | -| 100k | LEA36 | 95 k | 0.22 | 0.009 | 448.5 k | -| 100k | LEA47 | 395 | 0.23 | 0.001 | 437.5 k | -| 100k | LEA47 | 95 k | 0.14 | 0.002 | 731.9 k | -| 100k | LEA58 | 395 | 0.16 | 0.001 | 607.1 k | -| 100k | LEA58 | 95 k | 0.11 | 0.002 | 941.9 k | -| 100k-fh | LEA25 | 2 k | 1.30 | 0.008 | 76.7 k | -| 100k-fh | LEA25 | 98 k | 0.47 | 0.006 | 214.4 k | -| 100k-fh | LEA36 | 2 k | 0.75 | 0.008 | 133.1 k | -| 100k-fh | LEA36 | 98 k | 0.29 | 0.009 | 343.9 k | -| 100k-fh | LEA47 | 2 k | 0.45 | 0.003 | 224.7 k | -| 100k-fh | LEA47 | 98 k | 0.16 | 0.003 | 628.0 k | -| 100k-fh | LEA58 | 2 k | 0.31 | 0.003 | 322.3 k | -| 100k-fh | LEA58 | 98 k | 0.10 | 0.002 | 1.0 M | -| 1M | LEA25 | 4 k | 13.60 | 0.073 | 73.5 k | -| 1M | LEA25 | 954 k | 11.87 | 0.027 | 84.2 k | -| 1M | LEA36 | 4 k | 7.24 | 0.009 | 138.1 k | -| 1M | LEA36 | 954 k | 6.09 | 0.035 | 164.3 k | -| 1M | LEA47 | 4 k | 2.11 | 0.005 | 473.3 k | -| 1M | LEA47 | 954 k | 1.18 | 0.003 | 846.4 k | -| 1M | LEA58 | 4 k | 1.39 | 0.003 | 719.2 k | -| 1M | LEA58 | 954 k | 0.95 | 0.004 | 1.053 M | +| 100k | LEA25 | 395 | 0.13 | 0.006 | 778.5 k | +| 100k | LEA25 | 95 k | 0.38 | 0.019 | 265.8 k | +| 100k | LEA36 | 395 | 0.07 | 0.003 | 1.488 M | +| 100k | LEA36 | 95 k | 0.21 | 0.007 | 471.0 k | +| 100k | LEA47 | 395 | 0.06 | 0.002 | 1.628 M | +| 100k | LEA47 | 95 k | 0.13 | 0.004 | 796.7 k | +| 100k | LEA58 | 395 | 0.06 | 0.002 | 1.705 M | +| 100k | LEA58 | 95 k | 0.09 | 0.002 | 1.170 M | +| 100k-fh | LEA25 | 2 k | 0.13 | 0.007 | 747.3 k | +| 100k-fh | LEA25 | 98 k | 0.35 | 0.019 | 282.9 k | +| 100k-fh | LEA36 | 2 k | 0.08 | 0.000 | 1.320 M | +| 100k-fh | LEA36 | 98 k | 0.18 | 0.003 | 547.0 k | +| 100k-fh | LEA47 | 2 k | 0.06 | 0.002 | 1.708 M | +| 100k-fh | LEA47 | 98 k | 0.09 | 0.002 | 1.171 M | +| 100k-fh | LEA58 | 2 k | 0.06 | 0.001 | 1.774 M | +| 100k-fh | LEA58 | 98 k | 0.08 | 0.001 | 1.178 M | +| 1M | LEA25 | 4 k | 1.14 | 0.243 | 873.6 k | +| 1M | LEA25 | 954 k | 17.57 | 0.530 | 56.9 k | +| 1M | LEA36 | 4 k | 0.52 | 0.021 | 1.905 M | +| 1M | LEA36 | 954 k | 5.52 | 0.034 | 181.2 k | +| 1M | LEA47 | 4 k | 0.50 | 0.004 | 2.002 M | +| 1M | LEA47 | 954 k | 0.82 | 0.008 | 1.217 M | +| 1M | LEA58 | 4 k | 0.51 | 0.014 | 1.963 M | +| 1M | LEA58 | 954 k | 0.73 | 0.008 | 1.378 M | ### CQL Query Frequent @@ -383,6 +383,56 @@ define InInitialPopulation: cql/search.sh condition-ten-rare ``` +## All Code Search + +### Data + +| Dataset | System | # Hits | Time (s) | StdDev | Pat./s | +|---------|--------|-------:|---------:|-------:|--------:| +| 100k | LEA25 | 99 k | 0.45 | 0.008 | 220.5 k | +| 100k | LEA36 | 99 k | 0.24 | 0.008 | 422.0 k | +| 100k | LEA47 | 99 k | 0.14 | 0.002 | 740.2 k | +| 100k | LEA58 | 99 k | 0.09 | 0.002 | 1.101 M | +| 100k-fh | LEA25 | 100 k | 0.38 | 0.014 | 263.0 k | +| 100k-fh | LEA36 | 100 k | 0.20 | 0.001 | 493.5 k | +| 100k-fh | LEA47 | 100 k | 0.10 | 0.001 | 1.046 M | +| 100k-fh | LEA58 | 100 k | 0.11 | 0.002 | 893.9 k | +| 1M | LEA25 | 995 k | 19.89 | 0.855 | 50.3 k | +| 1M | LEA36 | 995 k | 5.97 | 0.020 | 167.5 k | +| 1M | LEA47 | 995 k | 1.06 | 0.012 | 947.4 k | +| 1M | LEA58 | 995 k | 0.74 | 0.003 | 1.344 M | + +### CQL Query + +```sh +cql/search.sh condition-all +``` + +## Inpatient Stress Search + +### Data + +| Dataset | System | # Hits | Time (s) | StdDev | Pat./s | +|---------|--------|-------:|---------:|-------:|--------:| +| 100k | LEA25 | 2 k | 0.69 | 0.027 | 144.9 k | +| 100k | LEA36 | 2 k | 0.39 | 0.007 | 256.6 k | +| 100k | LEA47 | 2 k | 0.24 | 0.005 | 422.0 k | +| 100k | LEA58 | 2 k | 0.16 | 0.002 | 619.2 k | +| 100k-fh | LEA25 | 2 k | 2.18 | 0.036 | 45.9 k | +| 100k-fh | LEA36 | 2 k | 1.40 | 0.014 | 71.2 k | +| 100k-fh | LEA47 | 2 k | 0.51 | 0.003 | 196.6 k | +| 100k-fh | LEA58 | 2 k | 0.53 | 0.003 | 187.9 k | +| 1M | LEA25 | 16 k | 8.79 | 0.613 | 113.8 k | +| 1M | LEA36 | 16 k | 3.76 | 0.029 | 265.7 k | +| 1M | LEA47 | 16 k | 1.82 | 0.009 | 549.2 k | +| 1M | LEA58 | 16 k | 1.14 | 0.005 | 876.3 k | + +### CQL Query + +```sh +cql/search.sh inpatient-stress +``` + ## Condition Code Stratification ### Data diff --git a/docs/performance/cql/code-value-search-100k-fh.png b/docs/performance/cql/code-value-search-100k-fh.png index b419e3171..72da531ba 100644 Binary files a/docs/performance/cql/code-value-search-100k-fh.png and b/docs/performance/cql/code-value-search-100k-fh.png differ diff --git a/docs/performance/cql/code-value-search-100k-fh.txt b/docs/performance/cql/code-value-search-100k-fh.txt index 18e4189df..c64d5ce97 100644 --- a/docs/performance/cql/code-value-search-100k-fh.txt +++ b/docs/performance/cql/code-value-search-100k-fh.txt @@ -1,12 +1,12 @@ -| 100k-fh | LEA25 | 29463-7 | 13.6 kg | 100 k | 1.40 | 0.046 | 71.3 k | -| 100k-fh | LEA25 | 29463-7 | 75.3 kg | 100 k | 0.86 | 0.007 | 116.4 k | -| 100k-fh | LEA25 | 29463-7 | 185 kg | 100 k | 0.44 | 0.018 | 228.2 k | -| 100k-fh | LEA36 | 29463-7 | 13.6 kg | 100 k | 0.78 | 0.003 | 127.9 k | -| 100k-fh | LEA36 | 29463-7 | 75.3 kg | 100 k | 0.49 | 0.015 | 202.8 k | -| 100k-fh | LEA36 | 29463-7 | 185 kg | 100 k | 0.25 | 0.006 | 397.9 k | -| 100k-fh | LEA47 | 29463-7 | 13.6 kg | 100 k | 0.45 | 0.005 | 222.7 k | -| 100k-fh | LEA47 | 29463-7 | 75.3 kg | 100 k | 0.29 | 0.005 | 349.2 k | -| 100k-fh | LEA47 | 29463-7 | 185 kg | 100 k | 0.15 | 0.005 | 663.4 k | -| 100k-fh | LEA58 | 29463-7 | 13.6 kg | 100 k | 0.30 | 0.002 | 331.9 k | -| 100k-fh | LEA58 | 29463-7 | 75.3 kg | 100 k | 0.19 | 0.002 | 536.2 k | -| 100k-fh | LEA58 | 29463-7 | 185 kg | 100 k | 0.10 | 0.004 | 976.5 k | +| 100k-fh | LEA25 | 29463-7 | 13.6 kg | 100 k | 6.40 | 0.072 | 15.6 k | +| 100k-fh | LEA25 | 29463-7 | 75.3 kg | 100 k | 3.23 | 0.037 | 31.0 k | +| 100k-fh | LEA25 | 29463-7 | 185 kg | 100 k | 1.18 | 0.017 | 84.7 k | +| 100k-fh | LEA36 | 29463-7 | 13.6 kg | 100 k | 2.45 | 0.023 | 40.8 k | +| 100k-fh | LEA36 | 29463-7 | 75.3 kg | 100 k | 1.27 | 0.020 | 78.6 k | +| 100k-fh | LEA36 | 29463-7 | 185 kg | 100 k | 0.50 | 0.005 | 199.6 k | +| 100k-fh | LEA47 | 29463-7 | 13.6 kg | 100 k | 0.78 | 0.021 | 128.3 k | +| 100k-fh | LEA47 | 29463-7 | 75.3 kg | 100 k | 0.45 | 0.006 | 221.1 k | +| 100k-fh | LEA47 | 29463-7 | 185 kg | 100 k | 0.17 | 0.005 | 572.4 k | +| 100k-fh | LEA58 | 29463-7 | 13.6 kg | 100 k | 0.74 | 0.022 | 134.4 k | +| 100k-fh | LEA58 | 29463-7 | 75.3 kg | 100 k | 0.43 | 0.007 | 234.5 k | +| 100k-fh | LEA58 | 29463-7 | 185 kg | 100 k | 0.18 | 0.004 | 565.8 k | diff --git a/docs/performance/cql/code-value-search-100k.png b/docs/performance/cql/code-value-search-100k.png index 6636334fc..e93ce7341 100644 Binary files a/docs/performance/cql/code-value-search-100k.png and b/docs/performance/cql/code-value-search-100k.png differ diff --git a/docs/performance/cql/code-value-search-100k.txt b/docs/performance/cql/code-value-search-100k.txt index 03650abb7..1a2ff1a5a 100644 --- a/docs/performance/cql/code-value-search-100k.txt +++ b/docs/performance/cql/code-value-search-100k.txt @@ -1,12 +1,12 @@ -| 100k | LEA25 | 29463-7 | 13.6 kg | 10 k | 2.64 | 0.033 | 37.9 k | -| 100k | LEA25 | 29463-7 | 75.3 kg | 50 k | 1.41 | 0.007 | 70.7 k | -| 100k | LEA25 | 29463-7 | 185 kg | 100 k | 0.35 | 0.018 | 285.1 k | -| 100k | LEA36 | 29463-7 | 13.6 kg | 10 k | 1.04 | 0.012 | 95.9 k | -| 100k | LEA36 | 29463-7 | 75.3 kg | 50 k | 0.71 | 0.006 | 141.3 k | -| 100k | LEA36 | 29463-7 | 185 kg | 100 k | 0.20 | 0.009 | 511.1 k | -| 100k | LEA47 | 29463-7 | 13.6 kg | 10 k | 0.68 | 0.004 | 147.2 k | -| 100k | LEA47 | 29463-7 | 75.3 kg | 50 k | 0.47 | 0.010 | 214.1 k | -| 100k | LEA47 | 29463-7 | 185 kg | 100 k | 0.12 | 0.004 | 855.7 k | -| 100k | LEA58 | 29463-7 | 13.6 kg | 10 k | 0.47 | 0.004 | 210.7 k | -| 100k | LEA58 | 29463-7 | 75.3 kg | 50 k | 0.31 | 0.002 | 320.2 k | -| 100k | LEA58 | 29463-7 | 185 kg | 100 k | 0.09 | 0.001 | 1057 k | +| 100k | LEA25 | 29463-7 | 13.6 kg | 10 k | 0.35 | 0.019 | 286.1 k | +| 100k | LEA25 | 29463-7 | 75.3 kg | 50 k | 0.84 | 0.025 | 118.8 k | +| 100k | LEA25 | 29463-7 | 185 kg | 100 k | 1.23 | 0.022 | 81.1 k | +| 100k | LEA36 | 29463-7 | 13.6 kg | 10 k | 0.14 | 0.007 | 698.5 k | +| 100k | LEA36 | 29463-7 | 75.3 kg | 50 k | 0.36 | 0.011 | 275.7 k | +| 100k | LEA36 | 29463-7 | 185 kg | 100 k | 0.57 | 0.016 | 176.4 k | +| 100k | LEA47 | 29463-7 | 13.6 kg | 10 k | 0.07 | 0.002 | 1410 k | +| 100k | LEA47 | 29463-7 | 75.3 kg | 50 k | 0.18 | 0.006 | 571.1 k | +| 100k | LEA47 | 29463-7 | 185 kg | 100 k | 0.27 | 0.014 | 374.8 k | +| 100k | LEA58 | 29463-7 | 13.6 kg | 10 k | 0.07 | 0.002 | 1409 k | +| 100k | LEA58 | 29463-7 | 75.3 kg | 50 k | 0.13 | 0.004 | 780.9 k | +| 100k | LEA58 | 29463-7 | 185 kg | 100 k | 0.18 | 0.006 | 543.8 k | diff --git a/docs/performance/cql/code-value-search-1M.png b/docs/performance/cql/code-value-search-1M.png index ddea316b6..19c1b2668 100644 Binary files a/docs/performance/cql/code-value-search-1M.png and b/docs/performance/cql/code-value-search-1M.png differ diff --git a/docs/performance/cql/code-value-search-1M.txt b/docs/performance/cql/code-value-search-1M.txt index bcf84b698..7c7b74c38 100644 --- a/docs/performance/cql/code-value-search-1M.txt +++ b/docs/performance/cql/code-value-search-1M.txt @@ -1,12 +1,12 @@ -| 1M | LEA25 | 29463-7 | 13.6 kg | 99 k | 719.84 | 3.734 | 1.4 k | -| 1M | LEA25 | 29463-7 | 75.3 kg | 500 k | 479.52 | 11.096 | 2.1 k | -| 1M | LEA25 | 29463-7 | 185 kg | 998 k | 103.51 | 40.442 | 9.7 k | -| 1M | LEA36 | 29463-7 | 13.6 kg | 99 k | 432.80 | 1.586 | 2.3 k | -| 1M | LEA36 | 29463-7 | 75.3 kg | 500 k | 265.29 | 1.618 | 3.8 k | -| 1M | LEA36 | 29463-7 | 185 kg | 998 k | 7.72 | 0.045 | 129.5 k | -| 1M | LEA47 | 29463-7 | 13.6 kg | 99 k | 138.82 | 1.378 | 7.2 k | -| 1M | LEA47 | 29463-7 | 75.3 kg | 500 k | 8.09 | 0.015 | 123.6 k | -| 1M | LEA47 | 29463-7 | 185 kg | 998 k | 1.03 | 0.004 | 973.7 k | -| 1M | LEA58 | 29463-7 | 13.6 kg | 99 k | 4.18 | 0.004 | 239.3 k | -| 1M | LEA58 | 29463-7 | 75.3 kg | 500 k | 2.67 | 0.008 | 374.2 k | -| 1M | LEA58 | 29463-7 | 185 kg | 998 k | 0.78 | 0.003 | 1288 k | +| 1M | LEA25 | 29463-7 | 13.6 kg | 99 k | 2.91 | 0.169 | 344.2 k | +| 1M | LEA25 | 29463-7 | 75.3 kg | 500 k | 15.89 | 1.056 | 62.9 k | +| 1M | LEA25 | 29463-7 | 185 kg | 998 k | 27.61 | 0.948 | 36.2 k | +| 1M | LEA36 | 29463-7 | 13.6 kg | 99 k | 1.11 | 0.012 | 901.9 k | +| 1M | LEA36 | 29463-7 | 75.3 kg | 500 k | 3.19 | 0.042 | 313.1 k | +| 1M | LEA36 | 29463-7 | 185 kg | 998 k | 10.61 | 0.030 | 94.3 k | +| 1M | LEA47 | 29463-7 | 13.6 kg | 99 k | 0.60 | 0.018 | 1664 k | +| 1M | LEA47 | 29463-7 | 75.3 kg | 500 k | 1.55 | 0.012 | 646.4 k | +| 1M | LEA47 | 29463-7 | 185 kg | 998 k | 2.17 | 0.022 | 460.9 k | +| 1M | LEA58 | 29463-7 | 13.6 kg | 99 k | 0.57 | 0.012 | 1754 k | +| 1M | LEA58 | 29463-7 | 75.3 kg | 500 k | 0.93 | 0.011 | 1078 k | +| 1M | LEA58 | 29463-7 | 185 kg | 998 k | 1.30 | 0.022 | 772.0 k | diff --git a/docs/performance/cql/condition-450-rare.cql b/docs/performance/cql/condition-450-rare.cql new file mode 100644 index 000000000..f2833c03d --- /dev/null +++ b/docs/performance/cql/condition-450-rare.cql @@ -0,0 +1,459 @@ +library "condition-450-rare" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '0' from sct] or + exists [Condition: Code '1' from sct] or + exists [Condition: Code '2' from sct] or + exists [Condition: Code '3' from sct] or + exists [Condition: Code '4' from sct] or + exists [Condition: Code '5' from sct] or + exists [Condition: Code '6' from sct] or + exists [Condition: Code '7' from sct] or + exists [Condition: Code '8' from sct] or + exists [Condition: Code '9' from sct] or + exists [Condition: Code '10' from sct] or + exists [Condition: Code '11' from sct] or + exists [Condition: Code '12' from sct] or + exists [Condition: Code '13' from sct] or + exists [Condition: Code '14' from sct] or + exists [Condition: Code '15' from sct] or + exists [Condition: Code '16' from sct] or + exists [Condition: Code '17' from sct] or + exists [Condition: Code '18' from sct] or + exists [Condition: Code '19' from sct] or + exists [Condition: Code '20' from sct] or + exists [Condition: Code '21' from sct] or + exists [Condition: Code '22' from sct] or + exists [Condition: Code '23' from sct] or + exists [Condition: Code '24' from sct] or + exists [Condition: Code '25' from sct] or + exists [Condition: Code '26' from sct] or + exists [Condition: Code '27' from sct] or + exists [Condition: Code '28' from sct] or + exists [Condition: Code '29' from sct] or + exists [Condition: Code '30' from sct] or + exists [Condition: Code '31' from sct] or + exists [Condition: Code '32' from sct] or + exists [Condition: Code '33' from sct] or + exists [Condition: Code '34' from sct] or + exists [Condition: Code '35' from sct] or + exists [Condition: Code '36' from sct] or + exists [Condition: Code '37' from sct] or + exists [Condition: Code '38' from sct] or + exists [Condition: Code '39' from sct] or + exists [Condition: Code '40' from sct] or + exists [Condition: Code '41' from sct] or + exists [Condition: Code '42' from sct] or + exists [Condition: Code '43' from sct] or + exists [Condition: Code '44' from sct] or + exists [Condition: Code '45' from sct] or + exists [Condition: Code '46' from sct] or + exists [Condition: Code '47' from sct] or + exists [Condition: Code '48' from sct] or + exists [Condition: Code '49' from sct] or + exists [Condition: Code '50' from sct] or + exists [Condition: Code '51' from sct] or + exists [Condition: Code '52' from sct] or + exists [Condition: Code '53' from sct] or + exists [Condition: Code '54' from sct] or + exists [Condition: Code '55' from sct] or + exists [Condition: Code '56' from sct] or + exists [Condition: Code '57' from sct] or + exists [Condition: Code '58' from sct] or + exists [Condition: Code '59' from sct] or + exists [Condition: Code '60' from sct] or + exists [Condition: Code '61' from sct] or + exists [Condition: Code '62' from sct] or + exists [Condition: Code '63' from sct] or + exists [Condition: Code '64' from sct] or + exists [Condition: Code '65' from sct] or + exists [Condition: Code '66' from sct] or + exists [Condition: Code '67' from sct] or + exists [Condition: Code '68' from sct] or + exists [Condition: Code '69' from sct] or + exists [Condition: Code '70' from sct] or + exists [Condition: Code '71' from sct] or + exists [Condition: Code '72' from sct] or + exists [Condition: Code '73' from sct] or + exists [Condition: Code '74' from sct] or + exists [Condition: Code '75' from sct] or + exists [Condition: Code '76' from sct] or + exists [Condition: Code '77' from sct] or + exists [Condition: Code '78' from sct] or + exists [Condition: Code '79' from sct] or + exists [Condition: Code '80' from sct] or + exists [Condition: Code '81' from sct] or + exists [Condition: Code '82' from sct] or + exists [Condition: Code '83' from sct] or + exists [Condition: Code '84' from sct] or + exists [Condition: Code '85' from sct] or + exists [Condition: Code '86' from sct] or + exists [Condition: Code '87' from sct] or + exists [Condition: Code '88' from sct] or + exists [Condition: Code '89' from sct] or + exists [Condition: Code '90' from sct] or + exists [Condition: Code '91' from sct] or + exists [Condition: Code '92' from sct] or + exists [Condition: Code '93' from sct] or + exists [Condition: Code '94' from sct] or + exists [Condition: Code '95' from sct] or + exists [Condition: Code '96' from sct] or + exists [Condition: Code '97' from sct] or + exists [Condition: Code '98' from sct] or + exists [Condition: Code '99' from sct] or + exists [Condition: Code '100' from sct] or + exists [Condition: Code '101' from sct] or + exists [Condition: Code '102' from sct] or + exists [Condition: Code '103' from sct] or + exists [Condition: Code '104' from sct] or + exists [Condition: Code '105' from sct] or + exists [Condition: Code '106' from sct] or + exists [Condition: Code '107' from sct] or + exists [Condition: Code '108' from sct] or + exists [Condition: Code '109' from sct] or + exists [Condition: Code '110' from sct] or + exists [Condition: Code '111' from sct] or + exists [Condition: Code '112' from sct] or + exists [Condition: Code '113' from sct] or + exists [Condition: Code '114' from sct] or + exists [Condition: Code '115' from sct] or + exists [Condition: Code '116' from sct] or + exists [Condition: Code '117' from sct] or + exists [Condition: Code '118' from sct] or + exists [Condition: Code '119' from sct] or + exists [Condition: Code '120' from sct] or + exists [Condition: Code '121' from sct] or + exists [Condition: Code '122' from sct] or + exists [Condition: Code '123' from sct] or + exists [Condition: Code '124' from sct] or + exists [Condition: Code '125' from sct] or + exists [Condition: Code '126' from sct] or + exists [Condition: Code '127' from sct] or + exists [Condition: Code '128' from sct] or + exists [Condition: Code '129' from sct] or + exists [Condition: Code '130' from sct] or + exists [Condition: Code '131' from sct] or + exists [Condition: Code '132' from sct] or + exists [Condition: Code '133' from sct] or + exists [Condition: Code '134' from sct] or + exists [Condition: Code '135' from sct] or + exists [Condition: Code '136' from sct] or + exists [Condition: Code '137' from sct] or + exists [Condition: Code '138' from sct] or + exists [Condition: Code '139' from sct] or + exists [Condition: Code '140' from sct] or + exists [Condition: Code '141' from sct] or + exists [Condition: Code '142' from sct] or + exists [Condition: Code '143' from sct] or + exists [Condition: Code '144' from sct] or + exists [Condition: Code '145' from sct] or + exists [Condition: Code '146' from sct] or + exists [Condition: Code '147' from sct] or + exists [Condition: Code '148' from sct] or + exists [Condition: Code '149' from sct] or + exists [Condition: Code '150' from sct] or + exists [Condition: Code '151' from sct] or + exists [Condition: Code '152' from sct] or + exists [Condition: Code '153' from sct] or + exists [Condition: Code '154' from sct] or + exists [Condition: Code '155' from sct] or + exists [Condition: Code '156' from sct] or + exists [Condition: Code '157' from sct] or + exists [Condition: Code '158' from sct] or + exists [Condition: Code '159' from sct] or + exists [Condition: Code '160' from sct] or + exists [Condition: Code '161' from sct] or + exists [Condition: Code '162' from sct] or + exists [Condition: Code '163' from sct] or + exists [Condition: Code '164' from sct] or + exists [Condition: Code '165' from sct] or + exists [Condition: Code '166' from sct] or + exists [Condition: Code '167' from sct] or + exists [Condition: Code '168' from sct] or + exists [Condition: Code '169' from sct] or + exists [Condition: Code '170' from sct] or + exists [Condition: Code '171' from sct] or + exists [Condition: Code '172' from sct] or + exists [Condition: Code '173' from sct] or + exists [Condition: Code '174' from sct] or + exists [Condition: Code '175' from sct] or + exists [Condition: Code '176' from sct] or + exists [Condition: Code '177' from sct] or + exists [Condition: Code '178' from sct] or + exists [Condition: Code '179' from sct] or + exists [Condition: Code '180' from sct] or + exists [Condition: Code '181' from sct] or + exists [Condition: Code '182' from sct] or + exists [Condition: Code '183' from sct] or + exists [Condition: Code '184' from sct] or + exists [Condition: Code '185' from sct] or + exists [Condition: Code '186' from sct] or + exists [Condition: Code '187' from sct] or + exists [Condition: Code '188' from sct] or + exists [Condition: Code '189' from sct] or + exists [Condition: Code '190' from sct] or + exists [Condition: Code '191' from sct] or + exists [Condition: Code '192' from sct] or + exists [Condition: Code '193' from sct] or + exists [Condition: Code '194' from sct] or + exists [Condition: Code '195' from sct] or + exists [Condition: Code '196' from sct] or + exists [Condition: Code '197' from sct] or + exists [Condition: Code '198' from sct] or + exists [Condition: Code '199' from sct] or + exists [Condition: Code '200' from sct] or + exists [Condition: Code '201' from sct] or + exists [Condition: Code '202' from sct] or + exists [Condition: Code '203' from sct] or + exists [Condition: Code '204' from sct] or + exists [Condition: Code '205' from sct] or + exists [Condition: Code '206' from sct] or + exists [Condition: Code '207' from sct] or + exists [Condition: Code '208' from sct] or + exists [Condition: Code '209' from sct] or + exists [Condition: Code '210' from sct] or + exists [Condition: Code '211' from sct] or + exists [Condition: Code '212' from sct] or + exists [Condition: Code '213' from sct] or + exists [Condition: Code '214' from sct] or + exists [Condition: Code '215' from sct] or + exists [Condition: Code '216' from sct] or + exists [Condition: Code '217' from sct] or + exists [Condition: Code '218' from sct] or + exists [Condition: Code '219' from sct] or + exists [Condition: Code '220' from sct] or + exists [Condition: Code '221' from sct] or + exists [Condition: Code '222' from sct] or + exists [Condition: Code '223' from sct] or + exists [Condition: Code '224' from sct] or + exists [Condition: Code '225' from sct] or + exists [Condition: Code '226' from sct] or + exists [Condition: Code '227' from sct] or + exists [Condition: Code '228' from sct] or + exists [Condition: Code '229' from sct] or + exists [Condition: Code '230' from sct] or + exists [Condition: Code '231' from sct] or + exists [Condition: Code '232' from sct] or + exists [Condition: Code '233' from sct] or + exists [Condition: Code '234' from sct] or + exists [Condition: Code '235' from sct] or + exists [Condition: Code '236' from sct] or + exists [Condition: Code '237' from sct] or + exists [Condition: Code '238' from sct] or + exists [Condition: Code '239' from sct] or + exists [Condition: Code '240' from sct] or + exists [Condition: Code '241' from sct] or + exists [Condition: Code '242' from sct] or + exists [Condition: Code '243' from sct] or + exists [Condition: Code '244' from sct] or + exists [Condition: Code '245' from sct] or + exists [Condition: Code '246' from sct] or + exists [Condition: Code '247' from sct] or + exists [Condition: Code '248' from sct] or + exists [Condition: Code '249' from sct] or + exists [Condition: Code '250' from sct] or + exists [Condition: Code '251' from sct] or + exists [Condition: Code '252' from sct] or + exists [Condition: Code '253' from sct] or + exists [Condition: Code '254' from sct] or + exists [Condition: Code '255' from sct] or + exists [Condition: Code '256' from sct] or + exists [Condition: Code '257' from sct] or + exists [Condition: Code '258' from sct] or + exists [Condition: Code '259' from sct] or + exists [Condition: Code '260' from sct] or + exists [Condition: Code '261' from sct] or + exists [Condition: Code '262' from sct] or + exists [Condition: Code '263' from sct] or + exists [Condition: Code '264' from sct] or + exists [Condition: Code '265' from sct] or + exists [Condition: Code '266' from sct] or + exists [Condition: Code '267' from sct] or + exists [Condition: Code '268' from sct] or + exists [Condition: Code '269' from sct] or + exists [Condition: Code '270' from sct] or + exists [Condition: Code '271' from sct] or + exists [Condition: Code '272' from sct] or + exists [Condition: Code '273' from sct] or + exists [Condition: Code '274' from sct] or + exists [Condition: Code '275' from sct] or + exists [Condition: Code '276' from sct] or + exists [Condition: Code '277' from sct] or + exists [Condition: Code '278' from sct] or + exists [Condition: Code '279' from sct] or + exists [Condition: Code '280' from sct] or + exists [Condition: Code '281' from sct] or + exists [Condition: Code '282' from sct] or + exists [Condition: Code '283' from sct] or + exists [Condition: Code '284' from sct] or + exists [Condition: Code '285' from sct] or + exists [Condition: Code '286' from sct] or + exists [Condition: Code '287' from sct] or + exists [Condition: Code '288' from sct] or + exists [Condition: Code '289' from sct] or + exists [Condition: Code '290' from sct] or + exists [Condition: Code '291' from sct] or + exists [Condition: Code '292' from sct] or + exists [Condition: Code '293' from sct] or + exists [Condition: Code '294' from sct] or + exists [Condition: Code '295' from sct] or + exists [Condition: Code '296' from sct] or + exists [Condition: Code '297' from sct] or + exists [Condition: Code '298' from sct] or + exists [Condition: Code '299' from sct] or + exists [Condition: Code '300' from sct] or + exists [Condition: Code '301' from sct] or + exists [Condition: Code '302' from sct] or + exists [Condition: Code '303' from sct] or + exists [Condition: Code '304' from sct] or + exists [Condition: Code '305' from sct] or + exists [Condition: Code '306' from sct] or + exists [Condition: Code '307' from sct] or + exists [Condition: Code '308' from sct] or + exists [Condition: Code '309' from sct] or + exists [Condition: Code '310' from sct] or + exists [Condition: Code '311' from sct] or + exists [Condition: Code '312' from sct] or + exists [Condition: Code '313' from sct] or + exists [Condition: Code '314' from sct] or + exists [Condition: Code '315' from sct] or + exists [Condition: Code '316' from sct] or + exists [Condition: Code '317' from sct] or + exists [Condition: Code '318' from sct] or + exists [Condition: Code '319' from sct] or + exists [Condition: Code '320' from sct] or + exists [Condition: Code '321' from sct] or + exists [Condition: Code '322' from sct] or + exists [Condition: Code '323' from sct] or + exists [Condition: Code '324' from sct] or + exists [Condition: Code '325' from sct] or + exists [Condition: Code '326' from sct] or + exists [Condition: Code '327' from sct] or + exists [Condition: Code '328' from sct] or + exists [Condition: Code '329' from sct] or + exists [Condition: Code '330' from sct] or + exists [Condition: Code '331' from sct] or + exists [Condition: Code '332' from sct] or + exists [Condition: Code '333' from sct] or + exists [Condition: Code '334' from sct] or + exists [Condition: Code '335' from sct] or + exists [Condition: Code '336' from sct] or + exists [Condition: Code '337' from sct] or + exists [Condition: Code '338' from sct] or + exists [Condition: Code '339' from sct] or + exists [Condition: Code '340' from sct] or + exists [Condition: Code '341' from sct] or + exists [Condition: Code '342' from sct] or + exists [Condition: Code '343' from sct] or + exists [Condition: Code '344' from sct] or + exists [Condition: Code '345' from sct] or + exists [Condition: Code '346' from sct] or + exists [Condition: Code '347' from sct] or + exists [Condition: Code '348' from sct] or + exists [Condition: Code '349' from sct] or + exists [Condition: Code '350' from sct] or + exists [Condition: Code '351' from sct] or + exists [Condition: Code '352' from sct] or + exists [Condition: Code '353' from sct] or + exists [Condition: Code '354' from sct] or + exists [Condition: Code '355' from sct] or + exists [Condition: Code '356' from sct] or + exists [Condition: Code '357' from sct] or + exists [Condition: Code '358' from sct] or + exists [Condition: Code '359' from sct] or + exists [Condition: Code '360' from sct] or + exists [Condition: Code '361' from sct] or + exists [Condition: Code '362' from sct] or + exists [Condition: Code '363' from sct] or + exists [Condition: Code '364' from sct] or + exists [Condition: Code '365' from sct] or + exists [Condition: Code '366' from sct] or + exists [Condition: Code '367' from sct] or + exists [Condition: Code '368' from sct] or + exists [Condition: Code '369' from sct] or + exists [Condition: Code '370' from sct] or + exists [Condition: Code '371' from sct] or + exists [Condition: Code '372' from sct] or + exists [Condition: Code '373' from sct] or + exists [Condition: Code '374' from sct] or + exists [Condition: Code '375' from sct] or + exists [Condition: Code '376' from sct] or + exists [Condition: Code '377' from sct] or + exists [Condition: Code '378' from sct] or + exists [Condition: Code '379' from sct] or + exists [Condition: Code '380' from sct] or + exists [Condition: Code '381' from sct] or + exists [Condition: Code '382' from sct] or + exists [Condition: Code '383' from sct] or + exists [Condition: Code '384' from sct] or + exists [Condition: Code '385' from sct] or + exists [Condition: Code '386' from sct] or + exists [Condition: Code '387' from sct] or + exists [Condition: Code '388' from sct] or + exists [Condition: Code '389' from sct] or + exists [Condition: Code '390' from sct] or + exists [Condition: Code '391' from sct] or + exists [Condition: Code '392' from sct] or + exists [Condition: Code '393' from sct] or + exists [Condition: Code '394' from sct] or + exists [Condition: Code '395' from sct] or + exists [Condition: Code '396' from sct] or + exists [Condition: Code '397' from sct] or + exists [Condition: Code '398' from sct] or + exists [Condition: Code '399' from sct] or + exists [Condition: Code '400' from sct] or + exists [Condition: Code '401' from sct] or + exists [Condition: Code '402' from sct] or + exists [Condition: Code '403' from sct] or + exists [Condition: Code '404' from sct] or + exists [Condition: Code '405' from sct] or + exists [Condition: Code '406' from sct] or + exists [Condition: Code '407' from sct] or + exists [Condition: Code '408' from sct] or + exists [Condition: Code '409' from sct] or + exists [Condition: Code '410' from sct] or + exists [Condition: Code '411' from sct] or + exists [Condition: Code '412' from sct] or + exists [Condition: Code '413' from sct] or + exists [Condition: Code '414' from sct] or + exists [Condition: Code '415' from sct] or + exists [Condition: Code '416' from sct] or + exists [Condition: Code '417' from sct] or + exists [Condition: Code '418' from sct] or + exists [Condition: Code '419' from sct] or + exists [Condition: Code '420' from sct] or + exists [Condition: Code '421' from sct] or + exists [Condition: Code '422' from sct] or + exists [Condition: Code '423' from sct] or + exists [Condition: Code '424' from sct] or + exists [Condition: Code '425' from sct] or + exists [Condition: Code '426' from sct] or + exists [Condition: Code '427' from sct] or + exists [Condition: Code '428' from sct] or + exists [Condition: Code '429' from sct] or + exists [Condition: Code '430' from sct] or + exists [Condition: Code '431' from sct] or + exists [Condition: Code '432' from sct] or + exists [Condition: Code '433' from sct] or + exists [Condition: Code '434' from sct] or + exists [Condition: Code '435' from sct] or + exists [Condition: Code '436' from sct] or + exists [Condition: Code '437' from sct] or + exists [Condition: Code '438' from sct] or + exists [Condition: Code '439' from sct] or + exists [Condition: Code '62718007' from sct] or + exists [Condition: Code '234466008' from sct] or + exists [Condition: Code '288959006' from sct] or + exists [Condition: Code '47505003' from sct] or + exists [Condition: Code '698754002' from sct] or + exists [Condition: Code '157265008' from sct] or + exists [Condition: Code '15802004' from sct] or + exists [Condition: Code '14760008' from sct] or + exists [Condition: Code '36923009' from sct] or + exists [Condition: Code '45816000' from sct] diff --git a/docs/performance/cql/condition-450-rare.yml b/docs/performance/cql/condition-450-rare.yml new file mode 100644 index 000000000..55b09b91c --- /dev/null +++ b/docs/performance/cql/condition-450-rare.yml @@ -0,0 +1,5 @@ +library: cql/condition-450-rare.cql +group: +- type: Patient + population: + - expression: InInitialPopulation diff --git a/docs/performance/cql/duration.jq b/docs/performance/cql/duration.jq index 9718df3c8..837808cc1 100644 --- a/docs/performance/cql/duration.jq +++ b/docs/performance/cql/duration.jq @@ -1 +1 @@ -.extension[0].valueQuantity.value +.extension[] | select(.url == "https://samply.github.io/blaze/fhir/StructureDefinition/eval-duration") | .valueQuantity.value diff --git a/docs/performance/cql/other.md b/docs/performance/cql/other.md index 7f0419d8f..7fcaf6462 100644 --- a/docs/performance/cql/other.md +++ b/docs/performance/cql/other.md @@ -18,42 +18,3 @@ ```sh cql/search.sh condition-50-rare ``` - -## All Code Search - -### Data - -| Dataset | System | # Hits | Time (s) | StdDev | Pat./s | -|---------|--------|-------:|---------:|-------:|--------:| -| 100k | LEA25 | 99 k | 2.51 | 0.015 | 39.8 k | -| 100k | LEA36 | 99 k | 1.55 | 0.018 | 64.5 k | -| 100k | LEA47 | 99 k | 0.93 | 0.021 | 107.8 k | -| 100k | LEA58 | 99 k | 0.63 | 0.009 | 159.1 k | -| 100k-fh | LEA58 | 100 k | 1.55 | 0.006 | 64.7 k | -| 1M | LEA47 | 995 k | 4.75 | 0.014 | 210.5 k | -| 1M | LEA58 | 995 k | 6.05 | 0.017 | 165.4 k | - -### CQL Query - -```sh -cql/search.sh condition-all -``` - -## Inpatient Stress Search - -### Data - -| Dataset | System | # Hits | Time (s) | StdDev | Pat./s | -|---------|--------|-------:|---------:|-------:|--------:| -| 100k | LEA25 | 2 k | 4.73 | 0.032 | 21.1 k | -| 100k | LEA36 | 2 k | 2.97 | 0.029 | 33.7 k | -| 100k | LEA47 | 2 k | 1.59 | 0.008 | 63.0 k | -| 100k | LEA58 | 2 k | 1.27 | 0.023 | 78.8 k | -| 100k-fh | LEA58 | 2 k | 4.41 | 0.041 | 22.7 k | -| 1M | LEA58 | 16 k | 11.08 | 0.044 | 90.2 k | - -### CQL Query - -```sh -cql/search.sh inpatient-stress -``` diff --git a/docs/performance/cql/result.jq b/docs/performance/cql/result.jq index f5222b4a5..d73f91851 100644 --- a/docs/performance/cql/result.jq +++ b/docs/performance/cql/result.jq @@ -1,4 +1,4 @@ { - duration: .extension[0].valueQuantity.value, + duration: .extension[] | select(.url == "https://samply.github.io/blaze/fhir/StructureDefinition/eval-duration") | .valueQuantity.value, result: .group[0].population[0].count } diff --git a/docs/performance/cql/search-all.sh b/docs/performance/cql/search-all.sh index 50f9c0e02..753e54b5c 100755 --- a/docs/performance/cql/search-all.sh +++ b/docs/performance/cql/search-all.sh @@ -65,12 +65,12 @@ cql/search.sh condition-ten-rare #cql/search.sh condition-50-rare #cql/search.sh condition-50-rare -#restart "$COMPOSE_FILE" -#cql/search.sh condition-all -#cql/search.sh condition-all -#cql/search.sh condition-all +restart "$COMPOSE_FILE" +cql/search.sh condition-all +cql/search.sh condition-all +cql/search.sh condition-all -#restart "$COMPOSE_FILE" -#cql/search.sh inpatient-stress -#cql/search.sh inpatient-stress -#cql/search.sh inpatient-stress +restart "$COMPOSE_FILE" +cql/search.sh inpatient-stress +cql/search.sh inpatient-stress +cql/search.sh inpatient-stress diff --git a/docs/performance/cql/search.sh b/docs/performance/cql/search.sh index 4648a95e8..9ab3d0898 100755 --- a/docs/performance/cql/search.sh +++ b/docs/performance/cql/search.sh @@ -10,7 +10,7 @@ FILE="$1" echo "Counting Patients with criteria from $FILE..." COUNT="$(blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/$FILE.yml" 2> /dev/null | jq -r '.group[0].population[0].count')" -for i in {0..6} +for i in {0..8} do blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/$FILE.yml" 2> /dev/null |\ jq -rf "$SCRIPT_DIR/duration.jq" >> "$START_EPOCH-$FILE.times" diff --git a/docs/performance/cql/simple-code-search-100k-fh.gnuplot b/docs/performance/cql/simple-code-search-100k-fh.gnuplot index 481e8678c..b9c5952eb 100644 --- a/docs/performance/cql/simple-code-search-100k-fh.gnuplot +++ b/docs/performance/cql/simple-code-search-100k-fh.gnuplot @@ -15,7 +15,7 @@ set title "Simple Code Search - Dataset 100k-fh" set xlabel 'System' set ylabel 'Patients/s' set format y "%.0f k" -set yrange [0:1800] +set yrange [0:2000] # Define grid set grid ytics diff --git a/docs/performance/cql/simple-code-search-100k-fh.png b/docs/performance/cql/simple-code-search-100k-fh.png index f2d5a24b7..11922fd63 100644 Binary files a/docs/performance/cql/simple-code-search-100k-fh.png and b/docs/performance/cql/simple-code-search-100k-fh.png differ diff --git a/docs/performance/cql/simple-code-search-100k-fh.txt b/docs/performance/cql/simple-code-search-100k-fh.txt index 3ce450754..e59726008 100644 --- a/docs/performance/cql/simple-code-search-100k-fh.txt +++ b/docs/performance/cql/simple-code-search-100k-fh.txt @@ -1,12 +1,12 @@ -| 100k-fh | LEA25 | 788-0 | 2 k | 0.21 | 0.009 | 475.3 k | -| 100k-fh | LEA25 | 44261-6 | 57 k | 0.30 | 0.012 | 331.4 k | -| 100k-fh | LEA25 | 72514-3 | 100 k | 0.38 | 0.021 | 265.8 k | -| 100k-fh | LEA36 | 788-0 | 2 k | 0.12 | 0.007 | 860.0 k | -| 100k-fh | LEA36 | 44261-6 | 57 k | 0.17 | 0.008 | 573.5 k | -| 100k-fh | LEA36 | 72514-3 | 100 k | 0.20 | 0.007 | 490.9 k | -| 100k-fh | LEA47 | 788-0 | 2 k | 0.07 | 0.002 | 1415 k | -| 100k-fh | LEA47 | 44261-6 | 57 k | 0.10 | 0.002 | 995.8 k | -| 100k-fh | LEA47 | 72514-3 | 100 k | 0.12 | 0.004 | 809.9 k | -| 100k-fh | LEA58 | 788-0 | 2 k | 0.06 | 0.003 | 1659 k | -| 100k-fh | LEA58 | 44261-6 | 57 k | 0.07 | 0.002 | 1521 k | -| 100k-fh | LEA58 | 72514-3 | 100 k | 0.08 | 0.001 | 1232 k | +| 100k-fh | LEA25 | 788-0 | 2 k | 0.08 | 0.005 | 1299 k | +| 100k-fh | LEA25 | 44261-6 | 57 k | 0.26 | 0.017 | 379.7 k | +| 100k-fh | LEA25 | 72514-3 | 100 k | 0.40 | 0.025 | 250.9 k | +| 100k-fh | LEA36 | 788-0 | 2 k | 0.05 | 0.001 | 1854 k | +| 100k-fh | LEA36 | 44261-6 | 57 k | 0.13 | 0.004 | 756.1 k | +| 100k-fh | LEA36 | 72514-3 | 100 k | 0.20 | 0.003 | 489.4 k | +| 100k-fh | LEA47 | 788-0 | 2 k | 0.05 | 0.001 | 1938 k | +| 100k-fh | LEA47 | 44261-6 | 57 k | 0.08 | 0.003 | 1332 k | +| 100k-fh | LEA47 | 72514-3 | 100 k | 0.09 | 0.002 | 1100 k | +| 100k-fh | LEA58 | 788-0 | 2 k | 0.05 | 0.002 | 1930 k | +| 100k-fh | LEA58 | 44261-6 | 57 k | 0.07 | 0.001 | 1385 k | +| 100k-fh | LEA58 | 72514-3 | 100 k | 0.09 | 0.001 | 1161 k | diff --git a/docs/performance/cql/simple-code-search-100k.gnuplot b/docs/performance/cql/simple-code-search-100k.gnuplot index 5c90a2b17..a57019841 100644 --- a/docs/performance/cql/simple-code-search-100k.gnuplot +++ b/docs/performance/cql/simple-code-search-100k.gnuplot @@ -15,7 +15,7 @@ set title "Simple Code Search - Dataset 100k" set xlabel 'System' set ylabel 'Patients/s' set format y "%.0f k" -set yrange [0:1800] +set yrange [0:2200] # Define grid set grid ytics diff --git a/docs/performance/cql/simple-code-search-100k.png b/docs/performance/cql/simple-code-search-100k.png index 90de7beb0..875073f83 100644 Binary files a/docs/performance/cql/simple-code-search-100k.png and b/docs/performance/cql/simple-code-search-100k.png differ diff --git a/docs/performance/cql/simple-code-search-100k.txt b/docs/performance/cql/simple-code-search-100k.txt index 2c2e801c0..c223300fa 100644 --- a/docs/performance/cql/simple-code-search-100k.txt +++ b/docs/performance/cql/simple-code-search-100k.txt @@ -1,12 +1,12 @@ -| 100k | LEA25 | 17861-6 | 2 k | 0.16 | 0.011 | 630.6 k | -| 100k | LEA25 | 8310-5 | 60 k | 0.25 | 0.017 | 393.1 k | -| 100k | LEA25 | 72514-3 | 100 k | 0.29 | 0.009 | 341.5 k | -| 100k | LEA36 | 17861-6 | 2 k | 0.09 | 0.006 | 1086 k | -| 100k | LEA36 | 8310-5 | 60 k | 0.12 | 0.003 | 807.1 k | -| 100k | LEA36 | 72514-3 | 100 k | 0.17 | 0.006 | 604.8 k | -| 100k | LEA47 | 17861-6 | 2 k | 0.06 | 0.002 | 1629 k | -| 100k | LEA47 | 8310-5 | 60 k | 0.08 | 0.004 | 1307 k | -| 100k | LEA47 | 72514-3 | 100 k | 0.09 | 0.003 | 1068 k | -| 100k | LEA58 | 17861-6 | 2 k | 0.07 | 0.002 | 1504 k | -| 100k | LEA58 | 8310-5 | 60 k | 0.08 | 0.001 | 1298 k | -| 100k | LEA58 | 72514-3 | 100 k | 0.08 | 0.002 | 1207 k | +| 100k | LEA25 | 17861-6 | 2 k | 0.08 | 0.005 | 1203 k | +| 100k | LEA25 | 8310-5 | 60 k | 0.29 | 0.008 | 342.8 k | +| 100k | LEA25 | 72514-3 | 100 k | 0.42 | 0.008 | 237.7 k | +| 100k | LEA36 | 17861-6 | 2 k | 0.05 | 0.001 | 2000 k | +| 100k | LEA36 | 8310-5 | 60 k | 0.15 | 0.003 | 667.8 k | +| 100k | LEA36 | 72514-3 | 100 k | 0.23 | 0.005 | 442.2 k | +| 100k | LEA47 | 17861-6 | 2 k | 0.05 | 0.002 | 1942 k | +| 100k | LEA47 | 8310-5 | 60 k | 0.08 | 0.002 | 1210 k | +| 100k | LEA47 | 72514-3 | 100 k | 0.13 | 0.002 | 794.3 k | +| 100k | LEA58 | 17861-6 | 2 k | 0.05 | 0.002 | 1845 k | +| 100k | LEA58 | 8310-5 | 60 k | 0.08 | 0.001 | 1315 k | +| 100k | LEA58 | 72514-3 | 100 k | 0.09 | 0.002 | 1116 k | diff --git a/docs/performance/cql/simple-code-search-1M.gnuplot b/docs/performance/cql/simple-code-search-1M.gnuplot index 0990cda91..4f08add5e 100644 --- a/docs/performance/cql/simple-code-search-1M.gnuplot +++ b/docs/performance/cql/simple-code-search-1M.gnuplot @@ -15,7 +15,7 @@ set title "Simple Code Search - Dataset 1M" set xlabel 'System' set ylabel 'Patients/s' set format y "%.0f k" -set yrange [0:2100] +set yrange [0:2300] # Define grid set grid ytics diff --git a/docs/performance/cql/simple-code-search-1M.png b/docs/performance/cql/simple-code-search-1M.png index 82dafd8e2..2429c70ea 100644 Binary files a/docs/performance/cql/simple-code-search-1M.png and b/docs/performance/cql/simple-code-search-1M.png differ diff --git a/docs/performance/cql/simple-code-search-1M.txt b/docs/performance/cql/simple-code-search-1M.txt index 1928b800e..bc944924d 100644 --- a/docs/performance/cql/simple-code-search-1M.txt +++ b/docs/performance/cql/simple-code-search-1M.txt @@ -1,12 +1,12 @@ -| 1M | LEA25 | 17861-6 | 25 k | 8.04 | 0.059 | 124.3 k | -| 1M | LEA25 | 8310-5 | 603 k | 11.40 | 0.043 | 87.7 k | -| 1M | LEA25 | 72514-3 | 998 k | 13.16 | 0.049 | 76.0 k | -| 1M | LEA36 | 17861-6 | 25 k | 3.90 | 0.009 | 256.1 k | -| 1M | LEA36 | 8310-5 | 603 k | 5.74 | 0.023 | 174.2 k | -| 1M | LEA36 | 72514-3 | 998 k | 6.68 | 0.036 | 149.6 k | -| 1M | LEA47 | 17861-6 | 25 k | 0.59 | 0.003 | 1705 k | -| 1M | LEA47 | 8310-5 | 603 k | 0.64 | 0.003 | 1557 k | -| 1M | LEA47 | 72514-3 | 998 k | 0.76 | 0.006 | 1324 k | -| 1M | LEA58 | 17861-6 | 25 k | 0.61 | 0.005 | 1633 k | -| 1M | LEA58 | 8310-5 | 603 k | 0.67 | 0.005 | 1495 k | -| 1M | LEA58 | 72514-3 | 998 k | 0.75 | 0.003 | 1336 k | +| 1M | LEA25 | 17861-6 | 25 k | 0.47 | 0.011 | 2148 k | +| 1M | LEA25 | 8310-5 | 603 k | 10.69 | 1.232 | 93.6 k | +| 1M | LEA25 | 72514-3 | 998 k | 16.74 | 1.959 | 59.8 k | +| 1M | LEA36 | 17861-6 | 25 k | 0.44 | 0.003 | 2283 k | +| 1M | LEA36 | 8310-5 | 603 k | 4.61 | 0.031 | 217.0 k | +| 1M | LEA36 | 72514-3 | 998 k | 7.15 | 0.018 | 139.8 k | +| 1M | LEA47 | 17861-6 | 25 k | 0.47 | 0.004 | 2138 k | +| 1M | LEA47 | 8310-5 | 603 k | 0.64 | 0.008 | 1555 k | +| 1M | LEA47 | 72514-3 | 998 k | 0.97 | 0.009 | 1032 k | +| 1M | LEA58 | 17861-6 | 25 k | 0.48 | 0.005 | 2069 k | +| 1M | LEA58 | 8310-5 | 603 k | 0.63 | 0.004 | 1587 k | +| 1M | LEA58 | 72514-3 | 998 k | 0.73 | 0.006 | 1375 k | diff --git a/docs/performance/cql/ten-code-search-100k-fh.png b/docs/performance/cql/ten-code-search-100k-fh.png index 67465fd83..9a411479d 100644 Binary files a/docs/performance/cql/ten-code-search-100k-fh.png and b/docs/performance/cql/ten-code-search-100k-fh.png differ diff --git a/docs/performance/cql/ten-code-search-100k-fh.txt b/docs/performance/cql/ten-code-search-100k-fh.txt index cf07e7acd..abcbbdd33 100644 --- a/docs/performance/cql/ten-code-search-100k-fh.txt +++ b/docs/performance/cql/ten-code-search-100k-fh.txt @@ -1,8 +1,8 @@ -| 100k-fh | LEA25 | 2 k | 1.30 | 0.008 | 76.7 k | -| 100k-fh | LEA25 | 98 k | 0.47 | 0.006 | 214.4 k | -| 100k-fh | LEA36 | 2 k | 0.75 | 0.008 | 133.1 k | -| 100k-fh | LEA36 | 98 k | 0.29 | 0.009 | 343.9 k | -| 100k-fh | LEA47 | 2 k | 0.45 | 0.003 | 224.7 k | -| 100k-fh | LEA47 | 98 k | 0.16 | 0.003 | 628.0 k | -| 100k-fh | LEA58 | 2 k | 0.31 | 0.003 | 322.3 k | -| 100k-fh | LEA58 | 98 k | 0.10 | 0.002 | 1000 k | +| 100k-fh | LEA25 | 2 k | 0.13 | 0.007 | 747.3 k | +| 100k-fh | LEA25 | 98 k | 0.35 | 0.019 | 282.9 k | +| 100k-fh | LEA36 | 2 k | 0.08 | 0.000 | 1320 k | +| 100k-fh | LEA36 | 98 k | 0.18 | 0.003 | 547.0 k | +| 100k-fh | LEA47 | 2 k | 0.06 | 0.002 | 1708 k | +| 100k-fh | LEA47 | 98 k | 0.09 | 0.002 | 1171 k | +| 100k-fh | LEA58 | 2 k | 0.06 | 0.001 | 1774 k | +| 100k-fh | LEA58 | 98 k | 0.08 | 0.001 | 1178 k | diff --git a/docs/performance/cql/ten-code-search-100k.gnuplot b/docs/performance/cql/ten-code-search-100k.gnuplot index ae278e9b3..5ed92909f 100644 --- a/docs/performance/cql/ten-code-search-100k.gnuplot +++ b/docs/performance/cql/ten-code-search-100k.gnuplot @@ -15,7 +15,7 @@ set title "Ten Code Search - Dataset 100k" set xlabel 'System' set ylabel 'Patients/s' set format y "%.0f k" -set yrange [0:] +set yrange [0:1900] # Define grid set grid ytics diff --git a/docs/performance/cql/ten-code-search-100k.png b/docs/performance/cql/ten-code-search-100k.png index 1a6bbe4e8..23559e5e6 100644 Binary files a/docs/performance/cql/ten-code-search-100k.png and b/docs/performance/cql/ten-code-search-100k.png differ diff --git a/docs/performance/cql/ten-code-search-100k.txt b/docs/performance/cql/ten-code-search-100k.txt index 5ff2fe72c..df927925f 100644 --- a/docs/performance/cql/ten-code-search-100k.txt +++ b/docs/performance/cql/ten-code-search-100k.txt @@ -1,8 +1,8 @@ -| 100k | LEA25 | 395 | 0.70 | 0.011 | 142.0 k | -| 100k | LEA25 | 95 k | 0.44 | 0.022 | 227.6 k | -| 100k | LEA36 | 395 | 0.40 | 0.010 | 249.3 k | -| 100k | LEA36 | 95 k | 0.22 | 0.009 | 448.5 k | -| 100k | LEA47 | 395 | 0.23 | 0.001 | 437.5 k | -| 100k | LEA47 | 95 k | 0.14 | 0.002 | 731.9 k | -| 100k | LEA58 | 395 | 0.16 | 0.001 | 607.1 k | -| 100k | LEA58 | 95 k | 0.11 | 0.002 | 941.9 k | +| 100k | LEA25 | 395 | 0.13 | 0.006 | 778.5 k | +| 100k | LEA25 | 95 k | 0.38 | 0.019 | 265.8 k | +| 100k | LEA36 | 395 | 0.07 | 0.003 | 1488 k | +| 100k | LEA36 | 95 k | 0.21 | 0.007 | 471.0 k | +| 100k | LEA47 | 395 | 0.06 | 0.002 | 1785 k | +| 100k | LEA47 | 95 k | 0.12 | 0.003 | 861.4 k | +| 100k | LEA58 | 395 | 0.06 | 0.002 | 1705 k | +| 100k | LEA58 | 95 k | 0.09 | 0.002 | 1170 k | diff --git a/docs/performance/cql/ten-code-search-1M.gnuplot b/docs/performance/cql/ten-code-search-1M.gnuplot index 1a554eb74..b62236bf4 100644 --- a/docs/performance/cql/ten-code-search-1M.gnuplot +++ b/docs/performance/cql/ten-code-search-1M.gnuplot @@ -15,7 +15,7 @@ set title "Ten Code Search - Dataset 1M" set xlabel 'System' set ylabel 'Patients/s' set format y "%.0f k" -set yrange [0:] +set yrange [0:2200] # Define grid set grid ytics diff --git a/docs/performance/cql/ten-code-search-1M.png b/docs/performance/cql/ten-code-search-1M.png index ce05adec8..0dad3d038 100644 Binary files a/docs/performance/cql/ten-code-search-1M.png and b/docs/performance/cql/ten-code-search-1M.png differ diff --git a/docs/performance/cql/ten-code-search-1M.txt b/docs/performance/cql/ten-code-search-1M.txt index 67dc32b7b..ab7974fe0 100644 --- a/docs/performance/cql/ten-code-search-1M.txt +++ b/docs/performance/cql/ten-code-search-1M.txt @@ -1,8 +1,8 @@ -| 1M | LEA25 | 4 k | 13.60 | 0.073 | 73.5 k | -| 1M | LEA25 | 954 k | 11.87 | 0.027 | 84.2 k | -| 1M | LEA36 | 4 k | 7.24 | 0.009 | 138.1 k | -| 1M | LEA36 | 954 k | 6.09 | 0.035 | 164.3 k | -| 1M | LEA47 | 4 k | 2.11 | 0.005 | 473.3 k | -| 1M | LEA47 | 954 k | 1.18 | 0.003 | 846.4 k | -| 1M | LEA58 | 4 k | 1.39 | 0.003 | 719.2 k | -| 1M | LEA58 | 954 k | 0.95 | 0.004 | 1053 k | +| 1M | LEA25 | 4 k | 1.14 | 0.243 | 873.6 k | +| 1M | LEA25 | 954 k | 17.57 | 0.530 | 56.9 k | +| 1M | LEA36 | 4 k | 0.52 | 0.021 | 1905 k | +| 1M | LEA36 | 954 k | 5.52 | 0.034 | 181.2 k | +| 1M | LEA47 | 4 k | 0.50 | 0.004 | 2002 k | +| 1M | LEA47 | 954 k | 0.82 | 0.008 | 1217 k | +| 1M | LEA58 | 4 k | 0.51 | 0.014 | 1963 k | +| 1M | LEA58 | 954 k | 0.73 | 0.008 | 1378 k | diff --git a/modules/admin-api/deps.edn b/modules/admin-api/deps.edn index ab342a678..fd2a5f9e8 100644 --- a/modules/admin-api/deps.edn +++ b/modules/admin-api/deps.edn @@ -4,6 +4,9 @@ {blaze/async {:local/root "../async"} + blaze/cql + {:local/root "../cql"} + blaze/interaction {:local/root "../interaction"} diff --git a/modules/admin-api/src/blaze/admin_api.clj b/modules/admin-api/src/blaze/admin_api.clj index b94cfd975..42c1760a0 100644 --- a/modules/admin-api/src/blaze/admin_api.clj +++ b/modules/admin-api/src/blaze/admin_api.clj @@ -4,8 +4,13 @@ [blaze.admin-api.validation] [blaze.anomaly :as ba :refer [if-ok]] [blaze.async.comp :as ac :refer [do-sync]] + [blaze.db.impl.index.patient-last-change :as plc] + [blaze.db.kv :as kv] [blaze.db.kv.rocksdb :as rocksdb] [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.spec] [blaze.fhir.response.create :as create-response] [blaze.fhir.spec :as fhir-spec] [blaze.handler.fhir.util :as fhir-util] @@ -86,7 +91,7 @@ (defn- column-family-data [db column-family] (let [long-property (partial rocksdb/long-property db column-family)] {:name (name column-family) - :estimate-num-keys (long-property "rocksdb.estimate-num-keys") + :estimate-num-keys (kv/estimate-num-keys db column-family) :estimate-live-data-size (long-property "rocksdb.estimate-live-data-size") :live-sst-files-size (long-property "rocksdb.live-sst-files-size") :size-all-mem-tables (long-property "rocksdb.size-all-mem-tables")})) @@ -97,6 +102,19 @@ (ring/response) (ac/completed-future)))) +(defn- column-family-state-not-found-msg [db-name column-family] + (format "The state of the column family `%s` in database `%s` was not found." column-family db-name)) + +(defn- column-family-state-not-found [db-name column-family] + (ring/not-found {:msg (column-family-state-not-found-msg db-name column-family)})) + +(def ^:private cf-state-handler + (fn [{:keys [db] {db-name :db :keys [column-family]} :path-params}] + (-> (if (and (= "index" db-name) (= "patient-last-change-index" column-family)) + (ring/response (plc/state db)) + (column-family-state-not-found db-name column-family)) + (ac/completed-future)))) + (defn- column-family-not-found-msg [db-name column-family] (format "The column family `%s` in database `%s` was not found." column-family db-name)) @@ -216,7 +234,8 @@ (defn- router [{:keys [context-path admin-node validator db-sync-timeout dbs create-job-handler read-job-handler search-type-job-handler - pause-job-handler resume-job-handler cancel-job-handler] + pause-job-handler resume-job-handler cancel-job-handler + cql-cache-stats-handler cql-bloom-filters-handler] :or {context-path "" db-sync-timeout 10000} :as context}] @@ -323,6 +342,10 @@ {:type "array"}}}}}}}}] ["/{column-family}" {} + ["/state" + {:get + {:handler cf-state-handler + :summary "Fetch the state of a column family of a database."}}] ["/metadata" {:get {:handler cf-metadata-handler @@ -385,7 +408,36 @@ {:handler resume-job-handler}}] ["/$cancel" {:post - {:handler cancel-job-handler}}]]]] + {:handler cancel-job-handler}}]]] + ["/cql" + {} + ["/cache-stats" + {:get + {:handler cql-cache-stats-handler + :summary "Fetch CQL cache stats." + :openapi + {:operation-id "cql-cache-stats" + :responses + {200 + {:description "CQL cache stats." + :content + {"application/json" + {:schema + {:type "object"}}}}}}}}] + ["/bloom-filters" + {:get + {:handler cql-bloom-filters-handler + :summary "Fetch the list of all CQL Bloom filters." + :openapi + {:operation-id "cql-bloom-filters" + :responses + {200 + {:description "A list of CQL Bloom filters." + :content + {"application/json" + {:schema + {:type "array" + :items {"$ref" "#/components/schemas/BloomFilter"}}}}}}}}}]]] {:path (str context-path "/__admin") :syntax :bracket})) @@ -453,6 +505,32 @@ (ring/header "Last-Modified" (fhir-util/last-modified tx)) (ring/header "ETag" (fhir-util/etag tx))))))) +(def ^:private bloom-filter-xf + (comp + (take 100) + (map + #(select-keys % [::bloom-filter/hash ::bloom-filter/t ::bloom-filter/expr-form + ::bloom-filter/patient-count ::bloom-filter/mem-size])) + (map #(update % ::bloom-filter/hash str)))) + +(defn- cql-cache-stats-handler [{::expr/keys [cache]}] + (if cache + (fn [_] + (-> (ring/response {:total (ec/total cache)}) + (ac/completed-future))) + (fn [_] + (-> (ring/not-found {:msg "The feature \"CQL Expression Cache\" is disabled."}) + (ac/completed-future))))) + +(defn- cql-bloom-filters-handler [{::expr/keys [cache]}] + (if cache + (fn [_] + (-> (ring/response (into [] bloom-filter-xf (ec/list-by-t cache))) + (ac/completed-future))) + (fn [_] + (-> (ring/not-found {:msg "The feature \"CQL Expression Cache\" is disabled."}) + (ac/completed-future))))) + (defmethod m/pre-init-spec :blaze/admin-api [_] (s/keys :req-un [:blaze/context-path ::admin-node :blaze/job-scheduler ::read-job-handler ::search-type-job-handler @@ -467,7 +545,9 @@ :create-job-handler (create-job-handler job-scheduler) :pause-job-handler (job-action-handler job-scheduler js/pause-job) :resume-job-handler (job-action-handler job-scheduler js/resume-job) - :cancel-job-handler (job-action-handler job-scheduler js/cancel-job))) + :cancel-job-handler (job-action-handler job-scheduler js/cancel-job) + :cql-cache-stats-handler (cql-cache-stats-handler context) + :cql-bloom-filters-handler (cql-bloom-filters-handler context))) ((wrap-json-output {}) (fn [{:keys [uri]}] (-> (ring/not-found {"uri" uri}) diff --git a/modules/admin-api/test/blaze/admin_api_test.clj b/modules/admin-api/test/blaze/admin_api_test.clj index 8e2f8b54a..d3629d89e 100644 --- a/modules/admin-api/test/blaze/admin_api_test.clj +++ b/modules/admin-api/test/blaze/admin_api_test.clj @@ -4,12 +4,16 @@ [blaze.async.comp :as ac :refer [do-sync]] [blaze.db.api :as d] [blaze.db.api-stub] + [blaze.db.impl.index.patient-last-change :as plc] [blaze.db.kv :as-alias kv] [blaze.db.kv.rocksdb :as rocksdb] [blaze.db.node :as node :refer [node?]] [blaze.db.resource-store :as rs] [blaze.db.resource-store.kv :as rs-kv] [blaze.db.tx-log :as tx-log] + [blaze.elm.compiler :as c] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as ec] [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type :as type] [blaze.fhir.test-util :refer [structure-definition-repo]] @@ -50,6 +54,7 @@ :kv-store (ig/ref :blaze.db.main/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.main/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} [:blaze.db/node :blaze.db.admin/node] @@ -60,6 +65,7 @@ :kv-store (ig/ref :blaze.db.admin/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.admin/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} [::tx-log/local :blaze.db.main/tx-log] @@ -116,7 +122,9 @@ :max-bytes-for-level-base-in-mb 1 :target-file-size-base-in-mb 1} :type-stats-index nil - :system-stats-index nil}} + :system-stats-index nil + :cql-bloom-filter nil + :cql-bloom-filter-by-t nil}} [::kv/mem :blaze.db.admin/index-kv-store] {:column-families @@ -199,6 +207,8 @@ :blaze.page-store/local {:secure-rng (ig/ref :blaze.test/fixed-rng)} + :blaze/scheduler {} + :blaze.test/fixed-rng {} :blaze.test/fixed-rng-fn {}}) @@ -418,7 +428,20 @@ ["content" "application/json" "schema" "type"] := "array" ["content" "application/json" "schema" "items" "type"] := "object" ["content" "application/json" "schema" "items" "properties" "dataSize" "type"] := "number" - ["content" "application/json" "schema" "items" "properties" "totalRawKeySize" "type"] := "number"))))))))) + ["content" "application/json" "schema" "items" "properties" "totalRawKeySize" "type"] := "number"))))) + + (testing "cql-bloom-filters" + (let [op (get-in body ["paths" "/fhir/__admin/cql/bloom-filters" "get"])] + (given op + "operationId" := "cql-bloom-filters" + "summary" := "Fetch the list of all CQL Bloom filters.") + + (testing "responses" + (testing "200" + (given (get-in op ["responses" "200"]) + "description" := "A list of CQL Bloom filters." + ["content" "application/json" "schema" "type"] := "array" + ["content" "application/json" "schema" "items" "$ref"] := "#/components/schemas/BloomFilter"))))))))) (deftest root-test (testing "without settings and features" @@ -454,7 +477,8 @@ (with-handler [handler] (assoc-in (config (new-temp-dir!)) [:blaze/admin-api :features] - [{:name "OpenID Authentication" + [{:key "open-id-authentication" + :name "OpenID Authentication" :toggle "OPENID_PROVIDER_URL" :enabled true}]) [] @@ -465,6 +489,8 @@ :status := 200 [:body "settings" count] := 0 [:body "features" count] := 1 + [:body "features" 0 count] := 4 + [:body "features" 0 "key"] := "open-id-authentication" [:body "features" 0 "name"] := "OpenID Authentication" [:body "features" 0 "toggle"] := "OPENID_PROVIDER_URL" [:body "features" 0 "enabled"] := true))))) @@ -545,6 +571,23 @@ [:body 0 "liveSstFilesSize"] := 0 [:body 0 "sizeAllMemTables"] := 2048))))) +(deftest column-family-state-test + (with-handler [handler] (config (new-temp-dir!)) [] + (testing "patient-last-change-index" + (with-redefs [plc/state (fn [_] {:type :current})] + (given @(handler + {:request-method :get + :uri "/fhir/__admin/dbs/index/column-families/patient-last-change-index/state"}) + :status := 200 + :body := {"type" "current"}))) + + (testing "other column-family" + (given @(handler + {:request-method :get + :uri "/fhir/__admin/dbs/index/column-families/default/state"}) + :status := 404 + [:body "msg"] := "The state of the column family `default` in database `index` was not found.")))) + (deftest column-family-metadata-test (with-handler [handler] (config (new-temp-dir!)) [] (testing "search-param-value-index in index database" @@ -581,7 +624,7 @@ [#fhir/Coding {:system #fhir/uri"system-192253" :code #fhir/code"code-192300"}]} - :subject (type/map->Reference {:reference (str "Patient/" pat-id)})}])) + :subject (type/reference {:reference (str "Patient/" pat-id)})}])) (range 120))) (mapv (range 100))) @@ -945,3 +988,41 @@ (given body "resourceType" := "Task" "status" := "cancelled"))))) + +(defn- with-cql-expr-cache [config] + (-> (assoc config + ::expr/cache + {:node (ig/ref :blaze.db.main/node) + :executor (ig/ref :blaze.test/executor)} + :blaze.test/executor {}) + (assoc-in [:blaze/admin-api ::expr/cache] (ig/ref ::expr/cache)))) + +(deftest cql-bloom-filters-test + (testing "without expression cache" + (with-handler [handler] (config (new-temp-dir!)) [] + (testing "not-found" + (given @(handler + {:request-method :get + :uri "/fhir/__admin/cql/bloom-filters"}) + :status := 404 + [:body "msg"] := "The feature \"CQL Expression Cache\" is disabled.")))) + + (with-handler [handler {::expr/keys [cache]}] (with-cql-expr-cache (config (new-temp-dir!))) [] + (let [elm {:type "Exists" + :operand {:type "Retrieve" :dataType "{http://hl7.org/fhir}Observation"}} + expr (c/compile {:eval-context "Patient"} elm)] + (ec/get cache expr)) + + (Thread/sleep 100) + + (testing "success" + (given @(handler + {:request-method :get + :uri "/fhir/__admin/cql/bloom-filters"}) + :status := 200 + [:body count] := 1 + [:body 0 "hash"] := "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985" + [:body 0 "t"] := 0 + [:body 0 "exprForm"] := "(exists (retrieve \"Observation\"))" + [:body 0 "patientCount"] := 0 + [:body 0 "memSize"] := 11981)))) diff --git a/modules/byte-buffer/deps.edn b/modules/byte-buffer/deps.edn index 9792f91bf..54828cb07 100644 --- a/modules/byte-buffer/deps.edn +++ b/modules/byte-buffer/deps.edn @@ -1,6 +1,6 @@ {:deps {com.google.protobuf/protobuf-java - {:mvn/version "4.27.1"}} + {:mvn/version "4.27.2"}} :aliases {:test diff --git a/modules/cache-collector/.clj-kondo/config.edn b/modules/cache-collector/.clj-kondo/config.edn new file mode 100644 index 000000000..035b03646 --- /dev/null +++ b/modules/cache-collector/.clj-kondo/config.edn @@ -0,0 +1,3 @@ +{:config-paths + ["../../../.clj-kondo/root" + "../../module-test-util/resources/clj-kondo.exports/blaze/module-test-util"]} diff --git a/modules/cache-collector/Makefile b/modules/cache-collector/Makefile new file mode 100644 index 000000000..2fff67259 --- /dev/null +++ b/modules/cache-collector/Makefile @@ -0,0 +1,25 @@ +fmt: + cljfmt check + +lint: + clj-kondo --lint src test deps.edn + +prep: + clojure -X:deps prep + +test: prep + clojure -M:test:kaocha --profile :ci + +test-coverage: prep + clojure -M:test:coverage + +deps-tree: + clojure -X:deps tree + +deps-list: + clojure -X:deps list + +clean: + rm -rf .clj-kondo/.cache .cpcache target + +.PHONY: fmt lint prep test test-coverage deps-tree deps-list clean diff --git a/modules/cache-collector/deps.edn b/modules/cache-collector/deps.edn new file mode 100644 index 000000000..9d0c7f829 --- /dev/null +++ b/modules/cache-collector/deps.edn @@ -0,0 +1,45 @@ +{:deps + {blaze/metrics + {:local/root "../metrics"} + + blaze/module-base + {:local/root "../module-base"} + + com.github.ben-manes.caffeine/caffeine + {:mvn/version "3.1.8"}} + + :aliases + {:test + {:extra-paths ["test"] + + :extra-deps + {blaze/module-test-util + {:local/root "../module-test-util"}}} + + :kaocha + {:extra-deps + {lambdaisland/kaocha + {:mvn/version "1.91.1392"}} + + :main-opts ["-m" "kaocha.runner"]} + + :test-perf + {:extra-paths ["test-perf"] + + :extra-deps + {blaze/fhir-test-util + {:local/root "../fhir-test-util"} + + criterium/criterium + {:mvn/version "0.4.6"} + + org.openjdk.jol/jol-core + {:mvn/version "0.17"}}} + + :coverage + {:extra-deps + {cloverage/cloverage + {:mvn/version "1.2.4"}} + + :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test" + "-e" ".+spec"]}}} diff --git a/modules/db/src/blaze/db/cache_collector.clj b/modules/cache-collector/src/blaze/cache_collector.clj similarity index 75% rename from modules/db/src/blaze/db/cache_collector.clj rename to modules/cache-collector/src/blaze/cache_collector.clj index 4db50791f..d78f1f5d9 100644 --- a/modules/db/src/blaze/db/cache_collector.clj +++ b/modules/cache-collector/src/blaze/cache_collector.clj @@ -1,13 +1,13 @@ -(ns blaze.db.cache-collector +(ns blaze.cache-collector (:require - [blaze.db.cache-collector.protocols :as p] - [blaze.db.cache-collector.spec] + [blaze.cache-collector.protocols :as p] + [blaze.cache-collector.spec] [blaze.metrics.core :as metrics] [blaze.module :as m] [clojure.spec.alpha :as s] [integrant.core :as ig]) (:import - [com.github.benmanes.caffeine.cache Cache] + [com.github.benmanes.caffeine.cache AsyncCache Cache] [com.github.benmanes.caffeine.cache.stats CacheStats])) (set! *warn-on-reflection* true) @@ -17,7 +17,12 @@ (-stats [cache] (.stats cache)) (-estimated-size [cache] - (.estimatedSize cache))) + (.estimatedSize cache)) + AsyncCache + (-stats [cache] + (.stats (.synchronous cache))) + (-estimated-size [cache] + (.estimatedSize (.synchronous cache)))) (defn- sample-xf [f] (map (fn [[name stats estimated-size]] {:label-values [name] :value (f stats estimated-size)}))) @@ -36,47 +41,47 @@ (when cache [name (p/-stats cache) (p/-estimated-size cache)])))) -(defmethod m/pre-init-spec :blaze.db/cache-collector [_] +(defmethod m/pre-init-spec :blaze/cache-collector [_] (s/keys :req-un [::caches])) -(defmethod ig/init-key :blaze.db/cache-collector +(defmethod ig/init-key :blaze/cache-collector [_ {:keys [caches]}] (metrics/collector (let [stats (into [] mapper caches)] [(counter-metric - "blaze_db_cache_hits_total" + "blaze_cache_hits_total" "Returns the number of times Cache lookup methods have returned a cached value." (fn [stats _] (.hitCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_misses_total" + "blaze_cache_misses_total" "Returns the number of times Cache lookup methods have returned an uncached (newly loaded) value, or null." (fn [stats _] (.missCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_successes_total" + "blaze_cache_load_successes_total" "Returns the number of times Cache lookup methods have successfully loaded a new value." (fn [stats _] (.loadSuccessCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_failures_total" + "blaze_cache_load_failures_total" "Returns the number of times Cache lookup methods failed to load a new value, either because no value was found or an exception was thrown while loading." (fn [stats _] (.loadFailureCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_seconds_total" + "blaze_cache_load_seconds_total" "Returns the total number of seconds the cache has spent loading new values." (fn [stats _] (/ (double (.totalLoadTime ^CacheStats stats)) 1e9)) stats) (counter-metric - "blaze_db_cache_evictions_total" + "blaze_cache_evictions_total" "Returns the number of times an entry has been evicted." (fn [stats _] (.evictionCount ^CacheStats stats)) stats) (gauge-metric - "blaze_db_cache_estimated_size" + "blaze_cache_estimated_size" "Returns the approximate number of entries in this cache." (fn [_ estimated-size] estimated-size) stats)]))) -(derive :blaze.db/cache-collector :blaze.metrics/collector) +(derive :blaze/cache-collector :blaze.metrics/collector) diff --git a/modules/db/src/blaze/db/cache_collector/protocols.clj b/modules/cache-collector/src/blaze/cache_collector/protocols.clj similarity index 61% rename from modules/db/src/blaze/db/cache_collector/protocols.clj rename to modules/cache-collector/src/blaze/cache_collector/protocols.clj index d2098c858..8b813df4b 100644 --- a/modules/db/src/blaze/db/cache_collector/protocols.clj +++ b/modules/cache-collector/src/blaze/cache_collector/protocols.clj @@ -1,4 +1,4 @@ -(ns blaze.db.cache-collector.protocols) +(ns blaze.cache-collector.protocols) (defprotocol StatsCache (-stats [_]) diff --git a/modules/cache-collector/src/blaze/cache_collector/spec.clj b/modules/cache-collector/src/blaze/cache_collector/spec.clj new file mode 100644 index 000000000..1eaceda59 --- /dev/null +++ b/modules/cache-collector/src/blaze/cache_collector/spec.clj @@ -0,0 +1,7 @@ +(ns blaze.cache-collector.spec + (:require + [blaze.cache-collector.protocols :as p] + [clojure.spec.alpha :as s])) + +(s/def :blaze.cache-collector/caches + (s/map-of string? (s/nilable #(satisfies? p/StatsCache %)))) diff --git a/modules/cache-collector/test/blaze/cache_collector_test.clj b/modules/cache-collector/test/blaze/cache_collector_test.clj new file mode 100644 index 000000000..fbc255a5b --- /dev/null +++ b/modules/cache-collector/test/blaze/cache_collector_test.clj @@ -0,0 +1,142 @@ +(ns blaze.cache-collector-test + (:require + [blaze.cache-collector] + [blaze.metrics.core :as metrics] + [blaze.module.test-util :refer [with-system]] + [blaze.test-util :as tu :refer [given-thrown]] + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest testing]] + [integrant.core :as ig] + [juxt.iota :refer [given]]) + (:import + [com.github.benmanes.caffeine.cache AsyncCache Cache Caffeine] + [java.util.function Function])) + +(set! *warn-on-reflection* true) +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(def ^Cache cache (-> (Caffeine/newBuilder) (.recordStats) (.build))) +(def ^AsyncCache async-cache (-> (Caffeine/newBuilder) (.recordStats) (.buildAsync))) + +(def config + {:blaze/cache-collector + {:caches + {"name-135224" cache + "name-145135" async-cache + "name-093214" nil}}}) + +(deftest init-test + (testing "nil config" + (given-thrown (ig/init {:blaze/cache-collector nil}) + :key := :blaze/cache-collector + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `map?)) + + (testing "missing config" + (given-thrown (ig/init {:blaze/cache-collector {}}) + :key := :blaze/cache-collector + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :caches)))) + + (testing "invalid caches" + (given-thrown (ig/init {:blaze/cache-collector {:caches ::invalid}}) + :key := :blaze/cache-collector + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `map? + [:cause-data ::s/problems 0 :val] := ::invalid))) + +(deftest cache-collector-test + (with-system [{collector :blaze/cache-collector} config] + + (testing "all zero on fresh cache" + (given (metrics/collect collector) + [0 :name] := "blaze_cache_hits" + [0 :type] := :counter + [0 :samples count] := 2 + [0 :samples 0 :value] := 0.0 + [0 :samples 0 :label-values] := ["name-135224"] + [0 :samples 1 :value] := 0.0 + [0 :samples 1 :label-values] := ["name-145135"] + [1 :name] := "blaze_cache_misses" + [1 :type] := :counter + [1 :samples count] := 2 + [1 :samples 0 :value] := 0.0 + [1 :samples 1 :value] := 0.0 + [2 :name] := "blaze_cache_load_successes" + [2 :type] := :counter + [2 :samples count] := 2 + [2 :samples 0 :value] := 0.0 + [2 :samples 1 :value] := 0.0 + [3 :name] := "blaze_cache_load_failures" + [3 :type] := :counter + [3 :samples count] := 2 + [3 :samples 0 :value] := 0.0 + [3 :samples 1 :value] := 0.0 + [4 :name] := "blaze_cache_load_seconds" + [4 :type] := :counter + [4 :samples count] := 2 + [4 :samples 0 :value] := 0.0 + [4 :samples 1 :value] := 0.0 + [5 :name] := "blaze_cache_evictions" + [5 :type] := :counter + [5 :samples count] := 2 + [5 :samples 0 :value] := 0.0 + [5 :samples 1 :value] := 0.0 + [6 :name] := "blaze_cache_estimated_size" + [6 :type] := :gauge + [6 :samples count] := 2 + [6 :samples 0 :value] := 0.0 + [6 :samples 1 :value] := 0.0)) + + (testing "one load" + (.get cache "1" (reify Function (apply [_ key] key))) + (.get async-cache "1" (reify Function (apply [_ key] key))) + (Thread/sleep 100) + + (given (metrics/collect collector) + [0 :name] := "blaze_cache_hits" + [0 :samples 0 :value] := 0.0 + [0 :samples 1 :value] := 0.0 + [1 :name] := "blaze_cache_misses" + [1 :samples 0 :value] := 1.0 + [1 :samples 1 :value] := 1.0 + [2 :name] := "blaze_cache_load_successes" + [2 :samples 0 :value] := 1.0 + [2 :samples 1 :value] := 1.0 + [3 :name] := "blaze_cache_load_failures" + [3 :samples 0 :value] := 0.0 + [3 :samples 1 :value] := 0.0 + [5 :name] := "blaze_cache_evictions" + [5 :samples 0 :value] := 0.0 + [5 :samples 1 :value] := 0.0 + [6 :name] := "blaze_cache_estimated_size" + [6 :samples 0 :value] := 1.0 + [6 :samples 1 :value] := 1.0)) + + (testing "one loads and one hit" + (.get cache "1" (reify Function (apply [_ key] key))) + (.get async-cache "1" (reify Function (apply [_ key] key))) + (Thread/sleep 100) + + (given (metrics/collect collector) + [0 :name] := "blaze_cache_hits" + [0 :samples 0 :value] := 1.0 + [0 :samples 1 :value] := 1.0 + [1 :name] := "blaze_cache_misses" + [1 :samples 0 :value] := 1.0 + [1 :samples 1 :value] := 1.0 + [2 :name] := "blaze_cache_load_successes" + [2 :samples 0 :value] := 1.0 + [2 :samples 1 :value] := 1.0 + [3 :name] := "blaze_cache_load_failures" + [3 :samples 0 :value] := 0.0 + [3 :samples 1 :value] := 0.0 + [5 :name] := "blaze_cache_evictions" + [5 :samples 0 :value] := 0.0 + [5 :samples 1 :value] := 0.0 + [6 :name] := "blaze_cache_estimated_size" + [6 :samples 0 :value] := 1.0 + [6 :samples 1 :value] := 1.0)))) diff --git a/modules/cache-collector/tests.edn b/modules/cache-collector/tests.edn new file mode 100644 index 000000000..94fe5636c --- /dev/null +++ b/modules/cache-collector/tests.edn @@ -0,0 +1,5 @@ +#kaocha/v1 + #merge + [{} + #profile {:ci {:reporter kaocha.report/documentation + :color? false}}] diff --git a/modules/coll/deps.edn b/modules/coll/deps.edn index 233518014..ea44c6e51 100644 --- a/modules/coll/deps.edn +++ b/modules/coll/deps.edn @@ -1,4 +1,6 @@ -{:aliases +{:paths ["src" "resources"] + + :aliases {:test {:extra-paths ["test"] diff --git a/modules/coll/resources/clj-kondo.exports/blaze/coll/config.edn b/modules/coll/resources/clj-kondo.exports/blaze/coll/config.edn new file mode 100644 index 000000000..626f56def --- /dev/null +++ b/modules/coll/resources/clj-kondo.exports/blaze/coll/config.edn @@ -0,0 +1,2 @@ +{:lint-as + {blaze.coll.core/with-open-coll clojure.core/with-open}} diff --git a/modules/coll/src/blaze/coll/core.clj b/modules/coll/src/blaze/coll/core.clj index 37957cfcd..e2b0c0582 100644 --- a/modules/coll/src/blaze/coll/core.clj +++ b/modules/coll/src/blaze/coll/core.clj @@ -57,3 +57,14 @@ (.nth ^Indexed coll i)) ([coll i not-found] (.nth ^Indexed coll i not-found))) + +(defmacro with-open-coll + "Like `clojure.core/with-open` but opens and closes the resources on every + reduce call to `coll`." + [bindings coll] + `(reify + Sequential + IReduceInit + (reduce [_ rf# init#] + (with-open ~bindings + (reduce rf# init# ~coll))))) diff --git a/modules/cql/.clj-kondo/config.edn b/modules/cql/.clj-kondo/config.edn index dc262ed13..ca7000645 100644 --- a/modules/cql/.clj-kondo/config.edn +++ b/modules/cql/.clj-kondo/config.edn @@ -2,17 +2,21 @@ ["../../../.clj-kondo/root" "../../anomaly/resources/clj-kondo.exports/blaze/anomaly" "../../async/resources/clj-kondo.exports/blaze/async" + "../../coll/resources/clj-kondo.exports/blaze/coll" "../../db-stub/resources/clj-kondo.exports/blaze/db-stub" + "../../module-base/resources/clj-kondo.exports/prom-metrics/prom-metrics" "../../module-test-util/resources/clj-kondo.exports/blaze/module-test-util"] :lint-as - {blaze.elm.compiler.macros/defunop clojure.core/defn + {blaze.db.impl.macros/with-open-coll clojure.core/with-open + blaze.elm.compiler.macros/defunop clojure.core/defn blaze.elm.compiler.macros/defbinop clojure.core/defn blaze.elm.compiler.macros/defternop clojure.core/defn blaze.elm.compiler.macros/defnaryop clojure.core/defn blaze.elm.compiler.macros/defaggop clojure.core/defn blaze.elm.compiler.macros/defbinopp clojure.core/defn - blaze.elm.compiler.macros/defunopp clojure.core/defn} + blaze.elm.compiler.macros/defunopp clojure.core/defn + blaze.elm.compiler.macros/reify-expr clojure.core/reify} :linters {;; because of macros in modules/cql/src/blaze/elm/compiler.clj diff --git a/modules/cql/deps.edn b/modules/cql/deps.edn index 89fefe625..4aa5f5e73 100644 --- a/modules/cql/deps.edn +++ b/modules/cql/deps.edn @@ -64,4 +64,4 @@ {:mvn/version "1.2.4"}} :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test" - "-e" ".*spec$"]}}} + "-e" ".*spec$" "-e" "blaze.elm.compiler.macros"]}}} diff --git a/modules/cql/src/blaze/cql.clj b/modules/cql/src/blaze/cql.clj new file mode 100644 index 000000000..85e543b59 --- /dev/null +++ b/modules/cql/src/blaze/cql.clj @@ -0,0 +1,7 @@ +(ns blaze.cql + (:require + [blaze.elm.compiler.external-data :as ed] + [blaze.module :refer [reg-collector]])) + +(reg-collector ::retrieve-total + ed/retrieve-total) diff --git a/modules/cql/src/blaze/elm/code.clj b/modules/cql/src/blaze/elm/code.clj index 1b0fb3272..4b1259bd0 100644 --- a/modules/cql/src/blaze/elm/code.clj +++ b/modules/cql/src/blaze/elm/code.clj @@ -26,6 +26,14 @@ core/Expression (-static [_] true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) (-eval [this _ _ _] this) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler.clj b/modules/cql/src/blaze/elm/compiler.clj index d823f0211..6b178a421 100644 --- a/modules/cql/src/blaze/elm/compiler.clj +++ b/modules/cql/src/blaze/elm/compiler.clj @@ -42,5 +42,25 @@ [context expression] (core/compile* context expression)) +(defn attach-cache + "Attaches expression `cache` to `expression` returning a tuple of expression + that uses `cache` in order to improve evaluation performance and list of + attached Bloom filters. + + Otherwise the semantics of the returned expression have to be the same as that + of `expression`." + [expression cache] + ((first (core/-attach-cache expression cache)))) + +(defn resolve-refs + "Resolves expressions defined in `expression-defs` in `expression`." + [expression expression-defs] + (core/-resolve-refs expression expression-defs)) + +(defn resolve-params + "Resolves `parameters` in `expression`." + [expression parameters] + (core/-resolve-params expression parameters)) + (defn form [expression] (core/-form expression)) diff --git a/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj b/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj index c63af970d..0a7cfc18d 100644 --- a/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj @@ -6,7 +6,7 @@ (:require [blaze.anomaly :as ba :refer [throw-anom]] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinop defunop]] + [blaze.elm.compiler.macros :refer [defbinop defunop reify-expr]] [blaze.elm.date-time :as date-time] [blaze.elm.decimal :as decimal] [blaze.elm.protocols :as p] @@ -115,22 +115,30 @@ (p/predecessor x)) ;; 16.19. Round +(defn- round-op [operand precision] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper round-op cache operand precision)) + (-resolve-refs [_ expression-defs] + (round-op (core/-resolve-refs operand expression-defs) + (core/-resolve-refs precision expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper round-op parameters operand precision)) + (-eval [_ context resource scope] + (p/round (core/-eval operand context resource scope) + (core/-eval precision context resource scope))) + (-form [_] + (->> (some-> (core/-form precision) list) + (cons (core/-form operand)) + (cons 'round))))) + (defmethod core/compile* :elm.compiler.type/round [context {:keys [operand precision]}] (let [operand (core/compile* context operand) precision (some->> precision (core/compile* context))] (if (and (core/static? operand) (core/static? precision)) (p/round operand precision) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (p/round (core/-eval operand context resource scope) - (core/-eval precision context resource scope))) - (-form [_] - (->> (some-> (core/-form precision) list) - (cons (core/-form operand)) - (cons 'round))))))) + (round-op operand precision)))) ;; 16.20. Subtract (defbinop subtract [x y] diff --git a/modules/cql/src/blaze/elm/compiler/clinical_operators.clj b/modules/cql/src/blaze/elm/compiler/clinical_operators.clj index 4c53f9b17..afac9e115 100644 --- a/modules/cql/src/blaze/elm/compiler/clinical_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/clinical_operators.clj @@ -5,6 +5,7 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.protocols :as p])) ;; 23.3. CalculateAge @@ -12,19 +13,31 @@ ;; see normalizer.clj ;; 23.4. CalculateAgeAt +(defn- calculate-age-at-op [birth-date date chrono-precision precision] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper-2 calculate-age-at-op cache birth-date + date chrono-precision precision)) + (-resolve-refs [_ expression-defs] + (calculate-age-at-op (core/-resolve-refs birth-date expression-defs) + (core/-resolve-refs date expression-defs) + chrono-precision precision)) + (-resolve-params [_ parameters] + (calculate-age-at-op (core/-resolve-params birth-date parameters) + (core/-resolve-params date parameters) + chrono-precision precision)) + (-eval [_ context resource scope] + (p/duration-between + (core/-eval birth-date context resource scope) + (core/-eval date context resource scope) + chrono-precision)) + (-form [_] + (list 'calculate-age-at (core/-form birth-date) (core/-form date) + precision)))) + (defmethod core/compile* :elm.compiler.type/calculate-age-at [context {[birth-date date] :operand precision :precision}] (when-let [birth-date (core/compile* context birth-date)] (when-let [date (core/compile* context date)] (let [chrono-precision (some-> precision core/to-chrono-unit)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (p/duration-between - (core/-eval birth-date context resource scope) - (core/-eval date context resource scope) - chrono-precision)) - (-form [_] - (list 'calculate-age-at (core/-form birth-date) (core/-form date) - precision))))))) + (calculate-age-at-op birth-date date chrono-precision precision))))) diff --git a/modules/cql/src/blaze/elm/compiler/conditional_operators.clj b/modules/cql/src/blaze/elm/compiler/conditional_operators.clj index 245d9f203..821dc1e54 100644 --- a/modules/cql/src/blaze/elm/compiler/conditional_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/conditional_operators.clj @@ -5,13 +5,50 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.protocols :as p])) ;; 15.1. Case +(defn- attach-cache [items cache] + (reduce + (fn [[items bfs] [when then]] + (let [[when w-bfs] ((first (core/-attach-cache when cache))) + [then t-bfs] ((first (core/-attach-cache then cache)))] + [(conj items [when then]) (into bfs (into w-bfs t-bfs))])) + [[] []] + items)) + +(defn- resolve-refs [items expression-defs] + (mapv + (fn [[when then]] + [(core/-resolve-refs when expression-defs) + (core/-resolve-refs then expression-defs)]) + items)) + +(defn- resolve-param-refs [items parameters] + (mapv + (fn [[when then]] + [(core/-resolve-params when parameters) + (core/-resolve-params then parameters)]) + items)) + (defn- comparand-case-op [comparand items else] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-attach-cache [_ cache] + (let [[comparand c-bfs] ((first (core/-attach-cache comparand cache))) + [items i-bfs] (attach-cache items cache) + [else e-bfs] ((first (core/-attach-cache else cache)))] + [(fn [] [(comparand-case-op comparand items else) (into c-bfs (into i-bfs e-bfs))])])) + (-resolve-refs [_ expression-defs] + (comparand-case-op + (core/-resolve-refs comparand expression-defs) + (resolve-refs items expression-defs) + (core/-resolve-refs else expression-defs))) + (-resolve-params [_ parameters] + (comparand-case-op + (core/-resolve-params comparand parameters) + (resolve-param-refs items parameters) + (core/-resolve-params else parameters))) (-eval [_ context resource scope] (let [comparand (core/-eval comparand context resource scope)] (loop [[[when then] & next-items] items] @@ -24,9 +61,19 @@ `(~'case ~(core/-form comparand) ~@(map core/-form (flatten items)) ~(core/-form else))))) (defn- multi-conditional-case-op [items else] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-attach-cache [_ cache] + (let [[items i-bfs] (attach-cache items cache) + [else e-bfs] ((first (core/-attach-cache else cache)))] + [(fn [] [(multi-conditional-case-op items else) (into i-bfs e-bfs)])])) + (-resolve-refs [_ expression-defs] + (multi-conditional-case-op + (resolve-refs items expression-defs) + (core/-resolve-refs else expression-defs))) + (-resolve-params [_ parameters] + (multi-conditional-case-op + (resolve-param-refs items parameters) + (core/-resolve-params else parameters))) (-eval [_ context resource scope] (loop [[[when then] & next-items] items] (if (core/-eval when context resource scope) @@ -41,7 +88,7 @@ [context {:keys [comparand else] items :caseItem}] (let [comparand (some->> comparand (core/compile* context)) items - (map + (mapv (fn [{:keys [when then]}] [(core/compile* context when) (core/compile* context then)]) @@ -53,17 +100,22 @@ ;; 15.2. If (defn- if-op [condition then else] - (reify - core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper if-op cache condition then else)) + (-resolve-refs [_ expression-defs] + (if-op + (core/-resolve-refs condition expression-defs) + (core/-resolve-refs then expression-defs) + (core/-resolve-refs else expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper if-op parameters condition then else)) (-eval [_ context resource scope] (if (core/-eval condition context resource scope) (core/-eval then context resource scope) (core/-eval else context resource scope))) (-form [_] - (list 'if (core/-form condition) (core/-form then) - (core/-form else))))) + (list 'if (core/-form condition) (core/-form then) (core/-form else))))) (defmethod core/compile* :elm.compiler.type/if [context {:keys [condition then else]}] diff --git a/modules/cql/src/blaze/elm/compiler/core.clj b/modules/cql/src/blaze/elm/compiler/core.clj index 1f29cf8c7..56728a034 100644 --- a/modules/cql/src/blaze/elm/compiler/core.clj +++ b/modules/cql/src/blaze/elm/compiler/core.clj @@ -12,6 +12,26 @@ (defprotocol Expression (-static [expression]) + (-attach-cache [expression cache] + "Attaches `cache` to `expression` so the expression can obtain a Bloom filter. + + Returns a vector with a function as first element that will return tuple of + expression and possible list of attached Bloom filters if called with no + argument. + + If the Bloom filter is available, returns a new expression holding the Bloom + filter, so it can be used to increase evaluation performance. + + Expressions that don't like to obtain a Bloom filter should call + `-attach-cache` on it's operands in order to allow them to possible obtain a + Bloom filter.") + (-patient-count [expression] + "Returns the number of patients from an attached Bloom filter. That patient + count can be used by other expressions (most likely and/or/case) to + reorder their operands so that expressions with less patients get evaluated + first. Returns nil if unknown.") + (-resolve-refs [expression expression-defs]) + (-resolve-params [expression parameters]) (-eval [expression context resource scope] "Evaluates `expression` on `resource` using `context` and optional `scope` for scoped expressions inside queries.") @@ -23,10 +43,84 @@ (defn static? [x] (-static x)) +(defn attach-cache-expressions [cache expressions] + (reduce + (fn [[expressions bfs] expression] + (let [[expression expression-bfs] ((first (-attach-cache expression cache)))] + [(conj expressions expression) (into bfs expression-bfs)])) + [[] []] + expressions)) + +(defn attach-cache-helper-list [constructor cache ops] + (let [[ops bfs] (attach-cache-expressions cache ops)] + [(fn [] [(constructor ops) bfs])])) + +(defn attach-cache-helper + ([constructor cache op] + (let [[op op-bfs] ((first (-attach-cache op cache)))] + [(fn [] [(constructor op) op-bfs])])) + ([constructor cache op-1 op-2] + (let [[op-1 op-1-bfs] ((first (-attach-cache op-1 cache))) + [op-2 op-2-bfs] ((first (-attach-cache op-2 cache)))] + [(fn [] [(constructor op-1 op-2) (into op-1-bfs op-2-bfs)])])) + ([constructor cache op-1 op-2 op-3] + (let [[op-1 op-1-bfs] ((first (-attach-cache op-1 cache))) + [op-2 op-2-bfs] ((first (-attach-cache op-2 cache))) + [op-3 op-3-bfs] ((first (-attach-cache op-3 cache)))] + [(fn [] [(constructor op-1 op-2 op-3) (into op-1-bfs (into op-2-bfs op-3-bfs))])]))) + +(defn attach-cache-helper-1 + ([constructor cache op arg] + (let [[op op-bfs] ((first (-attach-cache op cache)))] + [(fn [] [(constructor op arg) op-bfs])])) + ([constructor cache op-1 op-2 arg] + (let [[op-1 op-1-bfs] ((first (-attach-cache op-1 cache))) + [op-2 op-2-bfs] ((first (-attach-cache op-2 cache)))] + [(fn [] [(constructor op-1 op-2 arg) (into op-1-bfs op-2-bfs)])]))) + +(defn attach-cache-helper-2 + ([constructor cache op arg-1 arg-2] + (let [[op op-bfs] ((first (-attach-cache op cache)))] + [(fn [] [(constructor op arg-1 arg-2) op-bfs])])) + ([constructor cache op-1 op-2 arg-1 arg-2] + (let [[op-1 op-1-bfs] ((first (-attach-cache op-1 cache))) + [op-2 op-2-bfs] ((first (-attach-cache op-2 cache)))] + [(fn [] [(constructor op-1 op-2 arg-1 arg-2) (into op-1-bfs op-2-bfs)])]))) + +(defn resolve-refs-helper + ([constructor expression-defs op-1] + (constructor (-resolve-refs op-1 expression-defs))) + ([constructor expression-defs op-1 op-2] + (constructor (-resolve-refs op-1 expression-defs) + (-resolve-refs op-2 expression-defs))) + ([constructor expression-defs op-1 op-2 op-3] + (constructor (-resolve-refs op-1 expression-defs) + (-resolve-refs op-2 expression-defs) + (-resolve-refs op-3 expression-defs)))) + +(defn resolve-params-helper + ([constructor parameters op-1] + (constructor (-resolve-params op-1 parameters))) + ([constructor parameters op-1 op-2] + (constructor (-resolve-params op-1 parameters) + (-resolve-params op-2 parameters))) + ([constructor parameters op-1 op-2 op-3] + (constructor (-resolve-params op-1 parameters) + (-resolve-params op-2 parameters) + (-resolve-params op-3 parameters)))) + (extend-protocol Expression nil (-static [_] true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) (-eval [expr _ _ _] expr) (-form [_] @@ -35,6 +129,14 @@ Object (-static [_] true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) (-eval [expr _ _ _] expr) (-form [expr] @@ -43,6 +145,14 @@ IReduceInit (-static [_] true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) (-eval [expr _ _ _] expr) (-form [expr] diff --git a/modules/cql/src/blaze/elm/compiler/date_time_operators.clj b/modules/cql/src/blaze/elm/compiler/date_time_operators.clj index a6f2347d4..26d9228c0 100644 --- a/modules/cql/src/blaze/elm/compiler/date_time_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/date_time_operators.clj @@ -5,7 +5,7 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinopp defunop defunopp]] + [blaze.elm.compiler.macros :refer [defbinopp defunop defunopp reify-expr]] [blaze.elm.date-time :as date-time] [blaze.elm.protocols :as p] [blaze.fhir.spec.type.system :as system]) @@ -26,6 +26,94 @@ (.toLocalDateTime))) ;; 18.6. Date +(defn- date-op + ([year] + (reify + system/SystemType + (-type [_] :system/date) + + core/Expression + (-static [_] + false) + (-attach-cache [_ cache] + (core/attach-cache-helper date-op cache year)) + (-patient-count [_] + nil) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-op expression-defs year)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-op parameters year)) + (-eval [_ context resource scope] + (some-> (core/-eval year context resource scope) system/date)) + (-form [_] + (list 'date (core/-form year))) + + Object + (equals [this other] + (.equals ^Object (core/-form this) (core/-form other))) + (hashCode [this] + (.hashCode ^Object (core/-form this))))) + ([year month] + (reify + system/SystemType + (-type [_] :system/date) + + core/Expression + (-static [_] + false) + (-attach-cache [_ cache] + (core/attach-cache-helper date-op cache year month)) + (-patient-count [_] + nil) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-op expression-defs year month)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-op parameters year month)) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (system/date year month) + (system/date year)))) + (-form [_] + (list 'date (core/-form year) (core/-form month))) + + Object + (equals [this other] + (.equals ^Object (core/-form this) (core/-form other))) + (hashCode [this] + (.hashCode ^Object (core/-form this))))) + ([year month day] + (reify + system/SystemType + (-type [_] :system/date) + + core/Expression + (-static [_] + false) + (-attach-cache [_ cache] + (core/attach-cache-helper date-op cache year month day)) + (-patient-count [_] + nil) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-op expression-defs year month day)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-op parameters year month day)) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (if-let [day (core/-eval day context resource scope)] + (system/date year month day) + (system/date year month)) + (system/date year)))) + (-form [_] + (list 'date (core/-form year) (core/-form month) (core/-form day))) + + Object + (equals [this other] + (.equals ^Object (core/-form this) (core/-form other))) + (hashCode [this] + (.hashCode ^Object (core/-form this)))))) + (defmethod core/compile* :elm.compiler.type/date [context {:keys [year month day]}] (let [year (some->> year (core/compile* context)) @@ -36,61 +124,209 @@ (system/date year month day) (some? day) - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (if-let [day (core/-eval day context resource scope)] - (system/date year month day) - (system/date year month)) - (system/date year)))) - (-form [_] - (list 'date (core/-form year) (core/-form month) (core/-form day)))) + (date-op year month day) (and (int? month) (int? year)) (system/date year month) (some? month) - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (system/date year month) - (system/date year)))) - (-form [_] - (list 'date (core/-form year) (core/-form month)))) + (date-op year month) (int? year) (system/date year) :else - (when year - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (some-> (core/-eval year context resource scope) system/date)) - (-form [_] - (list 'date (core/-form year)))))))) + (some-> year date-op)))) ;; 18.7. DateFrom (defunop date-from [x] (p/date-from x)) ;; 18.8. DateTime +(defn- date-time-static-op + [year month day hour minute second millisecond timezone-offset] + (reify-expr core/Expression + (-eval [_ {:keys [now]} _ _] + (to-local-date-time-with-offset + now year month day hour minute second millisecond timezone-offset)) + (-form [_] + (list 'date-time (core/-form year) (core/-form month) + (core/-form day) (core/-form hour) (core/-form minute) + (core/-form second) (core/-form millisecond) + (core/-form timezone-offset))))) + +(defn- date-time-dynamic-op + ([year month day hour minute second millisecond] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (date-time-dynamic-op + (core/-resolve-refs year expression-defs) + (core/-resolve-refs month expression-defs) + (core/-resolve-refs day expression-defs) + (core/-resolve-refs hour expression-defs) + (core/-resolve-refs minute expression-defs) + (core/-resolve-refs second expression-defs) + (core/-resolve-refs millisecond expression-defs))) + (-resolve-params [_ parameters] + (date-time-dynamic-op + (core/-resolve-params year parameters) + (core/-resolve-params month parameters) + (core/-resolve-params day parameters) + (core/-resolve-params hour parameters) + (core/-resolve-params minute parameters) + (core/-resolve-params second parameters) + (core/-resolve-params millisecond parameters))) + (-eval [_ context resource scope] + (system/date-time + (core/-eval year context resource scope) + (core/-eval month context resource scope) + (core/-eval day context resource scope) + (core/-eval hour context resource scope) + (or (core/-eval minute context resource scope) 0) + (or (core/-eval second context resource scope) 0) + (or (core/-eval millisecond context resource scope) 0))) + (-form [_] + (list 'date-time (core/-form year) (core/-form month) + (core/-form day) (core/-form hour) (core/-form minute) + (core/-form second) (core/-form millisecond))))) + ([year month day hour minute second millisecond timezone-offset] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (date-time-dynamic-op + (core/-resolve-refs year expression-defs) + (core/-resolve-refs month expression-defs) + (core/-resolve-refs day expression-defs) + (core/-resolve-refs hour expression-defs) + (core/-resolve-refs minute expression-defs) + (core/-resolve-refs second expression-defs) + (core/-resolve-refs millisecond expression-defs) + (core/-resolve-refs timezone-offset expression-defs))) + (-resolve-params [_ parameters] + (date-time-dynamic-op + (core/-resolve-params year parameters) + (core/-resolve-params month parameters) + (core/-resolve-params day parameters) + (core/-resolve-params hour parameters) + (core/-resolve-params minute parameters) + (core/-resolve-params second parameters) + (core/-resolve-params millisecond parameters) + (core/-resolve-params timezone-offset parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (to-local-date-time-with-offset + now + (core/-eval year context resource scope) + (core/-eval month context resource scope) + (core/-eval day context resource scope) + (core/-eval hour context resource scope) + (or (core/-eval minute context resource scope) 0) + (or (core/-eval second context resource scope) 0) + (or (core/-eval millisecond context resource scope) 0) + timezone-offset)) + (-form [_] + (list 'date-time (core/-form year) (core/-form month) + (core/-form day) (core/-form hour) (core/-form minute) + (core/-form second) (core/-form millisecond) + (core/-form timezone-offset)))))) + +(defn- date-time-dynamic-timezone-offset-op + [year month day hour minute second millisecond timezone-offset] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (date-time-dynamic-timezone-offset-op + (core/-resolve-refs year expression-defs) + (core/-resolve-refs month expression-defs) + (core/-resolve-refs day expression-defs) + (core/-resolve-refs hour expression-defs) + (core/-resolve-refs minute expression-defs) + (core/-resolve-refs second expression-defs) + (core/-resolve-refs millisecond expression-defs) + (core/-resolve-refs timezone-offset expression-defs))) + (-resolve-params [_ parameters] + (let [timezone-offset (core/-resolve-params timezone-offset parameters)] + (if (number? timezone-offset) + (date-time-dynamic-op + (core/-resolve-params year parameters) + (core/-resolve-params month parameters) + (core/-resolve-params day parameters) + (core/-resolve-params hour parameters) + (core/-resolve-params minute parameters) + (core/-resolve-params second parameters) + (core/-resolve-params millisecond parameters) + timezone-offset) + (date-time-dynamic-timezone-offset-op + (core/-resolve-params year parameters) + (core/-resolve-params month parameters) + (core/-resolve-params day parameters) + (core/-resolve-params hour parameters) + (core/-resolve-params minute parameters) + (core/-resolve-params second parameters) + (core/-resolve-params millisecond parameters) + timezone-offset)))) + (-eval [_ {:keys [now] :as context} resource scope] + (to-local-date-time-with-offset + now + (core/-eval year context resource scope) + (core/-eval month context resource scope) + (core/-eval day context resource scope) + (core/-eval hour context resource scope) + (or (core/-eval minute context resource scope) 0) + (or (core/-eval second context resource scope) 0) + (or (core/-eval millisecond context resource scope) 0) + (core/-eval timezone-offset context resource scope))) + (-form [_] + (list 'date-time (core/-form year) (core/-form month) + (core/-form day) (core/-form hour) (core/-form minute) + (core/-form second) (core/-form millisecond) + (core/-form timezone-offset))))) + +(defn- date-time-date-op [year month day] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper date-time-date-op cache year month day)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-time-date-op expression-defs year month day)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-time-date-op parameters year month day)) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (if-let [day (core/-eval day context resource scope)] + (system/date-time year month day) + (system/date-time year month)) + (system/date-time year)))) + (-form [_] + (list 'date-time (core/-form year) (core/-form month) + (core/-form day))))) + +(defn- date-time-year-month-op [year month] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper date-time-year-month-op cache year month)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-time-year-month-op expression-defs year month)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-time-year-month-op parameters year month)) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (system/date-time year month) + (system/date-time year)))) + (-form [_] + (list 'date-time (core/-form year) (core/-form month))))) + +(defn- date-time-year-op [year] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper date-time-year-op cache year)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper date-time-year-op expression-defs year)) + (-resolve-params [_ parameters] + (core/resolve-params-helper date-time-year-op parameters year)) + (-eval [_ context resource scope] + (some-> (core/-eval year context resource scope) system/date-time)) + (-form [_] + (list 'date-time (core/-form year))))) + (defmethod core/compile* :elm.compiler.type/date-time [context {:keys [year month day hour minute second millisecond] timezone-offset :timezoneOffset @@ -108,38 +344,12 @@ (cond (and (int? millisecond) (int? second) (int? minute) (int? hour) (int? day) (int? month) (int? year)) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now]} _ _] - (to-local-date-time-with-offset - now year month day hour minute second millisecond timezone-offset)) - (-form [_] - (list 'date-time (core/-form year) (core/-form month) - (core/-form day) (core/-form hour) (core/-form minute) - (core/-form second) (core/-form millisecond) - (core/-form timezone-offset)))) + (date-time-static-op year month day hour minute second millisecond + timezone-offset) (some? hour) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (to-local-date-time-with-offset - now - (core/-eval year context resource scope) - (core/-eval month context resource scope) - (core/-eval day context resource scope) - (core/-eval hour context resource scope) - (or (core/-eval minute context resource scope) 0) - (or (core/-eval second context resource scope) 0) - (or (core/-eval millisecond context resource scope) 0) - timezone-offset)) - (-form [_] - (list 'date-time (core/-form year) (core/-form month) - (core/-form day) (core/-form hour) (core/-form minute) - (core/-form second) (core/-form millisecond) - (core/-form timezone-offset)))) + (date-time-dynamic-op year month day hour minute second millisecond + timezone-offset) :else (throw (ex-info "Need at least an hour if timezone offset is given." @@ -147,25 +357,8 @@ (some? timezone-offset) (if (some? hour) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (to-local-date-time-with-offset - now - (core/-eval year context resource scope) - (core/-eval month context resource scope) - (core/-eval day context resource scope) - (core/-eval hour context resource scope) - (or (core/-eval minute context resource scope) 0) - (or (core/-eval second context resource scope) 0) - (or (core/-eval millisecond context resource scope) 0) - (core/-eval timezone-offset context resource scope))) - (-form [_] - (list 'date-time (core/-form year) (core/-form month) - (core/-form day) (core/-form hour) (core/-form minute) - (core/-form second) (core/-form millisecond) - (core/-form timezone-offset)))) + (date-time-dynamic-timezone-offset-op year month day hour minute second + millisecond timezone-offset) (throw (ex-info "Need at least an hour if timezone offset is given." {:expression expression}))) @@ -176,68 +369,25 @@ (system/date-time year month day hour minute second millisecond) (some? hour) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (system/date-time - (core/-eval year context resource scope) - (core/-eval month context resource scope) - (core/-eval day context resource scope) - (core/-eval hour context resource scope) - (or (core/-eval minute context resource scope) 0) - (or (core/-eval second context resource scope) 0) - (or (core/-eval millisecond context resource scope) 0))) - (-form [_] - (list 'date-time (core/-form year) (core/-form month) - (core/-form day) (core/-form hour) (core/-form minute) - (core/-form second) (core/-form millisecond)))) + (date-time-dynamic-op year month day hour minute second millisecond) (and (int? day) (int? month) (int? year)) (system/date-time year month day) (some? day) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (if-let [day (core/-eval day context resource scope)] - (system/date-time year month day) - (system/date-time year month)) - (system/date-time year)))) - (-form [_] - (list 'date-time (core/-form year) (core/-form month) - (core/-form day)))) + (date-time-date-op year month day) (and (int? month) (int? year)) (system/date-time year month) (some? month) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (system/date-time year month) - (system/date-time year)))) - (-form [_] - (list 'date-time (core/-form year) (core/-form month)))) + (date-time-year-month-op year month) (int? year) (system/date-time year) :else - (when year - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (some-> (core/-eval year context resource scope) system/date-time)) - (-form [_] - (list 'date-time (core/-form year))))))))) + (some-> year date-time-year-op))))) ;; 18.9. DateTimeComponentFrom (defunopp date-time-component-from [x precision] @@ -252,14 +402,12 @@ (p/duration-between operand-1 operand-2 precision)) ;; 18.13. Now -(defrecord NowExpression [] - core/Expression - (-static [_] - false) - (-eval [_ {:keys [now]} _ _] - now)) - -(def now-expression (->NowExpression)) +(def ^:private now-expression + (reify-expr core/Expression + (-eval [_ {:keys [now]} _ _] + now) + (-form [_] + 'now))) (defmethod core/compile* :elm.compiler.type/now [_ _] now-expression) @@ -277,6 +425,66 @@ (p/same-or-after x y precision)) ;; 18.18. Time +(defn- time-op + ([hour] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper time-op expression-defs hour)) + (-resolve-params [_ parameters] + (time-op (core/-resolve-params hour parameters))) + (-eval [_ context resource scope] + (date-time/local-time (core/-eval hour context resource scope))) + (-form [_] + (list 'time (core/-form hour))))) + ([hour minute] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper time-op expression-defs hour minute)) + (-resolve-params [_ parameters] + (core/resolve-params-helper time-op parameters hour minute)) + (-eval [_ context resource scope] + (date-time/local-time + (core/-eval hour context resource scope) + (core/-eval minute context resource scope))) + (-form [_] + (list 'time (core/-form hour) (core/-form minute))))) + ([hour minute second] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper time-op expression-defs hour minute second)) + (-resolve-params [_ parameters] + (core/resolve-params-helper time-op parameters hour minute second)) + (-eval [_ context resource scope] + (date-time/local-time + (core/-eval hour context resource scope) + (core/-eval minute context resource scope) + (core/-eval second context resource scope))) + (-form [_] + (list 'time (core/-form hour) (core/-form minute) (core/-form second))))) + ([hour minute second millisecond] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (time-op + (core/-resolve-refs hour expression-defs) + (core/-resolve-refs minute expression-defs) + (core/-resolve-refs second expression-defs) + (core/-resolve-refs millisecond expression-defs))) + (-resolve-params [_ parameters] + (time-op + (core/-resolve-params hour parameters) + (core/-resolve-params minute parameters) + (core/-resolve-params second parameters) + (core/-resolve-params millisecond parameters))) + (-eval [_ context resource scope] + (date-time/local-time + (core/-eval hour context resource scope) + (core/-eval minute context resource scope) + (core/-eval second context resource scope) + (core/-eval millisecond context resource scope))) + (-form [_] + (list 'time (core/-form hour) (core/-form minute) (core/-form second) + (core/-form millisecond)))))) + (defmethod core/compile* :elm.compiler.type/time [context {:keys [hour minute second millisecond]}] (let [hour (some->> hour (core/compile* context)) @@ -288,62 +496,28 @@ (date-time/local-time hour minute second millisecond) (some? millisecond) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (date-time/local-time (core/-eval hour context resource scope) - (core/-eval minute context resource scope) - (core/-eval second context resource scope) - (core/-eval millisecond context resource scope))) - (-form [_] - (list 'time (core/-form hour) (core/-form minute) (core/-form second) - (core/-form millisecond)))) + (time-op hour minute second millisecond) (and (int? second) (int? minute) (int? hour)) (date-time/local-time hour minute second) (some? second) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (date-time/local-time (core/-eval hour context resource scope) - (core/-eval minute context resource scope) - (core/-eval second context resource scope))) - (-form [_] - (list 'time (core/-form hour) (core/-form minute) (core/-form second)))) + (time-op hour minute second) (and (int? minute) (int? hour)) (date-time/local-time hour minute) (some? minute) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (date-time/local-time (core/-eval hour context resource scope) - (core/-eval minute context resource scope))) - (-form [_] - (list 'time (core/-form hour) (core/-form minute)))) + (time-op hour minute) (int? hour) (date-time/local-time hour) :else - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (date-time/local-time (core/-eval hour context resource scope))) - (-form [_] - (list 'time (core/-form hour))))))) + (time-op hour)))) (def ^:private time-of-day-expr - (reify - core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [now]} _ _] (.toLocalTime ^OffsetDateTime now)) (-form [_] @@ -355,9 +529,7 @@ time-of-day-expr) (def ^:private today-expr - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [now]} _ _] (DateDate/fromLocalDate (.toLocalDate ^OffsetDateTime now))) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/external_data.clj b/modules/cql/src/blaze/elm/compiler/external_data.clj index a7c657f1d..ab0cef999 100644 --- a/modules/cql/src/blaze/elm/compiler/external_data.clj +++ b/modules/cql/src/blaze/elm/compiler/external_data.clj @@ -7,48 +7,24 @@ [blaze.anomaly :as ba :refer [if-ok]] [blaze.coll.core :as coll] [blaze.db.api :as d] - [blaze.db.impl.index.resource-handle :as rh] [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.compiler.structured-values] + [blaze.elm.resource :as cr] [blaze.elm.spec] [blaze.elm.util :as elm-util] - [blaze.fhir.spec.type.protocols :as p] - [clojure.string :as str]) + [blaze.fhir.spec.references :as fsr] + [prometheus.alpha :as prom :refer [defcounter]]) (:import [blaze.elm.compiler.structured_values SourcePropertyExpression] - [clojure.lang ILookup] [java.util List])) (set! *warn-on-reflection* true) -;; A resource that is a wrapper of a resource-handle that will lazily pull the -;; resource content if some property other than :id is accessed. -(deftype Resource [db handle content] - p/FhirType - (-type [_] - (p/-type handle)) - - ILookup - (valAt [r key] - (.valAt r key nil)) - (valAt [_ key not-found] - (case key - :id (rh/id handle) - (-> (or @content (vreset! content @(d/pull-content db handle))) - (get key not-found)))) - - Object - (toString [_] - (.toString handle))) - -(defn resource? [x] - (instance? Resource x)) - -(defn mk-resource [db handle] - (Resource. db handle (volatile! nil))) - -(defn resource-mapper [db] - (map (partial mk-resource db))) +(defcounter retrieve-total + "Number of times a retrieve expression was evaluated." + {:namespace "blaze" + :subsystem "cql"}) (defn- code->clause-value [{:keys [system code]}] (str system "|" code)) @@ -79,36 +55,27 @@ [node context data-type property codes] (let [clauses (-to-clauses codes property) query (d/compile-compartment-query node context data-type clauses)] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [db]} {:keys [id]} _] - (coll/eduction (resource-mapper db) (d/execute-query db query id))) + (prom/inc! retrieve-total) + (coll/eduction (cr/resource-mapper db) (d/execute-query db query id))) (-form [_] `(~'retrieve ~data-type ~(d/query-clauses query)))))) -(defn- split-reference [s] - (when-let [idx (str/index-of s \/)] - [(subs s 0 idx) (subs s (inc idx))])) - ;; TODO: find a better solution than hard coding this case -(defrecord SpecimenPatientExpression [] - core/Expression - (-static [_] - false) - (-eval [_ {:keys [db]} resource _] - (let [{{:keys [reference]} :subject} resource] - (when reference - (when-let [[type id] (split-reference reference)] - (when (and (= "Patient" type) (string? id)) - (let [{:keys [op] :as handle} (d/resource-handle db "Patient" id)] - (when-not (identical? :delete op) - [(mk-resource db handle)]))))))) - (-form [_] - '(retrieve (Specimen) "Patient"))) - (def ^:private specimen-patient-expr - (->SpecimenPatientExpression)) + (reify-expr core/Expression + (-eval [_ {:keys [db]} resource _] + (prom/inc! retrieve-total) + (let [{{:keys [reference]} :subject} resource] + (when reference + (when-let [[type id] (fsr/split-literal-ref reference)] + (when (and (= "Patient" type) (string? id)) + (let [{:keys [op] :as handle} (d/resource-handle db "Patient" id)] + (when-not (identical? :delete op) + [(cr/mk-resource db handle)]))))))) + (-form [_] + '(retrieve (Specimen) "Patient")))) (defn- context-expr "Returns an expression which, when evaluated, returns all resources of type @@ -119,20 +86,17 @@ (case data-type "Patient" specimen-patient-expr) - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [db]} {:keys [id]} _] + (prom/inc! retrieve-total) (coll/eduction - (resource-mapper db) + (cr/resource-mapper db) (d/list-compartment-resource-handles db context id data-type))) (-form [_] `(~'retrieve ~data-type))))) (def ^:private resource-expr - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ _ resource _] [resource]) (-form [_] @@ -145,10 +109,12 @@ (ba/unsupported "Unsupported related context retrieve expression without result type.")) (defn- related-context-expr-without-codes [related-context-expr data-type] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (related-context-expr-without-codes + (core/-resolve-refs related-context-expr expression-defs) data-type)) (-eval [_ context resource scope] + (prom/inc! retrieve-total) (when-let [context-resource (core/-eval related-context-expr context resource scope)] (core/-eval (context-expr (-> context-resource :fhir/type name) data-type) @@ -158,6 +124,21 @@ (-form [_] (list 'retrieve (core/-form related-context-expr) data-type)))) +(defn- related-context-expr-with-codes [related-context-expr data-type query] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (related-context-expr-with-codes + (core/-resolve-refs related-context-expr expression-defs) data-type query)) + (-eval [_ {:keys [db] :as context} resource scope] + (prom/inc! retrieve-total) + (when-let [{:keys [id]} (core/-eval related-context-expr context resource scope)] + (when (string? id) + (coll/eduction + (cr/resource-mapper db) + (d/execute-query db query id))))) + (-form [_] + (list 'retrieve (core/-form related-context-expr) data-type (d/query-clauses query))))) + (defn- related-context-expr [node context-expr data-type code-property codes] (if (seq codes) @@ -166,17 +147,7 @@ (if (= "http://hl7.org/fhir" value-type-ns) (let [clauses [(into [code-property] (map code->clause-value) codes)]] (if-ok [query (d/compile-compartment-query node context-type data-type clauses)] - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [db] :as context} resource scope] - (when-let [{:keys [id]} (core/-eval context-expr context resource scope)] - (when (string? id) - (coll/eduction - (resource-mapper db) - (d/execute-query db query id))))) - (-form [_] - (list 'retrieve (core/-form context-expr) data-type (d/query-clauses query)))) + (related-context-expr-with-codes context-expr data-type query) ba/throw-anom)) (ba/throw-anom (unsupported-type-ns-anom value-type-ns)))) (ba/throw-anom unsupported-related-context-expr-without-type-anom)) @@ -184,20 +155,18 @@ (defn- unfiltered-context-expr [node data-type code-property codes] (if (empty? codes) - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [db]} _ _] - (coll/eduction (resource-mapper db) (d/type-list db data-type))) + (prom/inc! retrieve-total) + (coll/eduction (cr/resource-mapper db) (d/type-list db data-type))) (-form [_] `(~'retrieve ~data-type))) (let [clauses [(into [code-property] (map code->clause-value) codes)]] (if-ok [query (d/compile-type-query node data-type clauses)] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [db]} _ _] - (coll/eduction (resource-mapper db) (d/execute-query db query))) + (prom/inc! retrieve-total) + (coll/eduction (cr/resource-mapper db) (d/execute-query db query))) (-form [_] `(~'retrieve ~data-type ~(d/query-clauses query)))) ba/throw-anom)))) diff --git a/modules/cql/src/blaze/elm/compiler/function.clj b/modules/cql/src/blaze/elm/compiler/function.clj index e0d922999..5ae3f7df7 100644 --- a/modules/cql/src/blaze/elm/compiler/function.clj +++ b/modules/cql/src/blaze/elm/compiler/function.clj @@ -1,11 +1,20 @@ (ns blaze.elm.compiler.function (:require - [blaze.elm.compiler.core :as core])) + [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]])) (defn arity-n [name fn-expr operand-names operands] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-attach-cache [_ cache] + (let [[fn-expr fn-expr-bfs] ((first (core/-attach-cache fn-expr cache))) + [operands operands-bfs] (core/attach-cache-expressions cache operands)] + [(fn [] [(arity-n name fn-expr operand-names operands) (into (or fn-expr-bfs []) operands-bfs)])])) + (-resolve-refs [_ expression-defs] + (arity-n name (core/-resolve-refs fn-expr expression-defs) operand-names + (map #(core/-resolve-refs % expression-defs) operands))) + (-resolve-params [_ parameters] + (arity-n name (core/-resolve-params fn-expr parameters) operand-names + (map #(core/-resolve-params % parameters) operands))) (-eval [_ context resource scope] (let [values (map #(core/-eval % context resource scope) operands)] (core/-eval fn-expr context resource (merge scope (zipmap operand-names values))))) diff --git a/modules/cql/src/blaze/elm/compiler/interval_operators.clj b/modules/cql/src/blaze/elm/compiler/interval_operators.clj index 9a2a33002..91f0192a4 100644 --- a/modules/cql/src/blaze/elm/compiler/interval_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/interval_operators.clj @@ -6,7 +6,7 @@ (:require [blaze.elm.compiler.arithmetic-operators :as ao] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinop defbinopp defunop]] + [blaze.elm.compiler.macros :refer [defbinop defbinopp defunop reify-expr]] [blaze.elm.interval :refer [interval]] [blaze.elm.protocols :as p])) @@ -16,34 +16,49 @@ asType (when (= "ToDateTime" type) "{urn:hl7-org:elm-types:r1}DateTime"))) -(defrecord IntervalExpression - [type low high low-closed-expression high-closed-expression low-closed - high-closed] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [low (core/-eval low context resource scope) - high (core/-eval high context resource scope) - low-closed (or (core/-eval low-closed-expression context resource - scope) - low-closed) - high-closed (or (core/-eval high-closed-expression context resource - scope) - high-closed)] - (interval - (if low-closed - (if (nil? low) - (ao/min-value type) - low) - (p/successor low)) - (if high-closed - (if (nil? high) - (ao/max-value type) - high) - (p/predecessor high))))) - (-form [_] - (list 'interval (core/-form low) (core/-form high)))) +(defn- interval-expr [type low high low-closed-expression + high-closed-expression low-closed high-closed] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (interval-expr + type + (core/-resolve-refs low expression-defs) + (core/-resolve-refs high expression-defs) + (core/-resolve-refs low-closed-expression expression-defs) + (core/-resolve-refs high-closed-expression expression-defs) + low-closed + high-closed)) + (-resolve-params [_ parameters] + (interval-expr + type + (core/-resolve-params low parameters) + (core/-resolve-params high parameters) + (core/-resolve-params low-closed-expression parameters) + (core/-resolve-params high-closed-expression parameters) + low-closed + high-closed)) + (-eval [_ context resource scope] + (let [low (core/-eval low context resource scope) + high (core/-eval high context resource scope) + low-closed (or (core/-eval low-closed-expression context resource + scope) + low-closed) + high-closed (or (core/-eval high-closed-expression context resource + scope) + high-closed)] + (interval + (if low-closed + (if (nil? low) + (ao/min-value type) + low) + (p/successor low)) + (if high-closed + (if (nil? high) + (ao/max-value type) + high) + (p/predecessor high))))) + (-form [_] + (list 'interval (core/-form low) (core/-form high))))) (defmethod core/compile* :elm.compiler.type/interval [context {:keys [low high] @@ -77,8 +92,8 @@ (ao/max-value type) high) (p/predecessor high)))) - (->IntervalExpression type low high low-closed-expression - high-closed-expression low-closed high-closed)))) + (interval-expr type low high low-closed-expression + high-closed-expression low-closed high-closed)))) ;; 19.2. After (defbinopp after [operand-1 operand-2 precision] diff --git a/modules/cql/src/blaze/elm/compiler/library.clj b/modules/cql/src/blaze/elm/compiler/library.clj index 66dceab92..47f2d11d2 100644 --- a/modules/cql/src/blaze/elm/compiler/library.clj +++ b/modules/cql/src/blaze/elm/compiler/library.clj @@ -3,6 +3,7 @@ [blaze.anomaly :as ba :refer [if-ok when-ok]] [blaze.elm.compiler :as c] [blaze.elm.compiler.function :as function] + [blaze.elm.compiler.library.resolve-refs :refer [resolve-refs]] [blaze.elm.normalizer :as normalizer])) (defn- compile-expression-def @@ -53,11 +54,12 @@ itself is returned. Returns an anomaly on errors." - {:arglists '([context expression-def])} + {:arglists '([context parameter-def])} [context {:keys [default] :as parameter-def}] (if (some? default) (-> (ba/try-anomaly - (assoc parameter-def :default (c/compile context default))) + (let [context (assoc context :eval-context "Unfiltered")] + (assoc parameter-def :default (c/compile context default)))) (ba/exceptionally #(assoc % :context context :elm/expression default))) parameter-def)) @@ -72,6 +74,23 @@ {} (-> library :parameters :def))) +(defn resolve-all-refs [expression-defs] + (resolve-refs #{} expression-defs)) + +(defn- unfiltered-expr-names [expression-defs] + (into + #{} + (keep (fn [[name {:keys [context]}]] (when (= "Unfiltered" context) name))) + expression-defs)) + +(defn- resolve-param-refs-xf [parameters] + (map + (fn [[name expr-def]] + [name (update expr-def :expression c/resolve-params parameters)]))) + +(defn resolve-param-refs [expression-defs parameters] + (into {} (resolve-param-refs-xf parameters) expression-defs)) + (defn compile-library "Compiles `library` using `node`. @@ -81,6 +100,7 @@ context (assoc opts :node node :library library)] (when-ok [{:keys [function-defs] :as context} (compile-function-defs context library) expression-defs (expression-defs context library) + expression-defs (resolve-refs (unfiltered-expr-names expression-defs) expression-defs) parameter-default-values (parameter-default-values context library)] {:expression-defs expression-defs :function-defs function-defs diff --git a/modules/cql/src/blaze/elm/compiler/library/resolve_refs.clj b/modules/cql/src/blaze/elm/compiler/library/resolve_refs.clj new file mode 100644 index 000000000..f02ef4874 --- /dev/null +++ b/modules/cql/src/blaze/elm/compiler/library/resolve_refs.clj @@ -0,0 +1,52 @@ +(ns blaze.elm.compiler.library.resolve-refs + (:require + [blaze.anomaly :as ba] + [blaze.elm.compiler :as c] + [clojure.string :as str] + [clojure.walk :as walk])) + +(defn- has-refs? [non-refs {:keys [expression]}] + (identical? + (walk/postwalk + #(if (sequential? %) + (if (and (= 'expr-ref (first %)) (not (non-refs (second %)))) + ::ref + (some #{::ref} %)) + %) + (c/form expression)) + ::ref)) + +(defn- split-by-having-refs [non-refs expression-defs] + (reduce-kv + (fn [[with without] name expression-def] + (if (has-refs? non-refs expression-def) + [(assoc with name expression-def) without] + [with (assoc without name expression-def)])) + [{} {}] + expression-defs)) + +(defn resolve-refs* [expr-def without-refs] + (update expr-def :expression c/resolve-refs without-refs)) + +(defn unresolvable-msg [with-refs] + (format "The following expression definitions contain unresolvable references: %s." + (str/join "," (map key with-refs)))) + +(defn resolve-refs [non-refs expression-defs] + (let [[with-refs without-refs] (split-by-having-refs non-refs expression-defs)] + (cond + (empty? with-refs) + without-refs + + (= (count with-refs) (count expression-defs)) + (ba/incorrect (unresolvable-msg with-refs)) + + :else + (let [resolvable-defs (apply dissoc without-refs non-refs)] + (recur + non-refs + (reduce-kv + (fn [ret name expr-def] + (assoc ret name (resolve-refs* expr-def resolvable-defs))) + without-refs + with-refs)))))) diff --git a/modules/cql/src/blaze/elm/compiler/library/spec.clj b/modules/cql/src/blaze/elm/compiler/library/spec.clj index 3ceccaddd..305659ae2 100644 --- a/modules/cql/src/blaze/elm/compiler/library/spec.clj +++ b/modules/cql/src/blaze/elm/compiler/library/spec.clj @@ -2,7 +2,6 @@ (:require [blaze.anomaly-spec] [blaze.elm.compiler :as-alias c] - [blaze.elm.compiler-spec] [blaze.elm.compiler.expression-def :as-alias expression-def] [blaze.elm.compiler.function-def :as-alias function-def] [blaze.elm.compiler.spec] @@ -36,5 +35,11 @@ (s/def ::c/parameter-default-values (s/map-of :elm/name ::c/expression)) +(s/def ::c/parameters + (s/map-of :elm/name ::c/expression)) + (s/def ::c/library (s/keys :req-un [::c/expression-defs ::c/function-defs ::c/parameter-default-values])) + +(s/def ::c/options + map?) diff --git a/modules/cql/src/blaze/elm/compiler/library_spec.clj b/modules/cql/src/blaze/elm/compiler/library_spec.clj index d1318e391..243a6a4e7 100644 --- a/modules/cql/src/blaze/elm/compiler/library_spec.clj +++ b/modules/cql/src/blaze/elm/compiler/library_spec.clj @@ -1,15 +1,21 @@ (ns blaze.elm.compiler.library-spec (:require [blaze.anomaly-spec] + [blaze.db.spec] [blaze.elm.compiler :as-alias c] - [blaze.elm.compiler-spec] [blaze.elm.compiler.library :as library] [blaze.elm.compiler.library.spec] - [blaze.elm.compiler.spec] - [blaze.fhir.spec.spec] [clojure.spec.alpha :as s] - [cognitect.anomalies :as anom])) + [cognitect.anomalies :as-alias anom])) + +(s/fdef library/resolve-all-refs + :args (s/cat :expression-defs ::c/expression-defs) + :ret (s/or :expression-defs ::c/expression-defs :anomaly ::anom/anomaly)) + +(s/fdef library/resolve-param-refs + :args (s/cat :expression-defs ::c/expression-defs :parameters ::c/parameters) + :ret ::c/expression-defs) (s/fdef library/compile-library - :args (s/cat :node :blaze.db/node :library :elm/library :opts map?) + :args (s/cat :node :blaze.db/node :library :elm/library :opts ::c/options) :ret (s/or :library ::c/library :anomaly ::anom/anomaly)) diff --git a/modules/cql/src/blaze/elm/compiler/list_operators.clj b/modules/cql/src/blaze/elm/compiler/list_operators.clj index db30ef07e..193cdda87 100644 --- a/modules/cql/src/blaze/elm/compiler/list_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/list_operators.clj @@ -7,7 +7,7 @@ [blaze.anomaly :as ba] [blaze.coll.core :as coll] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinop defunop]] + [blaze.elm.compiler.macros :refer [defbinop defunop reify-expr]] [blaze.elm.compiler.queries :as queries] [blaze.elm.protocols :as p] [blaze.util :refer [conj-vec]] @@ -19,9 +19,13 @@ ;; 20.1. List (defn list-op [elements] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper-list list-op cache elements)) + (-resolve-refs [_ expression-defs] + (list-op (mapv #(core/-resolve-refs % expression-defs) elements))) + (-resolve-params [_ parameters] + (list-op (mapv #(core/-resolve-params % parameters) elements))) (-eval [_ context resource scope] (mapv #(core/-eval % context resource scope) elements)) (-form [_] @@ -36,16 +40,12 @@ (defmethod core/compile* :elm.compiler.type/current [_ {:keys [scope]}] (if scope - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ _ _ scopes] (get scopes scope)) (-form [_] (list 'current scope))) - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ _ _ scope] scope) (-form [_] @@ -66,51 +66,63 @@ ;; 20.8. Exists (defunop exists - {:optimizations #{:first :non-distinct}} + {:optimizations #{:first :non-distinct} + :cache true} [list] (not (coll/empty? list))) ;; 20.9. Filter +(defn- scoped-filter-op [source condition scope] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (scoped-filter-op + (core/-resolve-refs source expression-defs) + (core/-resolve-refs condition expression-defs) + scope)) + (-resolve-params [_ parameters] + (scoped-filter-op + (core/-resolve-params source parameters) + (core/-resolve-params condition parameters) + scope)) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (filterv + (fn [x] + (core/-eval condition context resource (assoc scopes scope x))) + source))) + (-form [_] + (list 'filter (core/-form source) (core/-form condition) scope)))) + +(defn- filter-op [source condition] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (filter-op + (core/-resolve-refs source expression-defs) + (core/-resolve-refs condition expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper filter-op parameters source condition)) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (filterv (partial core/-eval condition context resource) source))) + (-form [_] + (list 'filter (core/-form source) (core/-form condition))))) + (defmethod core/compile* :elm.compiler.type/filter [context {:keys [source condition scope]}] (let [source (core/compile* context source) condition (core/compile* context condition)] (if scope - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (filterv - (fn [x] - (core/-eval condition context resource (assoc scopes scope x))) - source))) - (-form [_] - (list 'filter (core/-form source) (core/-form condition) scope))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (filterv (partial core/-eval condition context resource) source))) - (-form [_] - (list 'filter (core/-form source) (core/-form condition))))))) + (scoped-filter-op source condition scope) + (filter-op source condition)))) ;; 20.10. First ;; ;; TODO: orderBy -(defmethod core/compile* :elm.compiler.type/first - [context {:keys [source]}] - (let [source (core/compile* (assoc context :optimizations #{:first :non-distinct}) source)] - (if (core/static? source) - (first source) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (coll/first (core/-eval source context resource scopes))) - (-form [_] - (list 'first (core/-form source))))))) +(defunop first + {:optimizations #{:first :non-distinct} + :operand-key :source} + [source] + (coll/first source)) ;; 20.11. Flatten (defunop flatten [list] @@ -126,69 +138,89 @@ (flatten [] list)))) ;; 20.12. ForEach +(defn- scoped-for-each [source element scope] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (scoped-for-each + (core/-resolve-refs source expression-defs) + (core/-resolve-refs element expression-defs) + scope)) + (-resolve-params [_ parameters] + (scoped-for-each + (core/-resolve-params source parameters) + (core/-resolve-params element parameters) + scope)) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (mapv + (fn [x] + (core/-eval element context resource (assoc scopes scope x))) + source))) + (-form [_] + (list 'for-each (core/-form source) (core/-form element) scope)))) + +(defn- for-each [source element] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (for-each + (core/-resolve-refs source expression-defs) + (core/-resolve-refs element expression-defs))) + (-resolve-params [_ parameters] + (for-each + (core/-resolve-params source parameters) + (core/-resolve-params element parameters))) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (mapv (partial core/-eval element context resource) source))) + (-form [_] + (list 'for-each (core/-form source) (core/-form element))))) + (defmethod core/compile* :elm.compiler.type/for-each [context {:keys [source element scope]}] (let [source (core/compile* context source) element (core/compile* context element)] (if scope - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (mapv - (fn [x] - (core/-eval element context resource (assoc scopes scope x))) - source))) - (-form [_] - (list 'for-each (core/-form source) (core/-form element) scope))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (mapv (partial core/-eval element context resource) source))) - (-form [_] - (list 'for-each (core/-form source) (core/-form element))))))) + (scoped-for-each source element scope) + (for-each source element)))) ;; 20.16. IndexOf +(defn- index-of-op [source element] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper index-of-op cache source element)) + (-resolve-refs [_ expression-defs] + (index-of-op (core/-resolve-refs source expression-defs) + (core/-resolve-refs element expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper index-of-op parameters source element)) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (when-let [element (core/-eval element context resource scopes)] + (or + (first + (keep-indexed + (fn [idx x] + (when + (p/equal element x) + idx)) + source)) + -1)))) + (-form [_] + (list 'index-of (core/-form source) (core/-form element))))) + (defmethod core/compile* :elm.compiler.type/index-of [context {:keys [source element]}] (let [source (core/compile* context source) element (core/compile* context element)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (when-let [element (core/-eval element context resource scopes)] - (or - (first - (keep-indexed - (fn [idx x] - (when - (p/equal element x) - idx)) - source)) - -1)))) - (-form [_] - (list 'index-of (core/-form source) (core/-form element)))))) + (index-of-op source element))) ;; 20.18. Last ;; ;; TODO: orderBy -(defmethod core/compile* :elm.compiler.type/last - [context {:keys [source]}] - (let [source (core/compile* context source)] - (if (core/static? source) - (peek source) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (peek (core/-eval source context resource scopes))) - (-form [_] - (list 'last (core/-form source))))))) +(defunop last + {:operand-key :source} + [source] + (peek source)) ;; 20.24. Repeat ;; @@ -204,23 +236,32 @@ (throw e))))) ;; 20.26. Slice +(defn- slice-op [source start-index end-index] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper slice-op cache source start-index end-index)) + (-resolve-refs [_ expression-defs] + (slice-op (core/-resolve-refs source expression-defs) + (core/-resolve-refs start-index expression-defs) + (core/-resolve-refs end-index expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper slice-op parameters source start-index end-index)) + (-eval [_ context resource scopes] + (when-let [source (core/-eval source context resource scopes)] + (let [start-index (or (core/-eval start-index context resource scopes) 0) + end-index (or (core/-eval end-index context resource scopes) (count source))] + (if (or (neg? start-index) (< end-index start-index)) + [] + (subvec source start-index end-index))))) + (-form [_] + (list 'slice (core/-form source) (core/-form start-index) (core/-form end-index))))) + (defmethod core/compile* :elm.compiler.type/slice [context {:keys [source] start-index :startIndex end-index :endIndex}] (let [source (core/compile* context source) start-index (core/compile* context start-index) end-index (core/compile* context end-index)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (when-let [source (core/-eval source context resource scopes)] - (let [start-index (or (core/-eval start-index context resource scopes) 0) - end-index (or (core/-eval end-index context resource scopes) (count source))] - (if (or (neg? start-index) (< end-index start-index)) - [] - (subvec source start-index end-index))))) - (-form [_] - (list 'slice (core/-form source) (core/-form start-index) (core/-form end-index)))))) + (slice-op source start-index end-index))) ;; 20.27. Sort (defmethod core/compile* :elm.compiler.type/sort @@ -233,9 +274,8 @@ (case type "ByDirection" (let [comp (queries/comparator direction)] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + ;; TODO: other methods (-eval [_ context resource scopes] (when-let [source (core/-eval source context resource scopes)] (sort-by identity comp source))) diff --git a/modules/cql/src/blaze/elm/compiler/logical_operators.clj b/modules/cql/src/blaze/elm/compiler/logical_operators.clj index 1785b883e..f2e4278aa 100644 --- a/modules/cql/src/blaze/elm/compiler/logical_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/logical_operators.clj @@ -5,13 +5,23 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defunop]])) + [blaze.elm.compiler.logical-operators.util :as u] + [blaze.elm.compiler.macros :refer [defunop reify-expr]] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [prometheus.alpha :as prom])) ;; 13.1. And -(defn- nil-and-expr [x] - (reify core/Expression - (-static [_] - false) +(defn- and-nil-op [x] + (reify-expr core/Expression + (-attach-cache [_ cache] + [(fn [] (and-nil-op ((first (core/-attach-cache x cache)))))]) + (-patient-count [_] + 0) + (-resolve-refs [_ expression-defs] + (and-nil-op (core/-resolve-refs x expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper and-nil-op parameters x)) (-eval [_ context resource scope] (when (false? (core/-eval x context resource scope)) false)) @@ -25,7 +35,83 @@ true nil false false nil nil - (nil-and-expr x))) + (and-nil-op x))) + +(defn- and-list-op [op ops] + (reify-expr core/Expression + (-attach-cache [_ _] + (Exception. "Can't attach a cache to `and-list-op`.")) + (-patient-count [_] + nil) + (-resolve-refs [_ _] + (Exception. "Can't resolve references in `and-list-op`.")) + (-resolve-params [_ _] + (Exception. "Can't resolve references in `and-list-op`.")) + (-eval [_ context resource scope] + (reduce + (fn [a op] + (if (false? a) + (reduced false) + (let [b (core/-eval op context resource scope)] + (cond + (false? b) (reduced false) + (and (true? a) (true? b)) true)))) + (core/-eval op context resource scope) + ops)) + (-form [_] + `(~'and ~(core/-form op) ~@(map core/-form ops))))) + +(defn- and-cmp [[a-op] [b-op]] + (let [a-count (or (core/-patient-count a-op) Long/MAX_VALUE) + b-count (or (core/-patient-count b-op) Long/MAX_VALUE)] + (- a-count b-count))) + +(defn and-op [a b] + (reify-expr core/Expression + (-attach-cache [_ cache] + (let [[fa a-kind a] (core/-attach-cache a cache) + [fb b-kind b] (core/-attach-cache b cache)] + (cond + (and (identical? :and a-kind) (identical? :and b-kind)) + (u/and-attach-cache-result + and-list-op + (u/merge-sorted and-cmp a b)) + + (identical? :and a-kind) + (u/and-attach-cache-result + and-list-op + (u/insert-sorted and-cmp a (fb))) + + (identical? :and b-kind) + (u/and-attach-cache-result + and-list-op + (u/insert-sorted and-cmp b (fa))) + + :else + (let [a (fa) + b (fb)] + (if (pos? (and-cmp a b)) + (u/and-attach-cache-result and-list-op [b a]) + (u/and-attach-cache-result and-list-op [a b])))))) + (-patient-count [_] + (let [count-a (core/-patient-count a) + count-b (core/-patient-count b)] + (when (and count-a count-b) + (min count-a count-b)))) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper and-op expression-defs a b)) + (-resolve-params [_ parameters] + (core/resolve-params-helper and-op parameters a b)) + (-eval [_ context resource scope] + (let [a (core/-eval a context resource scope)] + (if (false? a) + false + (let [b (core/-eval b context resource scope)] + (cond + (false? b) false + (and (true? a) (true? b)) true))))) + (-form [_] + (list 'and (core/-form a) (core/-form b))))) (defn- dynamic-and "Creates an and-expression where `a` is known to be dynamic and `b` could be @@ -34,20 +120,8 @@ (condp identical? b true a false false - nil (nil-and-expr a) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (if (false? a) - false - (let [b (core/-eval b context resource scope)] - (cond - (false? b) false - (and (true? a) (true? b)) true))))) - (-form [_] - (list 'and (core/-form a) (core/-form b)))))) + nil (and-nil-op a) + (and-op a b))) (defmethod core/compile* :elm.compiler.type/and [context {[a b] :operand}] @@ -64,15 +138,23 @@ (throw (Exception. "Unsupported Implies expression. Please normalize the ELM tree before compiling."))) ;; 13.3 Not +(declare not-op) + (defunop not [operand] (when (some? operand) (not operand))) ;; 13.4. Or -(defn- nil-or-expr [x] - (reify core/Expression - (-static [_] - false) +(defn- or-nil-op [x] + (reify-expr core/Expression + (-attach-cache [_ cache] + [(fn [] (or-nil-op ((first (core/-attach-cache x cache)))))]) + (-patient-count [_] + (core/-patient-count x)) + (-resolve-refs [_ expression-defs] + (or-nil-op (core/-resolve-refs x expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper or-nil-op parameters x)) (-eval [_ context resource scope] (when (true? (core/-eval x context resource scope)) true)) @@ -86,7 +168,90 @@ true true false nil nil nil - (nil-or-expr x))) + (or-nil-op x))) + +(defn- or-list-op [tuples] + (reify-expr core/Expression + (-attach-cache [_ _] + (Exception. "Can't attach a cache to `or-list-op`.")) + (-patient-count [_] + nil) + (-resolve-refs [_ _] + (Exception. "Can't resolve references in `or-list-op`.")) + (-resolve-params [_ _] + (Exception. "Can't resolve references in `or-list-op`.")) + (-eval [_ context resource scope] + (reduce + (fn [_ [op bf]] + ;; TODO: handle nil + (if bf + (if (bloom-filter/might-contain? bf resource) + (do (prom/inc! ec/bloom-filter-not-useful-total "or") + (if (core/-eval op context resource scope) + (reduced true) + (do (prom/inc! ec/bloom-filter-false-positive-total "or") + false))) + (do (prom/inc! ec/bloom-filter-useful-total "or") + (reduced false))) + (if (core/-eval op context resource scope) + (reduced true) + false))) + false + tuples)) + (-form [_] + `(~'or ~@(map (comp core/-form first) tuples))))) + +(defn- or-cmp [[a-op] [b-op]] + (let [a-count (or (core/-patient-count a-op) Long/MAX_VALUE) + b-count (or (core/-patient-count b-op) Long/MAX_VALUE)] + (- b-count a-count))) + +(defn or-op [a b] + (reify-expr core/Expression + (-attach-cache [_ cache] + (let [[fa a-kind a] (core/-attach-cache a cache) + [fb b-kind b] (core/-attach-cache b cache)] + (cond + (and (identical? :or a-kind) (identical? :or b-kind)) + (u/or-attach-cache-result + or-list-op + (u/merge-sorted or-cmp a b)) + + (identical? :or a-kind) + (u/or-attach-cache-result + or-list-op + (u/insert-sorted or-cmp a (fb))) + + (identical? :or b-kind) + (u/or-attach-cache-result + or-list-op + (u/insert-sorted or-cmp b (fa))) + + :else + (let [a (fa) + b (fb)] + (if (pos? (or-cmp a b)) + (u/or-attach-cache-result or-list-op [b a]) + (u/or-attach-cache-result or-list-op [a b])))))) + (-patient-count [_] + (let [count-a (core/-patient-count a) + count-b (core/-patient-count b)] + (when (and count-a count-b) + (max count-a count-b)))) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper or-op expression-defs a b)) + (-resolve-params [_ parameters] + (core/resolve-params-helper or-op parameters a b)) + (-eval [_ context resource scope] + (let [a (core/-eval a context resource scope)] + (if (true? a) + true + (let [b (core/-eval b context resource scope)] + (cond + (true? b) true + (and (false? a) (false? b)) false))))) + (-form [_] + (list 'or (core/-form a) (core/-form b))))) (defn- dynamic-or "Creates an or-expression where `a` is known to be dynamic and `b` could be @@ -95,20 +260,8 @@ (condp identical? b true true false a - nil (nil-or-expr a) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (if (true? a) - true - (let [b (core/-eval b context resource scope)] - (cond - (true? b) true - (and (false? a) (false? b)) false))))) - (-form [_] - (list 'or (core/-form a) (core/-form b)))))) + nil (or-nil-op a) + (or-op a b))) (defmethod core/compile* :elm.compiler.type/or [context {[a b] :operand}] @@ -120,32 +273,31 @@ (dynamic-or a (core/compile* context b))))) ;; 13.5 Xor +(defn- xor-op [a b] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper xor-op cache a b)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper xor-op expression-defs a b)) + (-resolve-params [_ parameters] + (core/resolve-params-helper xor-op parameters a b)) + (-eval [_ context resource scope] + (when-some [a (core/-eval a context resource scope)] + (when-some [b (core/-eval b context resource scope)] + (if a (not b) b)))) + (-form [_] + (list 'xor (core/-form a) (core/-form b))))) + (defn- dynamic-xor "Creates an xor-expression where `a` is known to be dynamic and `b` could be static or dynamic." [a b] (condp identical? b true - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (when (some? a) - (not a)))) - (-form [_] - (list 'not (core/-form a)))) + (not-op a) false a nil nil - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-some [a (core/-eval a context resource scope)] - (when-some [b (core/-eval b context resource scope)] - (if a (not b) b)))) - (-form [_] - (list 'xor (core/-form a) (core/-form b)))))) + (xor-op a b))) (defmethod core/compile* :elm.compiler.type/xor [context {[a b] :operand}] diff --git a/modules/cql/src/blaze/elm/compiler/logical_operators/util.clj b/modules/cql/src/blaze/elm/compiler/logical_operators/util.clj new file mode 100644 index 000000000..a0fb4659f --- /dev/null +++ b/modules/cql/src/blaze/elm/compiler/logical_operators/util.clj @@ -0,0 +1,55 @@ +(ns blaze.elm.compiler.logical-operators.util + (:require + [blaze.elm.expression.cache.bloom-filter :as bloom-filter])) + +(defn insert-sorted [cmp coll x] + (loop [coll (seq coll) result []] + (cond + (empty? coll) + (conj result x) + + (neg? (cmp x (first coll))) + (into result (cons x coll)) + + :else + (recur (rest coll) (conj result (first coll)))))) + +(defn merge-sorted [cmp coll1 coll2] + (loop [result [] + coll1 (seq coll1) + coll2 (seq coll2)] + (cond + (and (empty? coll1) (empty? coll2)) result + (empty? coll1) (into result coll2) + (empty? coll2) (into result coll1) + :else (let [x1 (first coll1) + x2 (first coll2)] + (if (neg? (cmp x2 x1)) + (recur (conj result x2) coll1 (rest coll2)) + (recur (conj result x1) (rest coll1) coll2)))))) + +(defn and-attach-cache-result [op triples] + [(fn [] + [(op (ffirst triples) (mapv first (rest triples))) + (into [] (mapcat second) triples)]) + :and + triples]) + +(defn or-attach-cache-result [op triples] + [(fn [] + [(let [triples (reverse triples)] + (op + (vec + (reduce + (fn [[[_ last-bf] :as r] [op _ bf]] + (cons + (if (and last-bf bf) + [op (bloom-filter/merge last-bf bf)] + [op]) + r)) + (let [[[op _ bf]] triples] + (when op (list [op bf]))) + (rest triples))))) + (into [] (mapcat second) triples)]) + :or + triples]) diff --git a/modules/cql/src/blaze/elm/compiler/macros.clj b/modules/cql/src/blaze/elm/compiler/macros.clj index df2512efe..ba5ec1a35 100644 --- a/modules/cql/src/blaze/elm/compiler/macros.clj +++ b/modules/cql/src/blaze/elm/compiler/macros.clj @@ -1,143 +1,411 @@ (ns blaze.elm.compiler.macros (:require - [blaze.elm.compiler.core :as core])) + [blaze.anomaly :as ba] + [blaze.elm.compiler.core :as core] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [blaze.elm.expression.cache.spec] + [clojure.spec.alpha :as s] + [prometheus.alpha :as prom] + [taoensso.timbre :as log])) (set! *warn-on-reflection* true) -(defn- compile-kw [name] - (keyword "elm.compiler.type" (clojure.core/name name))) +(defn- find-form [form body] + (some #(when (= form (first %)) %) body)) + +(def ^:private ^:const unknown nil) + +(defmacro reify-expr [_ & body] + `(reify + core/Expression + (~'-static [~'_] + false) + ~(if-let [form (find-form '-attach-cache body)] + form + (list '-attach-cache ['expr '_] `[(fn [] [~'expr])])) + ~(if-let [form (find-form '-patient-count body)] + form + (list '-patient-count ['_] unknown)) + ~(if-let [form (find-form '-resolve-refs body)] + form + (list '-resolve-refs ['expr '_] 'expr)) + ~(if-let [form (find-form '-resolve-params body)] + form + (list '-resolve-params ['expr '_] 'expr)) + ~(if-let [form (find-form '-eval body)] + form + (list '-eval ['expr '_ '_ '_] 'expr)) + ~(if-let [form (find-form '-form body)] + form + (list '-form ['_] 'nil)) -(defn generate-binding-vector + Object + (~'equals [~'this ~'other] + (.equals ^Object (core/-form ~'this) (core/-form ~'other))) + (~'hashCode [~'this] + (.hashCode ^Object (core/-form ~'this))))) + +(defn- generate-binding-vector "Creates a binding vector of at least `[operand-binding operand]` and - optionally `[operand-binding operand expr-binding expr-sym]` if `expr-binding` - is given." - [operand-binding operand expr-binding expr-sym] - (cond-> [operand-binding operand] expr-binding (conj expr-binding expr-sym))) - -(defn generate-unop [name operand-sym operand-binding expr-binding expr-sym body] - (let [context-sym (gensym "context") - resource-sym (gensym "resource") - scope-sym (gensym "scope")] - `(reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ ~context-sym ~resource-sym ~scope-sym] - (let ~(generate-binding-vector - operand-binding `(core/-eval ~operand-sym ~context-sym ~resource-sym ~scope-sym) - expr-binding expr-sym) - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form ~operand-sym)))))) + optionally `[operand-binding operand elm-elm-expr-binding elm-expr]` if + `elm-elm-expr-binding` is given." + [operand-binding operand elm-expr-binding elm-expr] + (cond-> [operand-binding operand] elm-expr-binding (conj elm-expr-binding elm-expr))) + +(defn- compile-kw [name] + (keyword "elm.compiler.type" (clojure.core/name name))) (defmacro defunop {:arglists '([name attr-map? bindings & body])} [name & more] (let [attr-map (when (map? (first more)) (first more)) more (if (map? (first more)) (next more) more) - [[operand-binding expr-binding] & body] more - context-sym (gensym "context") - operand-sym (gensym "operand") - expr-sym (gensym "expr")] - `(defmethod core/compile* ~(compile-kw name) - [~context-sym ~expr-sym] - (let [~operand-sym (core/compile* (merge ~context-sym ~(dissoc attr-map :cache)) (:operand ~expr-sym))] - (if (core/static? ~operand-sym) - (let [~operand-binding ~operand-sym - ~(or expr-binding '_) ~expr-sym] - ~@body) - ~(generate-unop name operand-sym operand-binding expr-binding expr-sym body)))))) + [[operand-binding elm-expr-binding] & body] more + operand-key (or (:operand-key attr-map) :operand) + attr-map (dissoc attr-map :operand-key) + op (symbol (str name "-op")) + cache-op (symbol (str name "-cache-op")) + caching-op (symbol (str name "-caching-op")) + elm-expr (gensym "elm-expr") + operand (gensym "operand") + context (gensym "context") + eval-context (gensym "eval-context") + resource (gensym "resource") + scope (gensym "scope") + bloom-filter (gensym "bloom-filter") + expr (gensym "expr")] + `(do + ~(when (:cache attr-map) + `(do + (declare ~caching-op) + + (s/fdef ~cache-op + :args ~(if elm-expr-binding + `(s/cat :operand core/expr? + :bloom-filter ::ec/bloom-filter + :expr :elm/expression) + `(s/cat :operand core/expr? + :bloom-filter ::ec/bloom-filter)) + :ret core/expr?) + + (defn ~cache-op + ~(str "Creates a " name " operator with attached Bloom filter that will be used to increase performance of evaluation.") + ~(cond-> [operand bloom-filter] elm-expr-binding (conj elm-expr)) + (log/trace (format "Create expression `%s` with attached Bloom filter." (list (quote ~name) (core/-form ~operand)))) + (reify-expr core/Expression + (~'-attach-cache [~expr ~'cache] + (if-let [~bloom-filter (ec/get ~'cache ~expr)] + ~(if elm-expr-binding + `[(fn [] + [(~cache-op ~operand ~bloom-filter ~elm-expr) + [~bloom-filter] + ~bloom-filter])] + `[(fn [] + [(~cache-op ~operand ~bloom-filter) + [~bloom-filter] + ~bloom-filter])]) + ~(if elm-expr-binding + `[(fn [] + [(~caching-op ~operand ~elm-expr) + [(ba/unavailable "No Bloom filter available.")]])] + `[(fn [] + [(~caching-op ~operand) + [(ba/unavailable "No Bloom filter available.")]])]))) + (~'-patient-count [~'_] + (::bloom-filter/patient-count ~bloom-filter)) + (~'-resolve-refs [~'_ ~'expression-defs] + ~(if elm-expr-binding + `(~caching-op + (core/-resolve-refs ~operand ~'expression-defs) + ~elm-expr) + `(~caching-op + (core/-resolve-refs ~operand ~'expression-defs)))) + (~'-resolve-params [~'_ ~'parameters] + ~(if elm-expr-binding + `(~caching-op + (core/-resolve-params ~operand ~'parameters) + ~elm-expr) + `(~caching-op + (core/-resolve-params ~operand ~'parameters)))) + (~'-eval [~'_ ~context ~resource ~scope] + (if (bloom-filter/might-contain? ~bloom-filter ~resource) + (let [res# (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand + ~context + ~resource + ~scope) + elm-expr-binding elm-expr) + ~@body)] + (prom/inc! ec/bloom-filter-not-useful-total ~(clojure.core/name name)) + (when-not res# + (prom/inc! ec/bloom-filter-false-positive-total ~(clojure.core/name name))) + res#) + (do (prom/inc! ec/bloom-filter-useful-total ~(clojure.core/name name)) + false))) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand))))) + + (s/fdef ~caching-op + :args ~(if elm-expr-binding + `(s/cat :operand core/expr? :expr :elm/expression) + `(s/cat :operand core/expr?)) + :ret core/expr?) + + (defn ~caching-op + ~(str "Creates a " name " operator that will handle cache attachment.") + ~(cond-> [operand] elm-expr-binding (conj elm-expr)) + (reify-expr core/Expression + (~'-attach-cache [~expr ~'cache] + (if-let [~bloom-filter (ec/get ~'cache ~expr)] + ;;TODO: add metric of how many times a bloom filter was available + ~(if elm-expr-binding + `[(fn [] + [(~cache-op ~operand ~bloom-filter ~elm-expr) + [~bloom-filter] + ~bloom-filter])] + `[(fn [] + [(~cache-op ~operand ~bloom-filter) + [~bloom-filter] + ~bloom-filter])]) + ~(if elm-expr-binding + `[(fn [] + [(~caching-op ~operand ~elm-expr) + [(ba/unavailable "No Bloom filter available.")]])] + `[(fn [] + [(~caching-op ~operand) + [(ba/unavailable "No Bloom filter available.")]])]))) + (~'-resolve-refs [~'_ ~'expression-defs] + ~(if elm-expr-binding + `(~caching-op + (core/-resolve-refs ~operand ~'expression-defs) + ~elm-expr) + `(~caching-op + (core/-resolve-refs ~operand ~'expression-defs)))) + (~'-resolve-params [~'_ ~'parameters] + ~(if elm-expr-binding + `(~caching-op + (core/-resolve-params ~operand ~'parameters) + ~elm-expr) + `(~caching-op + (core/-resolve-params ~operand ~'parameters)))) + (~'-eval [~'_ ~context ~resource ~scope] + (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand ~context ~resource ~scope) + elm-expr-binding elm-expr) + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand))))))) + + (defn ~op + ~(str "Creates a " name " operator that will only delegate cache attachment.") + ~(cond-> [operand] elm-expr-binding (conj elm-expr)) + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + ~(if elm-expr-binding + `(core/attach-cache-helper-1 ~op ~'cache ~operand ~elm-expr) + `(core/attach-cache-helper ~op ~'cache ~operand))) + (~'-resolve-refs [~'_ ~'expression-defs] + ~(if elm-expr-binding + `(~op + (core/-resolve-refs ~operand ~'expression-defs) + ~elm-expr) + `(~op + (core/-resolve-refs ~operand ~'expression-defs)))) + (~'-resolve-params [~'_ ~'parameters] + ~(if elm-expr-binding + `(~op + (core/-resolve-params ~operand ~'parameters) + ~elm-expr) + `(~op + (core/-resolve-params ~operand ~'parameters)))) + (~'-eval [~'_ ~context ~resource ~scope] + (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand ~context ~resource ~scope) + elm-expr-binding elm-expr) + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand))))) + + (defmethod core/compile* ~(compile-kw name) + [{~eval-context :eval-context :as ~context} + {~operand ~operand-key :as ~elm-expr}] + (let [~operand (core/compile* (merge ~context ~(dissoc attr-map :cache)) ~operand)] + (if (core/static? ~operand) + (let ~(generate-binding-vector + operand-binding operand + elm-expr-binding elm-expr) + ~@body) + ~(if (:cache attr-map) + `(if (= "Patient" ~eval-context) + ~(if elm-expr-binding + `(~caching-op ~operand ~elm-expr) + `(~caching-op ~operand)) + ~(if elm-expr-binding + `(~op ~operand ~elm-expr) + `(~op ~operand))) + (if elm-expr-binding + `(~op ~operand ~elm-expr) + `(~op ~operand))))))))) (defmacro defbinop {:arglists '([name attr-map? bindings & body])} [name & more] (let [attr-map (when (map? (first more)) (first more)) more (if (map? (first more)) (next more) more) - [[op-1-binding op-2-binding] & body] more] - `(defmethod core/compile* ~(compile-kw name) - [context# {[operand-1# operand-2#] :operand}] - (let [context# (merge context# ~attr-map) - operand-1# (core/compile* context# operand-1#) - operand-2# (core/compile* context# operand-2#)] - (if (and (core/static? operand-1#) (core/static? operand-2#)) - (let [~op-1-binding operand-1# - ~op-2-binding operand-2#] - ~@body) - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~op-1-binding (core/-eval operand-1# context# resource# scope#) - ~op-2-binding (core/-eval operand-2# context# resource# scope#)] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form operand-1#) (core/-form operand-2#))))))))) + [[op-1-binding op-2-binding] & body] more + context (gensym "context") + op (symbol (str name "-op")) + op-1 (gensym "op-1") + op-2 (gensym "op-2")] + `(do + (defn ~op [~op-1 ~op-2] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper ~op ~'cache ~op-1 ~op-2)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op + (core/-resolve-refs ~op-1 ~'expression-defs) + (core/-resolve-refs ~op-2 ~'expression-defs))) + (~'-resolve-params [~'_ ~'parameters] + (core/resolve-params-helper ~op ~'parameters ~op-1 ~op-2)) + (~'-eval [~'_ context# resource# scope#] + (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) + ~op-2-binding (core/-eval ~op-2 context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~op-1) (core/-form ~op-2))))) + + (defmethod core/compile* ~(compile-kw name) + [~context {[~op-1 ~op-2] :operand}] + (let [context# ~(if attr-map `(merge ~context ~attr-map) context) + ~op-1 (core/compile* context# ~op-1) + ~op-2 (core/compile* context# ~op-2)] + (if (and (core/static? ~op-1) (core/static? ~op-2)) + (let [~op-1-binding ~op-1 + ~op-2-binding ~op-2] + ~@body) + (~op ~op-1 ~op-2))))))) (defmacro defternop {:arglists '([name bindings & body])} [name [op-1-binding op-2-binding op-3-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {[operand-1# operand-2# operand-3#] :operand}] - (let [operand-1# (core/compile* context# operand-1#) - operand-2# (core/compile* context# operand-2#) - operand-3# (core/compile* context# operand-3#)] - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~op-1-binding (core/-eval operand-1# context# resource# scope#) - ~op-2-binding (core/-eval operand-2# context# resource# scope#) - ~op-3-binding (core/-eval operand-3# context# resource# scope#)] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form operand-1#) (core/-form operand-2#) - (core/-form operand-3#))))))) + (let [op (symbol (str name "-op")) + op-1 (gensym "op-1") + op-2 (gensym "op-2") + op-3 (gensym "op-3")] + `(do + (defn ~op [~op-1 ~op-2 ~op-3] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper ~op ~'cache ~op-1 ~op-2 ~op-3)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op + (core/-resolve-refs ~op-1 ~'expression-defs) + (core/-resolve-refs ~op-2 ~'expression-defs) + (core/-resolve-refs ~op-3 ~'expression-defs))) + (~'-resolve-params [~'_ ~'parameters] + (core/resolve-params-helper ~op ~'parameters ~op-1 ~op-2 ~op-3)) + (~'-eval [~'_ context# resource# scope#] + (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) + ~op-2-binding (core/-eval ~op-2 context# resource# scope#) + ~op-3-binding (core/-eval ~op-3 context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~op-1) (core/-form ~op-2) + (core/-form ~op-3))))) + (defmethod core/compile* ~(compile-kw name) + [context# {[~op-1 ~op-2 ~op-3] :operand}] + (let [~op-1 (core/compile* context# ~op-1) + ~op-2 (core/compile* context# ~op-2) + ~op-3 (core/compile* context# ~op-3)] + (if (and (core/static? ~op-1) (core/static? ~op-2) (core/static? ~op-3)) + (let [~op-1-binding ~op-1 + ~op-2-binding ~op-2 + ~op-3-binding ~op-3] + ~@body) + (~op ~op-1 ~op-2 ~op-3))))))) (defmacro defnaryop {:arglists '([name bindings & body])} [name [operands-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {operands# :operand}] - (let [operands# (mapv #(core/compile* context# %) operands#)] - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~operands-binding (mapv #(core/-eval % context# resource# scope#) operands#)] - ~@body)) - (~'-form [~'_] - (cons (quote ~name) (map core/-form operands#))))))) + (let [op (symbol (str name "-op"))] + `(do + (defn ~op [~operands-binding] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper-list ~op ~'cache ~operands-binding)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op + (mapv #(core/-resolve-refs % ~'expression-defs) ~operands-binding))) + (~'-resolve-params [~'_ ~'parameters] + (~op + (mapv #(core/-resolve-params % ~'parameters) ~operands-binding))) + (~'-eval [~'_ context# resource# scope#] + (let [~operands-binding (mapv #(core/-eval % context# resource# scope#) ~operands-binding)] + ~@body)) + (~'-form [~'_] + (cons (quote ~name) (map core/-form ~operands-binding))))) + + (defmethod core/compile* ~(compile-kw name) + [context# {operands# :operand}] + (~op (mapv #(core/compile* context# %) operands#)))))) (defmacro defaggop {:arglists '([name bindings & body])} [name [source-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {source# :source}] - (let [source# (core/compile* context# source#)] - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~source-binding (core/-eval source# context# resource# scope#)] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form source#))))))) + (let [op (symbol (str name "-op"))] + `(do + (defn ~op [~source-binding] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper ~op ~'cache ~source-binding)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op (core/-resolve-refs ~source-binding ~'expression-defs))) + (~'-resolve-params [~'_ ~'parameters] + (~op (core/-resolve-params ~source-binding ~'parameters))) + (~'-eval [~'_ context# resource# scope#] + (let [~source-binding (core/-eval ~source-binding context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~source-binding))))) + + (defmethod core/compile* ~(compile-kw name) + [context# {source# :source}] + (~op (core/compile* context# source#)))))) (defmacro defunopp {:arglists '([name bindings & body])} - [name [operand-binding precision-binding expr-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {operand# :operand precision# :precision :as expr#}] - (let [operand# (core/compile* context# operand#) - ~precision-binding (some-> precision# core/to-chrono-unit) - ~(or expr-binding '_) expr#] - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~operand-binding (core/-eval operand# context# resource# scope#)] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form operand#) precision#)))))) + [name [operand-binding precision-binding] & body] + (let [op (symbol (str name "-op")) + operand (gensym "operand") + precision (gensym "precision")] + `(do + (defn ~op [~operand ~precision-binding ~precision] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper-2 ~op ~'cache ~operand ~precision-binding + ~precision)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op + (core/-resolve-refs ~operand ~'expression-defs) + ~precision-binding ~precision)) + (~'-resolve-params [~'_ ~'parameters] + (~op + (core/-resolve-params ~operand ~'parameters) + ~precision-binding ~precision)) + (~'-eval [~'_ context# resource# scope#] + (let [~operand-binding (core/-eval ~operand context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand) ~precision)))) + + (defmethod core/compile* ~(compile-kw name) + [context# {operand# :operand precision# :precision}] + (~op + (core/compile* context# operand#) + (some-> precision# core/to-chrono-unit) + precision#))))) (defmacro defbinopp {:arglists '([name attr-map? bindings & body])} @@ -147,15 +415,23 @@ [[op-1-binding op-2-binding precision-binding] & body] more precision-required (:required (meta precision-binding)) context (gensym "context") + op (symbol (str name "-op")) + precision-op (symbol (str name "-precision-op")) op-1 (gensym "op-1") op-2 (gensym "op-2") precision (gensym "precision")] `(do ~(when-not precision-required - `(defn ~(symbol (str name "-op")) [~op-1 ~op-2] - (reify core/Expression - (~'-static [~'_] - false) + `(defn ~op [~op-1 ~op-2] + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper ~op ~'cache ~op-1 ~op-2)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~op + (core/-resolve-refs ~op-1 ~'expression-defs) + (core/-resolve-refs ~op-2 ~'expression-defs))) + (~'-resolve-params [~'_ ~'parameters] + (core/resolve-params-helper ~op ~'parameters ~op-1 ~op-2)) (~'-eval [~'_ context# resource# scope#] (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) ~op-2-binding (core/-eval ~op-2 context# resource# scope#) @@ -164,11 +440,22 @@ (~'-form [~'_] (list (quote ~name) (core/-form ~op-1) (core/-form ~op-2)))))) - (defn ~(symbol (str name "-precision-op")) + (defn ~precision-op [~op-1 ~op-2 ~precision-binding ~precision] - (reify core/Expression - (~'-static [~'_] - false) + (reify-expr core/Expression + (~'-attach-cache [~'_ ~'cache] + (core/attach-cache-helper-2 ~precision-op ~'cache ~op-1 ~op-2 + ~precision-binding ~precision)) + (~'-resolve-refs [~'_ ~'expression-defs] + (~precision-op + (core/-resolve-refs ~op-1 ~'expression-defs) + (core/-resolve-refs ~op-2 ~'expression-defs) + ~precision-binding ~precision)) + (~'-resolve-params [~'_ ~'parameters] + (~precision-op + (core/-resolve-params ~op-1 ~'parameters) + (core/-resolve-params ~op-2 ~'parameters) + ~precision-binding ~precision)) (~'-eval [~'_ context# resource# scope#] (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) ~op-2-binding (core/-eval ~op-2 context# resource# scope#)] @@ -188,8 +475,7 @@ (let [~op-1-binding ~op-1 ~op-2-binding ~op-2] ~@body) - (~(symbol (str name "-precision-op")) ~op-1 ~op-2 - ~precision-binding ~precision)))) + (~precision-op ~op-1 ~op-2 ~precision-binding ~precision)))) `(defmethod core/compile* ~(compile-kw name) [~context {[~op-1 ~op-2] :operand ~precision :precision}] @@ -202,6 +488,5 @@ ~op-2-binding ~op-2] ~@body) (if ~precision - (~(symbol (str name "-precision-op")) ~op-1 ~op-2 - ~precision-binding ~precision) - (~(symbol (str name "-op")) ~op-1 ~op-2))))))))) + (~precision-op ~op-1 ~op-2 ~precision-binding ~precision) + (~op ~op-1 ~op-2))))))))) diff --git a/modules/cql/src/blaze/elm/compiler/nullological_operators.clj b/modules/cql/src/blaze/elm/compiler/nullological_operators.clj index ba911eb59..f462f57dc 100644 --- a/modules/cql/src/blaze/elm/compiler/nullological_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/nullological_operators.clj @@ -5,7 +5,7 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defunop]])) + [blaze.elm.compiler.macros :refer [defunop reify-expr]])) ;; 14.1. Null (defmethod core/compile* :elm.compiler.type/null @@ -19,41 +19,50 @@ ;; subsequent arguments must be of that same type. ;; ;; TODO: The list type argument is missing in the doc. +(defn- coalesce-op [operands] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper-list coalesce-op cache operands)) + (-resolve-refs [_ expression-defs] + (coalesce-op (mapv #(core/-resolve-refs % expression-defs) operands))) + (-resolve-params [_ parameters] + (coalesce-op (mapv #(core/-resolve-params % parameters) operands))) + (-eval [_ context resource scope] + (reduce + (fn [_ operand] + (let [operand (core/-eval operand context resource scope)] + (when (some? operand) + (reduced operand)))) + nil + operands)) + (-form [_] + `(~'coalesce ~@(map core/-form operands))))) + +(defn coalesce-list-op [list] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper coalesce-op cache list)) + (-resolve-refs [_ expression-defs] + (coalesce-op (core/-resolve-refs list expression-defs))) + (-resolve-params [_ parameters] + (coalesce-op (core/-resolve-params list parameters))) + (-eval [_ context resource scope] + (reduce + (fn [_ elem] + (let [elem (core/-eval elem context resource scope)] + (when (some? elem) + (reduced elem)))) + nil + (core/-eval list context resource scope))))) + (defmethod core/compile* :elm.compiler.type/coalesce [context {operands :operand}] (if (= 1 (count operands)) (let [operand (first operands)] - (if (= "List" (:type operand)) - (let [operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (reduce - (fn [_ elem] - (let [elem (core/-eval elem context resource scope)] - (when (some? elem) - (reduced elem)))) - nil - (core/-eval operand context resource scope))))) - (let [operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (core/-eval operand context resource scope)))))) - (let [operands (mapv #(core/compile* context %) operands)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (reduce - (fn [_ operand] - (let [operand (core/-eval operand context resource scope)] - (when (some? operand) - (reduced operand)))) - nil - operands)))))) + (cond-> (core/compile* context operand) + (= "List" (:type operand)) + coalesce-list-op)) + (coalesce-op (mapv #(core/compile* context %) operands)))) ;; 14.3. IsFalse (defunop is-false [operand] diff --git a/modules/cql/src/blaze/elm/compiler/parameters.clj b/modules/cql/src/blaze/elm/compiler/parameters.clj index 97e91221c..5fa9e863f 100644 --- a/modules/cql/src/blaze/elm/compiler/parameters.clj +++ b/modules/cql/src/blaze/elm/compiler/parameters.clj @@ -5,7 +5,8 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.anomaly :as ba :refer [throw-anom]] - [blaze.elm.compiler.core :as core])) + [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]])) ;; 7.1. ParameterDef ;; @@ -17,17 +18,20 @@ (format "Value of parameter `%s` not found." name) :context context)) -(defrecord ParameterRef [name] - core/Expression - (-static [_] - false) - (-eval [_ {:keys [parameters] :as context} _ _] - (let [value (get parameters name ::not-found)] - (if (identical? ::not-found value) - (throw-anom (parameter-value-not-found-anom context name)) - value))) - (-form [_] - `(~'param-ref ~name))) +(defn- param-ref [name] + (reify-expr core/Expression + (-resolve-params [expr parameters] + (let [value (get parameters name ::not-found)] + (if (identical? ::not-found value) + expr + value))) + (-eval [_ {:keys [parameters] :as context} _ _] + (let [value (get parameters name ::not-found)] + (if (identical? ::not-found value) + (throw-anom (parameter-value-not-found-anom context name)) + value))) + (-form [_] + `(~'param-ref ~name)))) (defn- find-parameter-def "Returns the parameter-def with `name` from `library` or nil if not found." @@ -44,5 +48,5 @@ [{:keys [library] :as context} {:keys [name]}] ;; TODO: look into other libraries (:libraryName) (if-let [{:keys [name]} (find-parameter-def library name)] - (->ParameterRef name) + (param-ref name) (throw-anom (parameter-def-not-found-anom context name)))) diff --git a/modules/cql/src/blaze/elm/compiler/queries.clj b/modules/cql/src/blaze/elm/compiler/queries.clj index 0df74039e..465bd2bc1 100644 --- a/modules/cql/src/blaze/elm/compiler/queries.clj +++ b/modules/cql/src/blaze/elm/compiler/queries.clj @@ -7,7 +7,7 @@ (:require [blaze.coll.core :as coll] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.structured-values :as structured-values] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.protocols :as p] [blaze.fhir.spec]) (:import @@ -18,12 +18,18 @@ (defprotocol XformFactory (-create [_ context resource scope] "Creates a xform which filters and/or shapes query sources.") + (-resolve-refs [_ expression-defs]) + (-resolve-params [_ parameters]) (-form [_])) (defn- where-xform-factory [alias expr] (reify XformFactory (-create [_ context resource scope] (filter #(core/-eval expr context resource (assoc scope alias %)))) + (-resolve-refs [_ expression-defs] + (where-xform-factory alias (core/-resolve-refs expr expression-defs))) + (-resolve-params [_ parameters] + (where-xform-factory alias (core/-resolve-params expr parameters))) (-form [_] `(~'filter (~'fn [~(symbol alias)] ~(core/-form expr)))))) @@ -31,6 +37,10 @@ (reify XformFactory (-create [_ context resource scope] (map #(core/-eval expr context resource (assoc scope alias %)))) + (-resolve-refs [_ expression-defs] + (return-xform-factory* alias (core/-resolve-refs expr expression-defs))) + (-resolve-params [_ parameters] + (return-xform-factory* alias (core/-resolve-params expr parameters))) (-form [_] `(~'map (~'fn [~(symbol alias)] ~(core/-form expr)))))) @@ -38,6 +48,10 @@ (reify XformFactory (-create [_ _ _ _] (distinct)) + (-resolve-refs [this _] + this) + (-resolve-params [this _] + this) (-form [_] 'distinct))) @@ -47,6 +61,10 @@ (comp (-create xform-factory context resource scope) (distinct))) + (-resolve-refs [_ expression-defs] + (composed-distinct-xform-factory (-resolve-refs xform-factory expression-defs))) + (-resolve-params [_ parameters] + (composed-distinct-xform-factory (-resolve-params xform-factory parameters))) (-form [_] `(~'comp ~(-form xform-factory) ~'distinct)))) @@ -60,6 +78,10 @@ (reify XformFactory (-create [_ context resource scope] (transduce (map #(-create % context resource scope)) comp factories)) + (-resolve-refs [_ expression-defs] + (composed-xform-factory (mapv #(-resolve-refs % expression-defs) factories))) + (-resolve-params [_ parameters] + (composed-xform-factory (mapv #(-resolve-params % parameters) factories))) (-form [_] `(~'comp ~@(map -form factories))))) @@ -77,9 +99,13 @@ return-xform-factory))) (defn- eduction-expr [xform-factory source] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (eduction-expr (-resolve-refs xform-factory expression-defs) + (core/-resolve-refs source expression-defs))) + (-resolve-params [_ parameters] + (eduction-expr (-resolve-params xform-factory parameters) + (core/-resolve-params source parameters))) (-eval [_ context resource scope] (coll/eduction (-create xform-factory context resource scope) @@ -88,9 +114,13 @@ `(~'eduction-query ~(-form xform-factory) ~(core/-form source))))) (defn- into-vector-expr [xform-factory source] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (into-vector-expr (-resolve-refs xform-factory expression-defs) + (core/-resolve-refs source expression-defs))) + (-resolve-params [_ parameters] + (into-vector-expr (-resolve-params xform-factory parameters) + (core/-resolve-params source parameters))) (-eval [_ context resource scope] (into [] @@ -134,9 +164,15 @@ (symbol (:direction sort-by-item)))) (defn- xform-sort-expr [xform-factory source sort-by-item] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (xform-sort-expr (-resolve-refs xform-factory expression-defs) + (core/-resolve-refs source expression-defs) + (update sort-by-item :expression core/-resolve-refs expression-defs))) + (-resolve-params [_ parameters] + (xform-sort-expr (-resolve-params xform-factory parameters) + (core/-resolve-params source parameters) + (update sort-by-item :expression core/-resolve-params parameters))) (-eval [_ context resource scope] ;; TODO: build a comparator of all sort by items (->> (into @@ -192,26 +228,51 @@ (throw (Exception. (str "Unsupported number of " (count sources) " sources in query."))))) ;; 10.3. AliasRef -(defrecord AliasRefExpression [key] - core/Expression - (-static [_] - false) - (-eval [_ _ _ scopes] - (get scopes key)) - (-form [_] - `(~'alias-ref ~(symbol key)))) - (defmethod core/compile* :elm.compiler.type/alias-ref [_ {:keys [name]}] - (->AliasRefExpression name)) + (reify-expr core/Expression + (-eval [_ _ _ scopes] + (get scopes name)) + (-form [_] + `(~'alias-ref ~(symbol name))))) ;; 10.7 IdentifierRef (defmethod core/compile* :elm.compiler.type/identifier-ref [_ {:keys [name]}] - (structured-values/->SingleScopePropertyExpression (keyword name))) + (let [key (keyword name)] + (reify-expr core/Expression + (-eval [_ _ _ value] + (p/get value key)) + (-form [_] + `(~key ~'default))))) ;; 10.14. With ;; 10.15. Without +(defn- relationship-clause-xform-factory [lhs-alias rhs-alias rhs such-that exists-fn form-sym] + (reify XformFactory + (-create [_ context resource scope] + (filter + (fn [lhs-item] + (let [scope (assoc scope lhs-alias lhs-item)] + (exists-fn + #(core/-eval such-that context resource (assoc scope rhs-alias %)) + (core/-eval rhs context resource scope)))))) + (-resolve-refs [_ expression-defs] + (relationship-clause-xform-factory + lhs-alias rhs-alias (core/-resolve-refs rhs expression-defs) + (core/-resolve-refs such-that expression-defs) exists-fn form-sym)) + (-resolve-params [_ parameters] + (relationship-clause-xform-factory + lhs-alias rhs-alias (core/-resolve-params rhs parameters) + (core/-resolve-params such-that parameters) exists-fn form-sym)) + (-form [_] + `(~'filter + (~'fn [~(symbol lhs-alias)] + (~form-sym + (~'fn [~(symbol rhs-alias)] + ~(core/-form such-that)) + ~(core/-form rhs))))))) + (defn compile-relationship-clause "We use the terms `lhs` and `rhs` for left-hand-side and right-hand-side of the semi-join here. @@ -223,18 +284,4 @@ such-that (core/compile* context such-that) exists-fn (if (= "With" type) coll/some (comp not coll/some)) form-sym (if (= "With" type) 'exists 'not-exists)] - (reify XformFactory - (-create [_ context resource scope] - (filter - (fn [lhs-item] - (let [scope (assoc scope lhs-alias lhs-item)] - (exists-fn - #(core/-eval such-that context resource (assoc scope rhs-alias %)) - (core/-eval rhs context resource scope)))))) - (-form [_] - `(~'filter - (~'fn [~(symbol lhs-alias)] - (~form-sym - (~'fn [~(symbol rhs-alias)] - ~(core/-form such-that)) - ~(core/-form rhs)))))))) + (relationship-clause-xform-factory lhs-alias rhs-alias rhs such-that exists-fn form-sym))) diff --git a/modules/cql/src/blaze/elm/compiler/reusing_logic.clj b/modules/cql/src/blaze/elm/compiler/reusing_logic.clj index 463fe96ec..dfa142ec8 100644 --- a/modules/cql/src/blaze/elm/compiler/reusing_logic.clj +++ b/modules/cql/src/blaze/elm/compiler/reusing_logic.clj @@ -8,6 +8,7 @@ [blaze.db.api :as d] [blaze.elm.code :as code] [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.interval :as interval] [blaze.elm.protocols :as p] [blaze.elm.quantity :as quantity] @@ -26,16 +27,16 @@ (defn- expression-not-found-anom [context name] (ba/incorrect (format "Expression `%s` not found." name) :context context)) -(defrecord ExpressionRef [name] - core/Expression - (-static [_] - false) - (-eval [_ {:keys [expression-defs] :as context} resource _] - (if-let [{:keys [expression]} (get expression-defs name)] - (core/-eval expression context resource nil) - (throw-anom (expression-not-found-anom context name)))) - (-form [_] - `(~'expr-ref ~name))) +(defn- expr-ref [name] + (reify-expr core/Expression + (-resolve-refs [expr expression-defs] + (or (:expression (get expression-defs name)) expr)) + (-eval [_ {:keys [expression-defs] :as context} resource _] + (if-let [{:keys [expression]} (get expression-defs name)] + (core/-eval expression context resource nil) + (throw-anom (expression-not-found-anom context name)))) + (-form [_] + (list 'expr-ref name)))) (defn- find-def "Returns the def with `name` from `library` or nil if not found." @@ -64,9 +65,7 @@ ;; The referenced expression has a concrete context but we are in the ;; Unfiltered context. So we map the referenced expression over all ;; concrete resources. - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ {:keys [db expression-defs] :as context} _ _] (if-some [{:keys [expression]} (get expression-defs name)] (mapv @@ -76,8 +75,8 @@ :else (if-let [result-type-name (:resultTypeName def)] - (vary-meta (->ExpressionRef name) assoc :result-type-name result-type-name) - (->ExpressionRef name))) + (vary-meta (expr-ref name) assoc :result-type-name result-type-name) + (expr-ref name))) (throw-anom (expression-def-not-found-anom context name))))) (defprotocol ToQuantity @@ -96,51 +95,71 @@ nil (-to-quantity [_])) -(defrecord ToQuantityFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (-to-quantity (core/-eval operand context resource scope))) - (-form [_] - `(~'call "ToQuantity" ~(core/-form operand)))) - -(defrecord ToCodeFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [{:keys [system version code]} (core/-eval operand context resource scope)] - (code/to-code (type/value system) (type/value version) (type/value code)))) - (-form [_] - `(~'call "ToCode" ~(core/-form operand)))) - -(defrecord ToDateFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (type/value (core/-eval operand context resource scope))) - (-form [_] - `(~'call "ToDate" ~(core/-form operand)))) - -(defrecord ToDateTimeFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (p/to-date-time (type/value (core/-eval operand context resource scope)) now)) - (-form [_] - `(~'call "ToDateTime" ~(core/-form operand)))) - -(defrecord ToStringFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (some-> (type/value (core/-eval operand context resource scope)) str)) - (-form [_] - `(~'call "ToString" ~(core/-form operand)))) +(defn- to-quantity-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-quantity-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-quantity-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper to-quantity-function-expr parameters operand)) + (-eval [_ context resource scope] + (-to-quantity (core/-eval operand context resource scope))) + (-form [_] + `(~'call "ToQuantity" ~(core/-form operand))))) + +(defn- to-code-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-code-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-code-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper to-code-function-expr parameters operand)) + (-eval [_ context resource scope] + (let [{:keys [system version code]} (core/-eval operand context resource scope)] + (code/to-code (type/value system) (type/value version) (type/value code)))) + (-form [_] + `(~'call "ToCode" ~(core/-form operand))))) + +(defn- to-date-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-date-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-date-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper to-date-function-expr parameters operand)) + (-eval [_ context resource scope] + (type/value (core/-eval operand context resource scope))) + (-form [_] + `(~'call "ToDate" ~(core/-form operand))))) + +(defn- to-date-time-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-date-time-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-date-time-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (to-date-time-function-expr (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (p/to-date-time (type/value (core/-eval operand context resource scope)) now)) + (-form [_] + `(~'call "ToDateTime" ~(core/-form operand))))) + +(defn- to-string-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-string-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-string-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper to-string-function-expr parameters operand)) + (-eval [_ context resource scope] + (some-> (type/value (core/-eval operand context resource scope)) str)) + (-form [_] + `(~'call "ToString" ~(core/-form operand))))) (defprotocol ToInterval (-to-interval [x context])) @@ -155,14 +174,18 @@ nil (-to-interval [_ _])) -(defrecord ToIntervalFunctionExpression [operand] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (-to-interval (core/-eval operand context resource scope) context)) - (-form [_] - `(~'call "ToInterval" ~(core/-form operand)))) +(defn- to-interval-function-expr [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-interval-function-expr cache operand)) + (-resolve-refs [_ expression-defs] + (to-interval-function-expr (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (to-interval-function-expr (core/-resolve-params operand parameters))) + (-eval [_ context resource scope] + (-to-interval (core/-eval operand context resource scope) context)) + (-form [_] + `(~'call "ToInterval" ~(core/-form operand))))) (defn- function-def-not-found-anom [context name] (ba/incorrect @@ -181,34 +204,32 @@ (let [operands (map #(core/compile* context %) operands)] (case name "ToQuantity" - (->ToQuantityFunctionExpression (first operands)) + (to-quantity-function-expr (first operands)) "ToDate" - (->ToDateFunctionExpression (first operands)) + (to-date-function-expr (first operands)) "ToDateTime" - (->ToDateTimeFunctionExpression (first operands)) + (to-date-time-function-expr (first operands)) "ToString" - (->ToStringFunctionExpression (first operands)) + (to-string-function-expr (first operands)) "ToCode" - (->ToCodeFunctionExpression (first operands)) + (to-code-function-expr (first operands)) "ToDecimal" (first operands) "ToInterval" - (->ToIntervalFunctionExpression (first operands)) + (to-interval-function-expr (first operands)) (compile-function context name operands)))) ;; 9.5 OperandRef (defmethod core/compile* :elm.compiler.type/operand-ref [_ {:keys [name]}] - (reify core/Expression - (-static [_] - false) + (reify-expr core/Expression (-eval [_ _ _ scope] (scope name)) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/spec.clj b/modules/cql/src/blaze/elm/compiler/spec.clj index 82ac73145..f78cc4b9f 100644 --- a/modules/cql/src/blaze/elm/compiler/spec.clj +++ b/modules/cql/src/blaze/elm/compiler/spec.clj @@ -1,15 +1,20 @@ (ns blaze.elm.compiler.spec (:require [blaze.db.spec] + [blaze.elm.compiler :as-alias c] [blaze.elm.compiler.core :as core] [blaze.elm.spec] + [blaze.fhir.spec.spec] [clojure.spec.alpha :as s])) -(s/def :blaze.elm.compiler/expression +(s/def ::c/expression core/expr?) -(s/def :blaze.elm.compiler/function +(s/def ::c/function fn?) +(s/def ::c/eval-context + (s/or :unfiltered #{"Unfiltered"} :resource-type :fhir.resource/type)) + (s/def :elm/compile-context - (s/keys :req-un [:elm/library :blaze.db/node])) + (s/keys :req-un [:elm/library ::c/eval-context :blaze.db/node])) diff --git a/modules/cql/src/blaze/elm/compiler/string_operators.clj b/modules/cql/src/blaze/elm/compiler/string_operators.clj index 8ac443d4d..9b9f714e5 100644 --- a/modules/cql/src/blaze/elm/compiler/string_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/string_operators.clj @@ -5,7 +5,7 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinop defnaryop defternop defunop]] + [blaze.elm.compiler.macros :refer [defbinop defnaryop defternop defunop reify-expr]] [blaze.elm.protocols :as p] [blaze.elm.string :as string] [clojure.string :as str])) @@ -13,28 +13,39 @@ (set! *warn-on-reflection* true) ;; 17.1. Combine +(defn combine-op + ([source] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (combine-op (core/-resolve-refs source expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper combine-op parameters source)) + (-eval [_ context resource scope] + (when-let [source (core/-eval source context resource scope)] + (string/combine source))) + (-form [_] + (list 'combine (core/-form source))))) + ([source separator] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (combine-op (core/-resolve-refs source expression-defs) + (core/-resolve-refs separator expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper combine-op parameters source separator)) + (-eval [_ context resource scope] + (when-let [source (core/-eval source context resource scope)] + (string/combine (core/-eval separator context resource scope) + source))) + (-form [_] + (list 'combine (core/-form source) (core/-form separator)))))) + (defmethod core/compile* :elm.compiler.type/combine [context {:keys [source separator]}] (let [source (core/compile* context source) separator (some->> separator (core/compile* context))] (if separator - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [source (core/-eval source context resource scope)] - (string/combine (core/-eval separator context resource scope) - source))) - (-form [_] - (list 'combine (core/-form source) (core/-form separator)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [source (core/-eval source context resource scope)] - (string/combine source))) - (-form [_] - (list 'combine (core/-form source))))))) + (combine-op source separator) + (combine-op source)))) ;; 17.2. Concatenate (defnaryop concatenate [strings] @@ -50,19 +61,26 @@ (p/indexer x index)) ;; 17.7. LastPositionOf +(defn last-position-of-op [pattern string] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper last-position-of-op cache pattern string)) + (-resolve-refs [_ expression-defs] + (last-position-of-op (core/-resolve-refs pattern expression-defs) + (core/-resolve-refs string expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper last-position-of-op parameters pattern string)) + (-eval [_ context resource scope] + (when-let [^String pattern (core/-eval pattern context resource scope)] + (when-let [^String string (core/-eval string context resource scope)] + (.lastIndexOf string pattern)))) + (-form [_] + (list 'last-position-of (core/-form pattern) (core/-form string))))) + (defmethod core/compile* :elm.compiler.type/last-position-of [context {:keys [pattern string]}] - (let [pattern (core/compile* context pattern) - string (core/compile* context string)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String pattern (core/-eval pattern context resource scope)] - (when-let [^String string (core/-eval string context resource scope)] - (.lastIndexOf string pattern)))) - (-form [_] - (list 'last-position-of (core/-form pattern) (core/-form string)))))) + (last-position-of-op (core/compile* context pattern) + (core/compile* context string))) ;; 17.8. Length (defunop length [x] @@ -78,19 +96,26 @@ (some? (re-matches (re-pattern pattern) s)))) ;; 17.12. PositionOf +(defn position-of-op [pattern string] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper position-of-op cache pattern string)) + (-resolve-refs [_ expression-defs] + (position-of-op (core/-resolve-refs pattern expression-defs) + (core/-resolve-refs string expression-defs))) + (-resolve-params [_ parameters] + (core/resolve-params-helper position-of-op parameters pattern string)) + (-eval [_ context resource scope] + (when-let [^String pattern (core/-eval pattern context resource scope)] + (when-let [^String string (core/-eval string context resource scope)] + (.indexOf string pattern)))) + (-form [_] + (list 'position-of (core/-form pattern) (core/-form string))))) + (defmethod core/compile* :elm.compiler.type/position-of [context {:keys [pattern string]}] - (let [pattern (core/compile* context pattern) - string (core/compile* context string)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String pattern (core/-eval pattern context resource scope)] - (when-let [^String string (core/-eval string context resource scope)] - (.indexOf string pattern)))) - (-form [_] - (list 'position-of (core/-form pattern) (core/-form string)))))) + (position-of-op (core/compile* context pattern) + (core/compile* context string))) ;; 17.13. ReplaceMatches (defternop replace-matches [s pattern substitution] @@ -98,46 +123,44 @@ (str/replace s (re-pattern pattern) substitution))) ;; 17.14. Split +(defn- split-op [string separator] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper split-op cache string separator)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper split-op expression-defs string separator)) + (-resolve-params [_ parameters] + (core/resolve-params-helper split-op parameters string separator)) + (-eval [_ context resource scope] + (when-let [string (core/-eval string context resource scope)] + (if (= "" string) + [string] + (if-let [separator (core/-eval separator context resource scope)] + (case (count separator) + 0 + [string] + 1 + (loop [[char & more] string + result [] + acc (StringBuilder.)] + (if (= (str char) separator) + (if more + (recur more (conj result (str acc)) (StringBuilder.)) + (conj result (str acc))) + (if more + (recur more result (.append acc char)) + (conj result (str (.append acc char)))))) + ;; TODO: implement split with more than one char. + (throw (Exception. "TODO: implement split with separators longer than one char."))) + [string])))) + (-form [_] + (list 'split (core/-form string) (core/-form separator))))) + (defmethod core/compile* :elm.compiler.type/split [context {string :stringToSplit :keys [separator]}] (let [string (core/compile* context string) separator (some->> separator (core/compile* context))] - (if separator - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [string (core/-eval string context resource scope)] - (if (= "" string) - [string] - (if-let [separator (core/-eval separator context resource scope)] - (case (count separator) - 0 - [string] - 1 - (loop [[char & more] string - result [] - acc (StringBuilder.)] - (if (= (str char) separator) - (if more - (recur more (conj result (str acc)) (StringBuilder.)) - (conj result (str acc))) - (if more - (recur more result (.append acc char)) - (conj result (str (.append acc char)))))) - ;; TODO: implement split with more than one char. - (throw (Exception. "TODO: implement split with separators longer than one char."))) - [string])))) - (-form [_] - (list 'split (core/-form string) (core/-form separator)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [string (core/-eval string context resource scope)] - [string])) - (-form [_] - (list 'split (core/-form string))))))) + (split-op string separator))) ;; 17.16. StartsWith (defbinop starts-with [s prefix] @@ -145,34 +168,48 @@ (str/starts-with? s prefix))) ;; 17.17. Substring +(defn substring-op + ([string start-index] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper substring-op cache string start-index)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper substring-op expression-defs string start-index)) + (-resolve-params [_ parameters] + (core/resolve-params-helper substring-op parameters string start-index)) + (-eval [_ context resource scope] + (when-let [^String string (core/-eval string context resource scope)] + (when-let [start-index (core/-eval start-index context resource scope)] + (when (and (<= 0 start-index) (< start-index (count string))) + (subs string start-index))))) + (-form [_] + (list 'substring (core/-form string) (core/-form start-index))))) + ([string start-index length] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper substring-op cache string start-index length)) + (-resolve-refs [_ expression-defs] + (core/resolve-refs-helper substring-op expression-defs string start-index length)) + (-resolve-params [_ parameters] + (core/resolve-params-helper substring-op parameters string start-index length)) + (-eval [_ context resource scope] + (when-let [^String string (core/-eval string context resource scope)] + (when-let [start-index (core/-eval start-index context resource scope)] + (when (and (<= 0 start-index) (< start-index (count string))) + (subs string start-index (min (+ start-index length) + (count string))))))) + (-form [_] + (list 'substring (core/-form string) (core/-form start-index) + (core/-form length)))))) + (defmethod core/compile* :elm.compiler.type/substring [context {string :stringToSub start-index :startIndex :keys [length]}] (let [string (core/compile* context string) start-index (core/compile* context start-index) length (some->> length (core/compile* context))] (if length - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String string (core/-eval string context resource scope)] - (when-let [start-index (core/-eval start-index context resource scope)] - (when (and (<= 0 start-index) (< start-index (count string))) - (subs string start-index (min (+ start-index length) - (count string))))))) - (-form [_] - (list 'substring (core/-form string) (core/-form start-index) - (core/-form length)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String string (core/-eval string context resource scope)] - (when-let [start-index (core/-eval start-index context resource scope)] - (when (and (<= 0 start-index) (< start-index (count string))) - (subs string start-index))))) - (-form [_] - (list 'substring (core/-form string) (core/-form start-index))))))) + (substring-op string start-index length) + (substring-op string start-index)))) ;; 17.18. Upper (defunop upper [s] diff --git a/modules/cql/src/blaze/elm/compiler/structured_values.clj b/modules/cql/src/blaze/elm/compiler/structured_values.clj index fe31d4094..fa11077e2 100644 --- a/modules/cql/src/blaze/elm/compiler/structured_values.clj +++ b/modules/cql/src/blaze/elm/compiler/structured_values.clj @@ -8,6 +8,7 @@ [blaze.coll.core :as coll] [blaze.elm.code :as code] [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.protocols :as p] [blaze.fhir.spec.type :as type] [clojure.string :as str]) @@ -41,26 +42,39 @@ {} elements)) +(defn tuple [elements] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (tuple + (reduce-kv + (fn [r key value] + (assoc r key (core/-resolve-refs value expression-defs))) + {} + elements))) + (-resolve-params [_ parameters] + (tuple + (reduce-kv + (fn [r key value] + (assoc r key (core/-resolve-params value parameters))) + {} + elements))) + (-eval [_ context resource scope] + (reduce-kv + (fn [r key value] + (assoc r key (core/-eval value context resource scope))) + {} + elements)) + (-form [_] + (reduce-kv + (fn [r key value] + (assoc r key (core/-form value))) + {} + elements)))) + (defmethod core/compile* :elm.compiler.type/tuple [context {elements :element}] (let [elements (compile-elements context elements)] - (if (every? core/static? (vals elements)) - elements - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (reduce-kv - (fn [r key value] - (assoc r key (core/-eval value context resource scope))) - {} - elements)) - (-form [_] - (reduce-kv - (fn [r key value] - (assoc r key (core/-form value))) - {} - elements)))))) + (cond-> elements (some (comp not core/static?) (vals elements)) tuple))) ;; 2.2. Instance (defmethod core/compile* :elm.compiler.type/instance @@ -77,46 +91,43 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [_ expression-defs] + (->SourcePropertyExpression (core/-resolve-refs source expression-defs) key)) + (-resolve-params [_ parameters] + (->SourcePropertyExpression (core/-resolve-params source parameters) key)) (-eval [_ context resource scope] (p/get (core/-eval source context resource scope) key)) (-form [_] `(~key ~(core/-form source)))) -(defrecord SourcePropertyValueExpression [source key] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (type/value (p/get (core/-eval source context resource scope) key))) - (-form [_] - `(:value (~key ~(core/-form source))))) - -(defrecord SingleScopePropertyExpression [key] - core/Expression - (-static [_] - false) - (-eval [_ _ _ value] - (p/get value key)) - (-form [_] - `(~key ~'default))) - -(defrecord ScopePropertyExpression [scope-key key] - core/Expression - (-static [_] - false) - (-eval [_ _ _ scope] - (p/get (get scope scope-key) key)) - (-form [_] - `(~key ~(symbol (name scope-key))))) - -(defrecord ScopePropertyValueExpression [scope-key key] - core/Expression - (-static [_] - false) - (-eval [_ _ _ scope] - (type/value (p/get (get scope scope-key) key))) - (-form [_] - `(:value (~key ~(symbol (name scope-key)))))) +(defn- source-property-value-expr [source key] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (source-property-value-expr (core/-resolve-refs source expression-defs) key)) + (-resolve-params [_ parameters] + (source-property-value-expr (core/-resolve-params source parameters) key)) + (-eval [_ context resource scope] + (type/value (p/get (core/-eval source context resource scope) key))) + (-form [_] + `(:value (~key ~(core/-form source)))))) + +(defn- scope-property-expr [scope-key key] + (reify-expr core/Expression + (-eval [_ _ _ scope] + (p/get (get scope scope-key) key)) + (-form [_] + `(~key ~(symbol (name scope-key)))))) + +(defn- scope-property-value-expr [scope-key key] + (reify-expr core/Expression + (-eval [_ _ _ scope] + (type/value (p/get (get scope scope-key) key))) + (-form [_] + `(:value (~key ~(symbol (name scope-key))))))) (defn- path->key [path] (let [[first-part more] (str/split path #"\." 2)] @@ -132,10 +143,10 @@ (cond source (if value? - (->SourcePropertyValueExpression (core/compile* context source) key) + (source-property-value-expr (core/compile* context source) key) (->SourcePropertyExpression (core/compile* context source) key)) scope (if value? - (->ScopePropertyValueExpression scope key) - (->ScopePropertyExpression scope key))))) + (scope-property-value-expr scope key) + (scope-property-expr scope key))))) diff --git a/modules/cql/src/blaze/elm/compiler/type_operators.clj b/modules/cql/src/blaze/elm/compiler/type_operators.clj index e5ee9a088..b37be11fc 100644 --- a/modules/cql/src/blaze/elm/compiler/type_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/type_operators.clj @@ -6,7 +6,7 @@ (:require [blaze.anomaly :as ba :refer [throw-anom]] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.macros :refer [defbinop defunop]] + [blaze.elm.compiler.macros :refer [defbinop defunop reify-expr]] [blaze.elm.date-time :as date-time] [blaze.elm.protocols :as p] [blaze.elm.quantity :as quantity] @@ -72,19 +72,24 @@ "Invalid As expression without `as-type` and `as-type-specifier`." :expression expression)))) +(defn as-op [type pred operand] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (as-op type pred (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (as-op type pred (core/-resolve-params operand parameters))) + (-eval [_ context resource scope] + (let [value (core/-eval operand context resource scope)] + (when (pred value) + value))) + (-form [_] + `(~'as ~type ~(core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/as [context {:keys [operand] :as expression}] (when-some [operand (core/compile* context operand)] (let [[type pred] (matches-type-fn expression)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [value (core/-eval operand context resource scope)] - (when (pred value) - value))) - (-form [_] - `(~'as ~type ~(core/-form operand))))))) + (as-op type pred operand)))) ;; TODO 22.2. CanConvert @@ -93,16 +98,10 @@ (p/can-convert-quantity x unit)) ;; 22.4. Children -(defmethod core/compile* :elm.compiler.type/children - [context {:keys [source]}] - (when-let [source (core/compile* context source)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (p/children (core/-eval source context resource scope))) - (-form [_] - (list 'children (core/-form source)))))) +(defunop children + {:operand-key :source} + [source] + (p/children source)) ;; TODO 22.5. Convert @@ -116,34 +115,47 @@ (some? (p/to-boolean operand)))) ;; 22.8. ConvertsToDate +(defn- converts-to-date-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper converts-to-date-op cache operand)) + (-resolve-refs [_ expression-defs] + (converts-to-date-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (converts-to-date-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (when-let [operand (core/-eval operand context resource scope)] + (when (some? operand) + (some? (p/to-date operand now))))) + (-form [_] + (list 'converts-to-date (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/converts-to-date [context {:keys [operand]}] - (when-let [operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (when-let [operand (core/-eval operand context resource scope)] - (when (some? operand) - (some? (p/to-date operand now))))) - (-form [_] - (list 'converts-to-date (core/-form operand)))))) + (some-> (core/compile* context operand) converts-to-date-op)) ;; 22.9. ConvertsToDateTime +(defn converts-to-date-time-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper converts-to-date-time-op cache operand)) + (-resolve-refs [_ expression-defs] + (converts-to-date-time-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (converts-to-date-time-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (when-let [operand (core/-eval operand context resource scope)] + (when (some? operand) + (some? (p/to-date-time operand now))))) + (-form [_] + (list 'converts-to-date-time (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/converts-to-date-time [context {:keys [operand]}] (when-let [operand (core/compile* context operand)] (if (system/date? operand) (some? (p/to-date-time operand nil)) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (when-let [operand (core/-eval operand context resource scope)] - (when (some? operand) - (some? (p/to-date-time operand now))))) - (-form [_] - (list 'converts-to-date-time (core/-form operand))))))) + (converts-to-date-time-op operand)))) ;; 22.10. ConvertsToDecimal (defunop converts-to-decimal [operand] @@ -176,29 +188,29 @@ (some? (p/to-string operand)))) ;; 22.16. ConvertsToTime +(defn- converts-to-time-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper converts-to-time-op cache operand)) + (-resolve-refs [_ expression-defs] + (converts-to-time-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (converts-to-time-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (when-some [operand (core/-eval operand context resource scope)] + (some? (p/to-time operand now)))) + (-form [_] + (list 'converts-to-time (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/converts-to-time [context {:keys [operand]}] - (when-let [operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (when-some [operand (core/-eval operand context resource scope)] - (some? (p/to-time operand now)))) - (-form [_] - (list 'converts-to-time (core/-form operand)))))) + (some-> (core/compile* context operand) converts-to-time-op)) ;; 22.17. Descendents -(defmethod core/compile* :elm.compiler.type/descendents - [context {:keys [source]}] - (when-let [source (core/compile* context source)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (p/descendents (core/-eval source context resource scope))) - (-form [_] - (list 'descendents (core/-form source)))))) +(defunop descendents + {:operand-key :source} + [source] + (p/descendents source)) ;; 22.18. Is (defn- matches-elm-named-type-is [type-name] @@ -242,17 +254,21 @@ is-type-specifier (matches-type-specifier-is is-type-specifier))) +(defn is-op [type pred operand] + (reify-expr core/Expression + (-resolve-refs [_ expression-defs] + (is-op type pred (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (is-op type pred (core/-resolve-params operand parameters))) + (-eval [_ context resource scope] + (pred (core/-eval operand context resource scope))) + (-form [_] + `(~'is ~type ~(core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/is [context {:keys [operand] :as expression}] - (let [[type pred] (matches-type-is expression) - operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (pred (core/-eval operand context resource scope))) - (-form [_] - `(~'is ~type ~(core/-form operand)))))) + (let [[type pred] (matches-type-is expression)] + (is-op type pred (core/compile* context operand)))) ;; 22.19. ToBoolean (defunop to-boolean [x] @@ -268,32 +284,46 @@ (p/to-concept x)) ;; 22.22. ToDate +(defn to-date-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-date-op cache operand)) + (-resolve-refs [_ expression-defs] + (to-date-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (to-date-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (p/to-date (core/-eval operand context resource scope) now)) + (-form [_] + (list 'to-date (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/to-date [context {:keys [operand]}] (when-let [operand (core/compile* context operand)] (if (system/date? operand) (p/to-date operand nil) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (p/to-date (core/-eval operand context resource scope) now)) - (-form [_] - (list 'to-date (core/-form operand))))))) + (to-date-op operand)))) ;; 22.23. ToDateTime +(defn to-date-time-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-date-time-op cache operand)) + (-resolve-refs [_ expression-defs] + (to-date-time-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (to-date-time-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (p/to-date-time (core/-eval operand context resource scope) now)) + (-form [_] + (list 'to-date-time (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/to-date-time [context {:keys [operand]}] (when-let [operand (core/compile* context operand)] (if (system/date? operand) (p/to-date-time operand nil) - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (p/to-date-time (core/-eval operand context resource scope) now)) - (-form [_] - (list 'to-date-time (core/-form operand))))))) + (to-date-time-op operand)))) ;; 22.24. ToDecimal (defunop to-decimal [x] @@ -324,13 +354,19 @@ (p/to-string x)) ;; 22.31. ToTime +(defn- to-time-op [operand] + (reify-expr core/Expression + (-attach-cache [_ cache] + (core/attach-cache-helper to-time-op cache operand)) + (-resolve-refs [_ expression-defs] + (to-time-op (core/-resolve-refs operand expression-defs))) + (-resolve-params [_ parameters] + (to-time-op (core/-resolve-params operand parameters))) + (-eval [_ {:keys [now] :as context} resource scope] + (p/to-time (core/-eval operand context resource scope) now)) + (-form [_] + (list 'to-time (core/-form operand))))) + (defmethod core/compile* :elm.compiler.type/to-time [context {:keys [operand]}] - (when-let [operand (core/compile* context operand)] - (reify core/Expression - (-static [_] - false) - (-eval [_ {:keys [now] :as context} resource scope] - (p/to-time (core/-eval operand context resource scope) now)) - (-form [_] - (list 'to-time (core/-form operand)))))) + (some-> (core/compile* context operand) to-time-op)) diff --git a/modules/cql/src/blaze/elm/compiler_spec.clj b/modules/cql/src/blaze/elm/compiler_spec.clj index 059bdfbf3..ff88b5040 100644 --- a/modules/cql/src/blaze/elm/compiler_spec.clj +++ b/modules/cql/src/blaze/elm/compiler_spec.clj @@ -2,11 +2,33 @@ (:require [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.library-spec] [blaze.elm.compiler.spec] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as-alias ec] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.expression.spec] [blaze.elm.spec] [blaze.fhir.spec-spec] - [clojure.spec.alpha :as s])) + [clojure.spec.alpha :as s] + [cognitect.anomalies :as anom])) (s/fdef c/compile :args (s/cat :context :elm/compile-context :expression :elm/expression) :ret core/expr?) + +(s/fdef c/attach-cache + :args (s/cat :expression core/expr? :cache ::expr/cache) + :ret (s/tuple core/expr? (s/coll-of (s/or :bloom-filter ::ec/bloom-filter :anomaly ::anom/anomaly)))) + +(s/fdef c/resolve-refs + :args (s/cat :expression core/expr? :expression-defs ::c/expression-defs) + :ret core/expr?) + +(s/fdef c/resolve-params + :args (s/cat :expression core/expr? :parameters ::c/parameters) + :ret core/expr?) + +(s/fdef c/form + :args (s/cat :expression core/expr?) + :ret list?) diff --git a/modules/cql/src/blaze/elm/date_time.clj b/modules/cql/src/blaze/elm/date_time.clj index eb0bd42c8..9756b068a 100644 --- a/modules/cql/src/blaze/elm/date_time.clj +++ b/modules/cql/src/blaze/elm/date_time.clj @@ -9,7 +9,9 @@ [blaze.fhir.spec.type.system :as system] [java-time.api :as time]) (:import - [blaze.fhir.spec.type.system Date DateDate DateTime DateTimeDate DateTimeYear DateTimeYearMonth DateYear DateYearMonth] + [blaze.fhir.spec.type.system + Date DateDate DateTime DateTimeDate DateTimeYear DateTimeYearMonth DateYear + DateYearMonth] [java.time DateTimeException LocalDateTime LocalTime OffsetDateTime] [java.time.temporal ChronoField ChronoUnit Temporal TemporalAccessor])) diff --git a/modules/cql/src/blaze/elm/decimal.clj b/modules/cql/src/blaze/elm/decimal.clj index c3b73107d..93a6abda4 100644 --- a/modules/cql/src/blaze/elm/decimal.clj +++ b/modules/cql/src/blaze/elm/decimal.clj @@ -13,7 +13,8 @@ [blaze.elm.protocols :as p] [clojure.math :as math]) (:import - [java.math RoundingMode])) + [java.math RoundingMode] + [tech.units.indriya.function RationalNumber])) (set! *warn-on-reflection* true) @@ -242,7 +243,10 @@ (extend-protocol p/ToDecimal BigDecimal (to-decimal [x] - (-> x constrain-scale check-overflow))) + (-> x constrain-scale check-overflow)) + RationalNumber + (to-decimal [x] + (.bigDecimalValue x))) (defn from-literal [s] (if-let [d (p/to-decimal s)] diff --git a/modules/cql/src/blaze/elm/expression.clj b/modules/cql/src/blaze/elm/expression.clj index ec16731c7..c670cf0ae 100644 --- a/modules/cql/src/blaze/elm/expression.clj +++ b/modules/cql/src/blaze/elm/expression.clj @@ -1,5 +1,5 @@ (ns blaze.elm.expression - (:refer-clojure :exclude [eval]) + (:refer-clojure :exclude [eval hash]) (:require [blaze.elm.compiler.core :as core])) diff --git a/modules/cql/src/blaze/elm/expression/cache.clj b/modules/cql/src/blaze/elm/expression/cache.clj new file mode 100644 index 000000000..ce0f04c27 --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache.clj @@ -0,0 +1,252 @@ +(ns blaze.elm.expression.cache + "Expression cache API." + (:refer-clojure :exclude [get list]) + (:require + [blaze.anomaly :as ba] + [blaze.async.comp :as ac] + [blaze.cache-collector.protocols :as ccp] + [blaze.coll.core :refer [with-open-coll]] + [blaze.db.impl.iterators :as i] + [blaze.db.kv :as kv] + [blaze.db.spec] + [blaze.elm.compiler.core :as core] + [blaze.elm.expression :as expr] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [blaze.elm.expression.cache.codec :as codec] + [blaze.elm.expression.cache.codec.by-t :as codec-by-t] + [blaze.elm.expression.cache.codec.form :as form] + [blaze.elm.expression.cache.protocols :as p] + [blaze.elm.expression.cache.spec] + [blaze.elm.expression.spec] + [blaze.executors :as ex] + [blaze.module :as m :refer [reg-collector]] + [clojure.spec.alpha :as s] + [integrant.core :as ig] + [java-time.api :as time] + [prometheus.alpha :refer [defcounter]] + [taoensso.timbre :as log]) + (:import + [com.github.benmanes.caffeine.cache + AsyncCache AsyncCacheLoader AsyncLoadingCache Caffeine Weigher] + [com.google.common.hash HashCode] + [java.lang AutoCloseable] + [java.util.concurrent TimeUnit])) + +(set! *warn-on-reflection* true) + +(defcounter bloom-filter-useful-total + "Number of times Bloom filter has avoided expression evaluation." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + "name") + +(defcounter bloom-filter-not-useful-total + "Number of times Bloom filter has not avoided expression evaluation." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + "name") + +(defcounter bloom-filter-false-positive-total + "Number of false positives reported by Bloom all filters." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + "name") + +(defn get + "Returns the Bloom filter of `expression` from `cache` or nil if it isn't + available yet." + [cache expression] + (p/-get cache expression)) + +(defn get-disk + "Returns the Bloom filter with `hash` from `cache` or an anomaly if the Bloom + filter was not found." + [cache hash] + (p/-get-disk cache hash)) + +(defn delete-disk! + "Deletes the Bloom filter with `hash` from `cache` or returns an anomaly if + the Bloom filter was not found." + [cache hash] + (p/-delete-disk cache hash)) + +(defn list-by-t + "Returns a reducible collection of all Bloom filters in `cache` ordered by t + descending." + [cache] + (p/-list-by-t cache)) + +(defn total + "Returns the total number of Bloom filters in `cache`." + [cache] + (p/-total cache)) + +(def ^:private weigher + (reify Weigher + (weigh [_ _ bloom-filter] + (::bloom-filter/mem-size bloom-filter)))) + +(defn- load-bloom-filter [kv-store hash] + (some->> (kv/get kv-store :cql-bloom-filter (.asBytes ^HashCode hash)) + (codec/decode-value hash))) + +(defn- load-bloom-filter-from-expr [kv-store expression] + (let [key (form/hash (pr-str (core/-form expression)))] + (load-bloom-filter kv-store key))) + +(defn- bloom-filter-creation-counter ^AutoCloseable [state] + (swap! state update :num-running-bloom-filter-creations inc) + (reify AutoCloseable + (close [_] + (swap! state update :num-running-bloom-filter-creations dec)))) + +(defn- mem-cache + [state {:keys [kv-store] :as node} executor max-size-in-mb refresh] + (-> (Caffeine/newBuilder) + (.weigher weigher) + (.maximumWeight (bit-shift-left max-size-in-mb 20)) + (.refreshAfterWrite refresh) + (.executor executor) + (.recordStats) + (.buildAsync + (reify AsyncCacheLoader + (asyncLoad [_ expression executor] + (if-let [bloom-filter (load-bloom-filter-from-expr kv-store expression)] + (ac/completed-future bloom-filter) + (ac/supply-async + #(with-open [_ (bloom-filter-creation-counter state)] + (let [bloom-filter (bloom-filter/create node expression)] + (kv/write! + kv-store + [(codec/put-entry bloom-filter) + (codec-by-t/put-entry bloom-filter)]) + bloom-filter)) + executor))) + (asyncReload [_ expression old-bloom-filter executor] + (ac/supply-async + #(with-open [_ (bloom-filter-creation-counter state)] + (let [bloom-filter (bloom-filter/recreate node old-bloom-filter + expression)] + (kv/write! + kv-store + [(codec/put-entry bloom-filter) + (codec-by-t/delete-entry old-bloom-filter) + (codec-by-t/put-entry bloom-filter)]) + bloom-filter)) + executor)))))) + +(def ^:private ^:const ^long expression-size-limit + "The limit of form size of cacheable expressions. + + Bigger expressions will not be cache to keep memory consumption under control. + + The current value is 10 kB." + (bit-shift-left 10 10)) + +(def ^:private ^:const ^long concurrent-bloom-filter-creation-limit + "Maximum number of concurrent Bloom filter creations allowed. + + This limit should prevent the over-saturation of Bloom filter creations." + 500) + +(defn concurrent-bloom-filter-creation-limit-reached? + [{:keys [num-running-bloom-filter-creations]}] + (< concurrent-bloom-filter-creation-limit num-running-bloom-filter-creations)) + +(defn- overly-large? [expression] + (< expression-size-limit (count (pr-str (core/-form expression))))) + +(defn- not-found-anom [hash] + (ba/not-found (format "The Bloom filter with hash `%s` was not found." hash))) + +(defrecord Cache [state ^AsyncLoadingCache mem-cache node kv-store] + p/Cache + (-get [_ expression] + (if (overly-large? expression) + (log/debug "Skip caching overly large CQL expression.") + (if-let [future (.getIfPresent mem-cache expression)] + (when (.isDone future) + (.get future)) + (if (concurrent-bloom-filter-creation-limit-reached? @state) + (log/debug "Skip caching CQL expression because the concurrent Bloom filter creation limit of" concurrent-bloom-filter-creation-limit "is reached.") + (let [future (.get mem-cache expression)] + (when (.isDone future) + (.get future))))))) + + (-get-disk [_ hash] + (or (load-bloom-filter kv-store hash) + (not-found-anom hash))) + + (-delete-disk [_ hash] + (if-let [bloom-filter (load-bloom-filter kv-store hash)] + (kv/write! + kv-store + [(codec/delete-entry bloom-filter) + (codec-by-t/delete-entry bloom-filter)]) + (not-found-anom hash))) + + (-list-by-t [_] + (with-open-coll [snapshot (kv/new-snapshot kv-store)] + (i/entries snapshot :cql-bloom-filter-by-t (map codec-by-t/decoder)))) + + (-total [_] + (kv/estimate-num-keys kv-store :cql-bloom-filter)) + + ccp/StatsCache + (-stats [_] + (.stats (.synchronous mem-cache))) + + (-estimated-size [_] + (.estimatedSize (.synchronous mem-cache)))) + +(defmethod m/pre-init-spec ::expr/cache [_] + (s/keys :req-un [:blaze.db/node ::executor] :opt-un [::max-size-in-mb ::refresh])) + +(defmethod ig/init-key ::expr/cache + [_ + {:keys [executor max-size-in-mb refresh] {:keys [kv-store] :as node} :node + :or {max-size-in-mb 100 refresh (time/hours 24)}}] + (log/info "Create CQL expression cache with a memory size of" max-size-in-mb "MiB and a refresh duration of" (str refresh)) + (let [state (atom {:num-running-bloom-filter-creations 0})] + (->Cache state (mem-cache state node executor max-size-in-mb refresh) node kv-store))) + +(defmethod ig/halt-key! ::expr/cache + [_ {:keys [mem-cache]}] + (log/info "Stopping CQL expression cache...") + (.cleanUp (.synchronous ^AsyncCache mem-cache))) + +(defmethod m/pre-init-spec ::executor [_] + (s/keys :opt-un [::num-threads])) + +(defn- executor-init-msg [num-threads] + (format "Init CQL expression cache executor with %d threads" num-threads)) + +(defmethod ig/init-key ::executor + [_ {:keys [num-threads] :or {num-threads 4}}] + (log/info (executor-init-msg num-threads)) + (ex/io-pool num-threads "cql-expr-cache-%d")) + +(defmethod ig/halt-key! ::executor + [_ executor] + (log/info "Stopping CQL expression cache executor...") + (ex/shutdown! executor) + (if (ex/await-termination executor 10 TimeUnit/SECONDS) + (log/info "CQL expression cache executor was stopped successfully") + (log/warn "Got timeout while stopping the CQL expression cache executor"))) + +(derive ::executor :blaze.metrics/thread-pool-executor) + +(reg-collector ::bloom-filter-creation-duration-seconds + bloom-filter/bloom-filter-creation-duration-seconds) + +(reg-collector ::bloom-filter-useful-total + bloom-filter-useful-total) + +(reg-collector ::bloom-filter-not-useful-total + bloom-filter-not-useful-total) + +(reg-collector ::bloom-filter-false-positive-total + bloom-filter-false-positive-total) + +(reg-collector ::bloom-filter-bytes + bloom-filter/bloom-filter-bytes) diff --git a/modules/cql/src/blaze/elm/expression/cache/bloom_filter.clj b/modules/cql/src/blaze/elm/expression/cache/bloom_filter.clj new file mode 100644 index 000000000..92dbac656 --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/bloom_filter.clj @@ -0,0 +1,92 @@ +(ns blaze.elm.expression.cache.bloom-filter + (:refer-clojure :exclude [merge]) + (:require + [blaze.db.api :as d] + [blaze.elm.compiler.core :as core] + [blaze.elm.expression :as expr] + [blaze.elm.expression.cache.codec :as codec] + [blaze.elm.expression.cache.codec.form :as form] + [blaze.elm.resource :as cr] + [prometheus.alpha :as prom :refer [defhistogram]] + [taoensso.timbre :as log]) + (:import + [blaze.elm.expression.cache.codec BloomFilterContainer] + [blaze.elm.resource Resource] + [com.google.common.hash BloomFilter] + [java.time OffsetDateTime])) + +(set! *warn-on-reflection* true) + +(defhistogram bloom-filter-bytes + "Bloom filter sizes in bytes." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + (take 12 (iterate #(* 4 %) 1))) + +(defhistogram bloom-filter-creation-duration-seconds + "Durations in Cassandra resource store." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + (take 14 (iterate #(* 2 %) 0.1))) + +(defn might-contain? + "Returns true if `resource` might have been put in `bloom-filter` or false if + this is definitely not the case." + {:arglists '([bloom-filter resource])} + [^BloomFilterContainer bloom-filter ^Resource resource] + (or (< (.-t bloom-filter) (.-lastChangeT resource)) + (.mightContain ^BloomFilter (.-filter bloom-filter) (.-id resource)))) + +(defn merge [bloom-filter-a bloom-filter-b] + (.merge ^BloomFilterContainer bloom-filter-a bloom-filter-b)) + +(defn- calc-mem-size [n p] + (long (/ (* (- n) (Math/log p)) (* (Math/log 2) (Math/log 2)) 8))) + +(defn build-bloom-filter [expression t resource-ids] + (let [n (count resource-ids) + p (double 0.01) + filter (BloomFilter/create codec/id-funnel (int (max 10000 n)) p) + mem-size (calc-mem-size (max 10000 n) p) + expr-form (pr-str (core/-form expression))] + (prom/observe! bloom-filter-bytes mem-size) + (run! #(.put filter %) resource-ids) + (BloomFilterContainer. (form/hash expr-form) t expr-form n filter mem-size))) + +(defn- calc-bloom-filter [db xform expression] + (with-open [batch-db (d/new-batch-db db) + _ (prom/timer bloom-filter-creation-duration-seconds)] + (build-bloom-filter + expression + (d/t batch-db) + (into + [] + (comp (map (partial cr/mk-resource batch-db)) + xform + (filter (partial expr/eval {:db batch-db :now (OffsetDateTime/now)} expression)) + (map :id)) + (d/type-list db "Patient"))))) + +(defn- create-bloom-filter-msg [expression db] + (format "Create Bloom filter for expression `%s` evaluating it for %d patients." + (core/-form expression) (d/type-total db "Patient"))) + +(defn create [node expression] + (let [db (d/db node)] + (log/debug (create-bloom-filter-msg expression db)) + (calc-bloom-filter db identity expression))) + +(defn recreate + {:arglists '([node old-bloom-filter expression])} + [node {::keys [t expr-form] :as old-bloom-filter} expression] + (let [db (d/db node)] + (log/debug "Recreate Bloom filter for expression" + expr-form + "last created at t =" t + "evaluating it for" + (d/type-total db "Patient") + "Patient resources") + (calc-bloom-filter + db + (filter (partial might-contain? old-bloom-filter)) + expression))) diff --git a/modules/cql/src/blaze/elm/expression/cache/bloom_filter/spec.clj b/modules/cql/src/blaze/elm/expression/cache/bloom_filter/spec.clj new file mode 100644 index 000000000..ea4126d28 --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/bloom_filter/spec.clj @@ -0,0 +1,29 @@ +(ns blaze.elm.expression.cache.bloom-filter.spec + (:require + [blaze.elm.expression.cache :as-alias ec] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [clojure.spec.alpha :as s]) + (:import + [blaze.elm.expression.cache.codec BloomFilterContainer] + [com.google.common.hash BloomFilter HashCode])) + +(s/def ::bloom-filter/hash + #(instance? HashCode %)) + +(s/def ::bloom-filter/t + :blaze.db/t) + +(s/def ::bloom-filter/expr-form + string?) + +(s/def ::bloom-filter/patient-count + nat-int?) + +(s/def ::bloom-filter/filter + #(instance? BloomFilter %)) + +(s/def ::bloom-filter/mem-size + nat-int?) + +(s/def ::ec/bloom-filter + #(instance? BloomFilterContainer %)) diff --git a/modules/cql/src/blaze/elm/expression/cache/bloom_filter_spec.clj b/modules/cql/src/blaze/elm/expression/cache/bloom_filter_spec.clj new file mode 100644 index 000000000..c5972ce85 --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/bloom_filter_spec.clj @@ -0,0 +1,26 @@ +(ns blaze.elm.expression.cache.bloom-filter-spec + (:require + [blaze.db.spec] + [blaze.elm.compiler.core :as core] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.resource :as cr] + [clojure.spec.alpha :as s])) + +(s/fdef bloom-filter/might-contain? + :args (s/cat :bloom-filter ::ec/bloom-filter :resource cr/resource?) + :ret boolean?) + +(s/fdef bloom-filter/merge + :args (s/cat :bloom-filter-a ::ec/bloom-filter :bloom-filter-b ::ec/bloom-filter) + :ret (s/nilable ::ec/bloom-filter)) + +(s/fdef bloom-filter/create + :args (s/cat :node :blaze.db/node :expression core/expr?) + :ret ::ec/bloom-filter) + +(s/fdef bloom-filter/recreate + :args (s/cat :node :blaze.db/node :old-bloom-filter ::ec/bloom-filter + :expression core/expr?) + :ret ::ec/bloom-filter) diff --git a/modules/cql/src/blaze/elm/expression/cache/codec.clj b/modules/cql/src/blaze/elm/expression/cache/codec.clj new file mode 100644 index 000000000..945acb4ba --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/codec.clj @@ -0,0 +1,80 @@ +(ns blaze.elm.expression.cache.codec + (:require + [blaze.byte-buffer :as bb] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.cache.codec.form :as form]) + (:import + [clojure.lang ILookup] + [com.fasterxml.jackson.databind.util ByteBufferBackedInputStream] + [com.google.common.hash BloomFilter Funnel Funnels HashCode] + [java.io ByteArrayOutputStream DataOutputStream] + [java.nio.charset StandardCharsets])) + +(set! *warn-on-reflection* true) + +(def ^:private ^:const ^long version 0) + +(definterface IBloomFilterContainer + (merge [other])) + +(deftype BloomFilterContainer [hash ^long t exprForm ^long patientCount + ^BloomFilter filter ^long memSize] + IBloomFilterContainer + (merge [_ other] + (when (.isCompatible filter (.-filter ^BloomFilterContainer other)) + (let [newFilter (doto (.copy filter) (.putAll (.-filter ^BloomFilterContainer other)))] + (when (< (.expectedFpp newFilter) 0.01) + (BloomFilterContainer. nil (min t (.-t ^BloomFilterContainer other)) nil + (.approximateElementCount newFilter) + newFilter memSize))))) + ILookup + (valAt [r key] + (.valAt r key nil)) + (valAt [_ key not-found] + (case key + ::bloom-filter/hash hash + ::bloom-filter/t t + ::bloom-filter/expr-form exprForm + ::bloom-filter/patient-count patientCount + ::bloom-filter/filter filter + ::bloom-filter/mem-size memSize + not-found))) + +(def ^Funnel id-funnel + (Funnels/stringFunnel StandardCharsets/ISO_8859_1)) + +(defn- encode-value [{::bloom-filter/keys [t expr-form filter]}] + (let [out (ByteArrayOutputStream.) + data-out (DataOutputStream. out) + form (.getBytes ^String expr-form StandardCharsets/UTF_8)] + (.writeByte data-out version) + (.writeLong data-out t) + (.writeInt data-out (alength form)) + (.write data-out ^bytes form) + (.writeTo ^BloomFilter filter data-out) + (.toByteArray out))) + +(defn put-entry + "Creates a put-entry for column-family `cql-bloom-filter` with the hash of + `bloom-filter` as key and `bloom-filter` as value." + {:arglists '([bloom-filter])} + [{::bloom-filter/keys [hash] :as bloom-filter}] + [:put :cql-bloom-filter (.asBytes ^HashCode hash) (encode-value bloom-filter)]) + +(defn delete-entry + "Creates a delete-entry for column-family `cql-bloom-filter` with the hash of + `bloom-filter` as key." + {:arglists '([bloom-filter])} + [{::bloom-filter/keys [hash]}] + [:delete :cql-bloom-filter (.asBytes ^HashCode hash)]) + +(defn- decode-value* [hash buf] + (assert (zero? (bb/get-byte! buf)) "assume version is always zero") + (let [t (bb/get-long! buf) + expr-form (form/decode! buf) + mem-size (bb/remaining buf) + filter (BloomFilter/readFrom (ByteBufferBackedInputStream. buf) id-funnel)] + (BloomFilterContainer. hash t expr-form (.approximateElementCount filter) filter mem-size))) + +(defn decode-value [hash byte-array] + (decode-value* hash (bb/wrap byte-array))) diff --git a/modules/cql/src/blaze/elm/expression/cache/codec/by_t.clj b/modules/cql/src/blaze/elm/expression/cache/codec/by_t.clj new file mode 100644 index 000000000..b27781bbc --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/codec/by_t.clj @@ -0,0 +1,47 @@ +(ns blaze.elm.expression.cache.codec.by-t + (:require + [blaze.byte-buffer :as bb] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.cache.codec.form :as form]) + (:import + [com.google.common.hash HashCode] + [java.nio.charset StandardCharsets])) + +(set! *warn-on-reflection* true) + +(defn- encode-key [{::bloom-filter/keys [t hash]}] + (-> (bb/allocate (+ Long/BYTES 32)) + (bb/put-long! t) + (bb/put-byte-array! (.asBytes ^HashCode hash)) + bb/array)) + +(defn- encode-value [{::bloom-filter/keys [expr-form patient-count mem-size]}] + (let [form (.getBytes ^String expr-form StandardCharsets/UTF_8)] + (-> (bb/allocate (+ Integer/BYTES (alength form) Long/BYTES Long/BYTES)) + (bb/put-int! (alength form)) + (bb/put-byte-array! form) + (bb/put-long! patient-count) + (bb/put-long! mem-size) + bb/array))) + +(defn put-entry [bloom-filter] + [:put :cql-bloom-filter-by-t (encode-key bloom-filter) + (encode-value bloom-filter)]) + +(defn delete-entry [bloom-filter] + [:delete :cql-bloom-filter-by-t (encode-key bloom-filter)]) + +(defn- decode-value* [buf] + (let [form (form/decode! buf) + patient-count (bb/get-long! buf) + mem-size (bb/get-long! buf)] + #::bloom-filter{:expr-form form :patient-count patient-count + :mem-size mem-size})) + +(defn decoder [[kb vb]] + (let [hash (byte-array 32) + t (bb/get-long! kb)] + (bb/copy-into-byte-array! kb hash) + (assoc (decode-value* vb) + ::bloom-filter/t t + ::bloom-filter/hash (HashCode/fromBytes hash)))) diff --git a/modules/cql/src/blaze/elm/expression/cache/codec/form.clj b/modules/cql/src/blaze/elm/expression/cache/codec/form.clj new file mode 100644 index 000000000..84c25849b --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/codec/form.clj @@ -0,0 +1,19 @@ +(ns blaze.elm.expression.cache.codec.form + (:refer-clojure :exclude [hash]) + (:require + [blaze.byte-buffer :as bb]) + (:import + [com.google.common.hash HashCode Hashing] + [java.nio.charset StandardCharsets])) + +(set! *warn-on-reflection* true) + +(defn hash ^HashCode [expr-form] + (-> (Hashing/sha256) + (.hashString expr-form StandardCharsets/UTF_8))) + +(defn decode! [buf] + (let [len (bb/get-int! buf) + bytes (byte-array len)] + (bb/copy-into-byte-array! buf bytes) + (String. bytes StandardCharsets/UTF_8))) diff --git a/modules/cql/src/blaze/elm/expression/cache/protocols.clj b/modules/cql/src/blaze/elm/expression/cache/protocols.clj new file mode 100644 index 000000000..450ccbdeb --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/protocols.clj @@ -0,0 +1,8 @@ +(ns blaze.elm.expression.cache.protocols) + +(defprotocol Cache + (-get [cache expression]) + (-get-disk [cache hash]) + (-delete-disk [cache hash]) + (-list-by-t [cache]) + (-total [cache])) diff --git a/modules/cql/src/blaze/elm/expression/cache/spec.clj b/modules/cql/src/blaze/elm/expression/cache/spec.clj new file mode 100644 index 000000000..9134bdf6c --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache/spec.clj @@ -0,0 +1,20 @@ +(ns blaze.elm.expression.cache.spec + (:require + [blaze.db.tx-log.spec] + [blaze.elm.expression.cache :as-alias ec] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.executors :as ex] + [clojure.spec.alpha :as s] + [java-time.api :as time])) + +(s/def ::ec/max-size-in-mb + nat-int?) + +(s/def ::ec/refresh + time/duration?) + +(s/def ::ec/executor + ex/executor?) + +(s/def ::ec/num-threads + pos-int?) diff --git a/modules/cql/src/blaze/elm/expression/cache_spec.clj b/modules/cql/src/blaze/elm/expression/cache_spec.clj new file mode 100644 index 000000000..67858ee16 --- /dev/null +++ b/modules/cql/src/blaze/elm/expression/cache_spec.clj @@ -0,0 +1,33 @@ +(ns blaze.elm.expression.cache-spec + (:require + [blaze.db.tx-log.spec] + [blaze.elm.compiler.core :as core] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.cache.bloom-filter-spec] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.expression.cache.spec] + [blaze.fhir.spec.spec] + [clojure.spec.alpha :as s] + [cognitect.anomalies :as anom])) + +(s/fdef ec/get + :args (s/cat :cache ::expr/cache :expression core/expr?) + :ret (s/nilable ::ec/bloom-filter)) + +(s/fdef ec/get-disk + :args (s/cat :cache ::expr/cache :hash ::bloom-filter/hash) + :ret (s/or :result ::ec/bloom-filter :anomaly ::anom/anomaly)) + +(s/fdef ec/delete-disk! + :args (s/cat :cache ::expr/cache :hash ::bloom-filter/hash) + :ret (s/nilable ::anom/anomaly)) + +(s/fdef ec/list-by-t + :args (s/cat :cache ::expr/cache) + :ret (s/nilable ::ec/bloom-filter)) + +(s/fdef ec/total + :args (s/cat :cache ::expr/cache) + :ret nat-int?) diff --git a/modules/cql/src/blaze/elm/expression/spec.clj b/modules/cql/src/blaze/elm/expression/spec.clj index 89a0d3995..746e78d03 100644 --- a/modules/cql/src/blaze/elm/expression/spec.clj +++ b/modules/cql/src/blaze/elm/expression/spec.clj @@ -3,6 +3,7 @@ [blaze.db.api-spec] [blaze.elm.compiler :as-alias c] [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache.protocols :as p] [blaze.elm.spec] [clojure.spec.alpha :as s] [java-time.api :as time])) @@ -10,6 +11,9 @@ (s/def ::now time/offset-date-time?) +(s/def ::expr/cache + #(satisfies? p/Cache %)) + (s/def ::parameters (s/map-of :elm/name ::c/expression)) diff --git a/modules/cql/src/blaze/elm/expression_spec.clj b/modules/cql/src/blaze/elm/expression_spec.clj index f50ccf60f..8e4de215f 100644 --- a/modules/cql/src/blaze/elm/expression_spec.clj +++ b/modules/cql/src/blaze/elm/expression_spec.clj @@ -2,15 +2,14 @@ (:require [blaze.db.api-spec] [blaze.elm.compiler :as-alias c] - [blaze.elm.compiler.external-data :as ed] - [blaze.elm.compiler.library-spec] [blaze.elm.compiler.spec] [blaze.elm.expression :as expr] [blaze.elm.expression.spec] + [blaze.elm.resource :as cr] [blaze.fhir.spec] [clojure.spec.alpha :as s])) (s/fdef expr/eval :args (s/cat :context ::expr/context :expression ::c/expression - :resource (s/nilable ed/resource?))) + :resource (s/nilable cr/resource?))) diff --git a/modules/cql/src/blaze/elm/interval.clj b/modules/cql/src/blaze/elm/interval.clj index ab362dae0..26c59d9c7 100644 --- a/modules/cql/src/blaze/elm/interval.clj +++ b/modules/cql/src/blaze/elm/interval.clj @@ -1,6 +1,7 @@ (ns blaze.elm.interval "Implementation of the interval type." (:require + [blaze.elm.compiler.core :as core] [blaze.elm.date-time :refer [temporal?]] [blaze.elm.protocols :as p])) @@ -99,7 +100,23 @@ (union [a b] (let [[left right] (if (p/less (:start a) (:start b)) [a b] [b a])] (when (p/greater-or-equal (:end left) (p/predecessor (:start right))) - (->Interval (:start left) (:end right)))))) + (->Interval (:start left) (:end right))))) + + core/Expression + (-static [_] + true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) + (-eval [this _ _ _] + this) + (-form [_] + (list 'interval (core/-form start) (core/-form end)))) (defn interval "Returns an interval with the given `start` and `end` bounds." diff --git a/modules/cql/src/blaze/elm/quantity.clj b/modules/cql/src/blaze/elm/quantity.clj index 764642c46..3e649ec21 100644 --- a/modules/cql/src/blaze/elm/quantity.clj +++ b/modules/cql/src/blaze/elm/quantity.clj @@ -68,10 +68,18 @@ Quantity (-static [_] true) + (-attach-cache [quantity _] + [quantity]) + (-patient-count [_] + nil) + (-resolve-refs [quantity _] + quantity) + (-resolve-params [quantity _] + quantity) (-eval [quantity _ _ _] quantity) (-form [quantity] - `(~'quantity ~(.getValue quantity) ~(format-unit (.getUnit quantity))))) + `(~'quantity ~(p/to-decimal (.getValue quantity)) ~(format-unit (.getUnit quantity))))) (defprotocol QuantityDivide (quantity-divide [divisor quantity])) diff --git a/modules/cql/src/blaze/elm/ratio.clj b/modules/cql/src/blaze/elm/ratio.clj index cc0a5aad1..c8ad66751 100644 --- a/modules/cql/src/blaze/elm/ratio.clj +++ b/modules/cql/src/blaze/elm/ratio.clj @@ -3,6 +3,7 @@ Section numbers are according to https://cql.hl7.org/04-logicalspecification.html." (:require + [blaze.elm.compiler.core :as core] [blaze.elm.protocols :as p] [clojure.string :as str])) @@ -19,7 +20,23 @@ (if other (p/equivalent (p/divide numerator denominator) (p/divide (:numerator other) (:denominator other))) - false))) + false)) + + core/Expression + (-static [_] + true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) + (-eval [this _ _ _] + this) + (-form [_] + (list 'ratio (core/-form numerator) (core/-form denominator)))) (defn ratio "Creates a ratio between two quantities." diff --git a/modules/cql/src/blaze/elm/resource.clj b/modules/cql/src/blaze/elm/resource.clj new file mode 100644 index 000000000..8d67ffefe --- /dev/null +++ b/modules/cql/src/blaze/elm/resource.clj @@ -0,0 +1,64 @@ +(ns blaze.elm.resource + (:require + [blaze.db.api :as d] + [blaze.db.impl.index.resource-handle :as rh] + [blaze.elm.compiler.core :as core] + [blaze.elm.spec] + [blaze.fhir.spec.type.protocols :as p]) + (:import + [clojure.lang ILookup])) + +(set! *warn-on-reflection* true) + +;; A resource that is a wrapper of a resource-handle that will lazily pull the +;; resource content if some property other than :id is accessed. +(deftype Resource [db id handle ^long lastChangeT content] + p/FhirType + (-type [_] + (p/-type handle)) + + ILookup + (valAt [r key] + (.valAt r key nil)) + (valAt [_ key not-found] + (case key + :id id + (-> (or @content (vreset! content @(d/pull-content db handle))) + (get key not-found)))) + + core/Expression + (-static [_] + true) + (-attach-cache [expr _] + [(fn [] [expr])]) + (-patient-count [_] + nil) + (-resolve-refs [expr _] + expr) + (-resolve-params [expr _] + expr) + (-eval [expr _ _ _] + expr) + (-form [_] + (list 'resource (name (p/-type handle)) id (rh/t handle))) + + Object + (toString [_] + (str (name (p/-type handle)) "[id = " id ", t = " (rh/t handle) ", last-change-t = " lastChangeT "]"))) + +(defn resource? [x] + (instance? Resource x)) + +(defn- patient-last-change-t [db handle] + (or (d/patient-compartment-last-change-t db (rh/id handle)) (rh/t handle))) + +(defn- last-change-t [db handle] + (if (identical? :fhir/Patient (p/-type handle)) + (patient-last-change-t db handle) + (d/t db))) + +(defn mk-resource [db handle] + (Resource. db (rh/id handle) handle (last-change-t db handle) (volatile! nil))) + +(defn resource-mapper [db] + (map (partial mk-resource db))) diff --git a/modules/cql/src/blaze/elm/compiler/external_data_spec.clj b/modules/cql/src/blaze/elm/resource_spec.clj similarity index 53% rename from modules/cql/src/blaze/elm/compiler/external_data_spec.clj rename to modules/cql/src/blaze/elm/resource_spec.clj index 0f9f98570..ba21278c5 100644 --- a/modules/cql/src/blaze/elm/compiler/external_data_spec.clj +++ b/modules/cql/src/blaze/elm/resource_spec.clj @@ -1,16 +1,16 @@ -(ns blaze.elm.compiler.external-data-spec +(ns blaze.elm.resource-spec (:require [blaze.db.spec] - [blaze.elm.compiler.external-data :as ed] + [blaze.elm.resource :as cr] [clojure.spec.alpha :as s])) -(s/fdef ed/resource? +(s/fdef cr/resource? :args (s/cat :x any?) :ret boolean?) -(s/fdef ed/mk-resource +(s/fdef cr/mk-resource :args (s/cat :db :blaze.db/db :handle :blaze.db/resource-handle) - :ret ed/resource?) + :ret cr/resource?) -(s/fdef ed/resource-mapper +(s/fdef cr/resource-mapper :args (s/cat :db :blaze.db/db)) diff --git a/modules/cql/test/blaze/elm/code_test.clj b/modules/cql/test/blaze/elm/code_test.clj new file mode 100644 index 000000000..17b69c6fd --- /dev/null +++ b/modules/cql/test/blaze/elm/code_test.clj @@ -0,0 +1,16 @@ +(ns blaze.elm.code-test + (:require + [blaze.elm.code :as code] + [blaze.elm.compiler :as c] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest is testing]])) + +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(deftest to-code-test + (testing "attach-cache" + (let [code (code/to-code "foo" "bar" "baz")] + (is (= [code] (st/with-instrument-disabled (c/attach-cache code ::cache))))))) diff --git a/modules/cql/test/blaze/elm/compiler/aggregate_operators_test.clj b/modules/cql/test/blaze/elm/compiler/aggregate_operators_test.clj index 659477b3c..c12010c69 100644 --- a/modules/cql/test/blaze/elm/compiler/aggregate_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/aggregate_operators_test.clj @@ -47,9 +47,7 @@ #elm/list [] true {:type "Null"} true)) - (ctu/testing-unary-dynamic elm/all-true) - - (ctu/testing-unary-form elm/all-true)) + (ctu/testing-unary-op elm/all-true)) ;; 21.2. AnyTrue ;; @@ -72,9 +70,7 @@ #elm/list [] false {:type "Null"} false)) - (ctu/testing-unary-dynamic elm/any-true) - - (ctu/testing-unary-form elm/any-true)) + (ctu/testing-unary-op elm/any-true)) ;; 21.3. Avg ;; @@ -97,9 +93,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/avg) - - (ctu/testing-unary-form elm/avg)) + (ctu/testing-unary-op elm/avg)) ;; 21.4. Count ;; @@ -124,9 +118,7 @@ #elm/list [] 0 {:type "Null"} 0)) - (ctu/testing-unary-dynamic elm/count) - - (ctu/testing-unary-form elm/count)) + (ctu/testing-unary-op elm/count)) ;; 21.5. GeometricMean ;; @@ -150,9 +142,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/geometric-mean) - - (ctu/testing-unary-form elm/geometric-mean)) + (ctu/testing-unary-op elm/geometric-mean)) ;; 21.6. Product ;; @@ -176,9 +166,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/product) - - (ctu/testing-unary-form elm/product)) + (ctu/testing-unary-op elm/product)) ;; 21.7. Max ;; @@ -203,9 +191,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/max) - - (ctu/testing-unary-form elm/max)) + (ctu/testing-unary-op elm/max)) ;; 21.8. Median ;; @@ -229,9 +215,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/median) - - (ctu/testing-unary-form elm/median)) + (ctu/testing-unary-op elm/median)) ;; 21.9. Min ;; @@ -256,9 +240,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/min) - - (ctu/testing-unary-form elm/min)) + (ctu/testing-unary-op elm/min)) ;; 21.10. Mode ;; @@ -282,9 +264,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/mode) - - (ctu/testing-unary-form elm/mode)) + (ctu/testing-unary-op elm/mode)) ;; 21.11. PopulationVariance ;; @@ -306,9 +286,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/population-variance) - - (ctu/testing-unary-form elm/population-variance)) + (ctu/testing-unary-op elm/population-variance)) ;; 21.12. PopulationStdDev ;; @@ -330,9 +308,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/population-std-dev) - - (ctu/testing-unary-form elm/population-std-dev)) + (ctu/testing-unary-op elm/population-std-dev)) ;; 21.13. Sum ;; @@ -355,9 +331,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/sum) - - (ctu/testing-unary-form elm/sum)) + (ctu/testing-unary-op elm/sum)) ;; 21.14. StdDev ;; @@ -379,9 +353,7 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/std-dev) - - (ctu/testing-unary-form elm/std-dev)) + (ctu/testing-unary-op elm/std-dev)) ;; 21.15. Variance ;; @@ -403,6 +375,4 @@ #elm/list [] nil {:type "Null"} nil)) - (ctu/testing-unary-dynamic elm/variance) - - (ctu/testing-unary-form elm/variance)) + (ctu/testing-unary-op elm/variance)) diff --git a/modules/cql/test/blaze/elm/compiler/arithmetic_operators_test.clj b/modules/cql/test/blaze/elm/compiler/arithmetic_operators_test.clj index 83fd2f26b..25d5cae43 100644 --- a/modules/cql/test/blaze/elm/compiler/arithmetic_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/arithmetic_operators_test.clj @@ -16,7 +16,7 @@ [blaze.elm.literal :as elm] [blaze.elm.literal-spec] [blaze.elm.protocols :as p] - [blaze.elm.quantity :as quantity] + [blaze.elm.quantity :refer [quantity]] [blaze.elm.util-spec] [blaze.fhir.spec.type.system :as system] [blaze.test-util :refer [satisfies-prop]] @@ -68,21 +68,21 @@ (testing "Quantity" (are [x res] (= res (ctu/compile-unop elm/abs elm/quantity x)) - [-1] (quantity/quantity 1 "1") - [0] (quantity/quantity 0 "1") - [1] (quantity/quantity 1 "1") + [-1] (quantity 1 "1") + [0] (quantity 0 "1") + [1] (quantity 1 "1") - [-1M] (quantity/quantity 1M "1") - [0M] (quantity/quantity 0M "1") - [1M] (quantity/quantity 1M "1") + [-1M] (quantity 1M "1") + [0M] (quantity 0M "1") + [1M] (quantity 1M "1") - [-1 "m"] (quantity/quantity 1 "m") - [0 "m"] (quantity/quantity 0 "m") - [1 "m"] (quantity/quantity 1 "m") + [-1 "m"] (quantity 1 "m") + [0 "m"] (quantity 0 "m") + [1 "m"] (quantity 1 "m") - [-1M "m"] (quantity/quantity 1M "m") - [0M "m"] (quantity/quantity 0M "m") - [1M "m"] (quantity/quantity 1M "m")))) + [-1M "m"] (quantity 1M "m") + [0M "m"] (quantity 0M "m") + [1M "m"] (quantity 1M "m")))) (testing "Dynamic" (are [elm res] (= res (ctu/dynamic-compile-eval (elm/abs elm))) @@ -91,9 +91,7 @@ (ctu/testing-unary-null elm/abs) - (ctu/testing-unary-dynamic elm/abs) - - (ctu/testing-unary-form elm/abs)) + (ctu/testing-unary-op elm/abs)) ;; 16.2. Add ;; @@ -231,8 +229,8 @@ (testing "UCUM quantity" (are [x y res] (p/equal res (ctu/compile-binop elm/add elm/quantity x y)) - [1 "m"] [1 "m"] (quantity/quantity 2 "m") - [1 "m"] [1 "cm"] (quantity/quantity 1.01M "m"))) + [1 "m"] [1 "m"] (quantity 2 "m") + [1 "m"] [1 "cm"] (quantity 1.01M "m"))) (testing "Incompatible UCUM Quantity Subtractions" (are [x y] (thrown? UnconvertibleException (ctu/compile-binop elm/add elm/quantity x y)) @@ -364,9 +362,7 @@ #elm/time "00:00:00" #elm/quantity [1 "minute"] (date-time/local-time 0 1 0) #elm/time "00:00:00" #elm/quantity [1 "second"] (date-time/local-time 0 0 1))) - (ctu/testing-binary-dynamic elm/add) - - (ctu/testing-binary-form elm/add)) + (ctu/testing-binary-op elm/add)) ;; 16.3. Ceiling ;; @@ -381,9 +377,7 @@ (ctu/testing-unary-null elm/ceiling) - (ctu/testing-unary-dynamic elm/ceiling) - - (ctu/testing-unary-form elm/ceiling)) + (ctu/testing-unary-op elm/ceiling)) ;; 16.4. Divide ;; @@ -445,24 +439,24 @@ (testing "Quantity" (testing "Static" (are [x y res] (p/equal res (ctu/compile-binop elm/divide elm/quantity x y)) - [1 "m"] [1 "s"] (quantity/quantity 1 "m/s") - [1M "m"] [1M "s"] (quantity/quantity 1M "m/s") + [1 "m"] [1 "s"] (quantity 1 "m/s") + [1M "m"] [1M "s"] (quantity 1M "m/s") - [12 "cm2"] [3 "cm"] (quantity/quantity 4 "cm"))) + [12 "cm2"] [3 "cm"] (quantity 4 "cm"))) (ctu/testing-binary-null elm/divide #elm/quantity [1])) (testing "Quantity/Integer" (testing "Static" (are [x y res] (p/equal res (ctu/compile-binop elm/divide elm/quantity elm/integer x y)) - [1M "m"] "2" (quantity/quantity 0.5M "m"))) + [1M "m"] "2" (quantity 0.5M "m"))) (ctu/testing-binary-null elm/divide #elm/quantity [1] #elm/integer "1")) (testing "Quantity/Decimal" (testing "Static" (are [x y res] (p/equal res (ctu/compile-binop elm/divide elm/quantity elm/decimal x y)) - [2.5M "m"] "2.5" (quantity/quantity 1M "m"))) + [2.5M "m"] "2.5" (quantity 1M "m"))) (ctu/testing-binary-null elm/divide #elm/quantity [1] #elm/decimal "1.1")) @@ -472,9 +466,7 @@ (let [elm (elm/equal [(elm/multiply [(elm/divide [decimal decimal]) decimal]) decimal])] (true? (core/-eval (c/compile {} elm) {} nil nil)))))) - (ctu/testing-binary-dynamic elm/divide) - - (ctu/testing-binary-form elm/divide)) + (ctu/testing-binary-op elm/divide)) ;; 16.5. Exp ;; @@ -488,9 +480,7 @@ (ctu/testing-unary-null elm/exp) - (ctu/testing-unary-dynamic elm/exp) - - (ctu/testing-unary-form elm/exp)) + (ctu/testing-unary-op elm/exp)) ;; 16.6. Floor ;; @@ -505,9 +495,7 @@ (ctu/testing-unary-null elm/floor) - (ctu/testing-unary-dynamic elm/floor) - - (ctu/testing-unary-form elm/floor)) + (ctu/testing-unary-op elm/floor)) ;; 16.7. HighBoundary ;; @@ -549,9 +537,7 @@ (ctu/testing-binary-null elm/log #elm/decimal "1.1")) - (ctu/testing-binary-dynamic elm/log) - - (ctu/testing-binary-form elm/log)) + (ctu/testing-binary-op elm/log)) ;; 16.9. LowBoundary ;; @@ -593,9 +579,7 @@ (ctu/testing-unary-null elm/ln) - (ctu/testing-unary-dynamic elm/ln) - - (ctu/testing-unary-form elm/ln)) + (ctu/testing-unary-op elm/ln)) ;; 16.11. MaxValue ;; @@ -719,9 +703,7 @@ #elm/integer "1" #elm/integer "0" nil #elm/decimal "1" #elm/decimal "0" nil)) - (ctu/testing-binary-dynamic elm/modulo) - - (ctu/testing-binary-form elm/modulo)) + (ctu/testing-binary-op elm/modulo)) ;; 16.14. Multiply ;; @@ -758,14 +740,12 @@ (testing "Quantity" (are [x y res] (p/equal res (core/-eval (c/compile {} (elm/multiply [x y])) {} nil nil)) - #elm/quantity [1 "m"] #elm/integer "2" (quantity/quantity 2 "m") - #elm/quantity [1 "m"] #elm/quantity [2 "m"] (quantity/quantity 2 "m2")) + #elm/quantity [1 "m"] #elm/integer "2" (quantity 2 "m") + #elm/quantity [1 "m"] #elm/quantity [2 "m"] (quantity 2 "m2")) (ctu/testing-binary-null elm/multiply #elm/quantity [1])) - (ctu/testing-binary-dynamic elm/multiply) - - (ctu/testing-binary-form elm/multiply)) + (ctu/testing-binary-op elm/multiply)) ;; 16.15. Negate ;; @@ -787,16 +767,14 @@ (testing "Quantity" (are [x res] (= res (c/compile {} (elm/negate x))) - #elm/quantity [1] (quantity/quantity -1 "1") - #elm/quantity [1M] (quantity/quantity -1M "1") - #elm/quantity [1 "m"] (quantity/quantity -1 "m") - #elm/quantity [1M "m"] (quantity/quantity -1M "m"))) + #elm/quantity [1] (quantity -1 "1") + #elm/quantity [1M] (quantity -1M "1") + #elm/quantity [1 "m"] (quantity -1 "m") + #elm/quantity [1M "m"] (quantity -1M "m"))) (ctu/testing-unary-null elm/negate) - (ctu/testing-unary-dynamic elm/negate) - - (ctu/testing-unary-form elm/negate)) + (ctu/testing-unary-op elm/negate)) ;; 16.16. Power ;; @@ -829,9 +807,7 @@ #elm/decimal "10" #elm/integer "2" 100M #elm/decimal "10" #elm/integer "2" 100M)) - (ctu/testing-binary-dynamic elm/power) - - (ctu/testing-binary-form elm/power)) + (ctu/testing-binary-op elm/power)) ;; 16.17. Precision ;; @@ -908,14 +884,12 @@ (testing "Quantity" (are [x res] (= res (c/compile {} (elm/predecessor x))) (elm/quantity [decimal/min]) nil - #_#_#elm/quantity [0 "m"] (quantity/quantity -1 "m") ; TODO: implement - #elm/quantity [0M "m"] (quantity/quantity -1E-8M "m"))) + #_#_#elm/quantity [0 "m"] (quantity -1 "m") ; TODO: implement + #elm/quantity [0M "m"] (quantity -1E-8M "m"))) (ctu/testing-unary-null elm/predecessor) - (ctu/testing-unary-dynamic elm/predecessor) - - (ctu/testing-unary-form elm/predecessor)) + (ctu/testing-unary-op elm/predecessor)) ;; 16.19. Round ;; @@ -929,7 +903,7 @@ ;; precision is not specified or null, 0 is assumed. (deftest compile-round-test (testing "without precision" - (testing "static" + (testing "Static" (are [x res] (= res (c/compile {} (elm/round [x]))) #elm/integer "1" 1M #elm/decimal "1" 1M @@ -965,13 +939,7 @@ (ctu/testing-unary-null elm/round) - (ctu/testing-unary-dynamic elm/round) - - (ctu/testing-unary-form elm/round) - - (ctu/testing-binary-dynamic elm/round) - - (ctu/testing-binary-form elm/round)) + (ctu/testing-unary-op elm/round)) ;; 16.20. Subtract ;; @@ -1079,8 +1047,8 @@ (testing "UCUM quantity" (are [x y res] (p/equal res (core/-eval (c/compile {} (elm/subtract [x y])) {} nil nil)) - #elm/quantity [1 "m"] #elm/quantity [1 "m"] (quantity/quantity 0 "m") - #elm/quantity [1 "m"] #elm/quantity [1 "cm"] (quantity/quantity 0.99 "m"))) + #elm/quantity [1 "m"] #elm/quantity [1 "m"] (quantity 0 "m") + #elm/quantity [1 "m"] #elm/quantity [1 "cm"] (quantity 0.99 "m"))) (testing "Incompatible UCUM Quantity Subtractions" (are [x y] (thrown? UnconvertibleException (core/-eval (c/compile {} (elm/subtract [x y])) {} nil nil)) @@ -1189,9 +1157,7 @@ #elm/time "00:00:00" #elm/quantity [1 "minute"] (date-time/local-time 23 59 0) #elm/time "00:00:00" #elm/quantity [1 "second"] (date-time/local-time 23 59 59))) - (ctu/testing-binary-dynamic elm/subtract) - - (ctu/testing-binary-form elm/subtract)) + (ctu/testing-binary-op elm/subtract)) ;; 16.21. Successor ;; @@ -1256,14 +1222,12 @@ (testing "Quantity" (are [x res] (= res (c/compile {} (elm/successor x))) (elm/quantity [decimal/max]) nil - #_#_#elm/quantity [0 "m"] (quantity/quantity 1 "m") ; TODO: implement - #elm/quantity [0M "m"] (quantity/quantity 1E-8M "m"))) + #_#_#elm/quantity [0 "m"] (quantity 1 "m") ; TODO: implement + #elm/quantity [0M "m"] (quantity 1E-8M "m"))) (ctu/testing-unary-null elm/successor) - (ctu/testing-unary-dynamic elm/successor) - - (ctu/testing-unary-form elm/successor)) + (ctu/testing-unary-op elm/successor)) ;; 16.22. Truncate ;; @@ -1278,9 +1242,7 @@ (ctu/testing-unary-null elm/truncate) - (ctu/testing-unary-dynamic elm/truncate) - - (ctu/testing-unary-form elm/truncate)) + (ctu/testing-unary-op elm/truncate)) ;; 16.23. TruncatedDivide ;; @@ -1340,6 +1302,4 @@ (ctu/testing-binary-null elm/truncated-divide #elm/integer "1" #elm/decimal "1.1")) - (ctu/testing-binary-dynamic elm/truncated-divide) - - (ctu/testing-binary-form elm/truncated-divide)) + (ctu/testing-binary-op elm/truncated-divide)) diff --git a/modules/cql/test/blaze/elm/compiler/clinical_operators_test.clj b/modules/cql/test/blaze/elm/compiler/clinical_operators_test.clj index 0bf5ec552..fa9f0deaa 100644 --- a/modules/cql/test/blaze/elm/compiler/clinical_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/clinical_operators_test.clj @@ -52,26 +52,36 @@ (deftest compile-calculate-age-at-test (testing "Static" (are [elm res] (= res (core/-eval (c/compile {} elm) {:now ctu/now} nil nil)) - #elm/calculate-age-at [#elm/date "2018" #elm/date "2019" "year"] 1 - #elm/calculate-age-at [#elm/date "2018" #elm/date "2018" "year"] 0 - #elm/calculate-age-at [#elm/date "2018" #elm/date "2019" "month"] nil + #elm/calculate-age-at [#elm/date"2018" #elm/date"2019" "year"] 1 + #elm/calculate-age-at [#elm/date"2018" #elm/date"2018" "year"] 0 + #elm/calculate-age-at [#elm/date"2018" #elm/date"2019" "month"] nil - #elm/calculate-age-at [#elm/date "2018-01" #elm/date "2019-02" "year"] 1 - #elm/calculate-age-at [#elm/date "2018-01" #elm/date "2018-12" "year"] 0 - #elm/calculate-age-at [#elm/date "2018-01" #elm/date "2018-12" "month"] 11 - #elm/calculate-age-at [#elm/date "2018-01" #elm/date "2018-12" "day"] nil + #elm/calculate-age-at [#elm/date"2018-01" #elm/date"2019-02" "year"] 1 + #elm/calculate-age-at [#elm/date"2018-01" #elm/date"2018-12" "year"] 0 + #elm/calculate-age-at [#elm/date"2018-01" #elm/date"2018-12" "month"] 11 + #elm/calculate-age-at [#elm/date"2018-01" #elm/date"2018-12" "day"] nil - #elm/calculate-age-at [#elm/date "2018-01-01" #elm/date "2019-02-02" "year"] 1 - #elm/calculate-age-at [#elm/date "2018-01" #elm/date "2018-12-15" "year"] 0 - #elm/calculate-age-at [#elm/date "2018-01-01" #elm/date "2018-12-02" "month"] 11 - #elm/calculate-age-at [#elm/date "2018-01-01" #elm/date "2018-02-01" "day"] 31 + #elm/calculate-age-at [#elm/date"2018-01-01" #elm/date"2019-02-02" "year"] 1 + #elm/calculate-age-at [#elm/date"2018-01" #elm/date"2018-12-15" "year"] 0 + #elm/calculate-age-at [#elm/date"2018-01-01" #elm/date"2018-12-02" "month"] 11 + #elm/calculate-age-at [#elm/date"2018-01-01" #elm/date"2018-02-01" "day"] 31 #elm/calculate-age-at [#elm/date-time"2018-01-01" #elm/date-time"2018-02-01" "day"] 31)) - (ctu/testing-binary-null elm/calculate-age-at #elm/date "2018") + (ctu/testing-binary-null elm/calculate-age-at #elm/date"2018") (ctu/testing-binary-null elm/calculate-age-at #elm/date-time"2018-01-01") - (ctu/testing-binary-dynamic elm/calculate-age-at) + (ctu/testing-binary-precision-dynamic elm/calculate-age-at "year" "month" "day") + + (ctu/testing-binary-precision-attach-cache elm/calculate-age-at "year" "month" "day") + + (ctu/testing-binary-precision-patient-count elm/calculate-age-at "year" "month" "day") + + (ctu/testing-binary-precision-resolve-refs elm/calculate-age-at "year" "month" "day") + + (ctu/testing-binary-precision-resolve-params elm/calculate-age-at "year" "month" "day") + + (ctu/testing-binary-precision-equals-hash-code elm/calculate-age-at "year" "month" "day") (ctu/testing-binary-precision-form elm/calculate-age-at "year" "month" "day")) diff --git a/modules/cql/test/blaze/elm/compiler/clinical_values_test.clj b/modules/cql/test/blaze/elm/compiler/clinical_values_test.clj index 855ce84eb..bc467be19 100644 --- a/modules/cql/test/blaze/elm/compiler/clinical_values_test.clj +++ b/modules/cql/test/blaze/elm/compiler/clinical_values_test.clj @@ -15,8 +15,8 @@ [blaze.elm.date-time :as date-time] [blaze.elm.literal] [blaze.elm.literal-spec] - [blaze.elm.quantity :as quantity] - [blaze.elm.ratio :as ratio] + [blaze.elm.quantity :refer [quantity]] + [blaze.elm.ratio :refer [ratio]] [blaze.test-util :refer [satisfies-prop]] [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] @@ -279,7 +279,7 @@ (testing "Examples" (are [elm res] (= res (c/compile {} elm)) {:type "Quantity"} nil - #elm/quantity [1] (quantity/quantity 1 "") + #elm/quantity [1] (quantity 1 "") #elm/quantity [1 "year"] (date-time/period 1 0 0) #elm/quantity [2 "years"] (date-time/period 2 0 0) #elm/quantity [1 "month"] (date-time/period 0 1 0) @@ -296,14 +296,14 @@ #elm/quantity [2 "seconds"] (date-time/period 0 0 2000) #elm/quantity [1 "millisecond"] (date-time/period 0 0 1) #elm/quantity [2 "milliseconds"] (date-time/period 0 0 2) - #elm/quantity [1 "s"] (quantity/quantity 1 "s") - #elm/quantity [1 "cm2"] (quantity/quantity 1 "cm2"))) + #elm/quantity [1 "s"] (quantity 1 "s") + #elm/quantity [1 "cm2"] (quantity 1 "cm2"))) (testing "form" (are [elm res] (= res (c/form (c/compile {} elm))) - #elm/quantity [1] '(quantity 1 "1") - #elm/quantity [1 "s"] '(quantity 1 "s") - #elm/quantity [2 "cm2"] '(quantity 2 "cm2"))) + #elm/quantity [1] '(quantity 1M "1") + #elm/quantity [1 "s"] '(quantity 1M "s") + #elm/quantity [2 "cm2"] '(quantity 2M "cm2"))) (testing "Periods" (satisfies-prop 100 @@ -318,12 +318,17 @@ (deftest compile-ratio-test (testing "Examples" (are [elm res] (= res (c/compile {} elm)) - #elm/ratio [[1 "s"] [1 "s"]] (ratio/ratio (quantity/quantity 1 "s") (quantity/quantity 1 "s")) - #elm/ratio [[1 ""] [128 ""]] (ratio/ratio (quantity/quantity 1 "") (quantity/quantity 128 "")) - #elm/ratio [[1 "s"] [1 ""]] (ratio/ratio (quantity/quantity 1 "s") (quantity/quantity 1 "")) - #elm/ratio [[1 ""] [1 "s"]] (ratio/ratio (quantity/quantity 1 "") (quantity/quantity 1 "s")) - #elm/ratio [[1 "cm2"] [1 "s"]] (ratio/ratio (quantity/quantity 1 "cm2") (quantity/quantity 1 "s")) - #elm/ratio [[1] [1]] (ratio/ratio (quantity/quantity 1 "") (quantity/quantity 1 "")) - #elm/ratio [[1] [1 "s"]] (ratio/ratio (quantity/quantity 1 "") (quantity/quantity 1 "s")) - #elm/ratio [[1 "s"] [1]] (ratio/ratio (quantity/quantity 1 "s") (quantity/quantity 1 "")) - #elm/ratio [[5 "mg"] [10 "g"]] (ratio/ratio (quantity/quantity 5 "mg") (quantity/quantity 10 "g"))))) + #elm/ratio [[1 "s"] [1 "s"]] (ratio (quantity 1 "s") (quantity 1 "s")) + #elm/ratio [[1 ""] [128 ""]] (ratio (quantity 1 "") (quantity 128 "")) + #elm/ratio [[1 "s"] [1 ""]] (ratio (quantity 1 "s") (quantity 1 "")) + #elm/ratio [[1 ""] [1 "s"]] (ratio (quantity 1 "") (quantity 1 "s")) + #elm/ratio [[1 "cm2"] [1 "s"]] (ratio (quantity 1 "cm2") (quantity 1 "s")) + #elm/ratio [[1] [1]] (ratio (quantity 1 "") (quantity 1 "")) + #elm/ratio [[1] [1 "s"]] (ratio (quantity 1 "") (quantity 1 "s")) + #elm/ratio [[1 "s"] [1]] (ratio (quantity 1 "s") (quantity 1 "")) + #elm/ratio [[5 "mg"] [10 "g"]] (ratio (quantity 5 "mg") (quantity 10 "g")))) + + (testing "form" + (has-form + (ratio (quantity 3 "m") (quantity 1 "s")) + '(ratio (quantity 3M "m") (quantity 1M "s"))))) diff --git a/modules/cql/test/blaze/elm/compiler/comparison_operators_test.clj b/modules/cql/test/blaze/elm/compiler/comparison_operators_test.clj index 5a87eb6b0..e80c420cd 100644 --- a/modules/cql/test/blaze/elm/compiler/comparison_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/comparison_operators_test.clj @@ -199,7 +199,7 @@ "2012" "2013" false? "2013" "2012" false?) - (ctu/testing-binary-null elm/equal #elm/date "2013")) + (ctu/testing-binary-null elm/equal #elm/date"2013")) (testing "Date with year-month precision" (are [x y pred] (pred (ctu/compile-binop elm/equal elm/date x y)) @@ -207,7 +207,7 @@ "2013-01" "2013-02" false? "2013-02" "2013-01" false?) - (ctu/testing-binary-null elm/equal #elm/date "2013-01")) + (ctu/testing-binary-null elm/equal #elm/date"2013-01")) (testing "Date with full precision" (are [x y pred] (pred (ctu/compile-binop elm/equal elm/date x y)) @@ -215,7 +215,7 @@ "2013-01-01" "2013-01-02" false? "2013-01-02" "2013-01-01" false?) - (ctu/testing-binary-null elm/equal #elm/date "2013-01-01")) + (ctu/testing-binary-null elm/equal #elm/date"2013-01-01")) (testing "Date with differing precisions" (are [x y pred] (pred (ctu/compile-binop elm/equal elm/date x y)) @@ -261,9 +261,7 @@ (ctu/testing-binary-null elm/equal (ctu/code "a" "0"))) - (ctu/testing-binary-dynamic elm/equal) - - (ctu/testing-binary-form elm/equal)) + (ctu/testing-binary-op elm/equal)) ;; 12.2. Equivalent ;; @@ -441,9 +439,7 @@ {:type "Null"} (ctu/code "a" "0") false? (ctu/code "a" "0") {:type "Null"} false?)) - (ctu/testing-binary-dynamic elm/equivalent) - - (ctu/testing-binary-form elm/equivalent)) + (ctu/testing-binary-op elm/equivalent)) ;; 12.3. Greater ;; @@ -510,7 +506,7 @@ "2014" "2013" true? "2013" "2013" false?) - (ctu/testing-binary-null elm/greater #elm/date "2013")) + (ctu/testing-binary-null elm/greater #elm/date"2013")) (testing "DateTime with year precision" (are [x y pred] (pred (ctu/compile-binop elm/greater elm/date-time x y)) @@ -562,9 +558,7 @@ (ctu/testing-binary-null elm/greater #elm/quantity [1])) - (ctu/testing-binary-dynamic elm/greater) - - (ctu/testing-binary-form elm/greater)) + (ctu/testing-binary-op elm/greater)) ;; 12.4. GreaterOrEqual ;; @@ -638,7 +632,7 @@ "2013-06-15" "2013-06-15" true? "2013-06-14" "2013-06-15" false?) - (ctu/testing-binary-null elm/greater-or-equal #elm/date "2013-06-15")) + (ctu/testing-binary-null elm/greater-or-equal #elm/date"2013-06-15")) (testing "DateTime with year precision" (are [x y pred] (pred (ctu/compile-binop elm/greater-or-equal elm/date-time x y)) @@ -646,7 +640,7 @@ "2013" "2013" true? "2012" "2013" false?) - (ctu/testing-binary-null elm/greater-or-equal #elm/date "2013")) + (ctu/testing-binary-null elm/greater-or-equal #elm/date"2013")) (testing "DateTime with year-month precision" (are [x y pred] (pred (ctu/compile-binop elm/greater-or-equal elm/date-time x y)) @@ -654,7 +648,7 @@ "2013-06" "2013-06" true? "2013-05" "2013-06" false?) - (ctu/testing-binary-null elm/greater-or-equal #elm/date "2013-06")) + (ctu/testing-binary-null elm/greater-or-equal #elm/date"2013-06")) (testing "DateTime with date precision" (are [x y pred] (pred (ctu/compile-binop elm/greater-or-equal elm/date-time x y)) @@ -662,7 +656,7 @@ "2013-06-15" "2013-06-15" true? "2013-06-14" "2013-06-15" false?) - (ctu/testing-binary-null elm/greater-or-equal #elm/date "2013-06-15")) + (ctu/testing-binary-null elm/greater-or-equal #elm/date"2013-06-15")) (testing "DateTime with mixed precision" (are [x y] (nil? (ctu/compile-binop elm/greater-or-equal elm/date-time x y)) @@ -695,9 +689,7 @@ (ctu/testing-binary-null elm/greater-or-equal #elm/quantity [1])) - (ctu/testing-binary-dynamic elm/greater-or-equal) - - (ctu/testing-binary-form elm/greater-or-equal)) + (ctu/testing-binary-op elm/greater-or-equal)) ;; 12.5. Less ;; @@ -762,7 +754,7 @@ "2012" "2013" true? "2013" "2013" false?) - (ctu/testing-binary-null elm/less #elm/date "2013")) + (ctu/testing-binary-null elm/less #elm/date"2013")) (testing "Comparing dates with mixed precisions (year and year-month) results in null." (are [x y pred] (pred (ctu/compile-binop elm/less elm/date x y)) @@ -774,7 +766,7 @@ "2013-06-14" "2013-06-15" true? "2013-06-15" "2013-06-15" false?) - (ctu/testing-binary-null elm/less #elm/date "2013-06-15")) + (ctu/testing-binary-null elm/less #elm/date"2013-06-15")) (testing "Comparing dates with mixed precisions (year-month and full) results in null." (are [x y pred] (pred (ctu/compile-binop elm/less elm/date x y)) @@ -832,9 +824,7 @@ (ctu/testing-binary-null elm/less #elm/quantity [1])) - (ctu/testing-binary-dynamic elm/less) - - (ctu/testing-binary-form elm/less)) + (ctu/testing-binary-op elm/less)) ;; 12.6. LessOrEqual ;; @@ -900,7 +890,7 @@ "2013-06-15" "2013-06-15" true? "2013-06-16" "2013-06-15" false?) - (ctu/testing-binary-null elm/less-or-equal #elm/date "2013-06-15")) + (ctu/testing-binary-null elm/less-or-equal #elm/date"2013-06-15")) (testing "Mixed Date and DateTime" (are [x y pred] (pred (c/compile {} (elm/less-or-equal [x y]))) @@ -913,7 +903,7 @@ "2013" "2013" true? "2014" "2013" false?) - (ctu/testing-binary-null elm/less-or-equal #elm/date "2013")) + (ctu/testing-binary-null elm/less-or-equal #elm/date"2013")) (testing "DateTime with year-month precision" (are [x y pred] (pred (ctu/compile-binop elm/less-or-equal elm/date-time x y)) @@ -921,7 +911,7 @@ "2013-06" "2013-06" true? "2013-07" "2013-06" false?) - (ctu/testing-binary-null elm/less-or-equal #elm/date "2013-06")) + (ctu/testing-binary-null elm/less-or-equal #elm/date"2013-06")) (testing "DateTime with date precision" (are [x y pred] (pred (ctu/compile-binop elm/less-or-equal elm/date-time x y)) @@ -929,7 +919,7 @@ "2013-06-15" "2013-06-15" true? "2013-06-16" "2013-06-15" false?) - (ctu/testing-binary-null elm/less-or-equal #elm/date "2013-06-15")) + (ctu/testing-binary-null elm/less-or-equal #elm/date"2013-06-15")) (testing "Time" (are [x y pred] (pred (ctu/compile-binop elm/less-or-equal elm/time x y)) @@ -958,9 +948,7 @@ (ctu/testing-binary-null elm/less-or-equal #elm/quantity [1])) - (ctu/testing-binary-dynamic elm/less-or-equal) - - (ctu/testing-binary-form elm/less-or-equal)) + (ctu/testing-binary-op elm/less-or-equal)) ;; 12.7. NotEqual ;; diff --git a/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj b/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj index 6fec8c56c..b87b2f5de 100644 --- a/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj @@ -7,9 +7,11 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] + [blaze.elm.expression.cache :as ec] [blaze.elm.literal-spec] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [are deftest is testing]])) + [clojure.test :as test :refer [are deftest is testing]] + [juxt.iota :refer [given]])) (st/instrument) (ctu/instrument-compile) @@ -22,6 +24,17 @@ (test/use-fixtures :each fixture) +(defn- from-names [names] + (mapv + (fn [n] + {:type "ExpressionDef" :name n + :expression n + :context "Unfiltered"}) + names)) + +(defn- index-by-name [expr-defs] + (into {} (map (fn [{:keys [name] :as expr-def}] [name expr-def])) expr-defs)) + ;; 15.1. Case ;; ;; The Case operator allows for multiple conditional expressions to be chained @@ -119,7 +132,116 @@ :caseItem [{:when #elm/parameter-ref "b" :then #elm/parameter-ref "1"}] - :else #elm/parameter-ref "2"}))))))) + :else #elm/parameter-ref "2"})))))) + + (testing "attach cache" + (testing "multi-conditional" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm {:type "Case" + :caseItem + [{:when #elm/exists #elm/retrieve{:type "Encounter"} + :then #elm/exists #elm/retrieve{:type "Observation"}}] + :else #elm/exists #elm/retrieve{:type "Condition"}} + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 3 + [1 0] := '(exists (retrieve "Encounter")) + [1 1] := '(exists (retrieve "Observation")) + [1 2] := '(exists (retrieve "Condition")))))) + + (testing "comparand-based" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm {:type "Case" + :comparand #elm/exists #elm/retrieve{:type "Encounter"} + :caseItem + [{:when #elm/exists #elm/retrieve{:type "Observation"} + :then #elm/exists #elm/retrieve{:type "Condition"}}] + :else #elm/exists #elm/retrieve{:type "MedicationAdministration"}} + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 4 + [1 0] := '(exists (retrieve "Encounter")) + [1 1] := '(exists (retrieve "Observation")) + [1 2] := '(exists (retrieve "Condition")) + [1 3] := '(exists (retrieve "MedicationAdministration"))))))) + + (testing "resolve expression references" + (testing "multi-conditional" + (let [elm {:type "Case" + :caseItem + [{:when #elm/expression-ref "w" + :then #elm/expression-ref "t"}] + :else #elm/expression-ref "e"} + expr-defs (from-names ["w" "t" "e"]) + ctx {:library {:statements {:def expr-defs}}} + expr (c/resolve-refs (c/compile ctx elm) (index-by-name expr-defs))] + (has-form expr '(case "w" "t" "e")))) + + (testing "comparand-based" + (let [elm {:type "Case" + :comparand #elm/expression-ref "c" + :caseItem + [{:when #elm/expression-ref "w" + :then #elm/expression-ref "t"}] + :else #elm/expression-ref "e"} + expr-defs (from-names ["c" "w" "t" "e"]) + ctx {:library {:statements {:def expr-defs}}} + expr (c/resolve-refs (c/compile ctx elm) (index-by-name expr-defs))] + (has-form expr '(case "c" "w" "t" "e"))))) + + (testing "resolve parameters" + (testing "multi-conditional" + (let [elm {:type "Case" + :caseItem + [{:when #elm/parameter-ref "w" + :then #elm/parameter-ref "t"}] + :else #elm/parameter-ref "e"} + ctx {:library {:parameters {:def [{:name "w"} {:name "t"} {:name "e"}]}}} + expr (c/resolve-params (c/compile ctx elm) {"w" "w" "t" "t" "e" "e"})] + (has-form expr '(case "w" "t" "e")))) + + (testing "comparand-based" + (let [elm {:type "Case" + :comparand #elm/parameter-ref "c" + :caseItem + [{:when #elm/parameter-ref "w" + :then #elm/parameter-ref "t"}] + :else #elm/parameter-ref "e"} + ctx {:library {:parameters {:def [{:name "c"} {:name "w"} {:name "t"} {:name "e"}]}}} + expr (c/resolve-params (c/compile ctx elm) {"c" "c" "w" "w" "t" "t" "e" "e"})] + (has-form expr '(case "c" "w" "t" "e"))))) + + (testing "equals/hashCode" + (testing "multi-conditional" + (let [elm {:type "Case" + :caseItem + [{:when #elm/parameter-ref "w" + :then #elm/parameter-ref "t"}] + :else #elm/parameter-ref "e"} + ctx {:library {:parameters {:def [{:name "w"} {:name "t"} {:name "e"}]}}} + expr-1 (c/compile ctx elm) + expr-2 (c/compile ctx elm)] + (is (= 1 (count (set [expr-1 expr-2])))))) + + (testing "comparand-based" + (let [elm {:type "Case" + :comparand #elm/parameter-ref "c" + :caseItem + [{:when #elm/parameter-ref "w" + :then #elm/parameter-ref "t"}] + :else #elm/parameter-ref "e"} + ctx {:library {:parameters {:def [{:name "c"} {:name "w"} {:name "t"} {:name "e"}]}}} + expr-1 (c/compile ctx elm) + expr-2 (c/compile ctx elm)] + (is (= 1 (count (set [expr-1 expr-2])))))))) ;; 15.2. If ;; @@ -156,4 +278,40 @@ (let [expr (ctu/dynamic-compile #elm/if [#elm/parameter-ref "x" #elm/parameter-ref "y" #elm/parameter-ref "z"])] - (has-form expr '(if (param-ref "x") (param-ref "y") (param-ref "z")))))) + (has-form expr '(if (param-ref "x") (param-ref "y") (param-ref "z"))))) + + (testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) %2)] + (let [elm #elm/if [#elm/exists #elm/retrieve{:type "Encounter"} + #elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 3 + [1 0 c/form] := '(exists (retrieve "Encounter")) + [1 1 c/form] := '(exists (retrieve "Observation")) + [1 2 c/form] := '(exists (retrieve "Condition")))))) + + (testing "resolve expression references" + (let [elm #elm/if [#elm/expression-ref "c" + #elm/expression-ref "t" + #elm/expression-ref "e"] + expr-defs (from-names ["c" "t" "e"]) + ctx {:library {:statements {:def expr-defs}}} + expr (c/resolve-refs (c/compile ctx elm) (index-by-name expr-defs))] + (has-form expr '(if "c" "t" "e")))) + + (testing "resolve parameters" + (let [elm #elm/if [#elm/parameter-ref "c" + #elm/parameter-ref "t" + #elm/parameter-ref "e"] + ctx {:library {:parameters {:def [{:name "c"} {:name "t"} {:name "e"}]}}} + expr (c/resolve-params (c/compile ctx elm) {"c" "c" "t" "t" "e" "e"})] + (has-form expr '(if "c" "t" "e")))) + + (ctu/testing-equals-hash-code #elm/if [#elm/parameter-ref "x" + #elm/parameter-ref "y" + #elm/parameter-ref "z"])) diff --git a/modules/cql/test/blaze/elm/compiler/date_time_operators_test.clj b/modules/cql/test/blaze/elm/compiler/date_time_operators_test.clj index 7fb98799e..e265cdbbf 100644 --- a/modules/cql/test/blaze/elm/compiler/date_time_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/date_time_operators_test.clj @@ -7,7 +7,8 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] - [blaze.elm.compiler.test-util :as ctu] + [blaze.elm.compiler.date-time-operators] + [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.date-time :as date-time] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] @@ -183,7 +184,7 @@ (let [elm #elm/date [#elm/parameter-ref "year"] expr (c/compile compile-ctx elm)] - (is (= '(date (param-ref "year")) (core/-form expr))) + (has-form expr '(date (param-ref "year"))) (is (false? (core/-static expr))))) @@ -192,8 +193,7 @@ #elm/parameter-ref "month"] expr (c/compile compile-ctx elm)] - (is (= '(date (param-ref "year") (param-ref "month")) - (core/-form expr))) + (has-form expr '(date (param-ref "year") (param-ref "month"))) (is (false? (core/-static expr))))) @@ -203,10 +203,55 @@ #elm/parameter-ref "day"] expr (c/compile compile-ctx elm)] - (is (= '(date (param-ref "year") (param-ref "month") (param-ref "day")) - (core/-form expr))) + (has-form expr '(date (param-ref "year") (param-ref "month") (param-ref "day"))) - (is (false? (core/-static expr)))))))) + (is (false? (core/-static expr))))))) + + (testing "resolve parameters" + (let [compile-ctx {:library + {:parameters + {:def + [{:name "year"} + {:name "month"} + {:name "day"} + {:name "x"}]}}}] + + (testing "year" + (testing "with parameter-ref in expression with unresolved parameter-ref" + (let [elm #elm/date [#elm/add [#elm/parameter-ref "year" #elm/parameter-ref "x"]] + expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date (add 2024 (param-ref "x"))))))) + + (testing "year-month" + (let [elm #elm/date [#elm/parameter-ref "year" #elm/parameter-ref "month"]] + + (testing "with only the year parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date 2024 (param-ref "month"))))) + + (testing "with only the month parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"month" 6})] + (has-form expr '(date (param-ref "year") 6)))))) + + (testing "date" + (let [elm #elm/date [#elm/parameter-ref "year" + #elm/parameter-ref "month" + #elm/parameter-ref "day"]] + + (testing "with only the year parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date 2024 (param-ref "month") (param-ref "day"))))) + + (testing "with only the month parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"month" 6})] + (has-form expr '(date (param-ref "year") 6 (param-ref "day"))))) + + (testing "with only the day parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"day" 15})] + (has-form expr '(date (param-ref "year") (param-ref "month") 15)))))))) + + (ctu/testing-binary-op elm/date) + (ctu/testing-ternary-op elm/date)) ;; 18.7. DateFrom ;; @@ -226,9 +271,7 @@ (ctu/testing-unary-null elm/date-from) - (ctu/testing-unary-dynamic elm/date-from) - - (ctu/testing-unary-form elm/date-from)) + (ctu/testing-unary-op elm/date-from)) ;; 18.8. DateTime ;; @@ -417,7 +460,7 @@ (let [elm #elm/date-time [#elm/parameter-ref "year"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year")) (core/-form expr))) + (has-form expr '(date-time (param-ref "year"))) (is (false? (core/-static expr))))) @@ -426,8 +469,7 @@ #elm/parameter-ref "month"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month")) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month"))) (is (false? (core/-static expr))))) @@ -437,9 +479,8 @@ #elm/parameter-ref "day"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day")) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day"))) (is (false? (core/-static expr))))) @@ -455,7 +496,7 @@ #elm/integer "8"] expr (c/compile compile-ctx elm)] - (is (= '(date-time 1 2 3 4 5 6 7 8) (core/-form expr))) + (has-form expr '(date-time 1 2 3 4 5 6 7 8)) (is (false? (core/-static expr))))) @@ -470,11 +511,10 @@ #elm/integer "1"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") - (param-ref "minute") (param-ref "second") - (param-ref "millisecond") 1) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + (param-ref "minute") (param-ref "second") + (param-ref "millisecond") 1)) (is (false? (core/-static expr))))) @@ -489,12 +529,11 @@ #elm/parameter-ref "timezone-offset"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") - (param-ref "minute") (param-ref "second") - (param-ref "millisecond") - (param-ref "timezone-offset")) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + (param-ref "minute") (param-ref "second") + (param-ref "millisecond") + (param-ref "timezone-offset"))) (is (false? (core/-static expr)))))) @@ -506,9 +545,9 @@ #elm/parameter-ref "hour"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") 0 0 0) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + 0 0 0)) (is (false? (core/-static expr))))) @@ -520,10 +559,9 @@ #elm/parameter-ref "minute"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") - (param-ref "minute") 0 0) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + (param-ref "minute") 0 0)) (is (false? (core/-static expr))))) @@ -536,10 +574,9 @@ #elm/parameter-ref "second"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") - (param-ref "minute") (param-ref "second") 0) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + (param-ref "minute") (param-ref "second") 0)) (is (false? (core/-static expr))))) @@ -553,13 +590,101 @@ #elm/parameter-ref "millisecond"] expr (c/compile compile-ctx elm)] - (is (= '(date-time (param-ref "year") (param-ref "month") - (param-ref "day") (param-ref "hour") - (param-ref "minute") (param-ref "second") - (param-ref "millisecond")) - (core/-form expr))) + (has-form expr '(date-time (param-ref "year") (param-ref "month") + (param-ref "day") (param-ref "hour") + (param-ref "minute") (param-ref "second") + (param-ref "millisecond"))) - (is (false? (core/-static expr))))))))) + (is (false? (core/-static expr)))))))) + + (testing "resolve parameters" + (let [compile-ctx {:library + {:parameters + {:def + [{:name "year"} + {:name "month"} + {:name "day"} + {:name "hour"} + {:name "minute"} + {:name "second"} + {:name "millisecond"} + {:name "timezone-offset"} + {:name "x"}]}}}] + + (testing "year" + (testing "with parameter-ref in expression with unresolved parameter-ref" + (let [elm #elm/date-time [#elm/add [#elm/parameter-ref "year" #elm/parameter-ref "x"]] + expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date-time (add 2024 (param-ref "x"))))))) + + (testing "year-month" + (let [elm #elm/date-time [#elm/parameter-ref "year" #elm/parameter-ref "month"]] + + (testing "with only the year parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date-time 2024 (param-ref "month"))))) + + (testing "with only the month parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"month" 6})] + (has-form expr '(date-time (param-ref "year") 6)))))) + + (testing "date" + (let [elm #elm/date-time [#elm/parameter-ref "year" + #elm/parameter-ref "month" + #elm/parameter-ref "day"]] + + (testing "with only the year parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date-time 2024 (param-ref "month") (param-ref "day"))))) + + (testing "with only the month parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"month" 6})] + (has-form expr '(date-time (param-ref "year") 6 (param-ref "day"))))) + + (testing "with only the day parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"day" 15})] + (has-form expr '(date-time (param-ref "year") (param-ref "month") 15)))))) + + (testing "with timezone offset" + (let [elm #elm/date-time [#elm/parameter-ref "year" + #elm/parameter-ref "month" + #elm/parameter-ref "day" + #elm/parameter-ref "hour" + #elm/parameter-ref "minute" + #elm/parameter-ref "second" + #elm/parameter-ref "millisecond" + #elm/parameter-ref "timezone-offset"]] + + (testing "with all parameter-refs resolved" + (let [params {"year" 2024 "month" 6 "day" 15 "hour" 16 + "minute" 50 "second" 23 "millisecond" 42 + "timezone-offset" 1.5M} + expr (c/resolve-params (c/compile compile-ctx elm) params)] + (has-form expr '(date-time 2024 6 15 16 50 23 42 1.5M)))) + + (testing "with only the year parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"year" 2024})] + (has-form expr '(date-time 2024 (param-ref "month") (param-ref "day") + (param-ref "hour") (param-ref "minute") + (param-ref "second") (param-ref "millisecond") + (param-ref "timezone-offset"))))) + + (testing "with only the month parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"month" 6})] + (has-form expr '(date-time (param-ref "year") 6 (param-ref "day") + (param-ref "hour") (param-ref "minute") + (param-ref "second") (param-ref "millisecond") + (param-ref "timezone-offset"))))) + + (testing "with only the day parameter-ref resolved" + (let [expr (c/resolve-params (c/compile compile-ctx elm) {"day" 15})] + (has-form expr '(date-time (param-ref "year") (param-ref "month") 15 + (param-ref "hour") (param-ref "minute") + (param-ref "second") (param-ref "millisecond") + (param-ref "timezone-offset"))))))))) + + (ctu/testing-binary-op elm/date-time) + (ctu/testing-ternary-op elm/date-time)) ;; 18.9. DateTimeComponentFrom ;; @@ -590,11 +715,8 @@ (are [x precision res] (= res (eval (compile elm/date-time x precision))) "2019-04-17T12:48" "Hour" 12)) - (ctu/testing-unary-precision-dynamic elm/date-time-component-from "Year" "Month" - "Day" "Hour" "Minute" "Second" "Millisecond") - - (ctu/testing-unary-precision-form elm/date-time-component-from "Year" "Month" - "Day" "Hour" "Minute" "Second" "Millisecond")) + (ctu/testing-unary-precision-op elm/date-time-component-from "Year" "Month" + "Day" "Hour" "Minute" "Second" "Millisecond")) ;; 18.10. DifferenceBetween ;; @@ -662,9 +784,7 @@ "2018-01" "2018-01" "Day" "2018-01-01" "2018-01-01" "Hour")))) - (ctu/testing-binary-precision-dynamic elm/difference-between "Year" "Month" "Day") - - (ctu/testing-binary-precision-form elm/difference-between "Year" "Month" "Day")) + (ctu/testing-binary-precision-only-op elm/difference-between "Year" "Month" "Day")) ;; 18.11. DurationBetween ;; @@ -731,9 +851,7 @@ "2018-01" "2018-01" "Day" "2018-01-01" "2018-01-01" "Hour")))) - (ctu/testing-binary-precision-dynamic elm/duration-between "Year" "Month" "Day") - - (ctu/testing-binary-precision-form elm/duration-between "Year" "Month" "Day")) + (ctu/testing-binary-precision-only-op elm/duration-between "Year" "Month" "Day")) ;; 18.12. Not Equal ;; @@ -802,9 +920,9 @@ "2019-04-17" "2019-04-17" true? "2019-04-17" "2019-04-18" false?) - (ctu/testing-binary-null elm/same-as #elm/date "2019") - (ctu/testing-binary-null elm/same-as #elm/date "2019-04") - (ctu/testing-binary-null elm/same-as #elm/date "2019-04-17") + (ctu/testing-binary-null elm/same-as #elm/date"2019") + (ctu/testing-binary-null elm/same-as #elm/date"2019-04") + (ctu/testing-binary-null elm/same-as #elm/date"2019-04-17") (testing "with year precision" (are [x y pred] (pred (ctu/compile-binop-precision elm/same-as elm/date x y "year")) @@ -837,13 +955,7 @@ "2019-04-17" "2019-04-17" true? "2019-04-17" "2019-04-18" true?))) - (ctu/testing-binary-dynamic elm/same-as) - - (ctu/testing-binary-precision-dynamic elm/same-as) - - (ctu/testing-binary-form elm/same-as) - - (ctu/testing-binary-precision-form elm/same-as)) + (ctu/testing-binary-precision-op elm/same-as)) ;; 18.15. SameOrBefore ;; @@ -906,9 +1018,9 @@ "2019-04-17" "2019-04-17" true? "2019-04-17" "2019-04-16" false?) - (ctu/testing-binary-null elm/same-or-before #elm/date "2019") - (ctu/testing-binary-null elm/same-or-before #elm/date "2019-04") - (ctu/testing-binary-null elm/same-or-before #elm/date "2019-04-17") + (ctu/testing-binary-null elm/same-or-before #elm/date"2019") + (ctu/testing-binary-null elm/same-or-before #elm/date"2019-04") + (ctu/testing-binary-null elm/same-or-before #elm/date"2019-04-17") (testing "with year precision" (are [x y pred] (pred (ctu/compile-binop-precision elm/same-or-before elm/date x y "year")) @@ -944,13 +1056,7 @@ "2019-04" "2019-04" true? "2019-04" "2019-03" true?))) - (ctu/testing-binary-dynamic elm/same-or-before) - - (ctu/testing-binary-precision-dynamic elm/same-or-before) - - (ctu/testing-binary-form elm/same-or-before) - - (ctu/testing-binary-precision-form elm/same-or-before)) + (ctu/testing-binary-precision-op elm/same-or-before)) ;; 18.15. SameOrAfter ;; @@ -1013,9 +1119,9 @@ "2019-04-17" "2019-04-17" true? "2019-04-17" "2019-04-18" false?) - (ctu/testing-binary-null elm/same-or-after #elm/date "2019") - (ctu/testing-binary-null elm/same-or-after #elm/date "2019-04") - (ctu/testing-binary-null elm/same-or-after #elm/date "2019-04-17") + (ctu/testing-binary-null elm/same-or-after #elm/date"2019") + (ctu/testing-binary-null elm/same-or-after #elm/date"2019-04") + (ctu/testing-binary-null elm/same-or-after #elm/date"2019-04-17") (testing "with year precision" (are [x y pred] (pred (ctu/compile-binop-precision elm/same-or-after elm/date x y "year")) @@ -1051,13 +1157,7 @@ "2019-04" "2019-04" true? "2019-04" "2019-05" true?))) - (ctu/testing-binary-dynamic elm/same-or-after) - - (ctu/testing-binary-precision-dynamic elm/same-or-after) - - (ctu/testing-binary-form elm/same-or-after) - - (ctu/testing-binary-precision-form elm/same-or-after)) + (ctu/testing-binary-precision-op elm/same-or-after)) ;; 18.18. Time ;; @@ -1144,7 +1244,7 @@ (let [elm #elm/time [#elm/parameter-ref "hour"] expr (c/compile compile-ctx elm)] - (is (= '(time (param-ref "hour")) (core/-form expr))) + (has-form expr '(time (param-ref "hour"))) (is (false? (core/-static expr))))) @@ -1153,8 +1253,7 @@ #elm/parameter-ref "minute"] expr (c/compile compile-ctx elm)] - (is (= '(time (param-ref "hour") (param-ref "minute")) - (core/-form expr))) + (has-form expr '(time (param-ref "hour") (param-ref "minute"))) (is (false? (core/-static expr))))) @@ -1164,9 +1263,8 @@ #elm/parameter-ref "second"] expr (c/compile compile-ctx elm)] - (is (= '(time (param-ref "hour") (param-ref "minute") - (param-ref "second")) - (core/-form expr))) + (has-form expr '(time (param-ref "hour") (param-ref "minute") + (param-ref "second"))) (is (false? (core/-static expr))))) @@ -1177,9 +1275,8 @@ #elm/parameter-ref "millisecond"] expr (c/compile compile-ctx elm)] - (is (= '(time (param-ref "hour") (param-ref "minute") - (param-ref "second") (param-ref "millisecond")) - (core/-form expr))) + (has-form expr '(time (param-ref "hour") (param-ref "minute") + (param-ref "second") (param-ref "millisecond"))) (is (false? (core/-static expr)))))))) diff --git a/modules/cql/test/blaze/elm/compiler/external_data_test.clj b/modules/cql/test/blaze/elm/compiler/external_data_test.clj index 9898f16ea..e33b488fa 100644 --- a/modules/cql/test/blaze/elm/compiler/external_data_test.clj +++ b/modules/cql/test/blaze/elm/compiler/external_data_test.clj @@ -10,13 +10,11 @@ [blaze.db.api-stub :refer [mem-node-config with-system-data]] [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.external-data :as ed] - [blaze.elm.compiler.external-data-spec] + [blaze.elm.compiler.external-data] [blaze.elm.compiler.library :as library] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.expression :as expr] [blaze.elm.expression-spec] - [blaze.elm.expression.cache :as-alias expr-cache] [blaze.elm.util-spec] [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type] @@ -43,16 +41,6 @@ (defn- eval-context [db] {:db db :now (OffsetDateTime/now)}) -(defn- resource [db type id] - (ed/mk-resource db (d/resource-handle db type id))) - -(deftest resource-test - (testing "toString" - (with-system-data [{:blaze.db/keys [node]} mem-node-config] - [[[:put {:fhir/type :fhir/Patient :id "0"}]]] - - (is (= "Patient[id = 0, t = 1]" (str (ctu/resource (d/db node) "Patient" "0"))))))) - ;; 11.1. Retrieve ;; ;; All access to external data within ELM is represented by Retrieve expressions. @@ -99,6 +87,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) '(retrieve-resource))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) '(retrieve-resource))) + (testing "form" (has-form expr '(retrieve-resource)))))) @@ -124,6 +124,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) '(retrieve "Observation"))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) '(retrieve "Observation"))) + (testing "form" (has-form expr '(retrieve "Observation"))))) @@ -166,6 +178,20 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve "Observation" [["code" "system-192253|code-192300"]]))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve "Observation" [["code" "system-192253|code-192300"]]))) + (testing "form" (has-form expr '(retrieve "Observation" [["code" "system-192253|code-192300"]])))))) @@ -220,6 +246,28 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve + "Observation" + [["code" + "system-192253|code-192300" + "system-192253|code-140541"]]))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve + "Observation" + [["code" + "system-192253|code-192300" + "system-192253|code-140541"]]))) + (testing "form" (has-form expr '(retrieve @@ -281,6 +329,28 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve + "Observation" + [["code" + "system-192253|code-192300" + "system-192253|code-140541"]]))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve + "Observation" + [["code" + "system-192253|code-192300" + "system-192253|code-140541"]]))) + (testing "form" (has-form expr '(retrieve @@ -313,6 +383,20 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve (Specimen) "Patient"))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve (Specimen) "Patient"))) + (testing "form" (has-form expr '(retrieve (Specimen) "Patient"))))))) @@ -352,6 +436,20 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve "Medication" [["code" "system-225806|code-225809"]]))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve "Medication" [["code" "system-225806|code-225809"]]))) + (testing "form" (has-form expr '(retrieve "Medication" [["code" "system-225806|code-225809"]])))))) @@ -396,9 +494,7 @@ define InInitialPopulation: [\"name-133756\" -> Observation] ") - compile-context {::expr-cache/enabled? false} - {:keys [expression-defs]} (library/compile-library - node library compile-context) + {:keys [expression-defs]} (library/compile-library node library {}) db (d/db node) patient (ctu/resource db "Patient" "0") eval-context (assoc (eval-context db) :expression-defs expression-defs) @@ -413,8 +509,23 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve (singleton-from (retrieve-resource)) "Observation"))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve (singleton-from (retrieve-resource)) "Observation"))) + (testing "form" - (has-form expr '(retrieve (expr-ref "name-133756") "Observation")))))) + (has-form expr + '(retrieve (singleton-from (retrieve-resource)) "Observation")))))) (testing "with pre-compiled database query" (with-system-data [{:blaze.db/keys [node]} mem-node-config] @@ -443,9 +554,7 @@ define InInitialPopulation: [\"name-133730\" -> Observation: Code 'code-133657' from sys] ") - compile-context {::expr-cache/enabled? false} - {:keys [expression-defs]} (library/compile-library - node library compile-context) + {:keys [expression-defs]} (library/compile-library node library {}) db (d/db node) patient (ctu/resource db "Patient" "0") eval-context (assoc (eval-context db) :expression-defs expression-defs) @@ -460,9 +569,25 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) + '(retrieve (singleton-from (retrieve-resource)) "Observation" + [["code" "system-133620|code-133657"]]))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) + '(retrieve (singleton-from (retrieve-resource)) "Observation" + [["code" "system-133620|code-133657"]]))) + (testing "form" (has-form expr - '(retrieve (expr-ref "name-133730") "Observation" + '(retrieve (singleton-from (retrieve-resource)) "Observation" [["code" "system-133620|code-133657"]])))))) (testing "unknown code property" diff --git a/modules/cql/test/blaze/elm/compiler/interval_operators_test.clj b/modules/cql/test/blaze/elm/compiler/interval_operators_test.clj index 41e81c40b..5152577b3 100644 --- a/modules/cql/test/blaze/elm/compiler/interval_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/interval_operators_test.clj @@ -68,20 +68,36 @@ (testing "Static" (testing "Integer" (are [s e res] (= res (ctu/compile-binop elm/interval elm/integer s e)) - "1" "2" (interval 1 2))) + "1" "2" (interval 1 2)) + + (testing "form" + (has-form (interval 1 2) '(interval 1 2)))) (testing "Decimal" (are [s e res] (= res (ctu/compile-binop elm/interval elm/decimal s e)) - "1" "2" (interval 1M 2M))) + "1" "2" (interval 1M 2M)) + + (testing "form" + (has-form (interval 1M 2M) '(interval 1M 2M)))) (testing "Date" (are [s e res] (= res (ctu/compile-binop elm/interval elm/date s e)) - "2020" "2021" (interval #system/date"2020" #system/date"2021"))) + "2020" "2021" (interval #system/date"2020" #system/date"2021")) + + (testing "form" + (has-form + (interval #system/date"2020" #system/date"2021") + '(interval #system/date"2020" #system/date"2021")))) (testing "DateTime" (are [s e res] (= res (ctu/compile-binop elm/interval elm/date-time s e)) "2020" "2021" (interval #system/date-time"2020" #system/date-time"2021")) + (testing "form" + (has-form + (interval #system/date-time"2020" #system/date-time"2021") + '(interval #system/date-time"2020" #system/date-time"2021"))) + (testing "with ToDateTime" (are [s e res] (= res (c/compile {} (elm/interval [(elm/to-date-time (elm/date s)) (elm/date-time e)]))) "2020" "2021" (interval #system/date-time"2020" #system/date-time"2021")))) @@ -113,6 +129,10 @@ (are [elm] (thrown? Exception (core/-eval (c/compile {} elm) {} nil nil)) #elm/interval [#elm/integer "5" #elm/integer "3"])) + (testing "attach-cache" + (let [interval (interval 1 1)] + (is (= [interval] (st/with-instrument-disabled (c/attach-cache interval ::cache)))))) + (testing "form" (let [elm# (elm/interval [(elm/as ["{urn:hl7-org:elm-types:r1}Integer" #elm/parameter-ref "x"]) (elm/as ["{urn:hl7-org:elm-types:r1}Integer" #elm/parameter-ref "y"])]) @@ -211,9 +231,9 @@ "2019-04-17" "2019-04-17" false? "2019-04-17" "2019-04-18" false?) - (ctu/testing-binary-null elm/after #elm/date "2019") - (ctu/testing-binary-null elm/after #elm/date "2019-04") - (ctu/testing-binary-null elm/after #elm/date "2019-04-17") + (ctu/testing-binary-null elm/after #elm/date"2019") + (ctu/testing-binary-null elm/after #elm/date"2019-04") + (ctu/testing-binary-null elm/after #elm/date"2019-04-17") (testing "with year precision" (are [x y pred] (pred (ctu/compile-binop-precision elm/after elm/date x y "year")) @@ -254,15 +274,9 @@ "2019-04" "2019-05" false? "2019-04-17" "2019-04-16" false? "2019-04-17" "2019-04-17" false? - "2019-04-17" "2019-04-18" false?))) + "2019-04-17" "2019-04-18" false?)) - (ctu/testing-binary-dynamic elm/after) - - (ctu/testing-binary-precision-dynamic elm/after) - - (ctu/testing-binary-form elm/after) - - (ctu/testing-binary-precision-form elm/after)) + (ctu/testing-binary-precision-op elm/after))) ;; 19.3. Before ;; @@ -352,9 +366,9 @@ "2019-04-17" "2019-04-17" false? "2019-04-17" "2019-04-16" false?) - (ctu/testing-binary-null elm/before #elm/date "2019") - (ctu/testing-binary-null elm/before #elm/date "2019-04") - (ctu/testing-binary-null elm/before #elm/date "2019-04-17") + (ctu/testing-binary-null elm/before #elm/date"2019") + (ctu/testing-binary-null elm/before #elm/date"2019-04") + (ctu/testing-binary-null elm/before #elm/date"2019-04-17") (testing "with year precision" (are [x y pred] (pred (ctu/compile-binop-precision elm/before elm/date x y @@ -398,13 +412,7 @@ "2019-04-17" "2019-04-17" false? "2019-04-17" "2019-04-16" false?))) - (ctu/testing-binary-dynamic elm/before) - - (ctu/testing-binary-precision-dynamic elm/before) - - (ctu/testing-binary-form elm/before) - - (ctu/testing-binary-precision-form elm/before)) + (ctu/testing-binary-precision-op elm/before)) ;; 19.4. Collapse ;; @@ -461,9 +469,7 @@ {:type "Null"} [(interval #system/date-time"2012-01-01" #system/date-time"2012-05-25")])) - (ctu/testing-binary-dynamic elm/collapse) - - (ctu/testing-binary-form elm/collapse)) + (ctu/testing-binary-op elm/collapse)) ;; 19.5. Contains ;; @@ -530,13 +536,7 @@ #elm/list [] {:type "Null"} nil?)) - (ctu/testing-binary-dynamic elm/contains) - - (ctu/testing-binary-precision-dynamic elm/contains) - - (ctu/testing-binary-form elm/contains) - - (ctu/testing-binary-precision-form elm/contains)) + (ctu/testing-binary-precision-op elm/contains)) ;; 19.6. End ;; @@ -573,9 +573,7 @@ (ctu/testing-unary-null elm/end) - (ctu/testing-unary-dynamic elm/end) - - (ctu/testing-unary-form elm/end)) + (ctu/testing-unary-op elm/end)) ;; 19.7. Ends ;; @@ -608,13 +606,7 @@ (ctu/testing-binary-null elm/ends interval-zero) - (ctu/testing-binary-dynamic elm/ends) - - (ctu/testing-binary-precision-dynamic elm/ends) - - (ctu/testing-binary-form elm/ends) - - (ctu/testing-binary-precision-form elm/ends)) + (ctu/testing-binary-precision-op elm/ends)) ;; 19.8. Equal ;; @@ -663,9 +655,7 @@ (ctu/testing-binary-null elm/except interval-zero)) - (ctu/testing-binary-dynamic elm/except) - - (ctu/testing-binary-form elm/except)) + (ctu/testing-binary-op elm/except)) ;; 19.11. Expand ;; @@ -755,13 +745,7 @@ (ctu/testing-binary-null elm/includes interval-zero)) - (ctu/testing-binary-dynamic elm/includes) - - (ctu/testing-binary-precision-dynamic elm/includes) - - (ctu/testing-binary-form elm/includes) - - (ctu/testing-binary-precision-form elm/includes)) + (ctu/testing-binary-precision-op elm/includes)) ;; 19.14. IncludedIn ;; @@ -824,9 +808,7 @@ (ctu/testing-binary-null elm/intersect interval-zero)) - (ctu/testing-binary-dynamic elm/intersect) - - (ctu/testing-binary-form elm/intersect)) + (ctu/testing-binary-op elm/intersect)) ;; 19.16. Meets ;; @@ -857,13 +839,7 @@ (ctu/testing-binary-null elm/meets-before interval-zero) - (ctu/testing-binary-dynamic elm/meets-before) - - (ctu/testing-binary-precision-dynamic elm/meets-before) - - (ctu/testing-binary-form elm/meets-before) - - (ctu/testing-binary-precision-form elm/meets-before)) + (ctu/testing-binary-precision-op elm/meets-before)) ;; 19.18. MeetsAfter ;; @@ -887,13 +863,7 @@ (ctu/testing-binary-null elm/meets-after interval-zero) - (ctu/testing-binary-dynamic elm/meets-after) - - (ctu/testing-binary-precision-dynamic elm/meets-after) - - (ctu/testing-binary-form elm/meets-after) - - (ctu/testing-binary-precision-form elm/meets-after)) + (ctu/testing-binary-precision-op elm/meets-after)) ;; 19.20. Overlaps ;; @@ -947,13 +917,7 @@ (ctu/testing-binary-null elm/overlaps interval-zero) - (ctu/testing-binary-dynamic elm/overlaps) - - (ctu/testing-binary-precision-dynamic elm/overlaps) - - (ctu/testing-binary-form elm/overlaps) - - (ctu/testing-binary-precision-form elm/overlaps)) + (ctu/testing-binary-precision-op elm/overlaps)) ;; 19.21. OverlapsBefore ;; @@ -1004,9 +968,7 @@ (ctu/testing-unary-null elm/point-from) - (ctu/testing-unary-dynamic elm/point-from) - - (ctu/testing-unary-form elm/point-from)) + (ctu/testing-unary-op elm/point-from)) ;; 19.24. ProperContains ;; @@ -1047,15 +1009,11 @@ #elm/list [#elm/integer "1"] #elm/integer "2" false? #elm/list [#elm/integer "1" #elm/integer "2"] #elm/integer "1" true? - #elm/list [#elm/integer "1" #elm/integer "2"] #elm/integer "2" true?))) + #elm/list [#elm/integer "1" #elm/integer "2"] #elm/integer "2" true?)) - (ctu/testing-binary-dynamic elm/proper-contains) + (ctu/testing-binary-null elm/proper-contains #elm/list [])) - (ctu/testing-binary-precision-dynamic elm/proper-contains) - - (ctu/testing-binary-form elm/proper-contains) - - (ctu/testing-binary-precision-form elm/proper-contains)) + (ctu/testing-binary-precision-op elm/proper-contains)) ;; 19.25. ProperIn ;; @@ -1094,13 +1052,7 @@ (ctu/testing-binary-null elm/proper-includes interval-zero)) - (ctu/testing-binary-dynamic elm/proper-includes) - - (ctu/testing-binary-precision-dynamic elm/proper-includes) - - (ctu/testing-binary-form elm/proper-includes) - - (ctu/testing-binary-precision-form elm/proper-includes)) + (ctu/testing-binary-precision-op elm/proper-includes)) ;; 19.27. ProperIncludedIn ;; @@ -1151,9 +1103,7 @@ (ctu/testing-unary-null elm/start) - (ctu/testing-unary-dynamic elm/start) - - (ctu/testing-unary-form elm/start)) + (ctu/testing-unary-op elm/start)) ;; 19.30. Starts ;; @@ -1179,13 +1129,7 @@ (ctu/testing-binary-null elm/starts interval-zero) - (ctu/testing-binary-dynamic elm/starts) - - (ctu/testing-binary-precision-dynamic elm/starts) - - (ctu/testing-binary-form elm/starts) - - (ctu/testing-binary-precision-form elm/starts)) + (ctu/testing-binary-precision-op elm/starts)) ;; 19.31. Union ;; @@ -1220,9 +1164,7 @@ (ctu/testing-binary-null elm/union interval-zero)) - (ctu/testing-binary-dynamic elm/union) - - (ctu/testing-binary-form elm/union)) + (ctu/testing-binary-op elm/union)) ;; 19.32. Width ;; @@ -1240,6 +1182,4 @@ (ctu/testing-unary-null elm/width) - (ctu/testing-unary-dynamic elm/width) - - (ctu/testing-unary-form elm/width)) + (ctu/testing-unary-op elm/width)) diff --git a/modules/cql/test/blaze/elm/compiler/library/resolve_refs_test.clj b/modules/cql/test/blaze/elm/compiler/library/resolve_refs_test.clj new file mode 100644 index 000000000..dd3362f03 --- /dev/null +++ b/modules/cql/test/blaze/elm/compiler/library/resolve_refs_test.clj @@ -0,0 +1,27 @@ +(ns blaze.elm.compiler.library.resolve-refs-test + (:require + [blaze.elm.compiler.core :as core] + [blaze.elm.compiler.library-spec] + [blaze.elm.compiler.library.resolve-refs :refer [resolve-refs]] + [blaze.elm.compiler.macros :refer [reify-expr]] + [blaze.fhir.spec.type.system] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest testing]] + [cognitect.anomalies :as anom] + [juxt.iota :refer [given]])) + +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(def ^:private expr-a-ref-b + (reify-expr core/Expression + (-form [_] + (list 'a (list 'expr-ref "b"))))) + +(deftest resolve-refs-test + (testing "with one unresolvable ref" + (given (resolve-refs #{} {"a" {:expression expr-a-ref-b}}) + ::anom/category := ::anom/incorrect + ::anom/message := "The following expression definitions contain unresolvable references: a."))) diff --git a/modules/cql/test/blaze/elm/compiler/library_test.clj b/modules/cql/test/blaze/elm/compiler/library_test.clj index a564b6e48..b16971a93 100644 --- a/modules/cql/test/blaze/elm/compiler/library_test.clj +++ b/modules/cql/test/blaze/elm/compiler/library_test.clj @@ -3,8 +3,10 @@ [blaze.cql-translator :as t] [blaze.db.api-stub :refer [mem-node-config]] [blaze.elm.compiler :as c] + [blaze.elm.compiler-spec] [blaze.elm.compiler.library :as library] [blaze.elm.compiler.library-spec] + [blaze.elm.expression.cache :as ec] [blaze.fhir.spec.type.system] [blaze.module.test-util :refer [with-system]] [blaze.test-util :as tu] @@ -17,7 +19,9 @@ (test/use-fixtures :each tu/fixture) -(def default-opts {}) +(def ^:private default-opts {}) + +(def ^:private expr-form (comp c/form :expression)) ;; 5.1. Library ;; @@ -65,9 +69,10 @@ (testing "one static expression" (let [library (t/translate "library Test define Foo: true")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library default-opts) - [:expression-defs "Foo" :context] := "Patient" - [:expression-defs "Foo" :expression] := true)))) + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["Foo" :context] := "Patient" + ["Foo" :expression] := true))))) (testing "one dynamic expression" (let [library (t/translate "library Test @@ -75,11 +80,10 @@ context Patient define Gender: Patient.gender")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library default-opts) - [:expression-defs "Patient" :context] := "Patient" - [:expression-defs "Patient" :expression c/form] := '(singleton-from (retrieve-resource)) - [:expression-defs "Gender" :context] := "Patient" - [:expression-defs "Gender" :expression c/form] := '(:gender (expr-ref "Patient")))))) + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["Gender" :context] := "Patient" + ["Gender" expr-form] := '(:gender (singleton-from (retrieve-resource)))))))) (testing "one function" (let [library (t/translate "library Test @@ -88,13 +92,16 @@ define function Gender(P Patient): P.gender define InInitialPopulation: Gender(Patient)")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library default-opts) - [:expression-defs "InInitialPopulation" :context] := "Patient" - [:expression-defs "InInitialPopulation" :resultTypeName] := "{http://hl7.org/fhir}AdministrativeGender" - [:expression-defs "InInitialPopulation" :expression c/form] := '(call "Gender" (expr-ref "Patient")) - [:function-defs "Gender" :context] := "Patient" - [:function-defs "Gender" :resultTypeName] := "{http://hl7.org/fhir}AdministrativeGender" - [:function-defs "Gender" :function] :? fn?)))) + (let [{:keys [expression-defs function-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["InInitialPopulation" :context] := "Patient" + ["InInitialPopulation" :resultTypeName] := "{http://hl7.org/fhir}AdministrativeGender" + ["InInitialPopulation" expr-form] := '(call "Gender" (singleton-from (retrieve-resource)))) + + (given function-defs + ["Gender" :context] := "Patient" + ["Gender" :resultTypeName] := "{http://hl7.org/fhir}AdministrativeGender" + ["Gender" :function] :? fn?))))) (testing "two functions, one calling the other" (let [library (t/translate "library Test @@ -104,31 +111,146 @@ define function Inc2(i System.Integer): Inc(i) + 1 define InInitialPopulation: Inc2(1)")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library default-opts) - [:expression-defs "InInitialPopulation" :context] := "Patient" - [:expression-defs "InInitialPopulation" :expression c/form] := '(call "Inc2" 1))))) + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["InInitialPopulation" :context] := "Patient" + ["InInitialPopulation" expr-form] := '(call "Inc2" 1)))))) + + (testing "expressions from Patient context are resolved" + (let [library (t/translate "library Test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define Female: + Patient.gender = 'female' + + define HasObservation: + exists [Observation] + + define HasCondition: + exists [Condition] + + define Inclusion: + Female and + HasObservation + + define Exclusion: + HasCondition + + define InInitialPopulation: + Inclusion and + not Exclusion")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["InInitialPopulation" :context] := "Patient" + ["InInitialPopulation" expr-form] := + '(and + (and + (equal + (call + "ToString" + (:gender + (singleton-from (retrieve-resource)))) + "female") + (exists + (retrieve + "Observation"))) + (not + (exists + (retrieve + "Condition"))))))))) (testing "expressions from Unfiltered context are not resolved" (let [library (t/translate "library Test - using FHIR version '4.0.0' - include FHIRHelpers version '4.0.0' + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' - codesystem atc: 'http://fhir.de/CodeSystem/dimdi/atc' + codesystem atc: 'http://fhir.de/CodeSystem/dimdi/atc' - context Unfiltered + context Unfiltered - define TemozolomidRefs: - [Medication: Code 'L01AX03' from atc] M return 'Medication/' + M.id + define TemozolomidRefs: + [Medication: Code 'L01AX03' from atc] M return 'Medication/' + M.id - context Patient + context Patient - define InInitialPopulation: - exists from [MedicationStatement] M - where M.medication.reference in TemozolomidRefs")] + define InInitialPopulation: + Patient.gender = 'female' and + exists from [MedicationStatement] M + where M.medication.reference in TemozolomidRefs")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library default-opts) - [:expression-defs "InInitialPopulation" :context] := "Patient" - [:expression-defs "InInitialPopulation" :expression c/form] := '(exists (eduction-query (comp (filter (fn [M] (contains (expr-ref "TemozolomidRefs") (call "ToString" (:reference (:medication M)))))) distinct) (retrieve "MedicationStatement"))))))) + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["TemozolomidRefs" :context] := "Unfiltered" + ["TemozolomidRefs" expr-form] := + '(vector-query + (comp + (map + (fn [M] + (concatenate "Medication/" (call "ToString" (:id M))))) + distinct) + (retrieve + "Medication" + [["code" + "http://fhir.de/CodeSystem/dimdi/atc|L01AX03"]])) + + ["InInitialPopulation" :context] := "Patient" + ["InInitialPopulation" expr-form] := + '(and + (equal + (call + "ToString" + (:gender + (singleton-from + (retrieve-resource)))) + "female") + (exists + (eduction-query + (comp + (filter + (fn + [M] + (contains + (expr-ref + "TemozolomidRefs") + (call + "ToString" + (:reference + (:medication + M)))))) + distinct) + (retrieve + "MedicationStatement"))))))))) + + (testing "expressions without refs are preserved" + (let [library (t/translate "library Retrieve + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define InInitialPopulation: + true + + define AllEncounters: + [Encounter] + + define Gender: + Patient.gender")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [{:keys [expression-defs]} (library/compile-library node library default-opts)] + (given expression-defs + ["Patient" :context] := "Patient" + ["Patient" expr-form] := '(singleton-from (retrieve-resource)) + ["InInitialPopulation" :context] := "Patient" + ["InInitialPopulation" expr-form] := true + ["AllEncounters" :context] := "Patient" + ["AllEncounters" expr-form] := '(retrieve "Encounter") + ["Gender" :context] := "Patient" + ["Gender" expr-form] := '(:gender (singleton-from (retrieve-resource)))))))) (testing "with compile-time error" (testing "function" @@ -177,7 +299,7 @@ (with-system [{:blaze.db/keys [node]} mem-node-config] (given (library/compile-library node library {}) [:expression-defs "InInitialPopulation" :context] := "Patient" - [:expression-defs "InInitialPopulation" :expression c/form] := + [:expression-defs "InInitialPopulation" expr-form] := '(exists (eduction-query (comp @@ -204,4 +326,280 @@ distinct) (retrieve "Observation"))) - [:function-defs "hasDiagnosis" :function] := nil))))) + [:function-defs "hasDiagnosis" :function] := nil)))) + + (testing "with related context" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define \"name-133756\": + singleton from ([Patient]) + + define InInitialPopulation: + [\"name-133756\" -> Observation]")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (given (library/compile-library node library {}) + [:expression-defs "InInitialPopulation" :context] := "Patient" + [:expression-defs "InInitialPopulation" expr-form] := + '(retrieve + (singleton-from + (retrieve-resource)) + "Observation"))))) + + (testing "and expression" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define InInitialPopulation: + exists [Observation] and + exists [Condition] and + exists [Encounter] and + exists [Specimen]")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (given (library/compile-library node library {}) + [:expression-defs "InInitialPopulation" :context] := "Patient" + [:expression-defs "InInitialPopulation" expr-form] := + '(and + (and + (and + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (exists + (retrieve + "Encounter"))) + (exists + (retrieve + "Specimen"))))))) + + (testing "and expression with named expressions" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define Criterion_1: + exists [Observation] and + exists [Condition] + + define Criterion_2: + exists [Encounter] and + exists [Specimen] + + define InInitialPopulation: + Criterion_1 and + Criterion_2")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [{:keys [expression-defs]} (library/compile-library node library default-opts) + in-initial-population (get expression-defs "InInitialPopulation")] + + (testing "after compilation the named expressions are resolved + the structure however is retained + so the and-expressions are still binary" + (given in-initial-population + :context := "Patient" + expr-form := + '(and + (and + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (and + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen")))))) + + (testing "after attaching the cache we get one single, flat and-expression" + (with-redefs [ec/get (fn [_ _])] + (let [{:keys [expression]} in-initial-population] + (given (st/with-instrument-disabled (c/attach-cache expression ::cache)) + [0 c/form] := + '(and + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition")) + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen"))))))))))) + + (testing "or expression" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define InInitialPopulation: + exists [Observation] or + exists [Condition] or + exists [Encounter] or + exists [Specimen]")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (given (library/compile-library node library {}) + [:expression-defs "InInitialPopulation" :context] := "Patient" + [:expression-defs "InInitialPopulation" expr-form] := + '(or + (or + (or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (exists + (retrieve + "Encounter"))) + (exists + (retrieve + "Specimen"))))))) + + (testing "or expression with named expressions" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define Criterion_1: + exists [Observation] or + exists [Condition] + + define Criterion_2: + exists [Encounter] or + exists [Specimen] + + define InInitialPopulation: + Criterion_1 or + Criterion_2")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [{:keys [expression-defs]} (library/compile-library node library default-opts) + in-initial-population (get expression-defs "InInitialPopulation")] + + (testing "after compilation the named expressions are resolved + the structure however is retained + so the or-expressions are still binary" + (given in-initial-population + :context := "Patient" + expr-form := + '(or + (or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (or + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen")))))) + + (testing "after attaching the cache we get one single, flat or-expression" + (with-redefs [ec/get (fn [_ _])] + (let [{:keys [expression]} in-initial-population] + (given (st/with-instrument-disabled (c/attach-cache expression ::cache)) + [0 c/form] := + '(or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition")) + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen"))))))))))) + + (testing "mixed and and or expressions" + (let [library (t/translate "library test + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define Criterion_1: + exists [Observation] or + exists [Condition] + + define Criterion_2: + exists [Encounter] or + exists [Specimen] + + define InInitialPopulation: + Criterion_1 and + Criterion_2")] + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [{:keys [expression-defs]} (library/compile-library node library default-opts) + in-initial-population (get expression-defs "InInitialPopulation")] + + (testing "after compilation the named expressions are resolved + the structure however is retained + so the or-expressions are still binary" + (given in-initial-population + :context := "Patient" + expr-form := + '(and + (or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (or + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen")))))) + + (testing "after attaching the cache the expressions don't change" + (with-redefs [ec/get (fn [_ _])] + (let [{:keys [expression]} in-initial-population] + (given (st/with-instrument-disabled (c/attach-cache expression ::cache)) + [0 c/form] := + '(and + (or + (exists + (retrieve + "Observation")) + (exists + (retrieve + "Condition"))) + (or + (exists + (retrieve + "Encounter")) + (exists + (retrieve + "Specimen"))))))))))))) diff --git a/modules/cql/test/blaze/elm/compiler/list_operators_test.clj b/modules/cql/test/blaze/elm/compiler/list_operators_test.clj index 4f3c7b306..eb5bc6b47 100644 --- a/modules/cql/test/blaze/elm/compiler/list_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/list_operators_test.clj @@ -4,23 +4,39 @@ Section numbers are according to https://cql.hl7.org/04-logicalspecification.html." (:require + [blaze.anomaly :as ba] [blaze.anomaly-spec] + [blaze.db.api :as d] + [blaze.db.api-stub :refer [mem-node-config with-system-data]] [blaze.elm.compiler :as c] [blaze.elm.compiler-spec] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] - [blaze.elm.compiler.list-operators] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] + [blaze.elm.expression :as expr] [blaze.elm.expression-spec] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache-spec] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] - [blaze.elm.quantity :as quantity] - [blaze.test-util :refer [satisfies-prop]] + [blaze.elm.quantity :refer [quantity]] + [blaze.elm.resource :as cr] + [blaze.elm.spec :as elm-spec] + [blaze.fhir.test-util] + [blaze.module.test-util :refer [with-system]] + [blaze.test-util :refer [given-thrown satisfies-prop]] [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [are deftest is testing]] - [clojure.test.check.properties :as prop])) + [clojure.test.check.properties :as prop] + [integrant.core :as ig] + [juxt.iota :refer [given]]) + (:import + [java.time OffsetDateTime])) +(set! *warn-on-reflection* true) (st/instrument) (ctu/instrument-compile) @@ -64,9 +80,53 @@ #elm/list [#elm/parameter-ref "1" #elm/parameter-ref "nil"] [1 nil] #elm/list [#elm/parameter-ref "1" #elm/parameter-ref "2"] [1 2]) - (testing "form" - (let [expr (ctu/dynamic-compile #elm/list [#elm/parameter-ref "x"])] - (has-form expr '(list (param-ref "x"))))))) + (let [expr (ctu/dynamic-compile #elm/list [#elm/parameter-ref "x"])] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {"x" 1}) '(list 1))) + + (testing "form" + (has-form expr '(list (param-ref "x")))))) + + (testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (testing "with one element" + (let [elm (elm/list [#elm/exists #elm/retrieve{:type "Observation"}]) + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 1 + [1 0] := '(exists (retrieve "Observation"))))) + + (testing "with two elements" + (let [elm (elm/list [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}]) + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 2 + [1 0] := '(exists (retrieve "Observation")) + [1 1] := '(exists (retrieve "Condition"))))))) + + (testing "resolve expression references" + (let [elm #elm/list [#elm/expression-ref "x"] + expr-def {:type "ExpressionDef" :name "x" :expression 1 + :context "Patient"} + ctx {:library {:statements {:def [expr-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-def})] + (has-form expr '(list 1)))) + + (ctu/testing-equals-hash-code #elm/list [#elm/parameter-ref "1"])) ;; 20.2. Contains ;; @@ -86,9 +146,24 @@ (prop/for-all [x (s/gen int?)] (= x (core/-eval (c/compile {} #elm/current nil) {} nil x)))) - (testing "form" - (let [expr (c/compile {} #elm/current nil)] - (has-form expr 'current)))) + (let [expr (c/compile {} #elm/current nil)] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) 'current)) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) 'current)) + + (testing "form" + (has-form expr 'current))) + + (ctu/testing-equals-hash-code #elm/current nil)) (testing "named scope" (satisfies-prop 100 @@ -97,9 +172,24 @@ (let [expr (c/compile {} (elm/current scope))] (= x (core/-eval expr {} nil {scope x}))))) - (testing "form" - (let [expr (c/compile {} #elm/current "x")] - (has-form expr '(current "x")))))) + (let [expr (c/compile {} #elm/current "x")] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (has-form (c/resolve-refs expr {}) '(current "x"))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {}) '(current "x"))) + + (testing "form" + (has-form expr '(current "x")))) + + (ctu/testing-equals-hash-code #elm/current "x"))) ;; 20.4. Distinct ;; @@ -121,14 +211,12 @@ #elm/list [{:type "Null"}] [nil] #elm/list [{:type "Null"} {:type "Null"}] [nil nil] #elm/list [{:type "Null"} {:type "Null"} {:type "Null"}] [nil nil nil] - #elm/list [#elm/quantity [100 "cm"] #elm/quantity [1 "m"]] [(quantity/quantity 100 "cm")] - #elm/list [#elm/quantity [1 "m"] #elm/quantity [100 "cm"]] [(quantity/quantity 1 "m")]) + #elm/list [#elm/quantity [100 "cm"] #elm/quantity [1 "m"]] [(quantity 100 "cm")] + #elm/list [#elm/quantity [1 "m"] #elm/quantity [100 "cm"]] [(quantity 1 "m")]) (ctu/testing-unary-null elm/distinct) - (ctu/testing-unary-dynamic elm/distinct) - - (ctu/testing-unary-form elm/distinct)) + (ctu/testing-unary-op elm/distinct)) ;; 20.5. Equal ;; @@ -144,9 +232,29 @@ ;; 20.8. Exists ;; -;; The Exists operator returns true if the list contains any elements. +;; The Exists operator returns true if the list contains any non-null elements. ;; ;; If the argument is null, the result is false. +(def ^:private exists-config + (assoc mem-node-config + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} + :blaze.test/executor {})) + +(defmethod elm-spec/expression :elm.spec.type/exists-test [_] + map?) + +(defmethod core/compile* :elm.compiler.type/exists-test + [_ _] + (reify-expr core/Expression + (-attach-cache [_ _] + [(reify-expr core/Expression + (-form [_] + 'exists-test-with-cache))]) + (-form [_] + 'exists-test))) + (deftest compile-exists-test (testing "Static" (are [list res] (= res (c/compile {} (elm/exists list))) @@ -166,7 +274,109 @@ {:type "Null"} false) - (ctu/testing-unary-form elm/exists))) + (testing "doesn't attach the cache downstream because there is no need for double caching" + (with-system [{::expr/keys [cache]} exists-config] + (let [elm #elm/exists {:type "ExistsTest"} + expr (c/compile {:eval-context "Patient"} elm) + ;; ensure Bloom filter is available + _ (do (c/attach-cache expr cache) (Thread/sleep 100)) + expr (first (c/attach-cache expr cache))] + + (has-form expr '(exists exists-test)) + + (testing "also not at a second cache attachment" + (has-form (first (c/attach-cache expr cache)) '(exists exists-test)))))) + + (testing "with caching expressions" + (with-system-data [{:blaze.db/keys [node] ::expr/keys [cache]} exists-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [db (d/db node) + patient (cr/mk-resource db (d/resource-handle db "Patient" "0")) + elm #elm/exists #elm/retrieve{:type "Observation"} + compile-context + {:node node + :eval-context "Patient" + :library {}} + expr (c/compile compile-context elm) + eval-context + {:db db + :now (OffsetDateTime/now)}] + + (testing "has no Observation at the beginning" + (let [[expr bloom-filters] (c/attach-cache expr cache)] + (is (every? ba/unavailable? bloom-filters)) + (is (false? (expr/eval eval-context expr patient))))) + + (Thread/sleep 100) + + (testing "has still no Observation after the Bloom filter is filled" + (let [[expr bloom-filters] (c/attach-cache expr cache)] + (given bloom-filters + count := 1 + [0 ::bloom-filter/expr-form] := (pr-str '(exists (retrieve "Observation"))) + [0 ::bloom-filter/patient-count] := 0) + (is (false? (expr/eval eval-context expr patient))))) + + (let [tx-op [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}] + db-after @(d/transact node [tx-op])] + + (testing "has an Observation after transaction" + (let [[expr bloom-filters] (c/attach-cache expr cache) + patient (cr/mk-resource db-after (d/resource-handle db "Patient" "0"))] + (testing "even so the bloom filter is still the old one" + (given bloom-filters + count := 1 + [0 ::bloom-filter/expr-form] := (pr-str '(exists (retrieve "Observation"))) + [0 ::bloom-filter/patient-count] := 0)) + (is (true? (expr/eval (assoc eval-context :db db-after) expr patient))))) + + (testing "has still no Observation at the old database" + (let [[expr] (c/attach-cache expr cache)] + (is (false? (expr/eval eval-context expr patient))))))))) + + (testing "without caching expressions" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [db (d/db node) + patient (cr/mk-resource db (d/resource-handle db "Patient" "0")) + elm #elm/exists #elm/retrieve{:type "Observation"} + compile-context + {:node node + :eval-context "Patient" + :library {}} + expr (c/compile compile-context elm) + eval-context + {:db db + :now (OffsetDateTime/now)}] + + (testing "has no Observation at the beginning" + (is (false? (expr/eval eval-context expr patient)))) + + (let [tx-op [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}] + db-after @(d/transact node [tx-op])] + + (testing "has an Observation after transaction" + (let [patient (cr/mk-resource db-after (d/resource-handle db "Patient" "0"))] + (is (true? (expr/eval (assoc eval-context :db db-after) expr patient))))) + + (testing "has still no Observation at the old database" + (is (false? (expr/eval eval-context expr patient))))))))) + + (ctu/testing-unary-dynamic elm/exists) + + (ctu/testing-unary-patient-count elm/exists) + + (ctu/testing-unary-resolve-refs elm/exists) + + (ctu/testing-unary-resolve-params elm/exists) + + (ctu/testing-unary-equals-hash-code elm/exists) + + (ctu/testing-unary-form elm/exists)) ;; 20.9. Filter ;; @@ -196,25 +406,82 @@ {:type "Null"} #elm/boolean "true" nil)))) - (testing "form and static" - (testing "with scope" - (let [expr (ctu/dynamic-compile {:type "Filter" - :source #elm/parameter-ref "x" - :condition #elm/parameter-ref "y" - :scope "A"})] - - (has-form expr '(filter (param-ref "x") (param-ref "y") "A")) - - (is (false? (core/-static expr))))) - - (testing "without scope" - (let [expr (ctu/dynamic-compile {:type "Filter" - :source #elm/parameter-ref "x" - :condition #elm/parameter-ref "y"})] - - (has-form expr '(filter (param-ref "x") (param-ref "y"))) - - (is (false? (core/-static expr))))))) + (testing "with scope" + (let [expr (ctu/dynamic-compile {:type "Filter" + :source #elm/parameter-ref "x" + :condition #elm/parameter-ref "y" + :scope "A"})] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm {:type "Filter" + :source #elm/expression-ref "x" + :condition #elm/expression-ref "y" + :scope "A"} + source-def {:type "ExpressionDef" :name "x" :expression [1] + :context "Patient"} + condition-def {:type "ExpressionDef" :name "y" :expression true + :context "Patient"} + ctx {:library {:statements {:def [source-def condition-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" source-def "y" condition-def})] + (has-form expr '(filter [1] true "A")))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {"x" [1]}) + '(filter [1] (param-ref "y") "A"))) + + (testing "form" + (has-form expr '(filter (param-ref "x") (param-ref "y") "A"))) + + (ctu/testing-equals-hash-code {:type "Filter" + :source #elm/parameter-ref "x" + :condition #elm/parameter-ref "y" + :scope "A"}))) + + (testing "without scope" + (let [expr (ctu/dynamic-compile {:type "Filter" + :source #elm/parameter-ref "x" + :condition #elm/parameter-ref "y"})] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm {:type "Filter" + :source #elm/expression-ref "x" + :condition #elm/expression-ref "y"} + source-def {:type "ExpressionDef" :name "x" :expression [1] + :context "Patient"} + condition-def {:type "ExpressionDef" :name "y" :expression true + :context "Patient"} + ctx {:library {:statements {:def [source-def condition-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" source-def "y" condition-def})] + (has-form expr '(filter [1] true)))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {"x" [1]}) + '(filter [1] (param-ref "y")))) + + (testing "form" + (has-form expr '(filter (param-ref "x") (param-ref "y")))) + + (ctu/testing-equals-hash-code {:type "Filter" + :source #elm/parameter-ref "x" + :condition #elm/parameter-ref "y"})))) ;; 20.10. First ;; @@ -236,9 +503,7 @@ (ctu/testing-unary-null elm/first) - (ctu/testing-unary-dynamic elm/first) - - (ctu/testing-unary-form elm/first)) + (ctu/testing-unary-op elm/first)) ;; 20.11. Flatten ;; @@ -256,9 +521,7 @@ (ctu/testing-unary-null elm/flatten) - (ctu/testing-unary-dynamic elm/flatten) - - (ctu/testing-unary-form elm/flatten)) + (ctu/testing-unary-op elm/flatten)) ;; 20.12. ForEach ;; @@ -293,25 +556,78 @@ {:type "Null"} {:type "Null"} nil)))) - (testing "form and static" - (testing "with scope" - (let [expr (ctu/dynamic-compile {:type "ForEach" - :source #elm/parameter-ref "x" - :element #elm/parameter-ref "y" - :scope "A"})] - - (has-form expr '(for-each (param-ref "x") (param-ref "y") "A")) - - (is (false? (core/-static expr))))) - - (testing "without scope" - (let [expr (ctu/dynamic-compile {:type "ForEach" - :source #elm/parameter-ref "x" - :element #elm/parameter-ref "y"})] - - (has-form expr '(for-each (param-ref "x") (param-ref "y"))) - - (is (false? (core/-static expr))))))) + (testing "with scope" + (let [expr (ctu/dynamic-compile {:type "ForEach" + :source #elm/parameter-ref "x" + :element #elm/parameter-ref "y" + :scope "A"})] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm {:type "ForEach" + :source #elm/expression-ref "x" + :element #elm/current "A" + :scope "A"} + expr-def {:type "ExpressionDef" :name "x" :expression [1] + :context "Patient"} + ctx {:library {:statements {:def [expr-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-def})] + (has-form expr '(for-each [1] (current "A") "A")))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {"x" [1]}) + '(for-each [1] (param-ref "y") "A"))) + + (testing "form" + (has-form expr '(for-each (param-ref "x") (param-ref "y") "A"))) + + (ctu/testing-equals-hash-code {:type "ForEach" + :source #elm/parameter-ref "x" + :element #elm/parameter-ref "y" + :scope "A"}))) + + (testing "without scope" + (let [expr (ctu/dynamic-compile {:type "ForEach" + :source #elm/parameter-ref "x" + :element #elm/parameter-ref "y"})] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm {:type "ForEach" + :source #elm/expression-ref "x" + :element #elm/current nil} + expr-defs {:type "ExpressionDef" :name "x" :expression [1] + :context "Patient"} + ctx {:library {:statements {:def [expr-defs]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-defs})] + (has-form expr '(for-each [1] current)))) + + (testing "resolve parameters" + (has-form (c/resolve-params expr {"x" [1]}) + '(for-each [1] (param-ref "y")))) + + (testing "form" + (has-form expr '(for-each (param-ref "x") (param-ref "y")))) + + (ctu/testing-equals-hash-code {:type "ForEach" + :source #elm/parameter-ref "x" + :element #elm/parameter-ref "y"})))) ;; 20.13. In ;; @@ -350,9 +666,7 @@ (ctu/testing-binary-dynamic-null elm/index-of #elm/list [] #elm/integer "1") - (ctu/testing-binary-dynamic elm/index-of) - - (ctu/testing-binary-form elm/index-of)) + (ctu/testing-binary-op elm/index-of)) ;; 20.17. Intersect ;; @@ -378,9 +692,7 @@ (ctu/testing-unary-null elm/last) - (ctu/testing-unary-dynamic elm/last) - - (ctu/testing-unary-form elm/last)) + (ctu/testing-unary-op elm/last)) ;; 20.19. Not Equal ;; @@ -430,14 +742,15 @@ #elm/list [#elm/integer "1"] 1 {:type "Null"} nil) - (are [list] (thrown? Exception (core/-eval (c/compile {} (elm/singleton-from list)) {} nil nil)) - #elm/list [#elm/integer "1" #elm/integer "1"]) + (let [expr #elm/singleton-from #elm/list [#elm/integer "1" #elm/integer "1"]] + (given-thrown (core/-eval (c/compile {} expr) {} nil nil) + :message := "More than one element in `SingletonFrom` expression." + [:expression :type] := "SingletonFrom" + [:expression :operand :type] := "List")) (ctu/testing-unary-null elm/singleton-from) - (ctu/testing-unary-dynamic elm/singleton-from) - - (ctu/testing-unary-form elm/singleton-from)) + (ctu/testing-unary-op elm/singleton-from)) ;; 20.26. Slice ;; @@ -466,17 +779,7 @@ {:type "Null"} #elm/integer "0" #elm/integer "0" nil {:type "Null"} {:type "Null"} {:type "Null"} nil) - (let [expr (ctu/dynamic-compile {:type "Slice" - :source #elm/parameter-ref "x" - :startIndex #elm/parameter-ref "y" - :endIndex #elm/parameter-ref "z"})] - - (testing "expression is dynamic" - (is (false? (core/-static expr)))) - - (testing "form" - (is (= '(slice (param-ref "x") (param-ref "y") (param-ref "z")) - (core/-form expr)))))) + (ctu/testing-ternary-op elm/slice)) ;; 20.27. Sort ;; @@ -506,7 +809,7 @@ (is (false? (core/-static expr)))) (testing "form" - (is (= '(sort (param-ref "x") :asc) (core/-form expr)))))) + (has-form expr '(sort (param-ref "x") :asc))))) ;; 20.28. Times ;; @@ -565,9 +868,7 @@ (ctu/testing-binary-null elm/times #elm/list[#elm/tuple{"name" #elm/string "hans"}]) - (ctu/testing-binary-dynamic elm/times) - - (ctu/testing-binary-form elm/times)) + (ctu/testing-binary-op elm/times)) ;; 20.29. Union ;; diff --git a/modules/cql/test/blaze/elm/compiler/logical_operators/util_test.clj b/modules/cql/test/blaze/elm/compiler/logical_operators/util_test.clj new file mode 100644 index 000000000..628405b9b --- /dev/null +++ b/modules/cql/test/blaze/elm/compiler/logical_operators/util_test.clj @@ -0,0 +1,132 @@ +(ns blaze.elm.compiler.logical-operators.util-test + (:require + [blaze.elm.compiler.logical-operators.util :as u] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [blaze.elm.literal-spec] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest is testing]])) + +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(deftest attach-cache-result-test + (testing "empty input" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity nil)))] + (is (empty? tuples)) + (is (empty? bfs)))) + + (testing "with one triple" + (testing "with direct Bloom filter" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op [::bf] ::bf]])))] + (testing "the direct Bloom filter of the triple is also the merged Bloom filter" + (is (= [[::op ::bf]] tuples))) + + (testing "the Bloom filter is returned in the collection of all Bloom filters" + (is (= [::bf] bfs))))) + + (testing "with one indirect Bloom filter" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op [::bf]]])))] + (testing "there is no merged Bloom filter" + (is (= [[::op nil]] tuples))) + + (testing "the Bloom filter is returned in the collection of all Bloom filters" + (is (= [::bf] bfs))))) + + (testing "with one direct and one indirect Bloom filter" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op [::indirect-bf ::direct-bf] ::direct-bf]])))] + (testing "the direct Bloom filter of the triple is also the merged Bloom filter" + (is (= [[::op ::direct-bf]] tuples))) + + (testing "the Bloom filter is returned in the collection of all Bloom filters" + (is (= [::indirect-bf ::direct-bf] bfs))))) + + (testing "with two indirect Bloom filters" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op [::bf-1 ::bf-2]]])))] + (testing "there is no merged Bloom filter" + (is (= [[::op nil]] tuples))) + + (testing "the Bloom filters are returned in the collection of all Bloom filters" + (is (= [::bf-1 ::bf-2] bfs)))))) + + (testing "with two triples" + (testing "only the second having a direct Bloom filter" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1]] [::op-2 [::bf-2] ::bf-2]])))] + (testing "there is no merged Bloom filter on the first tuple" + (is (= [[::op-1] [::op-2 ::bf-2]] tuples))) + + (testing "the indirect Bloom filter is also part of all Bloom filters" + (is (= [::bf-1 ::bf-2] bfs))))) + + (testing "only the first having a direct Bloom filter" + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2]]])))] + (testing "there are no merged Bloom filters because we need the second having one" + (is (= [[::op-1] [::op-2 nil]] tuples))) + + (testing "the indirect Bloom filter is also part of all Bloom filters" + (is (= [::bf-1 ::bf-2] bfs))))) + + (testing "with both having direct Bloom filters" + (with-redefs [bloom-filter/merge + (fn [bf-2 bf-1] + (assert (= ::bf-1 bf-1)) + (assert (= ::bf-2 bf-2)) + ::merged-bf)] + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2] ::bf-2]])))] + (testing "the first tuple has the merged Bloom filter while the second starts with the direct Bloom filter" + (is (= [[::op-1 ::merged-bf] [::op-2 ::bf-2]] tuples))) + + (testing "the merged Bloom filter is not part of all Bloom filters" + (is (= [::bf-1 ::bf-2] bfs))))) + + (testing "but merge returns nil" + (with-redefs [bloom-filter/merge + (fn [bf-2 bf-1] + (assert (= ::bf-1 bf-1)) + (assert (= ::bf-2 bf-2)) + nil)] + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2] ::bf-2]])))] + (testing "there is no merged Bloom filter at the first tuple" + (is (= [[::op-1 nil] [::op-2 ::bf-2]] tuples))) + + (testing "the merged Bloom filter is not part of all Bloom filters" + (is (= [::bf-1 ::bf-2] bfs)))))))) + + (testing "with three triples" + (testing "with all having direct Bloom filters" + (with-redefs [bloom-filter/merge + (fn [bf-a bf-b] + (cond + (and (= ::bf-3 bf-a) (= ::bf-2 bf-b)) + ::bf-m1 + (and (= ::bf-m1 bf-a) (= ::bf-1 bf-b)) + ::bf-m2))] + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2] ::bf-2] [::op-3 [::bf-3] ::bf-3]])))] + (testing "the first two tuples have the merged Bloom filters while the last starts with the direct Bloom filter" + (is (= [[::op-1 ::bf-m2] [::op-2 ::bf-m1] [::op-3 ::bf-3]] tuples))) + + (testing "the merged Bloom filter is not part of all Bloom filters" + (is (= [::bf-1 ::bf-2 ::bf-3] bfs))))) + + (testing "merge returns nil on the second merge" + (with-redefs [bloom-filter/merge + (fn [bf-a bf-b] + (cond + (and (= ::bf-3 bf-a) (= ::bf-2 bf-b)) + ::bf-m1))] + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2] ::bf-2] [::op-3 [::bf-3] ::bf-3]])))] + (testing "there is no merged Bloom filter at the first tuple" + (is (= [[::op-1 nil] [::op-2 ::bf-m1] [::op-3 ::bf-3]] tuples))) + + (testing "the merged Bloom filter is not part of all Bloom filters" + (is (= [::bf-1 ::bf-2 ::bf-3] bfs)))))) + + (testing "merge returns nil" + (with-redefs [bloom-filter/merge (fn [_ _])] + (let [[tuples bfs] ((first (u/or-attach-cache-result identity [[::op-1 [::bf-1] ::bf-1] [::op-2 [::bf-2] ::bf-2] [::op-3 [::bf-3] ::bf-3]])))] + (testing "there are no merged Bloom filter at the first two tuples" + (is (= [[::op-1] [::op-2 nil] [::op-3 ::bf-3]] tuples))) + + (testing "the merged Bloom filter is not part of all Bloom filters" + (is (= [::bf-1 ::bf-2 ::bf-3] bfs))))))))) diff --git a/modules/cql/test/blaze/elm/compiler/logical_operators_test.clj b/modules/cql/test/blaze/elm/compiler/logical_operators_test.clj index 9c230ae14..b21d40207 100644 --- a/modules/cql/test/blaze/elm/compiler/logical_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/logical_operators_test.clj @@ -4,14 +4,19 @@ Section numbers are according to https://cql.hl7.org/04-logicalspecification.html." (:require + [blaze.anomaly :as ba] [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] - [blaze.elm.compiler.logical-operators] + [blaze.elm.compiler.logical-operators :as ops] + [blaze.elm.compiler.macros :refer [reify-expr]] [blaze.elm.compiler.test-util :as ctu] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [are deftest testing]])) + [clojure.test :as test :refer [are deftest is testing]] + [juxt.iota :refer [given]])) (st/instrument) (ctu/instrument-compile) @@ -90,7 +95,7 @@ #elm/parameter-ref "a" {:type "Null"} '(and nil (param-ref "a")) #elm/parameter-ref "a" #elm/parameter-ref "b" '(and (param-ref "a") (param-ref "b")))) - (testing "static" + (testing "Static" (are [x y pred] (pred (core/-static (ctu/dynamic-compile (elm/and [x y])))) #elm/boolean "true" #elm/boolean "true" true? #elm/boolean "true" #elm/boolean "false" true? @@ -110,7 +115,174 @@ #elm/parameter-ref "a" #elm/boolean "true" false? #elm/parameter-ref "a" #elm/boolean "false" true? #elm/parameter-ref "a" {:type "Null"} false? - #elm/parameter-ref "a" #elm/parameter-ref "b" false?))) + #elm/parameter-ref "a" #elm/parameter-ref "b" false?)) + + (testing "attach cache" + (testing "only one bloom filter available" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (when-not (= '(exists (retrieve "Observation")) (c/form expr)) + (bloom-filter/build-bloom-filter expr 0 nil)))] + (let [elm #elm/and [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(and (exists (retrieve "Condition")) + (exists (retrieve "Observation"))) + [1 count] := 2 + [1 0 ::bloom-filter/expr-form] := "(exists (retrieve \"Condition\"))" + [1 1] := (ba/unavailable "No Bloom filter available."))))) + + (testing "both bloom filters available with first having more patients" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (cond + (= (c/form expr) '(exists (retrieve "Observation"))) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"]) + (= (c/form expr) '(exists (retrieve "Condition"))) + (bloom-filter/build-bloom-filter expr 0 ["0"])))] + (let [elm #elm/and [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Observation"}] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(and (exists (retrieve "Condition")) + (exists (retrieve "Observation"))) + [1 count] := 2 + [1 0 ::bloom-filter/patient-count] := 1 + [1 1 ::bloom-filter/patient-count] := 2)))) + + (testing "with three expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"]) + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0"]) + '(exists (retrieve "Specimen")) + nil))] + (let [elm #elm/and [#elm/exists #elm/retrieve{:type "Observation"} + #elm/and [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Specimen"}]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(and (exists (retrieve "Condition")) + (exists (retrieve "Observation")) + (exists (retrieve "Specimen"))) + [1 count] := 3 + [1 0 ::bloom-filter/patient-count] := 1 + [1 1 ::bloom-filter/patient-count] := 2 + [1 2 ::bloom-filter/patient-count] := nil)))) + + (testing "with four expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + nil + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2"]) + '(exists (retrieve "Specimen")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"]) + '(exists (retrieve "MedicationAdministration")) + (bloom-filter/build-bloom-filter expr 0 ["0"])))] + (let [elm #elm/and [#elm/and [#elm/exists #elm/retrieve{:type "MedicationAdministration"} + #elm/exists #elm/retrieve{:type "Specimen"}] + #elm/and [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Observation"}]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(and (exists (retrieve "MedicationAdministration")) + (exists (retrieve "Specimen")) + (exists (retrieve "Condition")) + (exists (retrieve "Observation"))) + [1 count] := 4 + [1 0 ::bloom-filter/patient-count] := 1 + [1 1 ::bloom-filter/patient-count] := 2 + [1 2 ::bloom-filter/patient-count] := 3 + [1 3 ::bloom-filter/patient-count] := nil)))) + + (testing "with five expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + nil + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2"]) + '(exists (retrieve "Specimen")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"]) + '(exists (retrieve "MedicationAdministration")) + (bloom-filter/build-bloom-filter expr 0 ["0"]) + '(exists (retrieve "Procedure")) + nil))] + (let [elm #elm/and [#elm/and [#elm/exists #elm/retrieve{:type "MedicationAdministration"} + #elm/and [#elm/exists #elm/retrieve{:type "Procedure"} + #elm/exists #elm/retrieve{:type "Specimen"}]] + #elm/and [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Observation"}]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(and (exists (retrieve "MedicationAdministration")) + (exists (retrieve "Specimen")) + (exists (retrieve "Condition")) + (exists (retrieve "Procedure")) + (exists (retrieve "Observation"))) + [1 count] := 5 + [1 0 ::bloom-filter/patient-count] := 1 + [1 1 ::bloom-filter/patient-count] := 2 + [1 2 ::bloom-filter/patient-count] := 3 + [1 3 ::bloom-filter/patient-count] := nil + [1 4 ::bloom-filter/patient-count] := nil))))) + + (ctu/testing-binary-op elm/and)) + +(deftest and-op-patient-count-test + (testing "both nil" + (let [op (ops/and-op + (reify-expr core/Expression) + (reify-expr core/Expression))] + (is (nil? (core/-patient-count op))))) + + (testing "first nil" + (let [op (ops/and-op + (reify-expr core/Expression) + (reify-expr core/Expression + (-patient-count [_] 0)))] + (is (nil? (core/-patient-count op))))) + + (testing "second nil" + (let [op (ops/and-op + (reify-expr core/Expression + (-patient-count [_] 0)) + (reify-expr core/Expression))] + (is (nil? (core/-patient-count op))))) + + (testing "first smaller" + (let [op (ops/and-op + (reify-expr core/Expression + (-patient-count [_] 1)) + (reify-expr core/Expression + (-patient-count [_] 2)))] + (is (= 1 (core/-patient-count op))))) + + (testing "second smaller" + (let [op (ops/and-op + (reify-expr core/Expression + (-patient-count [_] 2)) + (reify-expr core/Expression + (-patient-count [_] 1)))] + (is (= 1 (core/-patient-count op)))))) ;; 13.2. Implies ;; @@ -136,9 +308,7 @@ (ctu/testing-unary-null elm/not) - (ctu/testing-unary-dynamic elm/not) - - (ctu/testing-unary-form elm/not)) + (ctu/testing-unary-op elm/not)) ;; 13.4. Or ;; @@ -204,7 +374,7 @@ #elm/parameter-ref "a" {:type "Null"} '(or nil (param-ref "a")) #elm/parameter-ref "a" #elm/parameter-ref "b" '(or (param-ref "a") (param-ref "b")))) - (testing "static" + (testing "Static" (are [x y pred] (pred (core/-static (ctu/dynamic-compile (elm/or [x y])))) #elm/boolean "true" #elm/boolean "true" true? #elm/boolean "true" #elm/boolean "false" true? @@ -224,7 +394,181 @@ #elm/parameter-ref "a" #elm/boolean "true" true? #elm/parameter-ref "a" #elm/boolean "false" false? #elm/parameter-ref "a" {:type "Null"} false? - #elm/parameter-ref "a" #elm/parameter-ref "b" false?))) + #elm/parameter-ref "a" #elm/parameter-ref "b" false?)) + + (ctu/testing-binary-dynamic elm/or) + + (testing "attach cache" + (testing "only one bloom filter available" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (when-not (= '(exists (retrieve "Observation")) (c/form expr)) + (bloom-filter/build-bloom-filter expr 0 nil)))] + (let [elm #elm/or [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 2 + [1 0] := (ba/unavailable "No Bloom filter available.") + [1 1 ::bloom-filter/expr-form] := "(exists (retrieve \"Condition\"))")))) + + (testing "both bloom filters available with second having more patients" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (cond + (= (c/form expr) '(exists (retrieve "Observation"))) + (bloom-filter/build-bloom-filter expr 0 ["0"]) + (= (c/form expr) '(exists (retrieve "Condition"))) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"])))] + (let [elm #elm/or [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(or (exists (retrieve "Condition")) + (exists (retrieve "Observation"))) + [1 count] := 2 + [1 0 ::bloom-filter/patient-count] := 2 + [1 1 ::bloom-filter/patient-count] := 1)))) + + (testing "with three expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1"]) + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0"]) + '(exists (retrieve "Specimen")) + nil))] + (let [elm #elm/or [#elm/exists #elm/retrieve{:type "Observation"} + #elm/or [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Specimen"}]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(or (exists (retrieve "Specimen")) + (exists (retrieve "Observation")) + (exists (retrieve "Condition"))) + [1 count] := 3 + [1 0 ::bloom-filter/patient-count] := nil + [1 1 ::bloom-filter/patient-count] := 2 + [1 2 ::bloom-filter/patient-count] := 1)))) + + (testing "with four expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2" "3"]) + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2"]) + '(exists (retrieve "Specimen")) + nil + '(exists (retrieve "MedicationAdministration")) + (bloom-filter/build-bloom-filter expr 0 ["0"])))] + (let [elm #elm/or [#elm/or [#elm/exists #elm/retrieve{:type "MedicationAdministration"} + #elm/exists #elm/retrieve{:type "Specimen"}] + #elm/or [#elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Observation"}]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(or (exists (retrieve "Specimen")) + (exists (retrieve "Observation")) + (exists (retrieve "Condition")) + (exists (retrieve "MedicationAdministration"))) + [1 count] := 4 + [1 0 ::bloom-filter/patient-count] := nil + [1 1 ::bloom-filter/patient-count] := 4 + [1 2 ::bloom-filter/patient-count] := 3 + [1 3 ::bloom-filter/patient-count] := 1)))) + + (testing "with five expressions" + (with-redefs [ec/get (fn [cache expr] + (assert (= ::cache cache)) + (condp = (c/form expr) + '(exists (retrieve "Observation")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2" "3"]) + '(exists (retrieve "Condition")) + (bloom-filter/build-bloom-filter expr 0 ["0" "1" "2"]) + '(exists (retrieve "Specimen")) + nil + '(exists (retrieve "Procedure")) + nil + '(exists (retrieve "MedicationAdministration")) + (bloom-filter/build-bloom-filter expr 0 ["0"])))] + (let [elm #elm/or [#elm/or [#elm/exists #elm/retrieve{:type "MedicationAdministration"} + #elm/exists #elm/retrieve{:type "Specimen"}] + #elm/or [#elm/exists #elm/retrieve{:type "Condition"} + #elm/or [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Procedure"}]]] + ctx {:eval-context "Patient"} + expr (c/compile ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0 c/form] := '(or (exists (retrieve "Specimen")) + (exists (retrieve "Procedure")) + (exists (retrieve "Observation")) + (exists (retrieve "Condition")) + (exists (retrieve "MedicationAdministration"))) + [1 count] := 5 + [1 0 ::bloom-filter/patient-count] := nil + [1 1 ::bloom-filter/patient-count] := nil + [1 2 ::bloom-filter/patient-count] := 4 + [1 3 ::bloom-filter/patient-count] := 3 + [1 4 ::bloom-filter/patient-count] := 1))))) + + (ctu/testing-binary-resolve-refs elm/or) + + (ctu/testing-binary-resolve-params elm/or) + + (ctu/testing-binary-equals-hash-code elm/or) + + (ctu/testing-binary-form elm/or)) + +(deftest or-op-patient-count-test + (testing "both nil" + (let [op (ops/or-op + (reify-expr core/Expression) + (reify-expr core/Expression))] + (is (nil? (core/-patient-count op))))) + + (testing "first nil" + (let [op (ops/or-op + (reify-expr core/Expression) + (reify-expr core/Expression + (-patient-count [_] 0)))] + (is (nil? (core/-patient-count op))))) + + (testing "second nil" + (let [op (ops/or-op + (reify-expr core/Expression + (-patient-count [_] 0)) + (reify-expr core/Expression))] + (is (nil? (core/-patient-count op))))) + + (testing "first larger" + (let [op (ops/or-op + (reify-expr core/Expression + (-patient-count [_] 2)) + (reify-expr core/Expression + (-patient-count [_] 1)))] + (is (= 2 (core/-patient-count op))))) + + (testing "second larger" + (let [op (ops/or-op + (reify-expr core/Expression + (-patient-count [_] 1)) + (reify-expr core/Expression + (-patient-count [_] 2)))] + (is (= 2 (core/-patient-count op)))))) ;; 13.5. Xor ;; @@ -293,7 +637,7 @@ #elm/parameter-ref "a" {:type "Null"} nil #elm/parameter-ref "a" #elm/parameter-ref "b" '(xor (param-ref "a") (param-ref "b")))) - (testing "static" + (testing "Static" (are [x y pred] (pred (core/-static (ctu/dynamic-compile (elm/xor [x y])))) #elm/boolean "true" #elm/boolean "true" true? #elm/boolean "true" #elm/boolean "false" true? @@ -313,4 +657,6 @@ #elm/parameter-ref "a" #elm/boolean "true" false? #elm/parameter-ref "a" #elm/boolean "false" false? #elm/parameter-ref "a" {:type "Null"} true? - #elm/parameter-ref "a" #elm/parameter-ref "b" false?))) + #elm/parameter-ref "a" #elm/parameter-ref "b" false?)) + + (ctu/testing-binary-op elm/xor)) diff --git a/modules/cql/test/blaze/elm/compiler/nullological_operators_test.clj b/modules/cql/test/blaze/elm/compiler/nullological_operators_test.clj index cb988959b..9bdc6bccc 100644 --- a/modules/cql/test/blaze/elm/compiler/nullological_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/nullological_operators_test.clj @@ -51,8 +51,9 @@ (testing "expression is dynamic" (are [elm] (false? (core/-static (ctu/dynamic-compile (elm/coalesce elm)))) [] - [{:type "Null"}] - [#elm/list []]))) + [#elm/list []])) + + (ctu/testing-binary-op elm/coalesce)) ;; 14.3. IsFalse ;; @@ -72,9 +73,7 @@ #elm/parameter-ref "false" true? #elm/parameter-ref "nil" false?)) - (ctu/testing-unary-dynamic elm/is-false) - - (ctu/testing-unary-form elm/is-false)) + (ctu/testing-unary-op elm/is-false)) ;; 14.4. IsNull ;; @@ -94,9 +93,7 @@ #elm/parameter-ref "false" false? #elm/parameter-ref "nil" true?)) - (ctu/testing-unary-dynamic elm/is-null) - - (ctu/testing-unary-form elm/is-null)) + (ctu/testing-unary-op elm/is-null)) ;; 14.5. IsTrue ;; @@ -116,6 +113,4 @@ #elm/parameter-ref "false" false? #elm/parameter-ref "nil" false?)) - (ctu/testing-unary-dynamic elm/is-true) - - (ctu/testing-unary-form elm/is-true)) + (ctu/testing-unary-op elm/is-true)) diff --git a/modules/cql/test/blaze/elm/compiler/parameters_test.clj b/modules/cql/test/blaze/elm/compiler/parameters_test.clj index ecac3d7fe..11d8c3fb5 100644 --- a/modules/cql/test/blaze/elm/compiler/parameters_test.clj +++ b/modules/cql/test/blaze/elm/compiler/parameters_test.clj @@ -9,12 +9,12 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] - [blaze.elm.compiler.parameters :refer [->ParameterRef]] + [blaze.elm.compiler.parameters] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.literal] [blaze.elm.literal-spec] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest is testing]] + [clojure.test :as test :refer [deftest testing]] [cognitect.anomalies :as anom] [juxt.iota :refer [given]])) @@ -41,7 +41,6 @@ {:def [{:name "parameter-def-101820"}]}}} expr (c/compile context #elm/parameter-ref "parameter-def-101820")] - (is (= (->ParameterRef "parameter-def-101820") expr)) (testing "form" (has-form expr '(param-ref "parameter-def-101820"))))) diff --git a/modules/cql/test/blaze/elm/compiler/queries_test.clj b/modules/cql/test/blaze/elm/compiler/queries_test.clj index 5a9451a4d..4ec18f3f6 100644 --- a/modules/cql/test/blaze/elm/compiler/queries_test.clj +++ b/modules/cql/test/blaze/elm/compiler/queries_test.clj @@ -14,7 +14,7 @@ [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.literal] [blaze.elm.literal-spec] - [blaze.elm.quantity :as quantity] + [blaze.elm.quantity :refer [quantity]] [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type] [clojure.spec.test.alpha :as st] @@ -75,13 +75,13 @@ expr (c/compile {} elm)] (testing "eval" - (is (= [(quantity/quantity 1 "m") (quantity/quantity 2 "m")] (core/-eval expr {} nil nil)))) + (is (= [(quantity 1 "m") (quantity 2 "m")] (core/-eval expr {} nil nil)))) (testing "form" (has-form expr '(sorted-vector-query distinct - [(quantity 2 "m") - (quantity 1 "m") - (quantity 1 "m")] + [(quantity 2M "m") + (quantity 1M "m") + (quantity 1M "m")] [asc (:value S)])))) (testing "with IdentifierRef" diff --git a/modules/cql/test/blaze/elm/compiler/reusing_logic_test.clj b/modules/cql/test/blaze/elm/compiler/reusing_logic_test.clj index 943f4518c..2268ddbf6 100644 --- a/modules/cql/test/blaze/elm/compiler/reusing_logic_test.clj +++ b/modules/cql/test/blaze/elm/compiler/reusing_logic_test.clj @@ -11,10 +11,11 @@ [blaze.elm.compiler.core-spec] [blaze.elm.compiler.function :as function] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] + [blaze.elm.expression.cache :as ec] [blaze.elm.interval :as interval] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] - [blaze.elm.quantity :as quantity] + [blaze.elm.quantity :refer [quantity]] [blaze.fhir.spec.type.system :as system] [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [are deftest is testing]] @@ -70,6 +71,27 @@ ;; ;; The FunctionRef type defines an expression that invokes a previously defined ;; function. The result of evaluating each operand is passed to the function. +(defmacro testing-function-ref-attach-cache [name] + `(testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm# #elm/function-ref [~name #elm/exists #elm/retrieve{:type "Observation"}] + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 1 + [1 0] := '(~'exists (~'retrieve "Observation"))))))) + +(defmacro testing-function-ref-resolve-refs [name] + `(testing "resolve expression references" + (let [elm# #elm/function-ref [~name #elm/expression-ref "x"] + expr-def# {:type "ExpressionDef" :name "x" :expression "y" + :context "Unfiltered"} + ctx# {:library {:statements {:def [expr-def#]}}} + expr# (c/resolve-refs (c/compile ctx# elm#) {"x" expr-def#})] + (has-form expr# '(~'call ~name "y"))))) + (deftest compile-function-ref-test (testing "Throws error on missing function" (given (ba/try-anomaly (c/compile {} #elm/function-ref ["name-175844"])) @@ -87,9 +109,23 @@ (testing "eval" (is (= 1 (core/-eval expr {} nil nil)))) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach cache" + (is (= [expr []] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {})] + (has-form expr (list 'call function-name)))) + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {}) + (list 'call function-name))) + (testing "form" (has-form expr (list 'call function-name))))) @@ -97,6 +133,7 @@ (let [function-name "name-180815" fn-expr (c/compile {} #elm/negate #elm/operand-ref"x") compile-ctx {:library {:parameters {:def [{:name "a"}]}} + :eval-context "Patient" :function-defs {function-name {:function (partial function/arity-n function-name fn-expr ["x"])}}} elm (elm/function-ref [function-name #elm/parameter-ref "a"]) expr (c/compile compile-ctx elm)] @@ -107,9 +144,37 @@ -1 1 0 0)) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm (elm/function-ref [function-name #elm/exists #elm/retrieve{:type "Observation"}]) + expr (c/compile compile-ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 1 + [1 0] := '(exists (retrieve "Observation")))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm (elm/function-ref [function-name #elm/expression-ref "x"]) + expr-def {:type "ExpressionDef" :name "x" :expression "a" + :context "Unfiltered"} + ctx (assoc compile-ctx :library {:statements {:def [expr-def]}}) + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-def})] + (has-form expr (list 'call function-name "a")))) + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {"a" 1}) + (list 'call function-name 1)) + + (has-form (core/-resolve-params expr {}) + (list 'call function-name '(param-ref "a")))) + (testing "form" (has-form expr (list 'call function-name '(param-ref "a")))))) @@ -117,6 +182,7 @@ (let [function-name "name-184652" fn-expr (c/compile {} #elm/add [#elm/operand-ref"x" #elm/operand-ref"y"]) compile-ctx {:library {:parameters {:def [{:name "a"} {:name "b"}]}} + :eval-context "Patient" :function-defs {function-name {:function (partial function/arity-n function-name fn-expr ["x" "y"])}}} elm (elm/function-ref [function-name #elm/parameter-ref "a" #elm/parameter-ref "b"]) expr (c/compile compile-ctx elm)] @@ -127,9 +193,44 @@ 1 0 1 0 1 1)) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm (elm/function-ref [function-name + #elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}]) + expr (c/compile compile-ctx elm)] + (given (st/with-instrument-disabled (c/attach-cache expr ::cache)) + count := 2 + [0] := expr + [1 count] := 2 + [1 0] := '(exists (retrieve "Observation")) + [1 1] := '(exists (retrieve "Condition")))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm (elm/function-ref [function-name + #elm/expression-ref "x" + #elm/expression-ref "y"]) + expr-defs [{:type "ExpressionDef" :name "x" :expression "a" + :context "Unfiltered"} + {:type "ExpressionDef" :name "y" :expression "b" + :context "Unfiltered"}] + ctx (assoc compile-ctx :library {:statements {:def expr-defs}}) + expr (c/resolve-refs (c/compile ctx elm) (zipmap ["x" "y"] expr-defs))] + (has-form expr (list 'call function-name "a" "b")))) + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {"a" 1 "b" 2}) + (list 'call function-name 1 2)) + + (has-form (core/-resolve-params expr {}) + (list 'call function-name '(param-ref "a") '(param-ref "b")))) + (testing "form" (has-form expr (list 'call function-name '(param-ref "a") '(param-ref "b")))))) @@ -140,13 +241,21 @@ (testing "eval" (are [x res] (= res (core/-eval expr {:parameters {"x" x}} nil nil)) - {:value 23M :code "kg"} (quantity/quantity 23M "kg") - {:value 42M} (quantity/quantity 42M "1") + {:value 23M :code "kg"} (quantity 23M "kg") + {:value 42M} (quantity 42M "1") {} nil)) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToQuantity") + + (testing-function-ref-resolve-refs "ToQuantity") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {}) + '(call "ToQuantity" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToQuantity" (param-ref "x")))))) @@ -164,9 +273,17 @@ #fhir/date"2023-05" #system/date"2023-05" #fhir/date"2023-05-07" #system/date"2023-05-07")) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToDate") + + (testing-function-ref-resolve-refs "ToDate") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {}) + '(call "ToDate" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToDate" (param-ref "x")))))) @@ -190,9 +307,20 @@ #fhir/instant"2021-02-23T15:12:45Z" #system/date-time"2021-02-23T15:12:45" #fhir/instant"2021-02-23T15:12:45+01:00" #system/date-time"2021-02-23T14:12:45")) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToDateTime") + + (testing-function-ref-resolve-refs "ToDateTime") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {"x" #fhir/dateTime"2022-02"}) + '(call "ToDateTime" #fhir/dateTime"2022-02")) + + (has-form (core/-resolve-params expr {}) + '(call "ToDateTime" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToDateTime" (param-ref "x")))))) @@ -208,9 +336,17 @@ #fhir/code{:id "foo" :value "code-211914"} "code-211914" #fhir/code{:id "foo"} nil)) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToString") + + (testing-function-ref-resolve-refs "ToString") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {}) + '(call "ToString" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToString" (param-ref "x")))))) @@ -226,9 +362,17 @@ :code "code-140828"} (code/to-code "system-140820" "version-140924" "code-140828"))) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToCode") + + (testing-function-ref-resolve-refs "ToCode") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {}) + '(call "ToCode" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToCode" (param-ref "x")))))) @@ -259,9 +403,24 @@ (system/date-time 2021 2 23 14 12 45) nil))) - (testing "static" + (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing-function-ref-attach-cache "ToInterval") + + (testing-function-ref-resolve-refs "ToInterval") + + (testing "resolve parameters" + (has-form (core/-resolve-params expr {"x" #fhir/Period + {:start #fhir/dateTime"2021-02-23T15:12:45+01:00" + :end #fhir/dateTime"2021-02-23T16:00:00+01:00"}}) + '(call "ToInterval" #fhir/Period + {:start #fhir/dateTime"2021-02-23T15:12:45+01:00" + :end #fhir/dateTime"2021-02-23T16:00:00+01:00"})) + + (has-form (core/-resolve-params expr {}) + '(call "ToInterval" (param-ref "x")))) + (testing "form" (has-form expr '(call "ToInterval" (param-ref "x"))))))) diff --git a/modules/cql/test/blaze/elm/compiler/string_operators_test.clj b/modules/cql/test/blaze/elm/compiler/string_operators_test.clj index 75a7cb641..01d737177 100644 --- a/modules/cql/test/blaze/elm/compiler/string_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/string_operators_test.clj @@ -9,6 +9,7 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] + [blaze.elm.compiler.string-operators] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] @@ -79,7 +80,7 @@ ;; ;; If any argument is null, the result is null. (deftest compile-concatenate-test - (are [args res] (= res (core/-eval (c/compile {} {:type "Concatenate" :operand args}) {} nil nil)) + (are [args res] (= res (core/-eval (c/compile {} (elm/concatenate args)) {} nil nil)) [#elm/string "a"] "a" [#elm/string "a" #elm/string "b"] "ab" @@ -87,16 +88,18 @@ [{:type "Null"}] nil) (testing "form" - (are [args form] (= form (c/form (c/compile {} {:type "Concatenate" :operand args}))) + (are [args form] (= form (c/form (c/compile {} (elm/concatenate args)))) [#elm/string "a"] '(concatenate "a") [#elm/string "a" #elm/string "b"] '(concatenate "a" "b") [#elm/string "a" {:type "Null"}] '(concatenate "a" nil))) - (testing "static" - (are [args] (false? (core/-static (c/compile {} {:type "Concatenate" :operand args}))) + (testing "Static" + (are [args] (false? (core/-static (c/compile {} (elm/concatenate args)))) [#elm/string "a"] [#elm/string "a" #elm/string "b"] - [#elm/string "a" {:type "Null"}]))) + [#elm/string "a" {:type "Null"}])) + + (ctu/testing-binary-op elm/concatenate)) ;; 17.3. EndsWith ;; @@ -107,7 +110,7 @@ ;; ;; If either argument is null, the result is null. (deftest compile-ends-with-test - (testing "static" + (testing "Static" (are [s suffix pred] (pred (c/compile {} (elm/ends-with [s suffix]))) #elm/string "a" #elm/string "a" true? #elm/string "ab" #elm/string "b" true? @@ -125,9 +128,7 @@ (ctu/testing-binary-null elm/ends-with #elm/string "a") - (ctu/testing-binary-dynamic elm/ends-with) - - (ctu/testing-binary-form elm/ends-with)) + (ctu/testing-binary-op elm/ends-with)) ;; 17.4. Equal ;; @@ -168,9 +169,7 @@ (ctu/testing-binary-null elm/indexer #elm/list [] #elm/integer "0")) - (ctu/testing-binary-dynamic elm/indexer) - - (ctu/testing-binary-form elm/indexer)) + (ctu/testing-binary-op elm/indexer)) ;; 17.7. LastPositionOf ;; @@ -189,9 +188,7 @@ (ctu/testing-binary-dynamic-null elm/last-position-of #elm/string "a" #elm/string "a") - (ctu/testing-binary-dynamic elm/last-position-of) - - (ctu/testing-binary-form elm/last-position-of)) + (ctu/testing-binary-op elm/last-position-of)) ;; 17.8. Length ;; @@ -205,7 +202,7 @@ (deftest compile-length-test ;; It's important to use identical? here because we like to test that length ;; returns a long instead of an integer. - (testing "static" + (testing "Static" (are [x res] (identical? res (c/compile {} (elm/length x))) #elm/string "" 0 #elm/string "a" 1 @@ -239,9 +236,7 @@ (is (identical? count (core/-eval expr {:db db} patient nil))))))) - (ctu/testing-unary-dynamic elm/length) - - (ctu/testing-unary-form elm/length)) + (ctu/testing-unary-op elm/length)) ;; 17.9. Lower ;; @@ -255,7 +250,7 @@ ;; ;; If the argument is null, the result is null. (deftest compile-lower-test - (testing "static" + (testing "Static" (are [s res] (= res (c/compile {} (elm/lower s))) #elm/string "" "" #elm/string "A" "a")) @@ -267,9 +262,7 @@ (ctu/testing-unary-null elm/lower) - (ctu/testing-unary-dynamic elm/lower) - - (ctu/testing-unary-form elm/lower)) + (ctu/testing-unary-op elm/lower)) ;; 17.10. Matches ;; @@ -292,9 +285,7 @@ (ctu/testing-binary-null elm/matches #elm/string "a") - (ctu/testing-binary-dynamic elm/matches) - - (ctu/testing-binary-form elm/matches)) + (ctu/testing-binary-op elm/matches)) ;; 17.11. NotEqual ;; @@ -317,9 +308,7 @@ (ctu/testing-binary-dynamic-null elm/position-of #elm/string "a" #elm/string "a") - (ctu/testing-binary-dynamic elm/position-of) - - (ctu/testing-binary-form elm/position-of)) + (ctu/testing-binary-op elm/position-of)) ;; 17.13. ReplaceMatches ;; @@ -337,14 +326,12 @@ ;; such, CQL does not prescribe a particular dialect, but recommends the use of ;; the PCRE dialect. (deftest compile-replace-matches-test - (are [s pattern substitution res] (= res (core/-eval (c/compile {} (elm/replace-matches [s pattern substitution])) {} nil nil)) + (are [s pattern substitution res] (= res (c/compile {} (elm/replace-matches [s pattern substitution]))) #elm/string "a" #elm/string "a" #elm/string "b" "b") (ctu/testing-ternary-dynamic-null elm/replace-matches #elm/string "a" #elm/string "a" #elm/string "a") - (ctu/testing-ternary-dynamic elm/replace-matches) - - (ctu/testing-ternary-form elm/replace-matches)) + (ctu/testing-ternary-op elm/replace-matches)) ;; 17.14. Split ;; @@ -356,39 +343,17 @@ ;; separator, the result is a list of strings containing one element that is the ;; value of the stringToSplit argument. (deftest compile-split-test - (testing "Without separator" - (are [s res] (= res (core/-eval (c/compile {} {:type "Split" :stringToSplit s}) {} nil nil)) - #elm/string "" [""] - #elm/string "a" ["a"] - - {:type "Null"} nil) - - (testing "form and static" - (let [expr (ctu/dynamic-compile {:type "Split" - :stringToSplit #elm/parameter-ref "x"})] - - (has-form expr '(split (param-ref "x"))) - - (is (false? (core/-static expr)))))) - - (testing "With separator" - (are [s separator res] (= res (core/-eval (c/compile {} {:type "Split" :stringToSplit s :separator separator}) {} nil nil)) - #elm/string "" #elm/string "," [""] - #elm/string "a,b" #elm/string "," ["a" "b"] - #elm/string "a,,b" #elm/string "," ["a" "" "b"] - - {:type "Null"} #elm/string "," nil - #elm/string "a" {:type "Null"} ["a"] - {:type "Null"} {:type "Null"} nil) - - (testing "form and static" - (let [expr (ctu/dynamic-compile {:type "Split" - :stringToSplit #elm/parameter-ref "x" - :separator #elm/parameter-ref "y"})] + (testing "Dynamic" + (are [s separator res] (= res (ctu/dynamic-compile-eval (elm/split [s separator]))) + #elm/parameter-ref "empty-string" #elm/string "," [""] + #elm/parameter-ref "a,b" #elm/string "," ["a" "b"] + #elm/parameter-ref "a,,b" #elm/string "," ["a" "" "b"] - (has-form expr '(split (param-ref "x") (param-ref "y"))) + #elm/parameter-ref "nil" #elm/string "," nil + #elm/parameter-ref "a" {:type "Null"} ["a"] + #elm/parameter-ref "nil" {:type "Null"} nil)) - (is (false? (core/-static expr))))))) + (ctu/testing-binary-op elm/split)) ;; 17.15. SplitOnMatches ;; @@ -413,7 +378,7 @@ ;; ;; If either argument is null, the result is null. (deftest compile-starts-with-test - (testing "static" + (testing "Static" (are [s prefix pred] (pred (c/compile {} (elm/starts-with [s prefix]))) #elm/string "a" #elm/string "a" true? #elm/string "ba" #elm/string "b" true? @@ -431,9 +396,7 @@ (ctu/testing-binary-null elm/starts-with #elm/string "a") - (ctu/testing-binary-dynamic elm/starts-with) - - (ctu/testing-binary-form elm/starts-with)) + (ctu/testing-binary-op elm/starts-with)) ;; 17.17. Substring ;; @@ -449,7 +412,7 @@ ;; TODO: what todo if the length is out of range? (deftest compile-substring-test (testing "Without length" - (are [s start-index res] (= res (core/-eval (c/compile {} {:type "Substring" :stringToSub s :startIndex start-index}) {} nil nil)) + (are [s start-index res] (= res (core/-eval (c/compile {} (elm/substring [s start-index])) {} nil nil)) #elm/string "ab" #elm/integer "1" "b" #elm/string "a" #elm/integer "-1" nil @@ -468,7 +431,7 @@ (is (false? (core/-static expr)))))) (testing "With length" - (are [s start-index length res] (= res (core/-eval (c/compile {} {:type "Substring" :stringToSub s :startIndex start-index :length length}) {} nil nil)) + (are [s start-index length res] (= res (core/-eval (c/compile {} (elm/substring [s start-index length])) {} nil nil)) #elm/string "a" #elm/integer "0" #elm/integer "1" "a" #elm/string "a" #elm/integer "0" #elm/integer "2" "a" #elm/string "abc" #elm/integer "1" #elm/integer "1" "b" @@ -487,7 +450,11 @@ (has-form expr '(substring (param-ref "x") (param-ref "y") (param-ref "z"))) - (is (false? (core/-static expr))))))) + (is (false? (core/-static expr)))))) + + (ctu/testing-binary-op elm/substring) + + (ctu/testing-ternary-op elm/substring)) ;; 17.18. Upper ;; @@ -501,7 +468,7 @@ ;; ;; If the argument is null, the result is null. (deftest compile-upper-test - (testing "static" + (testing "Static" (are [s res] (= res (c/compile {} (elm/upper s))) #elm/string "" "" #elm/string "a" "A")) @@ -511,8 +478,4 @@ #elm/parameter-ref "empty-string" "" #elm/parameter-ref "a" "A")) - (ctu/testing-unary-null elm/upper) - - (ctu/testing-unary-dynamic elm/upper) - - (ctu/testing-unary-form elm/upper)) + (ctu/testing-unary-op elm/upper)) diff --git a/modules/cql/test/blaze/elm/compiler/structured_values_test.clj b/modules/cql/test/blaze/elm/compiler/structured_values_test.clj index 96111250e..232cf2f22 100644 --- a/modules/cql/test/blaze/elm/compiler/structured_values_test.clj +++ b/modules/cql/test/blaze/elm/compiler/structured_values_test.clj @@ -9,6 +9,7 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] + [blaze.elm.compiler.structured-values] [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.literal] [blaze.elm.literal-spec] @@ -54,12 +55,12 @@ #elm/tuple{"id" #elm/parameter-ref "1" "name" #elm/parameter-ref "a"} {:id 1 :name "a"}) - (testing "static" + (testing "Static" (is (false? (core/-static (ctu/dynamic-compile #elm/tuple{"id" #elm/parameter-ref "1"}))))) (testing "form" - (is (= '{:id (param-ref "x")} - (core/-form (ctu/dynamic-compile #elm/tuple{"id" #elm/parameter-ref "x"}))))))) + (let [expr (ctu/dynamic-compile #elm/tuple{"id" #elm/parameter-ref "x"})] + (has-form expr '{:id (param-ref "x")}))))) ;; 2.2. Instance ;; @@ -109,10 +110,8 @@ entity {:fhir/type :fhir/Patient :id "0" :identifier [identifier]} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "identifier"])] + elm #elm/scope-property ["R" "identifier"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= identifier (coll/first (core/-eval expr nil nil {"R" entity}))))) @@ -120,6 +119,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:identifier R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:identifier R))))) @@ -131,10 +142,8 @@ entity {:fhir/type :fhir/Patient :id "0" :identifier [identifier]} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "identifier"])] + elm #elm/scope-property ["R" "identifier"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= identifier (coll/first (core/-eval expr nil nil {"R" entity}))))) @@ -142,6 +151,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:identifier R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:identifier R)))))) @@ -154,10 +175,8 @@ entity {:fhir/type :fhir/Patient :id "0" :extension [extension]} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "extension"])] + elm #elm/scope-property ["R" "extension"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= extension (coll/first (core/-eval expr nil nil {"R" entity}))))) @@ -165,6 +184,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:extension R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:extension R)))))) @@ -173,10 +204,8 @@ (let [entity {:fhir/type :fhir/Patient :id "0" :gender #fhir/code"male"} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "gender"])] + elm #elm/scope-property ["R" "gender"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= #fhir/code"male" (core/-eval expr nil nil {"R" entity})))) @@ -184,6 +213,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:gender R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:gender R))))) @@ -191,10 +232,8 @@ (let [entity {:fhir/type :fhir/Patient :id "0" :gender #fhir/code"male"} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "gender"])] + elm #elm/scope-property ["R" "gender"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= #fhir/code"male" (core/-eval expr nil nil {"R" entity})))) @@ -202,6 +241,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:gender R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:gender R)))))) @@ -210,10 +261,8 @@ (fn [x] {:fhir/type :fhir/Patient :id "0" :birthDate x}) - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "birthDate.value"])] + elm #elm/scope-property ["R" "birthDate.value"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (are [birth-date res] (= res (core/-eval expr nil nil {"R" (entity birth-date)})) @@ -225,6 +274,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value (:birthDate R))))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:value (:birthDate R)))))) @@ -233,10 +294,8 @@ (let [entity {:fhir/type :fhir/Observation :id "0" :value "value-114318"} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "value"])] + elm #elm/scope-property ["R" "value"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= "value-114318" (core/-eval expr nil nil {"R" entity})))) @@ -244,6 +303,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:value R))))) @@ -251,10 +322,8 @@ (let [entity {:fhir/type :fhir/Observation :id "0" :value "value-114318"} - expr - (c/compile - {:eval-context "Patient"} - #elm/scope-property ["R" "value"])] + elm #elm/scope-property ["R" "value"] + expr (c/compile {:eval-context "Patient"} elm)] (testing "eval" (is (= "value-114318" (core/-eval expr nil nil {"R" entity})))) @@ -262,6 +331,18 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value R)))) + + (ctu/testing-equals-hash-code elm) + (testing "form" (has-form expr '(:value R)))))))) @@ -279,14 +360,32 @@ source {:fhir/type :fhir/Patient :id "0" :identifier [identifier]} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Patient" + :expression source}] (testing "eval" - (is (= identifier (coll/first (core/-eval expr {:expression-defs {"Patient" {:expression source}}} nil nil))))) + (is (= identifier (coll/first (core/-eval expr {:expression-defs {"Patient" expr-def}} nil nil))))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Patient" expr-def})] + (has-form expr (list :identifier source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:identifier (expr-ref "Patient"))))) + (testing "form" (has-form expr '(:identifier (expr-ref "Patient")))))) @@ -302,14 +401,32 @@ source {:fhir/type :fhir/Patient :id "0" :identifier [identifier]} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Patient" + :expression source}] (testing "eval" - (is (= identifier (coll/first (core/-eval expr {:expression-defs {"Patient" {:expression source}}} nil nil))))) + (is (= identifier (coll/first (core/-eval expr {:expression-defs {"Patient" expr-def}} nil nil))))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Patient" expr-def})] + (has-form expr (list :identifier source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:identifier (expr-ref "Patient"))))) + (testing "form" (has-form expr '(:identifier (expr-ref "Patient"))))))) @@ -322,14 +439,32 @@ source {:fhir/type :fhir/Patient :id "0" :gender #fhir/code"male"} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Patient" + :expression source}] (testing "eval" - (is (= #fhir/code"male" (core/-eval expr {:expression-defs {"Patient" {:expression source}}} nil nil)))) + (is (= #fhir/code"male" (core/-eval expr {:expression-defs {"Patient" expr-def}} nil nil)))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Patient" expr-def})] + (has-form expr (list :gender source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:gender (expr-ref "Patient"))))) + (testing "form" (has-form expr '(:gender (expr-ref "Patient")))))) @@ -341,14 +476,32 @@ source {:fhir/type :fhir/Patient :id "0" :gender #fhir/code"male"} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Patient" + :expression source}] (testing "eval" - (is (= #fhir/code"male" (core/-eval expr {:expression-defs {"Patient" {:expression source}}} nil nil)))) + (is (= #fhir/code"male" (core/-eval expr {:expression-defs {"Patient" expr-def}} nil nil)))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Patient" expr-def})] + (has-form expr (list :gender source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:gender (expr-ref "Patient"))))) + (testing "form" (has-form expr '(:gender (expr-ref "Patient"))))))) @@ -373,6 +526,24 @@ (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr-def {:type "ExpressionDef" + :context "Patient" + :name "Patient" + :expression (source #fhir/date"2023-05-07")} + expr (c/resolve-refs expr {"Patient" expr-def})] + (has-form expr (list :value (list :birthDate (source #fhir/date"2023-05-07")))))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value (:birthDate (expr-ref "Patient")))))) + (testing "form" (has-form expr '(:value (:birthDate (expr-ref "Patient"))))))) @@ -385,14 +556,32 @@ source {:fhir/type :fhir/Observation :id "0" :value "value-114318"} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Observation" + :expression source}] (testing "eval" - (is (= "value-114318" (core/-eval expr {:expression-defs {"Observation" {:expression source}}} nil nil)))) + (is (= "value-114318" (core/-eval expr {:expression-defs {"Observation" expr-def}} nil nil)))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Observation" expr-def})] + (has-form expr (list :value source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value (expr-ref "Observation"))))) + (testing "form" (has-form expr '(:value (expr-ref "Observation")))))) @@ -404,14 +593,32 @@ source {:fhir/type :fhir/Observation :id "0" :value "value-114318"} - expr (c/compile {:library library :eval-context "Patient"} elm)] + expr (c/compile {:library library :eval-context "Patient"} elm) + expr-def {:type "ExpressionDef" + :context "Patient" + :name "Observation" + :expression source}] (testing "eval" - (is (= "value-114318" (core/-eval expr {:expression-defs {"Observation" {:expression source}}} nil nil)))) + (is (= "value-114318" (core/-eval expr {:expression-defs {"Observation" expr-def}} nil nil)))) (testing "expression is dynamic" (is (false? (core/-static expr)))) + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [expr (c/resolve-refs expr {"Observation" expr-def})] + (has-form expr (list :value source)))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {})] + (has-form expr '(:value (expr-ref "Observation"))))) + (testing "form" (has-form expr '(:value (expr-ref "Observation"))))))) diff --git a/modules/cql/test/blaze/elm/compiler/test_util.clj b/modules/cql/test/blaze/elm/compiler/test_util.clj index 792bc4ded..468f8ddb0 100644 --- a/modules/cql/test/blaze/elm/compiler/test_util.clj +++ b/modules/cql/test/blaze/elm/compiler/test_util.clj @@ -1,17 +1,21 @@ (ns blaze.elm.compiler.test-util (:require + [blaze.anomaly :as ba] [blaze.db.api :as d] [blaze.elm.compiler :as c] + [blaze.elm.compiler-spec] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] - [blaze.elm.compiler.external-data :as ed] + [blaze.elm.expression.cache :as ec] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] + [blaze.elm.resource :as cr] [blaze.elm.spec] [blaze.fhir.spec.type.system :as system] [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] - [clojure.test :refer [is testing]]) + [clojure.test :refer [is testing]] + [juxt.iota :refer [given]]) (:import [java.time OffsetDateTime ZoneOffset])) @@ -43,7 +47,8 @@ (def now (OffsetDateTime/now (ZoneOffset/ofHours 0))) (def dynamic-compile-ctx - {:library + {:eval-context "Patient" + :library {:parameters {:def [{:name "true"} @@ -62,6 +67,8 @@ {:name "ab"} {:name "b"} {:name "ba"} + {:name "a,b"} + {:name "a,,b"} {:name "A"} {:name "2019"} {:name "2020"} @@ -75,7 +82,8 @@ (def dynamic-eval-ctx {:parameters {"true" true "false" false "nil" nil "-1" -1 "1" 1 "2" 2 "3" 3 "4" 4 - "empty-string" "" "x" "x" "y" "y" "z" "z" "a" "a" "ab" "ab" "b" "b" "ba" "ba" "A" "A" + "empty-string" "" "x" "x" "y" "y" "z" "z" + "a" "a" "ab" "ab" "b" "b" "ba" "ba" "a,b" "a,b" "a,,b" "a,,b" "A" "A" "2019" (system/date 2019) "2020" (system/date 2020) "2021" (system/date 2021) @@ -190,6 +198,128 @@ (let [expr# (dynamic-compile ~elm-constructor)] (has-form expr# (quote ~form-name)))))) +(defmacro testing-unary-dynamic [elm-constructor] + `(testing "expression is dynamic" + (is (false? (core/-static (dynamic-compile (~elm-constructor + #elm/parameter-ref "x"))))))) + +(defmacro testing-unary-precision-dynamic + [elm-constructor & precisions] + `(testing "expression is dynamic" + (doseq [precision# ~(vec precisions)] + (is (false? (core/-static (dynamic-compile (~elm-constructor + [#elm/parameter-ref "x" + precision#])))))))) + +(defmacro testing-unary-attach-cache + "Works with unary and aggregate operators." + [elm-constructor] + `(testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (let [elm# (~elm-constructor #elm/exists #elm/retrieve{:type "Observation"}) + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 1 + [1 0] := '(~'exists (~'retrieve "Observation"))))))) + +(defmacro testing-unary-precision-attach-cache + [elm-constructor & precisions] + `(testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor [#elm/exists #elm/retrieve{:type "Observation"} precision#]) + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 1 + [1 0] := '(~'exists (~'retrieve "Observation")))))))) + +(defmacro testing-unary-patient-count [elm-constructor] + `(testing "patient count" + (let [expr# (dynamic-compile (~elm-constructor #elm/parameter-ref "x"))] + (is (nil? (core/-patient-count expr#)))))) + +(defmacro testing-unary-precision-patient-count [elm-constructor & precisions] + `(testing "patient count" + (doseq [precision# ~(vec precisions)] + (let [expr# (dynamic-compile (~elm-constructor [#elm/parameter-ref "x" + precision#]))] + (is (nil? (core/-patient-count expr#))))))) + +(defmacro testing-unary-resolve-refs + "Works with unary and aggregate operators." + [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve expression references" + (let [elm# (~elm-constructor #elm/expression-ref "x") + expr-def# {:type "ExpressionDef" :name "x" :expression "y" + :context "Unfiltered"} + ctx# {:library {:statements {:def [expr-def#]}}} + expr# (c/resolve-refs (c/compile ctx# elm#) {"x" expr-def#})] + (has-form expr# '(~form-name "y")))))) + +(defmacro testing-unary-precision-resolve-refs + "Works with unary and aggregate operators." + [elm-constructor & precisions] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve expression references" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor [#elm/expression-ref "x" precision#]) + expr-def# {:type "ExpressionDef" :name "x" :expression "y" + :context "Unfiltered"} + ctx# {:library {:statements {:def [expr-def#]}}} + expr# (c/resolve-refs (c/compile ctx# elm#) {"x" expr-def#})] + (has-form expr# (list '~form-name "y" precision#))))))) + +(defmacro testing-unary-resolve-params + "Works with unary and aggregate operators." + [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve parameters" + (let [elm# (~elm-constructor #elm/parameter-ref "x") + ctx# {:library {:parameters {:def [{:name "x"}]}}} + expr# (c/resolve-params (c/compile ctx# elm#) {"x" "y"})] + (has-form expr# '(~form-name "y")))))) + +(defmacro testing-unary-precision-resolve-params + "Works with unary and aggregate operators." + [elm-constructor & precisions] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve parameters" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor [#elm/parameter-ref "x" precision#]) + ctx# {:library {:parameters {:def [{:name "x"}]}}} + expr# (c/resolve-params (c/compile ctx# elm#) {"x" "y"})] + (has-form expr# (list '~form-name "y" precision#))))))) + +(defmacro testing-unary-equals-hash-code + [elm-constructor] + `(testing "equals/hashCode" + (let [elm# (~elm-constructor #elm/parameter-ref "x") + ctx# {:library {:parameters {:def [{:name "x"}]}}} + expr-1# (c/compile ctx# elm#) + expr-2# (c/compile ctx# elm#)] + (is (and (.equals ^Object expr-1# expr-2#) + (= (.hashCode ^Object expr-1#) + (.hashCode ^Object expr-2#))))))) + +(defmacro testing-unary-precision-equals-hash-code + [elm-constructor & precisions] + `(testing "equals/hashCode" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor [#elm/parameter-ref "x" precision#]) + ctx# {:library {:parameters {:def [{:name "x"}]}}} + expr-1# (c/compile ctx# elm#) + expr-2# (c/compile ctx# elm#)] + (is (and (.equals ^Object expr-1# expr-2#) + (= (.hashCode ^Object expr-1#) + (.hashCode ^Object expr-2#)))))))) + (defmacro testing-unary-form "Works with unary and aggregate operators." [elm-constructor] @@ -211,6 +341,43 @@ (has-form expr# (list '~form-name '(~'param-ref "x") precision#)))))))) +(defmacro testing-binary-equals-hash-code + [elm-constructor] + `(testing "equals/hashCode" + (let [elm# (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y"]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"}]}}} + expr-1# (c/compile ctx# elm#) + expr-2# (c/compile ctx# elm#)] + (is (and (.equals ^Object expr-1# expr-2#) + (= (.hashCode ^Object expr-1#) + (.hashCode ^Object expr-2#))))))) + +(defmacro testing-binary-precision-equals-hash-code + ([elm-constructor] + `(testing-binary-precision-equals-hash-code ~elm-constructor "year" "month")) + ([elm-constructor & precisions] + `(testing "equals/hashCode" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y" precision#]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"}]}}} + expr-1# (c/compile ctx# elm#) + expr-2# (c/compile ctx# elm#)] + (is (and (.equals ^Object expr-1# expr-2#) + (= (.hashCode ^Object expr-1#) + (.hashCode ^Object expr-2#))))))))) + +(defmacro testing-ternary-equals-hash-code + [elm-constructor] + `(testing "equals/hashCode" + (let [elm# (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y" + #elm/parameter-ref "z"]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"} {:name "z"}]}}} + expr-1# (c/compile ctx# elm#) + expr-2# (c/compile ctx# elm#)] + (is (and (.equals ^Object expr-1# expr-2#) + (= (.hashCode ^Object expr-1#) + (.hashCode ^Object expr-2#))))))) + (defmacro testing-binary-form [elm-constructor] (let [form-name (symbol (name elm-constructor))] `(testing "form" @@ -250,25 +417,144 @@ `(testing "expression is dynamic" (is (false? (core/-static (dynamic-compile ~elm-constructor)))))) -(defmacro testing-unary-dynamic [elm-constructor] - `(testing "expression is dynamic" - (is (false? (core/-static (dynamic-compile (~elm-constructor - #elm/parameter-ref "x"))))))) - -(defmacro testing-unary-precision-dynamic - [elm-constructor & precisions] - `(testing "expression is dynamic" - (doseq [precision# ~(vec precisions)] - (is (false? (core/-static (dynamic-compile (~elm-constructor - [#elm/parameter-ref "x" - precision#])))))))) - (defmacro testing-binary-dynamic [elm-constructor] `(testing "expression is dynamic" (is (false? (core/-static (dynamic-compile (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y"]))))))) +(defn mock-cache-get [cache expr] + (assert (= ::cache cache)) + (when (= '(exists (retrieve "Observation")) (c/form expr)) + (c/form expr))) + +(defmacro testing-binary-attach-cache + [elm-constructor] + `(testing "attach cache" + (with-redefs [ec/get mock-cache-get] + (let [elm# (~elm-constructor + [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"}]) + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 2 + [1 0] := '(~'exists (~'retrieve "Observation")) + [1 1] := (ba/unavailable "No Bloom filter available.")))))) + +(defmacro testing-ternary-attach-cache + [elm-constructor] + `(testing "attach cache" + (with-redefs [ec/get mock-cache-get] + (let [elm# (~elm-constructor + [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"} + #elm/exists #elm/retrieve{:type "Specimen"}]) + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 3 + [1 0] := '(~'exists (~'retrieve "Observation")) + [1 1] := (ba/unavailable "No Bloom filter available.") + [1 2] := (ba/unavailable "No Bloom filter available.")))))) + +(defmacro testing-binary-precision-attach-cache + ([elm-constructor] + `(testing-binary-precision-attach-cache ~elm-constructor "year" "month")) + ([elm-constructor & precisions] + `(testing "attach cache" + (with-redefs [ec/get #(do (assert (= ::cache %1)) (c/form %2))] + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor + [#elm/exists #elm/retrieve{:type "Observation"} + #elm/exists #elm/retrieve{:type "Condition"} + precision#]) + ctx# {:eval-context "Patient"} + expr# (c/compile ctx# elm#)] + (given (st/with-instrument-disabled (c/attach-cache expr# ::cache)) + count := 2 + [0] := expr# + [1 count] := 2 + [1 0] := '(~'exists (~'retrieve "Observation")) + [1 1] := '(~'exists (~'retrieve "Condition"))))))))) + +(defmacro testing-binary-patient-count [elm-constructor] + `(testing "patient count" + (is (nil? (core/-patient-count (dynamic-compile (~elm-constructor + [#elm/parameter-ref "x" + #elm/parameter-ref "y"]))))))) + +(defmacro testing-binary-precision-patient-count + ([elm-constructor] + `(testing-binary-precision-patient-count ~elm-constructor "year" "month")) + ([elm-constructor & precisions] + `(testing "patient count" + (doseq [precision# ~(vec precisions)] + (is (nil? (core/-patient-count (dynamic-compile (~elm-constructor + [#elm/parameter-ref "x" + #elm/parameter-ref "y" + precision#]))))))))) + +(defmacro testing-ternary-patient-count [elm-constructor] + `(testing "patient count" + (is (nil? (core/-patient-count (dynamic-compile (~elm-constructor + [#elm/parameter-ref "x" + #elm/parameter-ref "y" + #elm/parameter-ref "z"]))))))) + +(defmacro testing-binary-resolve-refs + [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve expression references" + (let [elm# (~elm-constructor + [#elm/expression-ref "x" + #elm/expression-ref "y"]) + expr-defs# [{:type "ExpressionDef" :name "x" :expression "a" + :context "Unfiltered"} + {:type "ExpressionDef" :name "y" :expression "b" + :context "Unfiltered"}] + ctx# {:library {:statements {:def expr-defs#}}} + expr# (c/resolve-refs (c/compile ctx# elm#) (zipmap ["x" "y"] expr-defs#))] + (has-form expr# '(~form-name "a" "b")))))) + +(defmacro testing-ternary-resolve-refs [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve expression references" + (let [elm# (~elm-constructor + [#elm/expression-ref "x" + #elm/expression-ref "y" + #elm/expression-ref "z"]) + expr-defs# [{:type "ExpressionDef" :name "x" :expression "a" + :context "Unfiltered"} + {:type "ExpressionDef" :name "y" :expression "b" + :context "Unfiltered"} + {:type "ExpressionDef" :name "z" :expression "c" + :context "Unfiltered"}] + ctx# {:library {:statements {:def expr-defs#}}} + expr# (c/resolve-refs (c/compile ctx# elm#) (zipmap ["x" "y" "z"] expr-defs#))] + (has-form expr# '(~form-name "a" "b" "c")))))) + +(defmacro testing-binary-resolve-params [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve parameters" + (let [elm# (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y"]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"}]}}} + expr# (c/resolve-params (c/compile ctx# elm#) {"x" "a" "y" "b"})] + (has-form expr# '(~form-name "a" "b")))))) + +(defmacro testing-ternary-resolve-params [elm-constructor] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve parameters" + (let [elm# (~elm-constructor [#elm/parameter-ref "x" #elm/parameter-ref "y" + #elm/parameter-ref "z"]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"} {:name "z"}]}}} + expr# (c/resolve-params (c/compile ctx# elm#) {"x" "a" "y" "b" "z" "c"})] + (has-form expr# '(~form-name "a" "b" "c")))))) + (defmacro testing-binary-precision-dynamic ([elm-constructor] `(testing-binary-precision-dynamic ~elm-constructor "year" "month")) @@ -280,6 +566,41 @@ #elm/parameter-ref "y" precision#]))))))))) +(defmacro testing-binary-precision-resolve-refs + ([elm-constructor] + `(testing-binary-precision-resolve-refs ~elm-constructor "year" "month")) + ([elm-constructor & precisions] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve expression references" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor + [#elm/expression-ref "x" + #elm/expression-ref "y" + precision#]) + expr-def-x# {:type "ExpressionDef" :name "x" :expression "a" + :context "Unfiltered"} + expr-def-y# {:type "ExpressionDef" :name "y" :expression "b" + :context "Unfiltered"} + ctx# {:library {:statements {:def [expr-def-x# expr-def-y#]}}} + expr# (c/resolve-refs (c/compile ctx# elm#) {"x" expr-def-x# + "y" expr-def-y#})] + (has-form expr# (list (quote ~form-name) "a" "b" precision#)))))))) + +(defmacro testing-binary-precision-resolve-params + ([elm-constructor] + `(testing-binary-precision-resolve-params ~elm-constructor "year" "month")) + ([elm-constructor & precisions] + (let [form-name (symbol (name elm-constructor))] + `(testing "resolve parameters" + (doseq [precision# ~(vec precisions)] + (let [elm# (~elm-constructor + [#elm/parameter-ref "x" + #elm/parameter-ref "y" + precision#]) + ctx# {:library {:parameters {:def [{:name "x"} {:name "y"}]}}} + expr# (c/resolve-params (c/compile ctx# elm#) {"x" "a" "y" "b"})] + (has-form expr# (list (quote ~form-name) "a" "b" precision#)))))))) + (defmacro testing-ternary-dynamic [elm-constructor] `(testing "expression is dynamic" (is (false? (core/-static (dynamic-compile (~elm-constructor @@ -287,8 +608,124 @@ #elm/parameter-ref "y" #elm/parameter-ref "z"]))))))) +(defmacro testing-unary-op [elm-constructor] + `(do + (testing-unary-dynamic ~elm-constructor) + + (testing-unary-attach-cache ~elm-constructor) + + (testing-unary-patient-count ~elm-constructor) + + (testing-unary-resolve-refs ~elm-constructor) + + (testing-unary-resolve-params ~elm-constructor) + + (testing-unary-equals-hash-code ~elm-constructor) + + (testing-unary-form ~elm-constructor))) + +(defmacro testing-unary-precision-op [elm-constructor & precisions] + `(do + (testing-unary-precision-dynamic ~elm-constructor ~@precisions) + + (testing-unary-precision-attach-cache ~elm-constructor ~@precisions) + + (testing-unary-precision-patient-count ~elm-constructor ~@precisions) + + (testing-unary-precision-resolve-refs ~elm-constructor ~@precisions) + + (testing-unary-precision-resolve-params ~elm-constructor ~@precisions) + + (testing-unary-precision-equals-hash-code ~elm-constructor ~@precisions) + + (testing-unary-precision-form ~elm-constructor ~@precisions))) + +(defmacro testing-binary-op [elm-constructor] + `(do + (testing-binary-dynamic ~elm-constructor) + + (testing-binary-attach-cache ~elm-constructor) + + (testing-binary-patient-count ~elm-constructor) + + (testing-binary-resolve-refs ~elm-constructor) + + (testing-binary-resolve-params ~elm-constructor) + + (testing-binary-equals-hash-code ~elm-constructor) + + (testing-binary-form ~elm-constructor))) + +(defmacro testing-binary-precision-op [elm-constructor] + `(do + (testing-binary-dynamic ~elm-constructor) + + (testing-binary-precision-dynamic ~elm-constructor) + + (testing-binary-attach-cache ~elm-constructor) + + (testing-binary-precision-attach-cache ~elm-constructor) + + (testing-binary-patient-count ~elm-constructor) + + (testing-binary-precision-patient-count ~elm-constructor) + + (testing-binary-resolve-refs ~elm-constructor) + + (testing-binary-precision-resolve-refs ~elm-constructor) + + (testing-binary-resolve-params ~elm-constructor) + + (testing-binary-precision-resolve-params ~elm-constructor) + + (testing-binary-equals-hash-code ~elm-constructor) + + (testing-binary-precision-equals-hash-code ~elm-constructor) + + (testing-binary-form ~elm-constructor) + + (testing-binary-precision-form ~elm-constructor))) + +(defmacro testing-binary-precision-only-op [elm-constructor & precisions] + `(do + (testing-binary-precision-dynamic ~elm-constructor ~@precisions) + + (testing-binary-precision-attach-cache ~elm-constructor ~@precisions) + + (testing-binary-precision-patient-count ~elm-constructor ~@precisions) + + (testing-binary-precision-resolve-refs ~elm-constructor ~@precisions) + + (testing-binary-precision-resolve-params ~elm-constructor ~@precisions) + + (testing-binary-precision-equals-hash-code ~elm-constructor ~@precisions) + + (testing-binary-precision-form ~elm-constructor ~@precisions))) + +(defmacro testing-ternary-op [elm-constructor] + `(do + (testing-ternary-dynamic ~elm-constructor) + + (testing-ternary-attach-cache ~elm-constructor) + + (testing-ternary-patient-count ~elm-constructor) + + (testing-ternary-resolve-refs ~elm-constructor) + + (testing-ternary-resolve-params ~elm-constructor) + + (testing-ternary-equals-hash-code ~elm-constructor) + + (testing-ternary-form ~elm-constructor))) + +(defmacro testing-equals-hash-code [elm] + `(testing "equals/hashCode" + (let [expr-1# (dynamic-compile ~elm) + expr-2# (dynamic-compile ~elm)] + (is (= 1 (count (set [expr-1# expr-2#]))))))) + (defn resource [db type id] - (ed/mk-resource db (d/resource-handle db type id))) + (cr/mk-resource db (d/resource-handle db type id))) (defn eval-unfiltered [elm] (core/-eval (c/compile {:eval-context "Unfiltered"} elm) {} nil nil)) diff --git a/modules/cql/test/blaze/elm/compiler/type_operators_test.clj b/modules/cql/test/blaze/elm/compiler/type_operators_test.clj index 2c7d1a746..0dd5160ee 100644 --- a/modules/cql/test/blaze/elm/compiler/type_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/type_operators_test.clj @@ -9,16 +9,16 @@ [blaze.elm.compiler.clinical-operators] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] - [blaze.elm.compiler.test-util :as ctu] + [blaze.elm.compiler.test-util :as ctu :refer [has-form]] [blaze.elm.compiler.type-operators] [blaze.elm.concept :as concept] [blaze.elm.decimal :as decimal] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] [blaze.elm.protocols :as p] - [blaze.elm.quantity :as quantity] + [blaze.elm.quantity :refer [quantity]] [blaze.elm.quantity-spec] - [blaze.elm.ratio :as ratio] + [blaze.elm.ratio :refer [ratio]] [blaze.elm.util-spec] [blaze.fhir.spec.type.system :as system] [clojure.spec.test.alpha :as st] @@ -100,10 +100,33 @@ #elm/as ["{urn:hl7-org:elm-types:r1}DateTime" #elm/date-time"2019-09-04"] (system/date-time 2019 9 4))) - (testing "expression is dynamic" - (is (false? (core/-static (ctu/dynamic-compile - #elm/as["{urn:hl7-org:elm-types:r1}Integer" - #elm/parameter-ref "x"]))))) + (let [expr (ctu/dynamic-compile #elm/as["{urn:hl7-org:elm-types:r1}Integer" + #elm/parameter-ref "x"])] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm #elm/as["{urn:hl7-org:elm-types:r1}Integer" + #elm/expression-ref "x"] + expr-def {:type "ExpressionDef" :name "x" :expression "y" + :context "Patient"} + ctx {:library {:statements {:def [expr-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-def})] + (has-form expr '(as elm/integer "y")))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {"x" "y"})] + (has-form expr '(as elm/integer "y"))))) + + (ctu/testing-equals-hash-code #elm/as["{urn:hl7-org:elm-types:r1}Integer" + #elm/parameter-ref "x"]) (testing "form" (are [elm form] (= form (c/form (c/compile {} elm))) @@ -176,9 +199,7 @@ (ctu/testing-binary-null elm/can-convert-quantity #elm/quantity [1 "m"] #elm/string "m") - (ctu/testing-binary-dynamic elm/can-convert-quantity) - - (ctu/testing-binary-form elm/can-convert-quantity)) + (ctu/testing-binary-op elm/can-convert-quantity)) ;; 22.4. Children ;; @@ -201,9 +222,7 @@ (ctu/testing-unary-null elm/children) - (ctu/testing-unary-dynamic elm/children) - - (ctu/testing-unary-form elm/children)) + (ctu/testing-unary-op elm/children)) ;; TODO 22.5. Convert ;; @@ -249,16 +268,14 @@ ;; If either argument is null, the result is null. (deftest compile-convert-quantity-test (are [argument unit res] (p/equal res (core/-eval (c/compile {} (elm/convert-quantity [argument unit])) {} nil nil)) - #elm/quantity [5 "mg"] #elm/string "g" (quantity/quantity 0.005 "g")) + #elm/quantity [5 "mg"] #elm/string "g" (quantity 0.005M "g")) (are [argument unit] (nil? (core/-eval (c/compile {} (elm/convert-quantity [argument unit])) {} nil nil)) #elm/quantity [5 "mg"] #elm/string "m") (ctu/testing-binary-null elm/convert-quantity #elm/quantity [5 "mg"] #elm/string "m") - (ctu/testing-binary-dynamic elm/convert-quantity) - - (ctu/testing-binary-form elm/convert-quantity)) + (ctu/testing-binary-op elm/convert-quantity)) ;; 22.7. ConvertsToBoolean ;; @@ -356,9 +373,7 @@ (ctu/testing-unary-null elm/converts-to-boolean) - (ctu/testing-unary-dynamic elm/converts-to-boolean) - - (ctu/testing-unary-form elm/converts-to-boolean)) + (ctu/testing-unary-op elm/converts-to-boolean)) ;; 22.8. ConvertsToDate ;; @@ -412,9 +427,7 @@ (ctu/testing-unary-null elm/converts-to-date) - (ctu/testing-unary-dynamic elm/converts-to-date) - - (ctu/testing-unary-form elm/converts-to-date)) + (ctu/testing-unary-op elm/converts-to-date)) ;; 22.9. ConvertsToDateTime ;; @@ -465,9 +478,7 @@ (ctu/testing-unary-null elm/converts-to-date-time) - (ctu/testing-unary-dynamic elm/converts-to-date-time) - - (ctu/testing-unary-form elm/converts-to-date-time)) + (ctu/testing-unary-op elm/converts-to-date-time)) ;; 22.10. ConvertsToDecimal ;; @@ -525,9 +536,7 @@ (ctu/testing-unary-null elm/converts-to-decimal) - (ctu/testing-unary-dynamic elm/converts-to-decimal) - - (ctu/testing-unary-form elm/converts-to-decimal)) + (ctu/testing-unary-op elm/converts-to-decimal)) ;; 22.11. ConvertsToLong ;; @@ -582,9 +591,7 @@ (ctu/testing-unary-null elm/converts-to-long) - (ctu/testing-unary-dynamic elm/converts-to-long) - - (ctu/testing-unary-form elm/converts-to-long)) + (ctu/testing-unary-op elm/converts-to-long)) ;; 22.12. ConvertsToInteger ;; @@ -638,9 +645,7 @@ (ctu/testing-unary-null elm/converts-to-integer) - (ctu/testing-unary-dynamic elm/converts-to-integer) - - (ctu/testing-unary-form elm/converts-to-integer)) + (ctu/testing-unary-op elm/converts-to-integer)) ;; 22.13. ConvertsToQuantity ;; @@ -710,9 +715,7 @@ (ctu/testing-unary-null elm/converts-to-quantity) - (ctu/testing-unary-dynamic elm/converts-to-quantity) - - (ctu/testing-unary-form elm/converts-to-quantity)) + (ctu/testing-unary-op elm/converts-to-quantity)) ;; 22.14. ConvertsToRatio ;; @@ -748,9 +751,7 @@ (ctu/testing-unary-null elm/converts-to-ratio) - (ctu/testing-unary-dynamic elm/converts-to-ratio) - - (ctu/testing-unary-form elm/converts-to-ratio)) + (ctu/testing-unary-op elm/converts-to-ratio)) ;; 22.15. ConvertsToString ;; @@ -822,9 +823,7 @@ (ctu/testing-unary-null elm/converts-to-string) - (ctu/testing-unary-dynamic elm/converts-to-string) - - (ctu/testing-unary-form elm/converts-to-string)) + (ctu/testing-unary-op elm/converts-to-string)) ;; 22.16. ConvertsToTime ;; @@ -891,9 +890,7 @@ (ctu/testing-unary-null elm/converts-to-time) - (ctu/testing-unary-dynamic elm/converts-to-time) - - (ctu/testing-unary-form elm/converts-to-time)) + (ctu/testing-unary-op elm/converts-to-time)) ;; 22.17. Descendents ;; @@ -907,11 +904,7 @@ ;; If the source is null, the result is null. (deftest compile-to-descendents-test (testing "Code" - (testing "expression is dynamic" - (is (not (core/static? (c/compile {} (elm/descendents (ctu/code "system-134534" "code-134551"))))))) - - (are [x res] (= res (core/-eval (c/compile {} (elm/descendents x)) - {:now ctu/now} nil nil)) + (are [x res] (= res (c/compile {} (elm/descendents x))) (ctu/code "system-134534" "code-134551") ["code-134551" nil "system-134534" nil])) @@ -919,9 +912,7 @@ (ctu/testing-unary-null elm/descendents) - (ctu/testing-unary-dynamic elm/descendents) - - (ctu/testing-unary-form elm/descendents)) + (ctu/testing-unary-op elm/descendents)) ;; 22.18. Is ;; @@ -1036,10 +1027,33 @@ #elm/is ["{urn:hl7-org:elm-types:r1}DateTime" #elm/string "2019-09-04"] #elm/is ["{urn:hl7-org:elm-types:r1}DateTime" {:type "Null"}])) - (testing "expression is dynamic" - (is (false? (core/-static (ctu/dynamic-compile - #elm/is["{urn:hl7-org:elm-types:r1}Integer" - #elm/parameter-ref "x"]))))) + (let [expr (ctu/dynamic-compile #elm/is["{urn:hl7-org:elm-types:r1}Integer" + #elm/parameter-ref "x"])] + + (testing "expression is dynamic" + (is (false? (core/-static expr)))) + + (testing "attach-cache" + (is (= [expr] (st/with-instrument-disabled (c/attach-cache expr ::cache))))) + + (testing "patient count" + (is (nil? (core/-patient-count expr)))) + + (testing "resolve expression references" + (let [elm #elm/is["{urn:hl7-org:elm-types:r1}Integer" + #elm/expression-ref "x"] + expr-def {:type "ExpressionDef" :name "x" :expression "y" + :context "Patient"} + ctx {:library {:statements {:def [expr-def]}}} + expr (c/resolve-refs (c/compile ctx elm) {"x" expr-def})] + (has-form expr '(is elm/integer "y")))) + + (testing "resolve parameters" + (let [expr (c/resolve-params expr {"x" "y"})] + (has-form expr '(is elm/integer "y"))))) + + (ctu/testing-equals-hash-code #elm/is["{urn:hl7-org:elm-types:r1}Integer" + #elm/parameter-ref "x"]) (testing "form" (are [elm form] (= form (c/form (c/compile {} elm))) @@ -1160,9 +1174,7 @@ (ctu/testing-unary-null elm/to-boolean) - (ctu/testing-unary-dynamic elm/to-boolean) - - (ctu/testing-unary-form elm/to-boolean)) + (ctu/testing-unary-op elm/to-boolean)) ;; 22.20. ToChars ;; @@ -1190,9 +1202,7 @@ (ctu/testing-unary-null elm/to-chars) - (ctu/testing-unary-dynamic elm/to-chars) - - (ctu/testing-unary-form elm/to-chars)) + (ctu/testing-unary-op elm/to-chars)) ;; 22.21. ToConcept ;; @@ -1219,9 +1229,7 @@ (ctu/testing-unary-null elm/to-concept) - (ctu/testing-unary-dynamic elm/to-concept) - - (ctu/testing-unary-form elm/to-concept)) + (ctu/testing-unary-op elm/to-concept)) ;; 22.22. ToDate ;; @@ -1297,9 +1305,7 @@ (ctu/testing-unary-null elm/to-date) - (ctu/testing-unary-dynamic elm/to-date) - - (ctu/testing-unary-form elm/to-date)) + (ctu/testing-unary-op elm/to-date)) ;; 22.23. ToDateTime ;; @@ -1363,9 +1369,7 @@ (ctu/testing-unary-null elm/to-date-time) - (ctu/testing-unary-dynamic elm/to-date-time) - - (ctu/testing-unary-form elm/to-date-time)) + (ctu/testing-unary-op elm/to-date-time)) ;; 22.24. ToDecimal ;; @@ -1413,9 +1417,7 @@ (ctu/testing-unary-null elm/to-decimal) - (ctu/testing-unary-dynamic elm/to-decimal) - - (ctu/testing-unary-form elm/to-decimal)) + (ctu/testing-unary-op elm/to-decimal)) ;; 22.25. ToInteger ;; @@ -1456,9 +1458,7 @@ (ctu/testing-unary-null elm/to-integer) - (ctu/testing-unary-dynamic elm/to-integer) - - (ctu/testing-unary-form elm/to-integer)) + (ctu/testing-unary-op elm/to-integer)) ;; 22.26. ToList ;; @@ -1490,9 +1490,7 @@ #elm/parameter-ref "nil" [] #elm/parameter-ref "a" ["a"])) - (ctu/testing-unary-dynamic elm/to-list) - - (ctu/testing-unary-form elm/to-list)) + (ctu/testing-unary-op elm/to-list)) ;; 22.27. ToLong ;; @@ -1537,9 +1535,7 @@ (ctu/testing-unary-null elm/to-long) - (ctu/testing-unary-dynamic elm/to-long) - - (ctu/testing-unary-form elm/to-long)) + (ctu/testing-unary-op elm/to-long)) ;; 22.28. ToQuantity ;; @@ -1574,16 +1570,16 @@ (deftest compile-to-quantity-test (testing "String" (are [x res] (p/equal res (ctu/compile-unop elm/to-quantity elm/string x)) - "-1" (quantity/quantity -1 "1") - "1" (quantity/quantity 1 "1") + "-1" (quantity -1 "1") + "1" (quantity 1 "1") - "1'm'" (quantity/quantity 1 "m") - "1 'm'" (quantity/quantity 1 "m") - "1 'm'" (quantity/quantity 1 "m") + "1'm'" (quantity 1 "m") + "1 'm'" (quantity 1 "m") + "1 'm'" (quantity 1 "m") - "10 'm'" (quantity/quantity 10 "m") + "10 'm'" (quantity 10 "m") - "1.1 'm'" (quantity/quantity 1.1M "m")) + "1.1 'm'" (quantity 1.1M "m")) (are [x] (nil? (ctu/compile-unop elm/to-quantity elm/string x)) (str (- decimal/min 1e-8M)) @@ -1595,30 +1591,28 @@ (testing "Integer" (are [x res] (= res (ctu/compile-unop elm/to-quantity elm/integer x)) - "1" (quantity/quantity 1 "1"))) + "1" (quantity 1 "1"))) (testing "Decimal" (are [x res] (p/equal res (ctu/compile-unop elm/to-quantity elm/decimal x)) - "1" (quantity/quantity 1 "1") - "1.1" (quantity/quantity 1.1M "1"))) + "1" (quantity 1 "1") + "1.1" (quantity 1.1M "1"))) (testing "Ratio" (are [x res] (p/equal res (ctu/compile-unop elm/to-quantity elm/ratio x)) - [[1] [1]] (quantity/quantity 1 "1") - [[-1] [1]] (quantity/quantity -1 "1") + [[1] [1]] (quantity 1 "1") + [[-1] [1]] (quantity -1 "1") - [[1 "s"] [1 "s"]] (quantity/quantity 1 "1") - [[1 "s"] [2 "s"]] (quantity/quantity 2 "1") + [[1 "s"] [1 "s"]] (quantity 1 "1") + [[1 "s"] [2 "s"]] (quantity 2 "1") - [[1 "m"] [1 "s"]] (quantity/quantity 1 "s/m") - [[1 "s"] [1 "m"]] (quantity/quantity 1 "m/s") - [[100 "cm"] [1 "m"]] (quantity/quantity 1 "1"))) + [[1 "m"] [1 "s"]] (quantity 1 "s/m") + [[1 "s"] [1 "m"]] (quantity 1 "m/s") + [[100 "cm"] [1 "m"]] (quantity 1 "1"))) (ctu/testing-unary-null elm/to-quantity) - (ctu/testing-unary-dynamic elm/to-quantity) - - (ctu/testing-unary-form elm/to-quantity)) + (ctu/testing-unary-op elm/to-quantity)) ;; 22.29. ToRatio ;; @@ -1638,24 +1632,24 @@ (deftest compile-to-ratio-test (testing "String" (are [x res] (p/equal res (ctu/compile-unop elm/to-ratio elm/string x)) - "-1:-1" (ratio/ratio (quantity/quantity -1 "1") (quantity/quantity -1 "1")) - "1:1" (ratio/ratio (quantity/quantity 1 "1") (quantity/quantity 1 "1")) - "1:100" (ratio/ratio (quantity/quantity 1 "1") (quantity/quantity 100 "1")) - "100:1" (ratio/ratio (quantity/quantity 100 "1") (quantity/quantity 1 "1")) + "-1:-1" (ratio (quantity -1 "1") (quantity -1 "1")) + "1:1" (ratio (quantity 1 "1") (quantity 1 "1")) + "1:100" (ratio (quantity 1 "1") (quantity 100 "1")) + "100:1" (ratio (quantity 100 "1") (quantity 1 "1")) - "1'm':1'm'" (ratio/ratio (quantity/quantity 1 "m") (quantity/quantity 1 "m")) - "1 'm':1 'm'" (ratio/ratio (quantity/quantity 1 "m") (quantity/quantity 1 "m")) - "1 'm':1 'm'" (ratio/ratio (quantity/quantity 1 "m") (quantity/quantity 1 "m")) + "1'm':1'm'" (ratio (quantity 1 "m") (quantity 1 "m")) + "1 'm':1 'm'" (ratio (quantity 1 "m") (quantity 1 "m")) + "1 'm':1 'm'" (ratio (quantity 1 "m") (quantity 1 "m")) - "2'm':1'm'" (ratio/ratio (quantity/quantity 2 "m") (quantity/quantity 1 "m")) - "1'm':2'm'" (ratio/ratio (quantity/quantity 1 "m") (quantity/quantity 2 "m")) + "2'm':1'm'" (ratio (quantity 2 "m") (quantity 1 "m")) + "1'm':2'm'" (ratio (quantity 1 "m") (quantity 2 "m")) - "1'cm':1'm'" (ratio/ratio (quantity/quantity 1 "cm") (quantity/quantity 1 "m")) - "1'm':1'cm'" (ratio/ratio (quantity/quantity 1 "m") (quantity/quantity 1 "cm")) + "1'cm':1'm'" (ratio (quantity 1 "cm") (quantity 1 "m")) + "1'm':1'cm'" (ratio (quantity 1 "m") (quantity 1 "cm")) - "10 'm':10 'm'" (ratio/ratio (quantity/quantity 10 "m") (quantity/quantity 10 "m")) + "10 'm':10 'm'" (ratio (quantity 10 "m") (quantity 10 "m")) - "1.1 'm':1.1 'm'" (ratio/ratio (quantity/quantity 1.1M "m") (quantity/quantity 1.1M "m")))) + "1.1 'm':1.1 'm'" (ratio (quantity 1.1M "m") (quantity 1.1M "m")))) (are [x] (nil? (ctu/compile-unop elm/to-ratio elm/string x)) ":" @@ -1667,9 +1661,7 @@ (ctu/testing-unary-null elm/to-ratio) - (ctu/testing-unary-dynamic elm/to-ratio) - - (ctu/testing-unary-form elm/to-ratio)) + (ctu/testing-unary-op elm/to-ratio)) ;; 22.30. ToString ;; @@ -1746,9 +1738,7 @@ (ctu/testing-unary-null elm/to-string) - (ctu/testing-unary-dynamic elm/to-string) - - (ctu/testing-unary-form elm/to-string)) + (ctu/testing-unary-op elm/to-string)) ;; 22.31. ToTime ;; @@ -1805,6 +1795,4 @@ (ctu/testing-unary-null elm/to-time) - (ctu/testing-unary-dynamic elm/to-time) - - (ctu/testing-unary-form elm/to-time)) + (ctu/testing-unary-op elm/to-time)) diff --git a/modules/cql/test/blaze/elm/expression/cache/bloom_filter_test.clj b/modules/cql/test/blaze/elm/expression/cache/bloom_filter_test.clj new file mode 100644 index 000000000..9d34f9232 --- /dev/null +++ b/modules/cql/test/blaze/elm/expression/cache/bloom_filter_test.clj @@ -0,0 +1,120 @@ +(ns blaze.elm.expression.cache.bloom-filter-test + (:require + [blaze.db.api :as d] + [blaze.db.api-stub :refer [mem-node-config with-system-data]] + [blaze.elm.compiler :as c] + [blaze.elm.compiler.test-util :as ctu] + [blaze.elm.expression.cache.bloom-filter :as bloom-filter] + [blaze.elm.expression.cache.codec-spec] + [blaze.elm.expression.cache.codec.by-t-spec] + [blaze.elm.expression.cache.codec.form-spec] + [blaze.elm.literal] + [blaze.fhir.test-util] + [blaze.module.test-util :refer [with-system]] + [blaze.test-util] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest testing]] + [juxt.iota :refer [given]] + [taoensso.timbre :as log])) + +(st/instrument) +(ctu/instrument-compile) +(log/set-min-level! :trace) + +(defn- fixture [f] + (st/instrument) + (ctu/instrument-compile) + (f) + (st/unstrument)) + +(test/use-fixtures :each fixture) + +(deftest create-test + (testing "with empty database" + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm)] + + (given (bloom-filter/create node expr) + ::bloom-filter/t := 0 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 0 + ::bloom-filter/mem-size := 11981)))) + + (testing "with one Patient with one Observation" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm)] + + (given (bloom-filter/create node expr) + ::bloom-filter/t := 1 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 1 + ::bloom-filter/mem-size := 11981)))) + + (testing "with two Patients on of which has one Observation" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}] + [:put {:fhir/type :fhir/Patient :id "1"}]]] + + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm)] + + (given (bloom-filter/create node expr) + ::bloom-filter/t := 1 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 1 + ::bloom-filter/mem-size := 11981))))) + +(deftest recreate-test + (testing "with empty database" + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm) + bloom-filter (bloom-filter/create node expr)] + + (given (bloom-filter/recreate node bloom-filter expr) + ::bloom-filter/t := 0 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 0 + ::bloom-filter/mem-size := 11981)))) + + (testing "with one Patient with one Observation added" + (with-system [{:blaze.db/keys [node]} mem-node-config] + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm) + bloom-filter (bloom-filter/create node expr)] + + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]) + + (given (bloom-filter/recreate node bloom-filter expr) + ::bloom-filter/t := 1 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 1 + ::bloom-filter/mem-size := 11981)))) + + (testing "with one additional Patient with one Observation added" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [elm #elm/exists #elm/retrieve{:type "Observation"} + expr (c/compile {:eval-context "Patient"} elm) + bloom-filter (bloom-filter/create node expr)] + + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "1"}] + [:put {:fhir/type :fhir/Observation :id "1" + :subject #fhir/Reference{:reference "Patient/1"}}]]) + + (given (bloom-filter/recreate node bloom-filter expr) + ::bloom-filter/t := 2 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 1 + ::bloom-filter/mem-size := 11981))))) diff --git a/modules/cql/test/blaze/elm/expression/cache/codec/by_t_spec.clj b/modules/cql/test/blaze/elm/expression/cache/codec/by_t_spec.clj new file mode 100644 index 000000000..7108cccad --- /dev/null +++ b/modules/cql/test/blaze/elm/expression/cache/codec/by_t_spec.clj @@ -0,0 +1,15 @@ +(ns blaze.elm.expression.cache.codec.by-t-spec + (:require + [blaze.db.kv.spec] + [blaze.elm.expression.cache :as-alias ec] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.expression.cache.codec.by-t :as codec-by-t] + [clojure.spec.alpha :as s])) + +(s/fdef codec-by-t/put-entry + :args (s/cat :bloom-filter ::ec/bloom-filter) + :ret :blaze.db.kv/write-entry) + +(s/fdef codec-by-t/delete-entry + :args (s/cat :bloom-filter ::ec/bloom-filter) + :ret :blaze.db.kv/write-entry) diff --git a/modules/cql/test/blaze/elm/expression/cache/codec/form_spec.clj b/modules/cql/test/blaze/elm/expression/cache/codec/form_spec.clj new file mode 100644 index 000000000..f54f6cd1b --- /dev/null +++ b/modules/cql/test/blaze/elm/expression/cache/codec/form_spec.clj @@ -0,0 +1,10 @@ +(ns blaze.elm.expression.cache.codec.form-spec + (:require + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.expression.cache.codec.form :as codec-form] + [clojure.spec.alpha :as s])) + +(s/fdef codec-form/hash + :args (s/cat :expr-form ::bloom-filter/expr-form) + :ret ::bloom-filter/hash) diff --git a/modules/cql/test/blaze/elm/expression/cache/codec_spec.clj b/modules/cql/test/blaze/elm/expression/cache/codec_spec.clj new file mode 100644 index 000000000..252234e0f --- /dev/null +++ b/modules/cql/test/blaze/elm/expression/cache/codec_spec.clj @@ -0,0 +1,15 @@ +(ns blaze.elm.expression.cache.codec-spec + (:require + [blaze.db.kv.spec] + [blaze.elm.expression.cache :as-alias ec] + [blaze.elm.expression.cache.bloom-filter.spec] + [blaze.elm.expression.cache.codec :as codec] + [clojure.spec.alpha :as s])) + +(s/fdef codec/put-entry + :args (s/cat :bloom-filter ::ec/bloom-filter) + :ret :blaze.db.kv/write-entry) + +(s/fdef codec/delete-entry + :args (s/cat :bloom-filter ::ec/bloom-filter) + :ret :blaze.db.kv/write-entry) diff --git a/modules/cql/test/blaze/elm/expression/cache_test.clj b/modules/cql/test/blaze/elm/expression/cache_test.clj new file mode 100644 index 000000000..b006372f7 --- /dev/null +++ b/modules/cql/test/blaze/elm/expression/cache_test.clj @@ -0,0 +1,322 @@ +(ns blaze.elm.expression.cache-test + (:require + [blaze.anomaly :as ba] + [blaze.cache-collector.protocols :as ccp] + [blaze.coll.core :as coll] + [blaze.db.api :as d] + [blaze.db.api-stub :refer [mem-node-config with-system-data]] + [blaze.elm.compiler :as c] + [blaze.elm.compiler.test-util :as ctu] + [blaze.elm.expression :as expr] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache-spec] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] + [blaze.elm.expression.cache.codec-spec] + [blaze.elm.expression.cache.codec.by-t-spec] + [blaze.elm.literal :as elm] + [blaze.executors :as ex] + [blaze.fhir.test-util] + [blaze.log] + [blaze.metrics.spec] + [blaze.module.test-util :refer [with-system]] + [blaze.test-util :refer [given-thrown]] + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest is testing]] + [integrant.core :as ig] + [java-time.api :as time] + [juxt.iota :refer [given]] + [taoensso.timbre :as log]) + (:import + [com.github.benmanes.caffeine.cache AsyncLoadingCache] + [com.github.benmanes.caffeine.cache.stats CacheStats] + [com.google.common.hash HashCode])) + +(set! *warn-on-reflection* true) +(st/instrument) +(ctu/instrument-compile) +(log/set-min-level! :trace) + +(defn- fixture [f] + (st/instrument) + (ctu/instrument-compile) + (f) + (st/unstrument)) + +(test/use-fixtures :each fixture) + +(def ^:private config + (assoc mem-node-config + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} + :blaze.test/executor {})) + +(deftest init-test + (testing "nil config" + (given-thrown (ig/init {::expr/cache nil}) + :key := ::expr/cache + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `map?)) + + (testing "missing config" + (given-thrown (ig/init {::expr/cache {}}) + :key := ::expr/cache + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :node)) + [:cause-data ::s/problems 1 :pred] := `(fn ~'[%] (contains? ~'% :executor)))) + + (testing "invalid max size" + (given-thrown (ig/init {::expr/cache {:max-size-in-mb ::invalid}}) + :key := ::expr/cache + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :node)) + [:cause-data ::s/problems 1 :pred] := `(fn ~'[%] (contains? ~'% :executor)) + [:cause-data ::s/problems 2 :pred] := `nat-int? + [:cause-data ::s/problems 2 :val] := ::invalid)) + + (testing "invalid refresh" + (given-thrown (ig/init {::expr/cache {:refresh ::invalid}}) + :key := ::expr/cache + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :node)) + [:cause-data ::s/problems 1 :pred] := `(fn ~'[%] (contains? ~'% :executor)) + [:cause-data ::s/problems 2 :pred] := `time/duration? + [:cause-data ::s/problems 2 :val] := ::invalid)) + + (testing "init" + (with-system [{::expr/keys [cache]} config] + (is (s/valid? ::expr/cache cache))))) + +(deftest executor-init-test + (testing "nil config" + (given-thrown (ig/init {::ec/executor nil}) + :key := ::ec/executor + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `map?)) + + (testing "invalid num-threads" + (given-thrown (ig/init {::ec/executor {:num-threads ::invalid}}) + :key := ::ec/executor + :reason := ::ig/build-failed-spec + [:cause-data ::s/problems 0 :pred] := `pos-int? + [:cause-data ::s/problems 0 :val] := ::invalid))) + +(deftest executor-shutdown-timeout-test + (let [{::ec/keys [executor] :as system} (ig/init {::ec/executor {}})] + + ;; will produce a timeout, because the function runs 11 seconds + (ex/execute! executor #(Thread/sleep 11000)) + + ;; ensure that the function is called before the scheduler is halted + (Thread/sleep 100) + + (ig/halt! system) + + ;; the scheduler is shut down + (is (ex/shutdown? executor)) + + ;; but it isn't terminated yet + (is (not (ex/terminated? executor))))) + +(deftest bloom-filter-creation-duration-seconds-collector-init-test + (with-system [{collector ::ec/bloom-filter-creation-duration-seconds} {::ec/bloom-filter-creation-duration-seconds {}}] + (is (s/valid? :blaze.metrics/collector collector)))) + +(deftest bloom-filter-useful-total-collector-init-test + (with-system [{collector ::ec/bloom-filter-useful-total} {::ec/bloom-filter-useful-total {}}] + (is (s/valid? :blaze.metrics/collector collector)))) + +(deftest bloom-filter-not-useful-total-collector-init-test + (with-system [{collector ::ec/bloom-filter-not-useful-total} {::ec/bloom-filter-not-useful-total {}}] + (is (s/valid? :blaze.metrics/collector collector)))) + +(deftest bloom-filter-false-positive-total-collector-init-test + (with-system [{collector ::ec/bloom-filter-false-positive-total} {::ec/bloom-filter-false-positive-total {}}] + (is (s/valid? :blaze.metrics/collector collector)))) + +(deftest bloom-filter-bytes-collector-init-test + (with-system [{collector ::ec/bloom-filter-bytes} {::ec/bloom-filter-bytes {}}] + (is (s/valid? :blaze.metrics/collector collector)))) + +(def ^:private config + (assoc mem-node-config + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} + :blaze.test/executor {})) + +(defn- compile-exists-expr [resource-type] + (let [elm (elm/exists (elm/retrieve {:type resource-type}))] + (c/compile {:eval-context "Patient"} elm))) + +(defn- create-bloom-filter! + "Creates the Bloom filters used in `expr` and wait's some time to ensure that + the creation is finished." + [expr cache] + (c/attach-cache expr cache) + (Thread/sleep 100)) + +(deftest get-test + (testing "one Bloom filter on empty database" + (with-system [{::expr/keys [cache]} config] + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (ec/get cache (compile-exists-expr "Observation")) + ::bloom-filter/t := 0 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 0 + ::bloom-filter/mem-size := 11981)))) + +(deftest get-disk-test + (testing "an empty database contains no Bloom filter" + (with-system [{::expr/keys [cache]} config] + (is (ba/not-found? (ec/get-disk cache (HashCode/fromString "d4fc6cde1636852f9e362a68ca7be027a66bf7cb38ebff9c256c3eb2179c2639")))))) + + (testing "one Bloom filter on empty database" + (with-system [{::expr/keys [cache]} config] + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (ec/get-disk cache (HashCode/fromString "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985")) + ::bloom-filter/t := 0 + ::bloom-filter/expr-form := "(exists (retrieve \"Observation\"))" + ::bloom-filter/patient-count := 0 + ::bloom-filter/mem-size := 11990)))) + +(deftest delete-test + (testing "an empty database contains no Bloom filter" + (with-system [{::expr/keys [cache]} config] + (is (ba/not-found? (ec/delete-disk! cache (HashCode/fromString "d4fc6cde1636852f9e362a68ca7be027a66bf7cb38ebff9c256c3eb2179c2639")))))) + + (testing "one Bloom filter on empty database" + (with-system [{::expr/keys [cache]} config] + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (let [hash (HashCode/fromString "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985")] + (is (not (ba/anomaly? (ec/get-disk cache hash)))) + + (ec/delete-disk! cache hash) + + (is (ba/not-found? (ec/get-disk cache hash))))))) + +(deftest list-by-t-test + (testing "an empty database contains zero Bloom filters" + (with-system [{::expr/keys [cache]} config] + (is (coll/empty? (ec/list-by-t cache))))) + + (testing "one Bloom filter on empty database" + (with-system [{::expr/keys [cache]} config] + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/hash str] := "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985" + [0 ::bloom-filter/t] := 0 + [0 ::bloom-filter/expr-form] := "(exists (retrieve \"Observation\"))" + [0 ::bloom-filter/patient-count] := 0 + [0 ::bloom-filter/mem-size] := 11981))) + + (testing "one Bloom filter on database with one patient" + (with-system-data [{::expr/keys [cache]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/hash str] := "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985" + [0 ::bloom-filter/t] := 1 + [0 ::bloom-filter/expr-form] := "(exists (retrieve \"Observation\"))" + [0 ::bloom-filter/patient-count] := 1 + [0 ::bloom-filter/mem-size] := 11981))) + + (testing "two Bloom filters on database with one patient" + (with-system-data [{::expr/keys [cache]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (create-bloom-filter! (compile-exists-expr "Observation") cache) + (create-bloom-filter! (compile-exists-expr "Condition") cache) + + (given (into [] (ec/list-by-t cache)) + count := 2 + [0 ::bloom-filter/hash str] := "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985" + [0 ::bloom-filter/t] := 1 + [0 ::bloom-filter/expr-form] := "(exists (retrieve \"Observation\"))" + [0 ::bloom-filter/patient-count] := 1 + [0 ::bloom-filter/mem-size] := 11981 + [1 ::bloom-filter/hash str] := "b24882a623bc9c78572630b7c5f288553a0c5e31d6c0d9a21e0c3ec43a0d78e7" + [1 ::bloom-filter/t] := 1 + [1 ::bloom-filter/expr-form] := "(exists (retrieve \"Condition\"))" + [1 ::bloom-filter/patient-count] := 0 + [1 ::bloom-filter/mem-size] := 11981))) + + (testing "Bloom filter updates are reflected in the list" + (with-system-data [{::expr/keys [cache] :blaze.db/keys [node]} + (assoc-in config [::expr/cache :refresh] (time/millis 1))] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "1"}] + [:put {:fhir/type :fhir/Observation :id "1" + :subject #fhir/Reference{:reference "Patient/1"}}]]) + + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/hash str] := "78c3f9b9e187480870ce815ad6d324713dfa2cbd12968c5b14727fef7377b985" + [0 ::bloom-filter/t] := 2 + [0 ::bloom-filter/expr-form] := "(exists (retrieve \"Observation\"))" + [0 ::bloom-filter/patient-count] := 2 + [0 ::bloom-filter/mem-size] := 11981))) + + (testing "an old Bloom filter is loaded from the store even if the t was increased in the meantime" + (with-system-data [{::expr/keys [cache] :blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (testing "creates the Bloom filter with t=1" + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/t] := 1)) + + ;; invalidates the cache + (.invalidateAll (.synchronous ^AsyncLoadingCache (:mem-cache cache))) + + ;; advances the database + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "1"}] + [:put {:fhir/type :fhir/Observation :id "1" + :subject #fhir/Reference{:reference "Patient/1"}}]]) + + (testing "doesn't create a new Bloom filter because the old one is still in the store" + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/t] := 1))))) + +(deftest total-test + (testing "an empty database contains zero Bloom filters" + (with-system [{::expr/keys [cache]} config] + (is (zero? (ec/total cache))))) + + (testing "one Bloom filter on empty database" + (with-system [{::expr/keys [cache]} config] + (create-bloom-filter! (compile-exists-expr "Observation") cache) + + (is (= 1 (ec/total cache)))))) + +(deftest stats-test + (with-system [{::expr/keys [cache]} config] + (is (zero? (.hitCount ^CacheStats (ccp/-stats cache)))) + (is (zero? (ccp/-estimated-size cache))))) diff --git a/modules/cql/test/blaze/elm/literal.clj b/modules/cql/test/blaze/elm/literal.clj index 6d39f94ce..45fd03868 100644 --- a/modules/cql/test/blaze/elm/literal.clj +++ b/modules/cql/test/blaze/elm/literal.clj @@ -394,11 +394,26 @@ {:type "ReplaceMatches" :operand ops}) +;; 17.14. Split +(defn split [[s separator]] + {:type "Split" + :stringToSplit s + :separator separator}) + ;; 17.16. StartsWith (defn starts-with [ops] {:type "StartsWith" :operand ops}) +;; 17.17. Substring +(defn substring [[s start-index length]] + (cond-> + {:type "Substring" + :stringToSub s + :startIndex start-index} + length + (assoc :length length))) + ;; 17.18. Upper (defn upper [x] {:type "Upper" @@ -671,6 +686,10 @@ (defn singleton-from [list] {:type "SingletonFrom" :operand list}) +;; 20.26. Slice +(defn slice [[source start-index end-index]] + {:type "Slice" :source source :startIndex start-index :endIndex end-index}) + ;; 20.28. Times (defn times [lists] {:type "Times" :operand lists}) diff --git a/modules/cql/test/blaze/elm/quantity_test.clj b/modules/cql/test/blaze/elm/quantity_test.clj index 9600a0f13..25b4e3fe3 100644 --- a/modules/cql/test/blaze/elm/quantity_test.clj +++ b/modules/cql/test/blaze/elm/quantity_test.clj @@ -1,7 +1,8 @@ (ns blaze.elm.quantity-test (:require + [blaze.elm.compiler.test-util :refer [has-form]] [blaze.elm.protocols :as p] - [blaze.elm.quantity :as quantity] + [blaze.elm.quantity :refer [quantity]] [blaze.test-util :as tu] [clojure.java.io :as io] [clojure.spec.test.alpha :as st] @@ -16,7 +17,7 @@ (deftest quantity-test (testing "Commonly Used UCUM Codes for Healthcare Units" (testing "special units" - (are [unit] (quantity/quantity 1 unit) + (are [unit] (quantity 1 unit) "U/L" "10*3/uL" "mm[Hg]")) @@ -26,22 +27,25 @@ (drop 1) (map #(str/split % #"\t")) (map first) - (map #(try (quantity/quantity 1 %) (catch Exception e (ex-data e)))) + (map #(try (quantity 1 %) (catch Exception e (ex-data e)))) (filter ::anom/category) (map :unit) (count) (= 20) - (is))))) + (is)))) + + (testing "form" + (has-form (quantity 1M "m") '(quantity 1M "m")))) ;; 2.3. Property (deftest property-test (testing "the value of a quantity is always a BigDecimal" (are [quantity] (= BigDecimal (class (p/get quantity :value))) - (quantity/quantity 1M "m") - (quantity/quantity 1 "m") - (quantity/quantity (int 1) "m") - (p/divide (quantity/quantity 1M "m") (quantity/quantity 1M "s")) - (p/divide (quantity/quantity 1M "m") (quantity/quantity 2M "s")))) + (quantity 1M "m") + (quantity 1 "m") + (quantity (int 1) "m") + (p/divide (quantity 1M "m") (quantity 1M "s")) + (p/divide (quantity 1M "m") (quantity 2M "s")))) (testing "get on unknown key returns nil" - (is (nil? (p/get (quantity/quantity 1M "m") ::unknown))))) + (is (nil? (p/get (quantity 1M "m") ::unknown))))) diff --git a/modules/cql/test/blaze/elm/ratio_spec.clj b/modules/cql/test/blaze/elm/ratio_spec.clj index 0043b358b..6a8480a7b 100644 --- a/modules/cql/test/blaze/elm/ratio_spec.clj +++ b/modules/cql/test/blaze/elm/ratio_spec.clj @@ -1,15 +1,10 @@ (ns blaze.elm.ratio-spec - (:refer-clojure :exclude [ratio?]) (:require [blaze.anomaly-spec] [blaze.elm.quantity :as quantity] + [blaze.elm.quantity-spec] [blaze.elm.ratio :as ratio] - [clojure.spec.alpha :as s]) - (:import - [blaze.elm.ratio Ratio])) - -(defn ratio? [x] - (instance? Ratio x)) + [clojure.spec.alpha :as s])) (s/fdef ratio/ratio :args (s/cat :numerator quantity/quantity? :denominator quantity/quantity?)) diff --git a/modules/cql/test/blaze/elm/ratio_test.clj b/modules/cql/test/blaze/elm/ratio_test.clj new file mode 100644 index 000000000..db2bb88b8 --- /dev/null +++ b/modules/cql/test/blaze/elm/ratio_test.clj @@ -0,0 +1,18 @@ +(ns blaze.elm.ratio-test + (:require + [blaze.elm.compiler :as c] + [blaze.elm.quantity :refer [quantity]] + [blaze.elm.ratio :refer [ratio]] + [blaze.elm.ratio-spec] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest is testing]])) + +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(deftest to-ratio-test + (testing "attach-cache" + (let [ratio (ratio (quantity 1 "1") (quantity 2 "1"))] + (is (= [ratio] (st/with-instrument-disabled (c/attach-cache ratio ::cache))))))) diff --git a/modules/cql/test/blaze/elm/resource_test.clj b/modules/cql/test/blaze/elm/resource_test.clj new file mode 100644 index 000000000..2c25c3e0a --- /dev/null +++ b/modules/cql/test/blaze/elm/resource_test.clj @@ -0,0 +1,30 @@ +(ns blaze.elm.resource-test + (:require + [blaze.db.api :as d] + [blaze.db.api-stub :refer [mem-node-config with-system-data]] + [blaze.elm.compiler :as c] + [blaze.elm.expression-spec] + [blaze.elm.resource :as cr] + [blaze.elm.resource-spec] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest is testing]])) + +(set! *warn-on-reflection* true) +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(defn- resource [db type id] + (cr/mk-resource db (d/resource-handle db type id))) + +(deftest resource-test + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [resource (resource (d/db node) "Patient" "0")] + (testing "attach-cache" + (is (= [resource] (st/with-instrument-disabled (c/attach-cache resource ::cache))))) + + (testing "toString" + (is (= "Patient[id = 0, t = 1, last-change-t = 1]" (str resource))))))) diff --git a/modules/cql/test/data_readers.clj b/modules/cql/test/data_readers.clj index b42869f8b..08a9d1aa3 100644 --- a/modules/cql/test/data_readers.clj +++ b/modules/cql/test/data_readers.clj @@ -29,6 +29,7 @@ elm/not blaze.elm.literal/not elm/or blaze.elm.literal/or elm/xor blaze.elm.literal/xor + elm/coalesce blaze.elm.literal/coalesce elm/is-false blaze.elm.literal/is-false elm/is-null blaze.elm.literal/is-null elm/is-true blaze.elm.literal/is-true @@ -42,6 +43,7 @@ elm/lower blaze.elm.literal/lower elm/matches blaze.elm.literal/matches elm/starts-with blaze.elm.literal/starts-with + elm/substring blaze.elm.literal/substring elm/upper blaze.elm.literal/upper elm/add blaze.elm.literal/add elm/ceiling blaze.elm.literal/ceiling @@ -76,8 +78,11 @@ elm/distinct blaze.elm.literal/distinct elm/exists blaze.elm.literal/exists elm/first blaze.elm.literal/first + elm/flatten blaze.elm.literal/flatten + elm/index-of blaze.elm.literal/index-of elm/last blaze.elm.literal/last elm/singleton-from blaze.elm.literal/singleton-from + elm/slice blaze.elm.literal/slice elm/all-true blaze.elm.literal/all-true elm/any-true blaze.elm.literal/any-true elm/avg blaze.elm.literal/avg diff --git a/modules/db-protocols/src/blaze/db/impl/protocols.clj b/modules/db-protocols/src/blaze/db/impl/protocols.clj index 0ada5fc25..1432c8d2e 100644 --- a/modules/db-protocols/src/blaze/db/impl/protocols.clj +++ b/modules/db-protocols/src/blaze/db/impl/protocols.clj @@ -23,6 +23,8 @@ [db compartment tid] [db compartment tid start-id]) + (-patient-compartment-last-change-t [db patient-id]) + (-count-query [db query] "Returns a CompletableFuture that will complete with the count of the matching resource handles.") @@ -110,4 +112,4 @@ (-list-by-type [_ type]) (-list-by-target [_ target]) (-linked-compartments [_ resource]) - (-compartment-resources [_ type])) + (-compartment-resources [_ compartment-type] [_ compartment-type type])) diff --git a/modules/db-stub/src/blaze/db/api_stub.clj b/modules/db-stub/src/blaze/db/api_stub.clj index 429d53d44..2719526a2 100644 --- a/modules/db-stub/src/blaze/db/api_stub.clj +++ b/modules/db-stub/src/blaze/db/api_stub.clj @@ -27,6 +27,7 @@ :kv-store (ig/ref :blaze.db/index-kv-store) :resource-indexer (ig/ref ::node/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} ::tx-log/local @@ -56,8 +57,11 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-last-change-index nil :type-stats-index nil - :system-stats-index nil}} + :system-stats-index nil + :cql-bloom-filter nil + :cql-bloom-filter-by-t nil}} ::rs/kv {:kv-store (ig/ref :blaze.db/resource-kv-store) @@ -77,7 +81,9 @@ :blaze.db.node.resource-indexer/executor {} :blaze.db/search-param-registry - {:structure-definition-repo structure-definition-repo}}) + {:structure-definition-repo structure-definition-repo} + + :blaze/scheduler {}}) (defmacro with-system-data "Runs `body` inside a system that is initialized from `config`, bound to diff --git a/modules/db/.clj-kondo/config.edn b/modules/db/.clj-kondo/config.edn index aa614a66c..3c358d626 100644 --- a/modules/db/.clj-kondo/config.edn +++ b/modules/db/.clj-kondo/config.edn @@ -2,12 +2,12 @@ ["../../../.clj-kondo/root" "../../anomaly/resources/clj-kondo.exports/blaze/anomaly" "../../async/resources/clj-kondo.exports/blaze/async" + "../../coll/resources/clj-kondo.exports/blaze/coll" "../../module-base/resources/clj-kondo.exports/prom-metrics/prom-metrics" "../../module-test-util/resources/clj-kondo.exports/blaze/module-test-util"] :lint-as {blaze.db.api-test-perf/with-system-data clojure.core/with-open - blaze.db.impl.db/with-open-coll clojure.core/with-open blaze.db.test-util/with-system-data clojure.core/with-open} :linters diff --git a/modules/db/NOTES.md b/modules/db/NOTES.md deleted file mode 100644 index dee7125e5..000000000 --- a/modules/db/NOTES.md +++ /dev/null @@ -1,125 +0,0 @@ -# Blaze - -## Features - -### Versioned read of resources - -* I need to get to a particular version of a resource -* versionId could be the content-hash - -### Normal read returning the last known Version of a Resource - -* need to get the content-hash of a resource given a t -* can be done with the ResourceAsOf index - -### Search - -* stable search also in recent past (at given t) -* - -## Principles - -* don't optimize for deleted resources because deleting a resource is not common - -## Indices - -### Independent from t - -| Name | Key Parts | Value | -|---|---|---| -| SVR | c-hash tid value id hash-prefix | - | -| RSV | tid id hash-prefix c-hash value | - | -| CSVR | co-c-hash co-res-id sp-c-hash tid value id hash-prefix | - | -| CompartmentResource | co-c-hash co-res-id tid id | - | -| SearchParam | code tid | id | -| ActiveSearchParams | id | - | - -### Depend on t - -| Name | Key Parts | Value | -|---|---|---| -| TxSuccess | t | transaction | -| TxError | t | anomaly | -| TByInstant | inst-ms (desc) | t | -| ResourceAsOf | tid id t | hash, state | -| TypeAsOf | tid t id | hash, state | -| SystemAsOf | t tid id | hash, state | -| TypeStats | tid t | total, num-changes | -| SystemStats | t | total, num-changes | - -We can make hashes in SearchParam indices shorter (4-bytes) because we only need to differentiate between the versions of a resource. The odds of a hash collision is 1 out of 10000 for about 1000 versions. In case of a hash collision we would produce a false positive query hit. So we would return more resources instead of less, which is considered fine in FHIR. - -### Search param Value Resource version (SVR) - -The key consists of: - -* c-hash - a 4-byte hash of the code of the search parameter -* tid - the 4-byte type id -* value - the value encoded depending on the search parameter -* id - the logical id of the resource -* hash-prefix - a 4-byte prefix of the resource content hash - -The total size of the key is 4 + 4 + value-size + id-size + 4 = 12 + value-size + id-size bytes. - -The value is empty. - -The key contains the id of the resource for two reasons, first we can skip to the next resource by seeking with max-hash, not having to test all versions of a resource against ResourceAsOf and second, going into ResourceAsOf will be local because it is sorted by id. - -The SVR index is comparable to the AVET index in Datomic. Search parameters are the equivalent of indexed attributes in Datomic. - -### Resource version Search param Value (RSV) - - - -### Compartment Search-param Value Resource (CSVR) - -Same as the SVR index but prefixed with a compartment the resource belongs to. This index is used in [variant searches][2] and in CQL evaluation within the Patient context. In the CQL Patient context all retrieves are relative to one patient. Using that patient as compartment in the CSVR index allows for efficient implementation of that retrieves. - -The key consists of: - -* co-c-hash - a 4-byte hash of the code of the compartment -* co-res-id - the logical id of the resource of the compartment -* c-hash - a 4-byte hash of the code of the search parameter -* tid - the 4-byte type id -* value - the value encoded depending on the search parameter -* id - the logical id of the resource -* hash-prefix - a 4-byte prefix of the resource content hash - -The total size of the key is 4 + co-res-id-size 4 + 4 + value-size + id-size + 4 = 16 + co-res-id-size 4 + value-size + id-size bytes. - -The value is empty. - -### TByInstant - -Provides access to t's by instant (point in time). It encodes the instant as milliseconds since epoch as descending long from Long/MAX_VALUE so that TODO WHY??? - -### ResourceAsOf - -The key consists of: - -* tid - the 4-byte type id -* id - the variable length id (max 64 byte) - - -### TxSuccess - -### TxError - -### TypeStats / SystemStats - -total = number of non-deleted resources -num-changes = total number of changes (creates, updates, deletes) - -## Search - -The FHIR search parameters have different types. The search implementation depends on that types. The following sections describe the implementation by type. - -### Date - -The date search parameter type is used for the data types date, dateTime, instant, Period and Timing. The search is always performed against a range. Both the value given in the search and the target value in resources have either an implicit or an explicit range. For example the range of a date like 2020-02-09 starts at 2020-02-09T00:00:00.000 and ends at 2020-02-09T23:59:59.999. - -By default the search is an equal search were the range of the search value have to fully contain the range of the target value. In addition to the equal search, other search operators are possible. - - -[1]: -[2]: diff --git a/modules/db/deps.edn b/modules/db/deps.edn index 53b6700e5..05ddc9e45 100644 --- a/modules/db/deps.edn +++ b/modules/db/deps.edn @@ -10,6 +10,9 @@ blaze/byte-string {:local/root "../byte-string"} + blaze/cache-collector + {:local/root "../cache-collector"} + blaze/coll {:local/root "../coll"} @@ -34,6 +37,9 @@ blaze/db-resource-store {:local/root "../db-resource-store"} + blaze/scheduler + {:local/root "../scheduler"} + blaze/spec {:local/root "../spec"} diff --git a/modules/db/src/blaze/db/api.clj b/modules/db/src/blaze/db/api.clj index 20641a401..dc4c02e15 100644 --- a/modules/db/src/blaze/db/api.clj +++ b/modules/db/src/blaze/db/api.clj @@ -264,6 +264,14 @@ [node-or-db code type clauses] (p/-compile-compartment-query-lenient node-or-db code type clauses)) +;; ---- Patient-Compartment-Level Functions ----------------------------------- + +(defn patient-compartment-last-change-t + "Returns the `t` of last change of any resource in the patient compartment or + nil if the patient has no resources." + [db patient-id] + (p/-patient-compartment-last-change-t db (codec/id-byte-string patient-id))) + ;; ---- Common Query Functions ------------------------------------------------ (defn count-query diff --git a/modules/db/src/blaze/db/api_spec.clj b/modules/db/src/blaze/db/api_spec.clj index 701774c8a..c2f1dec3e 100644 --- a/modules/db/src/blaze/db/api_spec.clj +++ b/modules/db/src/blaze/db/api_spec.clj @@ -130,6 +130,10 @@ :clauses :blaze.db.query/clauses) :ret (s/or :query :blaze.db/query :anomaly ::anom/anomaly)) +(s/fdef d/patient-compartment-last-change-t + :args (s/cat :db :blaze.db/db :patient-id :blaze.resource/id) + :ret (s/nilable :blaze.db/t)) + ;; ---- Common Query Functions ------------------------------------------------ (s/fdef d/execute-query diff --git a/modules/db/src/blaze/db/cache_collector/spec.clj b/modules/db/src/blaze/db/cache_collector/spec.clj deleted file mode 100644 index fec1f8f2c..000000000 --- a/modules/db/src/blaze/db/cache_collector/spec.clj +++ /dev/null @@ -1,7 +0,0 @@ -(ns blaze.db.cache-collector.spec - (:require - [blaze.db.cache-collector.protocols :as p] - [clojure.spec.alpha :as s])) - -(s/def :blaze.db.cache-collector/caches - (s/map-of string? (s/nilable #(satisfies? p/StatsCache %)))) diff --git a/modules/db/src/blaze/db/impl/batch_db.clj b/modules/db/src/blaze/db/impl/batch_db.clj index 06963e211..cc6fdd39d 100644 --- a/modules/db/src/blaze/db/impl/batch_db.clj +++ b/modules/db/src/blaze/db/impl/batch_db.clj @@ -13,6 +13,7 @@ [blaze.db.impl.codec :as codec] [blaze.db.impl.index :as index] [blaze.db.impl.index.compartment.resource :as cr] + [blaze.db.impl.index.patient-last-change :as plc] [blaze.db.impl.index.resource-as-of :as rao] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.search-param-value-resource :as sp-vr] @@ -108,6 +109,12 @@ (-compartment-resource-handles [db compartment tid start-id] (cr/resource-handles db compartment tid start-id)) + ;; ---- Patient-Compartment-Level Functions --------------------------------- + + (-patient-compartment-last-change-t [_ patient-id] + (with-open [plci (kv/new-iterator snapshot :patient-last-change-index)] + (plc/last-change-t plci patient-id t))) + ;; ---- Common Query Functions ---------------------------------------------- (-count-query [db query] diff --git a/modules/db/src/blaze/db/impl/db.clj b/modules/db/src/blaze/db/impl/db.clj index 16aee6b42..51f0683aa 100644 --- a/modules/db/src/blaze/db/impl/db.clj +++ b/modules/db/src/blaze/db/impl/db.clj @@ -2,29 +2,19 @@ "Primary Database Implementation" (:require [blaze.async.comp :as ac :refer [do-sync]] + [blaze.coll.core :refer [with-open-coll]] [blaze.db.impl.batch-db :as batch-db] + [blaze.db.impl.index.patient-last-change :as plc] [blaze.db.impl.index.system-stats :as system-stats] [blaze.db.impl.index.type-stats :as type-stats] [blaze.db.impl.protocols :as p] [blaze.db.kv :as kv]) (:import - [clojure.lang IReduceInit Sequential] [java.io Writer])) (set! *warn-on-reflection* true) (set! *unchecked-math* :warn-on-boxed) -(defmacro with-open-coll - "Like `clojure.core/with-open` but opens and closes the resources on every - reduce call to `coll`." - [bindings coll] - `(reify - Sequential - IReduceInit - (reduce [_ rf# init#] - (with-open ~bindings - (reduce rf# init# ~coll))))) - (deftype Db [node kv-store basis-t t] p/Db (-node [_] @@ -84,6 +74,13 @@ (with-open-coll [batch-db (batch-db/new-batch-db node basis-t t)] (p/-compartment-resource-handles batch-db compartment tid start-id))) + ;; ---- Patient-Compartment-Level Functions --------------------------------- + + (-patient-compartment-last-change-t [_ patient-id] + (with-open [snapshot (kv/new-snapshot kv-store) + plci (kv/new-iterator snapshot :patient-last-change-index)] + (plc/last-change-t plci patient-id t))) + ;; ---- Common Query Functions ---------------------------------------------- (-count-query [_ query] diff --git a/modules/db/src/blaze/db/impl/index/patient_last_change.clj b/modules/db/src/blaze/db/impl/index/patient_last_change.clj new file mode 100644 index 000000000..8f5421678 --- /dev/null +++ b/modules/db/src/blaze/db/impl/index/patient_last_change.clj @@ -0,0 +1,64 @@ +(ns blaze.db.impl.index.patient-last-change + "Functions for accessing the PatientLastChange index." + (:require + [blaze.byte-buffer :as bb] + [blaze.byte-string :as bs] + [blaze.db.impl.bytes :as bytes] + [blaze.db.impl.codec :as codec] + [blaze.db.kv :as kv]) + (:import + [java.nio.charset StandardCharsets])) + +(set! *warn-on-reflection* true) +(set! *unchecked-math* :warn-on-boxed) + +(defn- encode-key [patient-id t] + (-> (bb/allocate (unchecked-add-int (bs/size patient-id) codec/t-size)) + (bb/put-byte-string! patient-id) + (bb/put-long! (codec/descending-long ^long t)) + bb/array)) + +(defn index-entry [patient-id t] + [:patient-last-change-index (encode-key patient-id t) bytes/empty]) + +(defn last-change-t + "Returns the `t` of last change of any resource in the patient compartment not + newer than `t` or nil if the patient has no resources." + [plci patient-id t] + (kv/seek! plci (encode-key patient-id t)) + (when (kv/valid? plci) + (let [bb (bb/wrap (kv/key plci)) + patient-id-size (bs/size patient-id)] + (when (and (< patient-id-size (bb/remaining bb)) + (= patient-id (bs/from-byte-buffer! bb patient-id-size))) + (-> (bb/get-long! bb) + (codec/descending-long)))))) + +(def ^:private state-key + (.getBytes "patient-last-change-state" StandardCharsets/ISO_8859_1)) + +(defn- encode-state [{:keys [type t]}] + (if (identical? :current type) + (byte-array [0]) + (-> (bb/allocate (inc Long/BYTES)) + (bb/put-byte! 1) + (bb/put-long! t) + bb/array))) + +(defn decode-state [bytes] + (let [buf (bb/wrap bytes)] + (if (zero? (bb/get-byte! buf)) + {:type :current} + {:type :building + :t (bb/get-long! buf)}))) + +(defn state + "Returns the state of the PatientLastChange index. + + The initial state is `{:type :building :t 0}`." + [kv-store] + (or (some-> (kv/get kv-store :default state-key) decode-state) + {:type :building :t 0})) + +(defn state-index-entry [state] + [:default state-key (encode-state state)]) diff --git a/modules/db/src/blaze/db/node.clj b/modules/db/src/blaze/db/node.clj index 0ef3b8124..fb071ee9e 100644 --- a/modules/db/src/blaze/db/node.clj +++ b/modules/db/src/blaze/db/node.clj @@ -9,12 +9,14 @@ [blaze.db.impl.codec :as codec] [blaze.db.impl.db :as db] [blaze.db.impl.index :as index] + [blaze.db.impl.index.patient-last-change :as plc] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.t-by-instant :as t-by-instant] [blaze.db.impl.index.tx-error :as tx-error] [blaze.db.impl.index.tx-success :as tx-success] [blaze.db.impl.protocols :as p] [blaze.db.kv :as kv] + [blaze.db.node.patient-last-change-index :as node-plc] [blaze.db.node.protocols :as np] [blaze.db.node.resource-indexer :as resource-indexer] [blaze.db.node.resource-indexer.spec] @@ -33,6 +35,7 @@ [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type :as type] [blaze.module :as m :refer [reg-collector]] + [blaze.scheduler :as sched] [blaze.spec] [blaze.util :refer [conj-vec]] [clojure.spec.alpha :as s] @@ -103,9 +106,9 @@ (remove-watch state future)))) future)) -(defn- index-tx [db-before tx-data] +(defn- index-tx [search-param-registry db-before tx-data] (with-open [_ (prom/timer duration-seconds "index-transactions")] - (tx-indexer/index-tx db-before tx-data))) + (tx-indexer/index-tx search-param-registry db-before tx-data))) (defn- advance-t! [state t] (log/trace "advance state to t =" t) @@ -150,13 +153,13 @@ "This is the main transaction handling function. If indexes resources and transaction data and commits either success or error." - [{:keys [resource-indexer kv-store] :as node} + [{:keys [resource-indexer search-param-registry kv-store] :as node} {:keys [t instant tx-cmds] :as tx-data}] (log/trace "index transaction with t =" t "and" (count tx-cmds) "command(s)") (prom/observe! transaction-sizes (count tx-cmds)) (let [timer (prom/timer duration-seconds "index-resources") future (resource-indexer/index-resources resource-indexer tx-data) - result (index-tx (np/-db node) tx-data)] + result (index-tx search-param-registry (np/-db node) tx-data)] (if (ba/anomaly? result) (commit-error! node t result) (do @@ -389,9 +392,9 @@ :error-t 0}) (defn- init-msg - [{:keys [enforce-referential-integrity] - :or {enforce-referential-integrity true}}] - (log/info "Open local database node with" + [key {:keys [enforce-referential-integrity] + :or {enforce-referential-integrity true}}] + (log/info "Open" (node-util/component-name key "local database node") "with" (if enforce-referential-integrity "enabled" "disabled") "referential integrity checks")) @@ -400,19 +403,6 @@ :or {enforce-referential-integrity true}}] {:blaze.db/enforce-referential-integrity enforce-referential-integrity}) -(defmethod m/pre-init-spec :blaze.db/node [_] - (s/keys - :req-un - [:blaze.db/tx-log - :blaze.db/tx-cache - ::indexer-executor - :blaze.db/kv-store - ::resource-indexer - :blaze.db/resource-store - :blaze.db/search-param-registry] - :opt-un - [:blaze.db/enforce-referential-integrity])) - (def ^:private expected-kv-store-version 0) (defn- kv-store-version [kv-store] @@ -450,12 +440,61 @@ (fn sync-standalone [^Node node] (ac/completed-future (db/db node (:t @(.-state node))))))) +(defn- index-patient-last-change-index! + [{:keys [kv-store] :as node} current-t {:keys [t] :as tx-data}] + (log/trace "Build PatientLastChange index with t =" t) + (when-ok [entries (node-plc/index-entries node tx-data)] + (store-tx-entries! kv-store entries)) + (vreset! current-t t)) + +(defn- poll-and-index-patient-last-change-index! + [node queue current-t poll-timeout] + (run! (partial index-patient-last-change-index! node current-t) + (poll-tx-queue! queue poll-timeout))) + +(defn build-patient-last-change-index + [key {:keys [tx-log kv-store run? state poll-timeout] :as node}] + (let [{:keys [type t]} (plc/state kv-store)] + (when (identical? :building type) + (let [start-t (inc t) + end-t (:t @state) + current-t (volatile! start-t)] + (log/info "Building PatientLastChange index of" (node-util/component-name key "node") "starting at t =" start-t) + (with-open [queue (tx-log/new-queue tx-log start-t)] + (while (and @run? (< @current-t end-t)) + (try + (poll-and-index-patient-last-change-index! node queue current-t poll-timeout) + (catch Exception e + (log/error (format "Error while building the PatientLastChange index of %s." + (node-util/component-name key "node")) e))))) + (if (>= @current-t end-t) + (do + (store-tx-entries! kv-store [(plc/state-index-entry {:type :current})]) + (log/info (format "Finished building PatientLastChange index of %s." (node-util/component-name key "node")))) + (log/info (format "Partially build PatientLastChange index of %s up to t =" + (node-util/component-name key "node")) @current-t + "at a goal of t =" end-t "Will continue at next start.")))))) + +(defmethod m/pre-init-spec :blaze.db/node [_] + (s/keys + :req-un + [:blaze.db/tx-log + :blaze.db/tx-cache + ::indexer-executor + :blaze.db/kv-store + ::resource-indexer + :blaze.db/resource-store + :blaze.db/search-param-registry + :blaze/scheduler] + :opt-un + [:blaze.db/enforce-referential-integrity])) + (defmethod ig/init-key :blaze.db/node - [_ {:keys [storage tx-log tx-cache indexer-executor kv-store resource-indexer - resource-store search-param-registry poll-timeout] - :or {poll-timeout (time/seconds 1)} - :as config}] - (init-msg config) + [key {:keys [storage tx-log tx-cache indexer-executor kv-store resource-indexer + resource-store search-param-registry scheduler poll-timeout] + :or {poll-timeout (time/seconds 1)} + :as config}] + (init-msg key config) (check-version! kv-store) (let [node (->Node (ctx config) tx-log tx-cache kv-store resource-store (sync-fn storage) search-param-registry resource-indexer @@ -463,12 +502,14 @@ (volatile! true) poll-timeout (ac/future))] + (when (= :building (:type (plc/state kv-store))) + (sched/submit scheduler #(build-patient-last-change-index key node))) (execute node indexer-executor) node)) (defmethod ig/halt-key! :blaze.db/node [_ node] - (log/info "Close local database node") + (log/info "Close" (node-util/component-name key "local database node")) (.close ^AutoCloseable node)) (defmethod ig/init-key ::indexer-executor @@ -478,11 +519,11 @@ (defmethod ig/halt-key! ::indexer-executor [_ executor] - (log/info "Stopping indexer executor...") + (log/info "Stopping" (node-util/component-name key "indexer executor...")) (ex/shutdown! executor) (if (ex/await-termination executor 10 TimeUnit/SECONDS) (log/info "Indexer executor was stopped successfully") - (log/warn "Got timeout while stopping the indexer executor"))) + (log/warn "Got timeout while stopping the" (node-util/component-name key "indexer executor")))) (reg-collector ::duration-seconds duration-seconds) diff --git a/modules/db/src/blaze/db/node/patient_last_change_index.clj b/modules/db/src/blaze/db/node/patient_last_change_index.clj new file mode 100644 index 000000000..3c634cddd --- /dev/null +++ b/modules/db/src/blaze/db/node/patient_last_change_index.clj @@ -0,0 +1,13 @@ +(ns blaze.db.node.patient-last-change-index + (:require + [blaze.anomaly :refer [when-ok]] + [blaze.db.impl.db :as db] + [blaze.db.impl.index.patient-last-change :as plc] + [blaze.db.node.tx-indexer :as tx-indexer])) + +(defn index-entries + {:arglists '([node tx-data])} + [{:keys [search-param-registry] :as node} {:keys [t] :as tx-data}] + (when-ok [entries (tx-indexer/index-tx search-param-registry (db/db node (dec t)) tx-data)] + (-> (filterv (comp #{:patient-last-change-index} first) entries) + (conj (plc/state-index-entry {:type :building :t t}))))) diff --git a/modules/db/src/blaze/db/node/tx_indexer.clj b/modules/db/src/blaze/db/node/tx_indexer.clj index 59a07178c..65d3b30ae 100644 --- a/modules/db/src/blaze/db/node/tx_indexer.clj +++ b/modules/db/src/blaze/db/node/tx_indexer.clj @@ -5,7 +5,7 @@ [taoensso.timbre :as log])) (defn index-tx - [db-before {:keys [t tx-cmds]}] + [search-param-registry db-before {:keys [t tx-cmds]}] (log/trace "verify transaction commands with t =" t "based on db with t =" (d/basis-t db-before)) - (verify/verify-tx-cmds db-before t tx-cmds)) + (verify/verify-tx-cmds search-param-registry db-before t tx-cmds)) diff --git a/modules/db/src/blaze/db/node/tx_indexer/verify.clj b/modules/db/src/blaze/db/node/tx_indexer/verify.clj index 0894e78d4..1a9ae55b5 100644 --- a/modules/db/src/blaze/db/node/tx_indexer/verify.clj +++ b/modules/db/src/blaze/db/node/tx_indexer/verify.clj @@ -3,11 +3,13 @@ [blaze.anomaly :as ba :refer [throw-anom]] [blaze.db.api :as d] [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.patient-last-change :as plc] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.rts-as-of :as rts] [blaze.db.impl.index.system-stats :as system-stats] [blaze.db.impl.index.type-stats :as type-stats] [blaze.db.kv.spec] + [blaze.db.search-param-registry :as sr] [blaze.fhir.hash :as hash] [blaze.util :as u] [clojure.string :as str] @@ -98,8 +100,8 @@ statistics of the transaction outcome. Throws an anomaly on conflicts." - {:arglists '([db-before t res cmd])} - (fn [_db-before _t _res {:keys [op]}] op)) + {:arglists '([search-param-registry db-before t res cmd])} + (fn [_search-param-registry _db-before _t _res {:keys [op]}] op)) (defn- verify-tx-cmd-create-msg [type id] (format "verify-tx-cmd :create %s/%s" type id)) @@ -111,18 +113,29 @@ (when (d/resource-handle db type id) (throw-anom (ba/conflict (id-collision-msg type id (d/t db)))))) -(defn- index-entries [tid id t hash num-changes op] - (rts/index-entries tid (codec/id-byte-string id) t hash num-changes op)) +(defn- index-entries + "Creates index entries for the resource with `tid` and `id`. + + `refs` are used to update the PatientLastChange index in case a Patient is + referenced." + [tid id t hash num-changes op refs] + (let [id (codec/id-byte-string id)] + (into + (rts/index-entries tid id t hash num-changes op) + (keep (fn [[ref-type ref-id]] + (when (= "Patient" ref-type) + (plc/index-entry (codec/id-byte-string ref-id) t)))) + refs))) (def ^:private inc-0 (fnil inc 0)) (defmethod verify-tx-cmd "create" - [db-before t res {:keys [type id hash]}] + [_search-param-registry db-before t res {:keys [type id hash refs]}] (log/trace (verify-tx-cmd-create-msg type id)) (with-open [_ (prom/timer duration-seconds "verify-create")] (check-id-collision! db-before type id) (let [tid (codec/tid type)] - (-> (update res :entries into (index-entries tid id t hash 1 :create)) + (-> (update res :entries into (index-entries tid id t hash 1 :create refs)) (update :new-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0) (update-in [:stats tid :total] inc-0))))) @@ -159,7 +172,8 @@ (ba/conflict (precondition-version-failed-msg type id if-none-match) :http/status 412)) (defmethod verify-tx-cmd "put" - [db-before t res {:keys [type id hash if-match if-none-match] :as tx-cmd}] + [_search-param-registry db-before t res + {:keys [type id hash if-match if-none-match refs] :as tx-cmd}] (log/trace (verify-tx-cmd-put-msg type id (u/to-seq if-match) if-none-match)) (with-open [_ (prom/timer duration-seconds "verify-put")] (let [tid (codec/tid type) @@ -181,7 +195,7 @@ :else (cond-> - (-> (update res :entries into (index-entries tid id t hash (inc num-changes) :put)) + (-> (update res :entries into (index-entries tid id t hash (inc num-changes) :put refs)) (update :new-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0)) (or (nil? old-t) (identical? :delete op)) @@ -193,7 +207,7 @@ (format "verify-tx-cmd :keep %s/%s" type id))) (defmethod verify-tx-cmd "keep" - [db-before _ res {:keys [type id hash if-match] :as tx-cmd}] + [_search-param-registry db-before _ res {:keys [type id hash if-match] :as tx-cmd}] (log/trace (verify-tx-cmd-keep-msg type id (u/to-seq if-match))) (with-open [_ (prom/timer duration-seconds "verify-keep")] (let [if-match (u/to-seq if-match) @@ -210,27 +224,37 @@ :else res)))) +(defn- patient-refs + "Returns references from `resource-handle` to Patient resources." + [search-param-registry db type resource-handle] + (into + [] + (comp (mapcat #(d/include db resource-handle % "Patient")) + (map (fn [{:keys [id]}] ["Patient" id]))) + (sr/compartment-resources search-param-registry "Patient" type))) + (defmethod verify-tx-cmd "delete" - [db-before t res {:keys [type id]}] + [search-param-registry db-before t res {:keys [type id]}] (log/trace "verify-tx-cmd :delete" (str type "/" id)) (with-open [_ (prom/timer duration-seconds "verify-delete")] (let [tid (codec/tid type) - {:keys [num-changes op] :or {num-changes 0}} - (d/resource-handle db-before type id)] + {:keys [num-changes op] :or {num-changes 0} :as old-resource-handle} + (d/resource-handle db-before type id) + refs (some->> old-resource-handle (patient-refs search-param-registry db-before type))] (cond-> - (-> (update res :entries into (index-entries tid id t hash/deleted-hash (inc num-changes) :delete)) + (-> (update res :entries into (index-entries tid id t hash/deleted-hash (inc num-changes) :delete refs)) (update :del-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0)) (and op (not (identical? :delete op))) (update-in [:stats tid :total] (fnil dec 0)))))) (defmethod verify-tx-cmd :default - [_db-before _t res _tx-cmd] + [_search-param-registry _db-before _t res _tx-cmd] res) -(defn- verify-tx-cmds** [db-before t tx-cmds] +(defn- verify-tx-cmds** [search-param-registry db-before t tx-cmds] (reduce - (partial verify-tx-cmd db-before t) + (partial verify-tx-cmd search-param-registry db-before t) {:entries [] :new-resources #{} :del-resources #{}} @@ -309,11 +333,11 @@ db new-resources del-resources type id refs)))) cmds)) -(defn- verify-tx-cmds* [db-before t cmds] +(defn- verify-tx-cmds* [search-param-registry db-before t cmds] (ba/try-anomaly (let [cmds (resolve-ids db-before cmds)] (detect-duplicate-commands! cmds) - (let [res (verify-tx-cmds** db-before t cmds)] + (let [res (verify-tx-cmds** search-param-registry db-before t cmds)] (check-referential-integrity! db-before res cmds) (post-process-res db-before t res))))) @@ -322,7 +346,7 @@ outcome if it is successful or an anomaly if it fails. The `t` is for the new transaction to commit." - [db-before t cmds] + [search-param-registry db-before t cmds] (with-open [_ (prom/timer duration-seconds "verify-tx-cmds") batch-db-before (d/new-batch-db db-before)] - (verify-tx-cmds* batch-db-before t cmds))) + (verify-tx-cmds* search-param-registry batch-db-before t cmds))) diff --git a/modules/db/src/blaze/db/resource_cache.clj b/modules/db/src/blaze/db/resource_cache.clj index 9453e7c30..9e090695c 100644 --- a/modules/db/src/blaze/db/resource_cache.clj +++ b/modules/db/src/blaze/db/resource_cache.clj @@ -4,7 +4,7 @@ Caffeine is used because it have better performance characteristics as a ConcurrentHashMap." (:require - [blaze.db.cache-collector.protocols :as ccp] + [blaze.cache-collector.protocols :as ccp] [blaze.db.resource-cache.spec] [blaze.db.resource-store :as rs] [blaze.db.resource-store.spec] diff --git a/modules/db/src/blaze/db/search_param_registry.clj b/modules/db/src/blaze/db/search_param_registry.clj index 832b31612..e3f1c68a0 100644 --- a/modules/db/src/blaze/db/search_param_registry.clj +++ b/modules/db/src/blaze/db/search_param_registry.clj @@ -52,15 +52,20 @@ (p/-linked-compartments search-param-registry resource)) (defn compartment-resources - "Returns a seq of `[type codes]` tuples of resources in compartment of `type`. + "Returns a seq of `[type codes]` tuples of resources in compartment of + `compartment-type` or a list of codes if the optional `type` is given. Example: - * `[\"Observation\" [\"subject\" \"performer\"]]` and others for \"Patient\"" - [search-param-registry type] - (p/-compartment-resources search-param-registry type)) + * `[\"Observation\" [\"subject\" \"performer\"]]` and others for \"Patient\" + * `[\"subject\"]` and others for \"Patient\" and \"Observation\"" + ([search-param-registry compartment-type] + (p/-compartment-resources search-param-registry compartment-type)) + ([search-param-registry compartment-type type] + (p/-compartment-resources search-param-registry compartment-type type))) (deftype MemSearchParamRegistry [url-index index target-index compartment-index - compartment-resource-index] + compartment-resource-index + compartment-resource-index-by-type] p/SearchParamRegistry (-get [_ code type] (or (get-in index [type code]) @@ -94,8 +99,11 @@ #{} (compartment-index (name (fhir-spec/fhir-type resource))))) - (-compartment-resources [_ type] - (compartment-resource-index type []))) + (-compartment-resources [_ compartment-type] + (compartment-resource-index compartment-type [])) + + (-compartment-resources [_ compartment-type type] + (get-in compartment-resource-index-by-type [compartment-type type] []))) (def ^:private object-mapper (j/object-mapper @@ -170,6 +178,14 @@ [res-type param-codes]))) resource-defs)}) +(defn- index-compartment-resources-by-type [{def-code :code resource-defs :resource}] + {def-code + (reduce + (fn [res {res-type :code param-codes :param}] + (cond-> res param-codes (assoc res-type param-codes))) + {} + resource-defs)}) + (def ^:private list-search-param {:type "special" :name "_list"}) @@ -257,7 +273,10 @@ patient-compartment (read-classpath-json-resource "blaze/db/compartment/patient.json")] (when-ok [url-index (build-url-index entries) index (build-index url-index entries)] - (->MemSearchParamRegistry url-index (add-special index) - (build-target-index url-index entries) - (index-compartment-def index patient-compartment) - (index-compartment-resources patient-compartment))))) + (->MemSearchParamRegistry + url-index + (add-special index) + (build-target-index url-index entries) + (index-compartment-def index patient-compartment) + (index-compartment-resources patient-compartment) + (index-compartment-resources-by-type patient-compartment))))) diff --git a/modules/db/src/blaze/db/search_param_registry_spec.clj b/modules/db/src/blaze/db/search_param_registry_spec.clj index d804451e4..0d6e879a5 100644 --- a/modules/db/src/blaze/db/search_param_registry_spec.clj +++ b/modules/db/src/blaze/db/search_param_registry_spec.clj @@ -38,5 +38,7 @@ (s/fdef sr/compartment-resources :args (s/cat :search-param-registry :blaze.db/search-param-registry - :type :fhir.resource/type) - :ret (s/coll-of (s/tuple :fhir.resource/type (s/coll-of string?)))) + :compartment-type :fhir.resource/type + :type (s/? :fhir.resource/type)) + :ret (s/or :all (s/coll-of (s/tuple :fhir.resource/type (s/coll-of string?))) + :by-type (s/coll-of string?))) diff --git a/modules/db/test-perf/blaze/db/api_test_perf.clj b/modules/db/test-perf/blaze/db/api_test_perf.clj index 972c292b0..f99f3cd74 100644 --- a/modules/db/test-perf/blaze/db/api_test_perf.clj +++ b/modules/db/test-perf/blaze/db/api_test_perf.clj @@ -58,6 +58,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-last-change-index nil :type-stats-index nil :system-stats-index nil}} diff --git a/modules/db/test/blaze/db/api_test.clj b/modules/db/test/blaze/db/api_test.clj index 749554af3..07cc98b7a 100644 --- a/modules/db/test/blaze/db/api_test.clj +++ b/modules/db/test/blaze/db/api_test.clj @@ -1400,7 +1400,7 @@ (with-system-data [{:blaze.db/keys [node]} config] [[[:put {:fhir/type :fhir/Patient :id "0" :active true - :name [(type/map->HumanName {:family (apply str (repeat 1000 "a"))})]}]]] + :name [(type/human-name {:family (apply str (repeat 1000 "a"))})]}]]] (testing "as first clause" (given (pull-type-query node "Patient" [["family" (apply str (repeat 1000 "a"))]]) @@ -1419,7 +1419,7 @@ (with-system-data [{:blaze.db/keys [node]} config] [[[:put {:fhir/type :fhir/Patient :id "0" :active true - :name [(type/map->HumanName {:family name})]}]]] + :name [(type/human-name {:family name})]}]]] (testing "as first clause" (given (pull-type-query node "Patient" [["family" name]]) @@ -4563,7 +4563,7 @@ (defn- patient-w-identifier [i] {:fhir/type :fhir/Patient :id (str i) - :identifier [(type/map->Identifier {:value (str i)})]}) + :identifier [(type/identifier {:value (str i)})]}) (deftest type-query-identifier-non-matching-test (st/unstrument) @@ -4586,15 +4586,15 @@ [[[:put {:fhir/type :fhir/Patient :id "0" :active true :identifier - [(type/map->Identifier {:system system :value "0"})]}] + [(type/identifier {:system system :value "0"})]}] [:put {:fhir/type :fhir/Patient :id "1" :active true :identifier - [(type/map->Identifier {:system system :value "0"})]}] + [(type/identifier {:system system :value "0"})]}] [:put {:fhir/type :fhir/Patient :id "2" :active true :identifier - [(type/map->Identifier {:system system :value "0"})]}]]] + [(type/identifier {:system system :value "0"})]}]]] (doseq [value (if system ["0" "foo|0"] ["0" "|0"])] (given (pull-type-query node "Patient" [["identifier" value]]) @@ -5230,6 +5230,49 @@ [0 :fhir/type] := :fhir/Observation [0 :id] := "0"))))) +(deftest patient-compartment-last-change-t-test + (testing "non-existing patient" + (with-system [{:blaze.db/keys [node]} config] + + (testing "just returns nil" + (is (nil? (d/patient-compartment-last-change-t (d/db node) "0")))))) + + (testing "single patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (testing "has no resources in its compartment" + (is (nil? (d/patient-compartment-last-change-t (d/db node) "0")))))) + + (testing "observation created in same transaction as patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (is (= 1 (d/patient-compartment-last-change-t (d/db node) "0"))))) + + (testing "observation created after the patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]] + [[:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (testing "the last change comes from the second transaction" + (is (= 2 (d/patient-compartment-last-change-t (d/db node) "0")))) + + (testing "at t=1 there was no change" + (is (nil? (d/patient-compartment-last-change-t (d/as-of (d/db node) 1) "0")))))) + + (testing "patient with last change in t=1 isn't affected by later patient added in t=2" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]] + [[:put {:fhir/type :fhir/Patient :id "1"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/1"}}]]] + + (is (nil? (d/patient-compartment-last-change-t (d/db node) "0")))))) + (defmethod ig/init-key ::defective-resource-store [_ {:keys [hashes-to-store]}] (let [store (atom {})] (reify @@ -6356,9 +6399,9 @@ (defn- observation-create-op [id] [:create {:fhir/type :fhir/Observation :id (format "%05d" id) :category - [(type/map->CodeableConcept + [(type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system #fhir/uri"system-141902" :code (type/code (format "%05d" id))})]})]}]) diff --git a/modules/db/test/blaze/db/cache_collector_test.clj b/modules/db/test/blaze/db/cache_collector_test.clj deleted file mode 100644 index 3f1adde1f..000000000 --- a/modules/db/test/blaze/db/cache_collector_test.clj +++ /dev/null @@ -1,108 +0,0 @@ -(ns blaze.db.cache-collector-test - (:require - [blaze.db.cache-collector] - [blaze.metrics.core :as metrics] - [blaze.module.test-util :refer [with-system]] - [blaze.test-util :as tu :refer [given-thrown]] - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest testing]] - [integrant.core :as ig] - [juxt.iota :refer [given]]) - (:import - [com.github.benmanes.caffeine.cache Cache Caffeine] - [java.util.function Function])) - -(set! *warn-on-reflection* true) -(st/instrument) - -(test/use-fixtures :each tu/fixture) - -(def ^Cache cache (-> (Caffeine/newBuilder) (.recordStats) (.build))) - -(def config - {:blaze.db/cache-collector - {:caches {"name-135224" cache "name-093214" nil}}}) - -(deftest init-test - (testing "nil config" - (given-thrown (ig/init {:blaze.db/cache-collector nil}) - :key := :blaze.db/cache-collector - :reason := ::ig/build-failed-spec - [:cause-data ::s/problems 0 :pred] := `map?)) - - (testing "missing config" - (given-thrown (ig/init {:blaze.db/cache-collector {}}) - :key := :blaze.db/cache-collector - :reason := ::ig/build-failed-spec - [:cause-data ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :caches)))) - - (testing "invalid caches" - (given-thrown (ig/init {:blaze.db/cache-collector {:caches ::invalid}}) - :key := :blaze.db/cache-collector - :reason := ::ig/build-failed-spec - [:cause-data ::s/problems 0 :pred] := `map? - [:cause-data ::s/problems 0 :val] := ::invalid))) - -(deftest cache-collector-test - (with-system [{collector :blaze.db/cache-collector} config] - - (testing "all zero on fresh cache" - (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" - [0 :type] := :counter - [0 :samples 0 :value] := 0.0 - [1 :name] := "blaze_db_cache_misses" - [1 :type] := :counter - [1 :samples 0 :value] := 0.0 - [2 :name] := "blaze_db_cache_load_successes" - [2 :type] := :counter - [2 :samples 0 :value] := 0.0 - [3 :name] := "blaze_db_cache_load_failures" - [3 :type] := :counter - [3 :samples 0 :value] := 0.0 - [4 :name] := "blaze_db_cache_load_seconds" - [4 :type] := :counter - [4 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" - [5 :type] := :counter - [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" - [6 :type] := :gauge - [6 :samples 0 :value] := 0.0)) - - (testing "one load" - (.get cache 1 (reify Function (apply [_ key] key))) - (Thread/sleep 100) - - (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" - [0 :samples 0 :value] := 0.0 - [1 :name] := "blaze_db_cache_misses" - [1 :samples 0 :value] := 1.0 - [2 :name] := "blaze_db_cache_load_successes" - [2 :samples 0 :value] := 1.0 - [3 :name] := "blaze_db_cache_load_failures" - [3 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" - [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" - [6 :samples 0 :value] := 1.0)) - - (testing "one loads and one hit" - (.get cache 1 (reify Function (apply [_ key] key))) - (Thread/sleep 100) - - (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" - [0 :samples 0 :value] := 1.0 - [1 :name] := "blaze_db_cache_misses" - [1 :samples 0 :value] := 1.0 - [2 :name] := "blaze_db_cache_load_successes" - [2 :samples 0 :value] := 1.0 - [3 :name] := "blaze_db_cache_load_failures" - [3 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" - [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" - [6 :samples 0 :value] := 1.0)))) diff --git a/modules/db/test/blaze/db/impl/batch_db_spec.clj b/modules/db/test/blaze/db/impl/batch_db_spec.clj index d7bd01719..6b9940f19 100644 --- a/modules/db/test/blaze/db/impl/batch_db_spec.clj +++ b/modules/db/test/blaze/db/impl/batch_db_spec.clj @@ -4,6 +4,7 @@ [blaze.db.impl.batch-db :as batch-db] [blaze.db.impl.batch-db.patient-everything-spec] [blaze.db.impl.index.compartment.resource-spec] + [blaze.db.impl.index.patient-last-change-spec] [blaze.db.impl.index.resource-as-of-spec] [blaze.db.impl.index.system-as-of-spec] [blaze.db.impl.index.type-as-of-spec] diff --git a/modules/db/test/blaze/db/impl/db_spec.clj b/modules/db/test/blaze/db/impl/db_spec.clj index c84f3335a..e4ee44ec4 100644 --- a/modules/db/test/blaze/db/impl/db_spec.clj +++ b/modules/db/test/blaze/db/impl/db_spec.clj @@ -5,6 +5,7 @@ [blaze.db.impl.codec-spec] [blaze.db.impl.db :as db] [blaze.db.impl.index-spec] + [blaze.db.impl.index.patient-last-change-spec] [blaze.db.impl.index.system-stats-spec] [blaze.db.impl.index.type-stats-spec] [blaze.db.impl.search-param-spec] diff --git a/modules/db/test/blaze/db/impl/index/patient_last_change/spec.clj b/modules/db/test/blaze/db/impl/index/patient_last_change/spec.clj new file mode 100644 index 000000000..e6d6d9da1 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/patient_last_change/spec.clj @@ -0,0 +1,12 @@ +(ns blaze.db.impl.index.patient-last-change.spec + (:require + [blaze.db.impl.index.patient-last-change :as-alias plc] + [blaze.db.impl.index.patient-last-change.state :as-alias state] + [blaze.db.tx-log.spec] + [clojure.spec.alpha :as s])) + +(s/def ::state/type + #{:building :current}) + +(s/def ::plc/state + (s/keys :req-un [::state/type] :opt-un [:blaze.db/t])) diff --git a/modules/db/test/blaze/db/impl/index/patient_last_change_spec.clj b/modules/db/test/blaze/db/impl/index/patient_last_change_spec.clj new file mode 100644 index 000000000..394025087 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/patient_last_change_spec.clj @@ -0,0 +1,27 @@ +(ns blaze.db.impl.index.patient-last-change-spec + (:require + [blaze.db.impl.codec-spec] + [blaze.db.impl.index.patient-last-change :as plc] + [blaze.db.impl.index.patient-last-change.spec] + [blaze.db.kv-spec] + [blaze.db.tx-log.spec] + [blaze.fhir.hash.spec] + [clojure.spec.alpha :as s])) + +(s/fdef plc/index-entry + :args (s/cat :patient-id :blaze.db/id-byte-string :t :blaze.db/t) + :ret :blaze.db.kv/put-entry) + +(s/fdef plc/last-change-t + :args (s/cat :plci :blaze.db.kv/iterator + :patient-id :blaze.db/id-byte-string + :t :blaze.db/t) + :ret (s/nilable :blaze.db/t)) + +(s/fdef plc/state + :args (s/cat :kv-store :blaze.db/kv-store) + :ret ::plc/state) + +(s/fdef plc/state-index-entry + :args (s/cat :state ::plc/state) + :ret :blaze.db.kv/put-entry) diff --git a/modules/db/test/blaze/db/impl/index/patient_last_change_test_util.clj b/modules/db/test/blaze/db/impl/index/patient_last_change_test_util.clj new file mode 100644 index 000000000..ffbcf4775 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/patient_last_change_test_util.clj @@ -0,0 +1,13 @@ +(ns blaze.db.impl.index.patient-last-change-test-util + (:require + [blaze.byte-buffer :as bb] + [blaze.byte-string :as bs] + [blaze.db.impl.codec :as codec])) + +(set! *unchecked-math* :warn-on-boxed) + +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array) + patient-id-len (- (bb/remaining buf) codec/t-size)] + {:patient-id (codec/id-string (bs/from-byte-buffer! buf patient-id-len)) + :t (codec/descending-long (bb/get-long! buf))})) diff --git a/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj index 3d3adbe6a..50ddb8a02 100644 --- a/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj +++ b/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj @@ -15,4 +15,4 @@ :hash :blaze.resource/hash :num-changes nat-int? :op keyword?) - :ret (s/coll-of :blaze.db.kv/put-entry-w-cf)) + :ret (s/coll-of :blaze.db.kv/put-entry)) diff --git a/modules/db/test/blaze/db/impl/search_param_spec.clj b/modules/db/test/blaze/db/impl/search_param_spec.clj index ea07cc361..48fbcb3f0 100644 --- a/modules/db/test/blaze/db/impl/search_param_spec.clj +++ b/modules/db/test/blaze/db/impl/search_param_spec.clj @@ -84,5 +84,5 @@ :linked-compartments (s/nilable (s/coll-of (s/tuple string? string?))) :hash :blaze.resource/hash :resource :blaze/resource) - :ret (s/or :entries (cs/coll-of :blaze.db.kv/put-entry-w-cf) + :ret (s/or :entries (cs/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/patient_last_change_index_spec.clj b/modules/db/test/blaze/db/node/patient_last_change_index_spec.clj new file mode 100644 index 000000000..038817cb2 --- /dev/null +++ b/modules/db/test/blaze/db/node/patient_last_change_index_spec.clj @@ -0,0 +1,14 @@ +(ns blaze.db.node.patient-last-change-index-spec + (:require + [blaze.db.kv.spec] + [blaze.db.node.patient-last-change-index :as node-plc] + [blaze.db.node.tx-indexer-spec] + [blaze.db.spec] + [blaze.db.tx-log.spec] + [clojure.spec.alpha :as s] + [cognitect.anomalies :as anom])) + +(s/fdef node-plc/index-entries + :args (s/cat :node :blaze.db/node + :tx-data :blaze.db/tx-data) + :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/patient_last_change_index_test.clj b/modules/db/test/blaze/db/node/patient_last_change_index_test.clj new file mode 100644 index 000000000..189b1f30a --- /dev/null +++ b/modules/db/test/blaze/db/node/patient_last_change_index_test.clj @@ -0,0 +1,42 @@ +(ns blaze.db.node.patient-last-change-index-test + (:require + [blaze.db.impl.index.patient-last-change :as plc] + [blaze.db.impl.index.patient-last-change-test-util :as plc-tu] + [blaze.db.node.patient-last-change-index :as node-plc] + [blaze.db.node.patient-last-change-index-spec] + [blaze.db.test-util :refer [config with-system-data]] + [blaze.fhir.hash :as hash] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest]] + [juxt.iota :refer [given]]) + (:import + [java.nio.charset StandardCharsets] + [java.time Instant])) + +(st/instrument) + +(test/use-fixtures :each tu/fixture) + +(def patient-0 {:fhir/type :fhir/Patient :id "0"}) +(def observation-0 {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}) +(def hash-observation-0 (hash/generate observation-0)) + +(deftest patient-last-change-index-entries-test + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0]]] + + (given (node-plc/index-entries + node + {:t 2 + :instant Instant/EPOCH + :tx-cmds [{:op "put" :type "Observation" :id "0" + :hash hash-observation-0 :refs [["Patient" "0"]]}]}) + count := 2 + [0 0] := :patient-last-change-index + [0 1 plc-tu/decode-key] := {:patient-id "0" :t 2} + + [1 0] := :default + [1 1 #(String. ^bytes % StandardCharsets/ISO_8859_1)] := "patient-last-change-state" + [1 2 plc/decode-state] := {:type :building :t 2}))) diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj index 88b251be4..50fb1139f 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj @@ -8,12 +8,16 @@ [blaze.db.impl.index.type-stats-spec] [blaze.db.kv.spec] [blaze.db.node.tx-indexer.verify :as verify] + [blaze.db.search-param-registry.spec] [blaze.db.spec] [blaze.db.tx-log.spec] [clojure.spec.alpha :as s] [cognitect.anomalies :as anom])) (s/fdef verify/verify-tx-cmds - :args (s/cat :db-before :blaze.db/db :t :blaze.db/t :cmds :blaze.db/tx-cmds) + :args (s/cat :search-param-registry :blaze.db/search-param-registry + :db-before :blaze.db/db + :t :blaze.db/t + :cmds :blaze.db/tx-cmds) :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj index 358cdb52b..9abc4b1ec 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj @@ -1,6 +1,8 @@ (ns blaze.db.node.tx-indexer.verify-test (:require + [blaze.byte-string :as bs] [blaze.db.api :as d] + [blaze.db.impl.index.patient-last-change-test-util :as plc-tu] [blaze.db.impl.index.resource-as-of-test-util :as rao-tu] [blaze.db.impl.index.rts-as-of-test-util :as rts-tu] [blaze.db.impl.index.system-as-of-test-util :as sao-tu] @@ -13,7 +15,7 @@ [blaze.db.node.tx-indexer.verify :as verify] [blaze.db.node.tx-indexer.verify-spec] [blaze.db.search-param-registry] - [blaze.db.test-util :refer [config with-system-data]] + [blaze.db.test-util :refer [config search-param-registry with-system-data]] [blaze.db.tx-cache] [blaze.db.tx-log.local] [blaze.fhir.hash :as hash] @@ -29,7 +31,7 @@ [taoensso.timbre :as log])) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -42,6 +44,9 @@ :subject #fhir/Reference{:reference "Patient/0"}}) (def observation-1 {:fhir/type :fhir/Observation :id "1" :subject #fhir/Reference{:reference "Patient/0"}}) +(def allergy-intolerance-0 {:fhir/type :fhir/AllergyIntolerance :id "0" + :patient #fhir/Reference{:reference "Patient/0"}}) + (deftest verify-tx-cmds-test (testing "adding one Patient to an empty store" (let [hash (hash/generate patient-0)] @@ -49,6 +54,7 @@ if-none-match [nil "*"]] (with-system [{:blaze.db/keys [node]} config] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [(cond-> {:op (name op) :type "Patient" :id "0" :hash hash} if-none-match @@ -83,6 +89,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [(cond-> {:op "put" :type "Patient" :id "0" :hash hash} if-match @@ -118,6 +125,7 @@ [[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 3 [(cond-> {:op "put" :type "Patient" :id "0" :hash hash} if-match @@ -150,6 +158,7 @@ [[[:put patient-0]]] (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0)}]))))) @@ -158,7 +167,10 @@ (with-system [{:blaze.db/keys [node]} config] (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0)}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Keep failed on `Patient/0`." :blaze.db/tx-cmd := tx-cmd)))) @@ -169,7 +181,10 @@ [[:put patient-0-v2]]] (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0)}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Keep failed on `Patient/0`." :blaze.db/tx-cmd := tx-cmd)))) @@ -185,6 +200,7 @@ :hash (hash/generate patient-0-v2) :if-match if-match}] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [tx-cmd]) ::anom/category := ::anom/conflict @@ -192,7 +208,7 @@ :http/status := 412 :blaze.db/tx-cmd := tx-cmd)))))) - (testing "keeping a non-matching hash and non-matching if-match patient fails" + (testing "keeping a non-matching hash and non-matching if-match Patient fails" (with-system-data [{:blaze.db/keys [node]} config] [[[:put patient-0]] [[:put patient-0-v2]]] @@ -202,7 +218,10 @@ (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0) :if-match if-match}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Precondition `W/\"3\"` failed on `Patient/0`." :http/status := 412 @@ -215,6 +234,7 @@ (testing "with different if-matches" (doseq [if-match [nil 1 [1] [1 2]]] (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [(cond-> {:op "keep" :type "Patient" :id "0" @@ -225,6 +245,7 @@ (testing "deleting a Patient from an empty store" (with-system [{:blaze.db/keys [node]} config] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [{:op "delete" :type "Patient" :id "0"}]) @@ -255,6 +276,7 @@ [[[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0"}]) @@ -285,6 +307,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0"}]) @@ -310,12 +333,85 @@ [4 1 ss-tu/decode-key] := {:t 2} [4 2 ss-tu/decode-val] := {:total 0 :num-changes 2}))) + (testing "deleting an existing Observation" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0] + [:put observation-0]]] + + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [{:op "delete" :type "Observation" :id "0"}]) + + count := 6 + + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "Observation" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "Observation" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [3 0] := :patient-last-change-index + [3 1 plc-tu/decode-key] := {:patient-id "0" :t 2} + [3 2 bs/from-byte-array] := bs/empty + + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [4 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 1 :num-changes 3}))) + + (testing "deleting an existing AllergyIntolerance" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0] + [:put allergy-intolerance-0]]] + + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [{:op "delete" :type "AllergyIntolerance" :id "0"}]) + + count := 6 + + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "AllergyIntolerance" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "AllergyIntolerance" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "AllergyIntolerance" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [3 0] := :patient-last-change-index + [3 1 plc-tu/decode-key] := {:patient-id "0" :t 2} + [3 2 bs/from-byte-array] := bs/empty + + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "AllergyIntolerance" :t 2} + [4 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 1 :num-changes 3}))) + (testing "adding a second Patient to a store containing already one" (let [hash (hash/generate patient-1)] (with-system-data [{:blaze.db/keys [node]} config] [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "1" :hash hash}]) @@ -343,34 +439,44 @@ (testing "adding an observation referring to an existing patient" (let [hash (hash/generate observation-0)] - (with-system-data [{:blaze.db/keys [node]} config] - [[[:put patient-0]]] + (doseq [op [:create :put] + if-none-match [nil "*"]] + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0]]] - (given (verify/verify-tx-cmds - (d/db node) 2 - [{:op "put" :type "Observation" :id "0" :hash hash :refs [["Patient" "0"]]}]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [(cond-> {:op (name op) :type "Observation" :id "0" + :hash hash :refs [["Patient" "0"]]} + if-none-match + (assoc :if-none-match if-none-match))]) - count := 5 + count := 6 - [0 0] := :resource-as-of-index - [0 1 rao-tu/decode-key] := {:type "Observation" :id "0" :t 2} - [0 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op :put} + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "Observation" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} - [1 0] := :type-as-of-index - [1 1 tao-tu/decode-key] := {:type "Observation" :t 2 :id "0"} - [1 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op :put} + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "Observation" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} - [2 0] := :system-as-of-index - [2 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} - [2 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op :put} + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} - [3 0] := :type-stats-index - [3 1 ts-tu/decode-key] := {:type "Observation" :t 2} - [3 2 ts-tu/decode-val] := {:total 1 :num-changes 1} + [3 0] := :patient-last-change-index + [3 1 plc-tu/decode-key] := {:patient-id "0" :t 2} + [3 2 bs/from-byte-array] := bs/empty - [4 0] := :system-stats-index - [4 1 ss-tu/decode-key] := {:t 2} - [4 2 ss-tu/decode-val] := {:total 2 :num-changes 2})))) + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [4 2 ts-tu/decode-val] := {:total 1 :num-changes 1} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 2 :num-changes 2}))))) (testing "update conflict" (testing "adding an observation referring to an empty store" @@ -378,6 +484,7 @@ (with-system [{:blaze.db/keys [node]} config] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Observation" :id "0" :hash hash :refs [["Patient" "0"]]}]) @@ -389,6 +496,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -402,6 +510,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -415,6 +524,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -430,6 +540,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0" :check-refs true} {:op "put" :type "Observation" :id "0" :hash hash :refs [["Patient" "0"]]}]) @@ -443,11 +554,12 @@ [:put observation-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0" :check-refs true} {:op "delete" :type "Observation" :id "0" :check-refs true}]) - count := 9 + count := 10 [0 0] := :resource-as-of-index [0 1 rao-tu/decode-key] := {:type "Patient" :id "0" :t 2} @@ -473,17 +585,21 @@ [5 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} [5 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} - [6 0] := :type-stats-index - [6 1 ts-tu/decode-key] := {:type "Patient" :t 2} - [6 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + [6 0] := :patient-last-change-index + [6 1 plc-tu/decode-key] := {:patient-id "0" :t 2} + [6 2 bs/from-byte-array] := bs/empty [7 0] := :type-stats-index - [7 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [7 1 ts-tu/decode-key] := {:type "Patient" :t 2} [7 2 ts-tu/decode-val] := {:total 0 :num-changes 2} - [8 0] := :system-stats-index - [8 1 ss-tu/decode-key] := {:t 2} - [8 2 ss-tu/decode-val] := {:total 0 :num-changes 4}))) + [8 0] := :type-stats-index + [8 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [8 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + + [9 0] := :system-stats-index + [9 1 ss-tu/decode-key] := {:t 2} + [9 2 ss-tu/decode-val] := {:total 0 :num-changes 4}))) (testing "deleting a patient which is referenced by a still existing observation" (with-system-data [{:blaze.db/keys [node]} config] @@ -491,6 +607,7 @@ [:put observation-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0" :check-refs true}]) @@ -503,6 +620,7 @@ [:put observation-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0"}]) @@ -535,6 +653,7 @@ [:put observation-1]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0" :check-refs true}]) @@ -550,6 +669,7 @@ :birthDate #fhir/date"2020"}]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "create" :type "Patient" :id "foo" :hash (hash/generate patient-0) @@ -565,6 +685,7 @@ (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "create" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -575,6 +696,7 @@ [[[:put patient-2]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "2"} {:op "create" :type "Patient" :id "0" @@ -590,6 +712,7 @@ [[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 3 [{:op "put" :type "Patient" :id "0" :hash hash}]) diff --git a/modules/db/test/blaze/db/node/tx_indexer_spec.clj b/modules/db/test/blaze/db/node/tx_indexer_spec.clj index beda8c044..703ab6d5a 100644 --- a/modules/db/test/blaze/db/node/tx_indexer_spec.clj +++ b/modules/db/test/blaze/db/node/tx_indexer_spec.clj @@ -3,11 +3,14 @@ [blaze.db.kv.spec] [blaze.db.node.tx-indexer :as tx-indexer] [blaze.db.node.tx-indexer.verify-spec] + [blaze.db.search-param-registry.spec] [blaze.db.spec] [blaze.db.tx-log.spec] [clojure.spec.alpha :as s] [cognitect.anomalies :as anom])) (s/fdef tx-indexer/index-tx - :args (s/cat :db-before :blaze.db/db :tx-data :blaze.db/tx-data) + :args (s/cat :search-param-registry :blaze.db/search-param-registry + :db-before :blaze.db/db + :tx-data :blaze.db/tx-data) :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node_test.clj b/modules/db/test/blaze/db/node_test.clj index a11da85ff..3555d43ef 100644 --- a/modules/db/test/blaze/db/node_test.clj +++ b/modules/db/test/blaze/db/node_test.clj @@ -6,11 +6,14 @@ [blaze.db.api :as d] [blaze.db.api-spec] [blaze.db.impl.db-spec] + [blaze.db.impl.index.patient-last-change :as plc] + [blaze.db.impl.index.patient-last-change-spec] [blaze.db.impl.index.tx-success :as tx-success] [blaze.db.kv :as kv] [blaze.db.kv.mem-spec] [blaze.db.node :as node] [blaze.db.node-spec] + [blaze.db.node.patient-last-change-index-spec] [blaze.db.node.resource-indexer :as resource-indexer] [blaze.db.node.tx-indexer :as-alias tx-indexer] [blaze.db.node.version :as version] @@ -42,7 +45,7 @@ (set! *warn-on-reflection* true) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -271,3 +274,12 @@ (deftest existing-data-with-compatible-version (with-system [{:blaze.db/keys [node]} (with-index-store-version config 0)] (is node))) + +(deftest patient-last-change-index-state-test + (testing "the state is set to current on a fresh start of the node" + (with-system [{:blaze.db/keys [node]} config] + ;; Wait for index building finished + (Thread/sleep 100) + + (given (plc/state (:kv-store node)) + :type := :current)))) diff --git a/modules/db/test/blaze/db/resource_cache_test.clj b/modules/db/test/blaze/db/resource_cache_test.clj index 598b4e040..acfd6e689 100644 --- a/modules/db/test/blaze/db/resource_cache_test.clj +++ b/modules/db/test/blaze/db/resource_cache_test.clj @@ -1,6 +1,6 @@ (ns blaze.db.resource-cache-test (:require - [blaze.db.cache-collector.protocols :as ccp] + [blaze.cache-collector.protocols :as ccp] [blaze.db.kv :as kv] [blaze.db.kv.mem] [blaze.db.resource-cache :as resource-cache] @@ -23,7 +23,7 @@ (set! *warn-on-reflection* true) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) diff --git a/modules/db/test/blaze/db/search_param_registry_test.clj b/modules/db/test/blaze/db/search_param_registry_test.clj index eb5c6b1c5..fd40cdbef 100644 --- a/modules/db/test/blaze/db/search_param_registry_test.clj +++ b/modules/db/test/blaze/db/search_param_registry_test.clj @@ -17,7 +17,7 @@ [taoensso.timbre :as log])) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -219,10 +219,15 @@ (deftest compartment-resources-test (testing "Patient" (with-system [{:blaze.db/keys [search-param-registry]} config] - (given (sr/compartment-resources search-param-registry "Patient") - count := 66 - [0] := ["Account" ["subject"]] - [1] := ["AdverseEvent" ["subject"]] - [2] := ["AllergyIntolerance" ["patient" "recorder" "asserter"]] - [3] := ["Appointment" ["actor"]] - [65] := ["VisionPrescription" ["patient"]])))) + (testing "all resource types" + (given (sr/compartment-resources search-param-registry "Patient") + count := 66 + [0] := ["Account" ["subject"]] + [1] := ["AdverseEvent" ["subject"]] + [2] := ["AllergyIntolerance" ["patient" "recorder" "asserter"]] + [3] := ["Appointment" ["actor"]] + [65] := ["VisionPrescription" ["patient"]])) + + (testing "only Observation codes" + (is (= (sr/compartment-resources search-param-registry "Patient" "Observation") + ["subject" "performer"])))))) diff --git a/modules/db/test/blaze/db/test_util.clj b/modules/db/test/blaze/db/test_util.clj index 6ad63556e..168aef74d 100644 --- a/modules/db/test/blaze/db/test_util.clj +++ b/modules/db/test/blaze/db/test_util.clj @@ -28,6 +28,7 @@ :kv-store (ig/ref :blaze.db/index-kv-store) :resource-indexer (ig/ref ::node/resource-indexer) :search-param-registry search-param-registry + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} ::tx-log/local @@ -58,6 +59,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-last-change-index nil :type-stats-index nil :system-stats-index nil}} @@ -76,7 +78,9 @@ :search-param-registry search-param-registry :executor (ig/ref :blaze.db.node.resource-indexer/executor)} - :blaze.db.node.resource-indexer/executor {}}) + :blaze.db.node.resource-indexer/executor {} + + :blaze/scheduler {}}) (defmacro with-system-data "Runs `body` inside a system that is initialized from `config`, bound to diff --git a/modules/fhir-structure/src/blaze/fhir/spec/impl.clj b/modules/fhir-structure/src/blaze/fhir/spec/impl.clj index 0fd7010ce..6d54377d5 100644 --- a/modules/fhir-structure/src/blaze/fhir/spec/impl.clj +++ b/modules/fhir-structure/src/blaze/fhir/spec/impl.clj @@ -311,6 +311,7 @@ :fhir/Coding :fhir/CodeableConcept :fhir/Quantity + :fhir/Ratio :fhir/Period :fhir/Identifier :fhir/HumanName @@ -425,6 +426,7 @@ :fhir.json/Meta :fhir.json/Attachment :fhir.json/Quantity + :fhir.json/Ratio :fhir.json/Period :fhir.json/Identifier :fhir.json/HumanName @@ -549,6 +551,7 @@ :fhir.xml/Coding :fhir.xml/CodeableConcept :fhir.xml/Quantity + :fhir.xml/Ratio :fhir.xml/Period :fhir.xml/Identifier :fhir.xml/HumanName @@ -590,6 +593,7 @@ :fhir.cbor/Meta :fhir.cbor/Attachment :fhir.cbor/Quantity + :fhir.cbor/Ratio :fhir.cbor/Period :fhir.cbor/Identifier :fhir.cbor/HumanName diff --git a/modules/fhir-structure/src/blaze/fhir/spec/spec.clj b/modules/fhir-structure/src/blaze/fhir/spec/spec.clj index a7d9a5392..d5d4958ca 100644 --- a/modules/fhir-structure/src/blaze/fhir/spec/spec.clj +++ b/modules/fhir-structure/src/blaze/fhir/spec/spec.clj @@ -37,6 +37,9 @@ (s/def :fhir/Task #(s2/valid? :fhir/Task %)) +(s/def :fhir/Measure + #(s2/valid? :fhir/Measure %)) + (s/def :fhir.Measure/group #(s2/valid? :fhir.Measure/group %)) diff --git a/modules/fhir-structure/src/blaze/fhir/spec/type.clj b/modules/fhir-structure/src/blaze/fhir/spec/type.clj index 77de41e01..0e94f40ed 100644 --- a/modules/fhir-structure/src/blaze/fhir/spec/type.clj +++ b/modules/fhir-structure/src/blaze/fhir/spec/type.clj @@ -1045,6 +1045,11 @@ :hash-num 40 :interned (and (nil? id) (p/-interned extension) (nil? value))) +(declare ratio) + +(def-complex-type Ratio [^String id extension numerator denominator] + :hash-num 48) + (declare period) (def-complex-type Period [^String id extension ^:primitive start ^:primitive end] diff --git a/modules/fhir-structure/src/blaze/fhir/spec/type/system_spec.clj b/modules/fhir-structure/src/blaze/fhir/spec/type/system_spec.clj index 2e9b3eeff..5284df41f 100644 --- a/modules/fhir-structure/src/blaze/fhir/spec/type/system_spec.clj +++ b/modules/fhir-structure/src/blaze/fhir/spec/type/system_spec.clj @@ -33,6 +33,10 @@ :args (s/cat :x any?) :ret boolean?) +(s/fdef system/date + :args (s/cat :year int? :month (s/? int?) :day (s/? int?)) + :ret system/date?) + ;; ---- System.DateTime ------------------------------------------------------- (s/fdef system/date-time? diff --git a/modules/fhir-structure/test-perf/blaze/fhir/spec/type_test_mem.clj b/modules/fhir-structure/test-perf/blaze/fhir/spec/type_test_mem.clj index cd94ee05b..dc9dce381 100644 --- a/modules/fhir-structure/test-perf/blaze/fhir/spec/type_test_mem.clj +++ b/modules/fhir-structure/test-perf/blaze/fhir/spec/type_test_mem.clj @@ -83,6 +83,8 @@ #fhir/Quantity{} 56 + #fhir/Ratio{} 48 + #fhir/Period{} 48 #fhir/Identifier{} 64 diff --git a/modules/fhir-structure/test/blaze/fhir/spec/generators.clj b/modules/fhir-structure/test/blaze/fhir/spec/generators.clj index 15ebeaf94..92e4e7cfd 100644 --- a/modules/fhir-structure/test/blaze/fhir/spec/generators.clj +++ b/modules/fhir-structure/test/blaze/fhir/spec/generators.clj @@ -265,7 +265,15 @@ ;; TODO: Range -;; TODO: Ratio +(defn ratio + [& {:keys [id extension numerator denominator] + :or {id (gen/return nil) + extension (extensions) + numerator (nilable (quantity)) + denominator (nilable (quantity))}}] + (->> (gen/tuple id extension numerator denominator) + (to-map [:id :extension :numerator :denominator]) + (gen/fmap type/ratio))) ;; TODO: RatioRange diff --git a/modules/fhir-structure/test/blaze/fhir/spec/type/system_test.clj b/modules/fhir-structure/test/blaze/fhir/spec/type/system_test.clj index 72f9a50ee..40f0353c5 100644 --- a/modules/fhir-structure/test/blaze/fhir/spec/type/system_test.clj +++ b/modules/fhir-structure/test/blaze/fhir/spec/type/system_test.clj @@ -175,6 +175,34 @@ nil #system/date-time"2020")) + (testing "date" + (testing "year" + (are [year date] (= date (system/date year)) + 1000 #system/date"1000" + 2024 #system/date"2024" + 9999 #system/date"9999") + + (given-thrown (system/date -1) + :message := "Invalid value for Year (valid values 1 - 9999): -1")) + + (testing "year-month" + (are [year month date] (= date (system/date year month)) + 1000 1 #system/date"1000-01" + 2024 6 #system/date"2024-06" + 9999 12 #system/date"9999-12") + + (given-thrown (system/date 2024 0) + :message := "Invalid value for MonthOfYear (valid values 1 - 12): 0")) + + (testing "year-month-day" + (are [year month day date] (= date (system/date year month day)) + 1000 1 1 #system/date"1000-01-01" + 2024 6 15 #system/date"2024-06-15" + 9999 12 31 #system/date"9999-12-31") + + (given-thrown (system/date 2023 2 29) + :message := "Invalid date 'February 29' as '2023' is not a leap year"))) + (testing "system equals" (testing "same precision" (testing "within date" diff --git a/modules/fhir-structure/test/blaze/fhir/spec/type_test.clj b/modules/fhir-structure/test/blaze/fhir/spec/type_test.clj index 0039fe286..46e93bd64 100644 --- a/modules/fhir-structure/test/blaze/fhir/spec/type_test.clj +++ b/modules/fhir-structure/test/blaze/fhir/spec/type_test.clj @@ -1986,6 +1986,10 @@ #fhir/Attachment{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]} #fhir/Attachment{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Attachment{}))) + (is (false? (p/-has-secondary-content #fhir/Attachment{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Attachment{} @@ -2062,6 +2066,10 @@ #fhir/Extension{:url "foo" :value "bar"} #fhir/Extension{:url "foo" :value "bar"}))) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Extension{}))) + (is (false? (p/-has-secondary-content #fhir/Extension{})))) + (testing "to-json" (are [code json] (= json (gen-json-string code)) #fhir/Extension{} "{}" @@ -2125,6 +2133,10 @@ (prop/for-all [x (fg/coding :extension (gen/vector string-extension-gen 1))] (not-interned? x (recreate type/coding x)))))) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Coding{}))) + (is (false? (p/-has-secondary-content #fhir/Coding{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Coding{} @@ -2177,6 +2189,10 @@ (prop/for-all [x (fg/codeable-concept :extension (gen/vector string-extension-gen 1))] (not-interned? x (recreate type/codeable-concept x)))))) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/CodeableConcept{}))) + (is (false? (p/-has-secondary-content #fhir/CodeableConcept{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/CodeableConcept{} @@ -2235,6 +2251,10 @@ #fhir/Quantity{:code #fhir/code"foo"} #fhir/Quantity{:code #fhir/code"foo"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Quantity{}))) + (is (false? (p/-has-secondary-content #fhir/Quantity{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Quantity{} @@ -2246,7 +2266,7 @@ #fhir/Quantity{:extension [#fhir/Extension{}]} "4f5028ac" - #fhir/Quantity{:value 1M} + #fhir/Quantity{:value #fhir/decimal 1M} "4adf97ab" #fhir/Quantity{:comparator #fhir/code"comparator-153342"} @@ -2271,10 +2291,88 @@ #fhir/Quantity{} "#fhir/Quantity{}" #fhir/Quantity{:id "212329"} "#fhir/Quantity{:id \"212329\"}"))) +(deftest ratio-test + (testing "type" + (is (= :fhir/Ratio (type/type #fhir/Ratio{})))) + + (testing "interned" + (are [x y] (not-interned? x y) + #fhir/Ratio{:id "foo"} + #fhir/Ratio{:id "foo"} + + #fhir/Ratio{:extension [#fhir/Extension{:url "foo" :value "bar"}]} + #fhir/Ratio{:extension [#fhir/Extension{:url "foo" :value "bar"}]} + + #fhir/Ratio{:numerator #fhir/Quantity{:value #fhir/decimal 1M}} + #fhir/Ratio{:numerator #fhir/Quantity{:value #fhir/decimal 1M}} + + #fhir/Ratio{:denominator #fhir/Quantity{:value #fhir/decimal 1M}} + #fhir/Ratio{:denominator #fhir/Quantity{:value #fhir/decimal 1M}}) + + (are [x y] (interned? x y) + #fhir/Ratio{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]} + #fhir/Ratio{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]} + + #fhir/Ratio{:numerator #fhir/Quantity{:code #fhir/code"foo"}} + #fhir/Ratio{:numerator #fhir/Quantity{:code #fhir/code"foo"}} + + #fhir/Ratio{:denominator #fhir/Quantity{:code #fhir/code"foo"}} + #fhir/Ratio{:denominator #fhir/Quantity{:code #fhir/code"foo"}})) + + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Ratio{}))) + (is (false? (p/-has-secondary-content #fhir/Ratio{})))) + + (testing "hash-into" + (are [x hex] (= hex (murmur3 x)) + #fhir/Ratio{} + "d271c07f" + + #fhir/Ratio{:id "id-130710"} + "e3c0ee3c" + + #fhir/Ratio{:extension [#fhir/Extension{}]} + "23473d24" + + #fhir/Ratio{:numerator #fhir/Quantity{:value #fhir/decimal 1M}} + "fbf83a67" + + #fhir/Ratio{:denominator #fhir/Quantity{:value #fhir/decimal 1M}} + "7f2075fb")) + + (testing "references" + (are [x refs] (= refs (type/references x)) + #fhir/Ratio{} + [])) + + (testing "print" + (are [v s] (= s (pr-str v)) + #fhir/Ratio{} "#fhir/Ratio{}" + #fhir/Ratio{:id "212329"} "#fhir/Ratio{:id \"212329\"}"))) + (deftest period-test (testing "type" (is (= :fhir/Period (type/type #fhir/Period{})))) + (testing "interned" + (are [x y] (not-interned? x y) + #fhir/Period{:id "foo"} + #fhir/Period{:id "foo"} + + #fhir/Period{:extension [#fhir/Extension{:url "foo" :value "bar"}]} + #fhir/Period{:extension [#fhir/Extension{:url "foo" :value "bar"}]} + + #fhir/Period{:start #fhir/dateTime"2020"} + #fhir/Period{:start #fhir/dateTime"2020"}) + + (are [x y] (interned? x y) + #fhir/Period{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]} + #fhir/Period{:extension [#fhir/Extension{:url "foo" :value #fhir/code"bar"}]})) + + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Period{}))) + (is (false? (p/-has-secondary-content #fhir/Period{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Period{} @@ -2324,6 +2422,10 @@ #fhir/Identifier{:use #fhir/code"foo"} #fhir/Identifier{:use #fhir/code"foo"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Identifier{}))) + (is (false? (p/-has-secondary-content #fhir/Identifier{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Identifier{} @@ -2388,6 +2490,10 @@ #fhir/HumanName{:use #fhir/code"foo"} #fhir/HumanName{:use #fhir/code"foo"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/HumanName{}))) + (is (false? (p/-has-secondary-content #fhir/HumanName{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/HumanName{} @@ -2464,6 +2570,10 @@ #fhir/Address{:type #fhir/code"foo"} #fhir/Address{:type #fhir/code"foo"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Address{}))) + (is (false? (p/-has-secondary-content #fhir/Address{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Address{} @@ -2546,6 +2656,10 @@ #fhir/Reference{:type #fhir/code"foo"} #fhir/Reference{:type #fhir/code"foo"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Reference{}))) + (is (false? (p/-has-secondary-content #fhir/Reference{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Reference{} @@ -2649,6 +2763,10 @@ #fhir/Meta{:tag [#fhir/Coding{:system #fhir/uri"foo" :code #fhir/code"bar"}]} #fhir/Meta{:tag [#fhir/Coding{:system #fhir/uri"foo" :code #fhir/code"bar"}]})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/Meta{}))) + (is (false? (p/-has-secondary-content #fhir/Meta{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/Meta{} @@ -2718,6 +2836,10 @@ #fhir/BundleEntrySearch{:mode #fhir/code"match"} #fhir/BundleEntrySearch{:mode #fhir/code"match"})) + (testing "primary/secondary content" + (is (true? (p/-has-primary-content #fhir/BundleEntrySearch{}))) + (is (false? (p/-has-secondary-content #fhir/BundleEntrySearch{})))) + (testing "hash-into" (are [x hex] (= hex (murmur3 x)) #fhir/BundleEntrySearch{} diff --git a/modules/fhir-structure/test/blaze/fhir/spec_test.clj b/modules/fhir-structure/test/blaze/fhir/spec_test.clj index a4083d827..3e73d10e6 100644 --- a/modules/fhir-structure/test/blaze/fhir/spec_test.clj +++ b/modules/fhir-structure/test/blaze/fhir/spec_test.clj @@ -3532,7 +3532,7 @@ #fhir/Quantity{} {:value 1M} - #fhir/Quantity{:value 1M} + #fhir/Quantity{:value #fhir/decimal 1M} {:value "1"} ::s2/invalid))) @@ -3552,7 +3552,7 @@ #fhir/Quantity{:extension [#fhir/Extension{} #fhir/Extension{}]} {:extension [{} {}]} - #fhir/Quantity{:value 1M} + #fhir/Quantity{:value #fhir/decimal 1M} {:value 1} #fhir/Quantity{:comparator #fhir/code"code-153342"} @@ -3567,6 +3567,83 @@ #fhir/Quantity{:code #fhir/code"code-153427"} {:code "code-153427"})))) +(deftest ratio-test + (testing "FHIR spec" + (testing "valid" + (satisfies-prop 1000 + (prop/for-all [x (fg/ratio)] + (s2/valid? :fhir/Ratio x)))) + + (testing "invalid" + (are [x] (not (s2/valid? :fhir/Ratio x)) + #fhir/Ratio{:numerator "1"}))) + + (testing "transforming" + (testing "JSON" + (satisfies-prop 1000 + (prop/for-all [x (fg/ratio)] + (= (->> x + fhir-spec/unform-json + fhir-spec/parse-json + (s2/conform :fhir.json/Ratio)) + x)))) + + (testing "XML" + (satisfies-prop 1000 + (prop/for-all [x (fg/ratio)] + (= (->> x + fhir-spec/unform-xml + (s2/conform :fhir.xml/Ratio)) + x)))) + + (testing "CBOR" + (satisfies-prop 1000 + (prop/for-all [x (fg/ratio)] + (= (->> x + fhir-spec/unform-cbor + fhir-spec/parse-cbor + (s2/conform :fhir.cbor/Ratio)) + x))))) + + (testing "conforming" + (testing "JSON" + (are [json fhir] (= fhir (s2/conform :fhir.json/Ratio json)) + {} + #fhir/Ratio{} + + {:id "id-151304"} + #fhir/Ratio{:id "id-151304"} + + {:extension [{}]} + #fhir/Ratio{:extension [#fhir/Extension{}]} + + {:numerator {:value 1M}} + #fhir/Ratio{:numerator #fhir/Quantity{:value #fhir/decimal 1M}} + + {:numerator "foo"} + ::s2/invalid))) + + (testing "unforming" + (testing "JSON" + (are [fhir json] (= json (fhir-spec/parse-json (fhir-spec/unform-json fhir))) + #fhir/Ratio{} + {} + + #fhir/Ratio{:id "id-134428"} + {:id "id-134428"} + + #fhir/Ratio{:extension [#fhir/Extension{}]} + {:extension [{}]} + + #fhir/Ratio{:extension [#fhir/Extension{} #fhir/Extension{}]} + {:extension [{} {}]} + + #fhir/Ratio{:numerator #fhir/Quantity{:value #fhir/decimal 1M}} + {:numerator {:value 1}} + + #fhir/Ratio{:denominator #fhir/Quantity{:value #fhir/decimal 1M}} + {:denominator {:value 1}})))) + (deftest period-test (testing "FHIR spec" (testing "valid" diff --git a/modules/frontend-e2e/package-lock.json b/modules/frontend-e2e/package-lock.json index 9eb8dacb1..f5223ea28 100644 --- a/modules/frontend-e2e/package-lock.json +++ b/modules/frontend-e2e/package-lock.json @@ -30,9 +30,9 @@ } }, "node_modules/@types/node": { - "version": "20.14.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz", - "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==", + "version": "20.14.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.8.tgz", + "integrity": "sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/modules/frontend/package-lock.json b/modules/frontend/package-lock.json index 77c059b20..4f58911fb 100644 --- a/modules/frontend/package-lock.json +++ b/modules/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "blaze-frontend", - "version": "0.27.1", + "version": "0.28.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "blaze-frontend", - "version": "0.27.1", + "version": "0.28.0", "dependencies": { "@auth/sveltekit": "^1.0.0", "@fontsource-variable/inter": "^5.0.8", @@ -1025,9 +1025,9 @@ "dev": true }, "node_modules/@sveltejs/adapter-auto": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-auto/-/adapter-auto-3.2.1.tgz", - "integrity": "sha512-/3xx8ZFCD5UBc/7AbyXkFF3HNCzWAp2xncH8HA4doGjoGQEN7PmwiRx4Y9nOzi4mqDqYYUic0gaIAE2khWWU4Q==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-auto/-/adapter-auto-3.2.2.tgz", + "integrity": "sha512-Mso5xPCA8zgcKrv+QioVlqMZkyUQ5MjDJiEPuG/Z7cV/5tmwV7LmcVWk5tZ+H0NCOV1x12AsoSpt/CwFwuVXMA==", "dev": true, "license": "MIT", "dependencies": { @@ -1053,18 +1053,19 @@ } }, "node_modules/@sveltejs/adapter-static": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.1.tgz", - "integrity": "sha512-6lMvf7xYEJ+oGeR5L8DFJJrowkefTK6ZgA4JiMqoClMkKq0s6yvsd3FZfCFvX1fQ0tpCD7fkuRVHsnUVgsHyNg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.2.tgz", + "integrity": "sha512-/EBFydZDwfwFfFEuF1vzUseBoRziwKP7AoHAwv+Ot3M084sE/HTVBHf9mCmXfdM9ijprY5YEugZjleflncX5fQ==", "dev": true, + "license": "MIT", "peerDependencies": { "@sveltejs/kit": "^2.0.0" } }, "node_modules/@sveltejs/kit": { - "version": "2.5.10", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.10.tgz", - "integrity": "sha512-OqoyTmFG2cYmCFAdBfW+Qxbg8m23H4dv6KqwEt7ofr/ROcfcIl3Z/VT56L22H9f0uNZyr+9Bs1eh2gedOCK9kA==", + "version": "2.5.17", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.17.tgz", + "integrity": "sha512-wiADwq7VreR3ctOyxilAZOfPz3Jiy2IIp2C8gfafhTdQaVuGIHllfqQm8dXZKADymKr3uShxzgLZFT+a+CM4kA==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -1178,9 +1179,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.14.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz", - "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==", + "version": "20.14.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.8.tgz", + "integrity": "sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==", "devOptional": true, "license": "MIT", "dependencies": { @@ -1717,11 +1718,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -2492,9 +2493,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4170,9 +4171,9 @@ } }, "node_modules/svelte": { - "version": "4.2.17", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.17.tgz", - "integrity": "sha512-N7m1YnoXtRf5wya5Gyx3TWuTddI4nAyayyIWFojiWV5IayDYNV5i2mRp/7qNGol4DtxEYxljmrbgp1HM6hUbmQ==", + "version": "4.2.18", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.18.tgz", + "integrity": "sha512-d0FdzYIiAePqRJEb90WlJDkjUEx42xhivxN8muUBmfZnP+tzUgz12DJ2hRJi8sIHCME7jeK1PTMgKPSfTd8JrA==", "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.2.1", diff --git a/modules/frontend/package.json b/modules/frontend/package.json index d5cb9e517..202db6d3a 100644 --- a/modules/frontend/package.json +++ b/modules/frontend/package.json @@ -1,6 +1,6 @@ { "name": "blaze-frontend", - "version": "0.27.1", + "version": "0.28.0", "private": true, "scripts": { "dev": "vite dev", diff --git a/modules/frontend/src/lib/resource/json/array.svelte b/modules/frontend/src/lib/resource/json/array.svelte index 2d7d6ea2d..46dfc6186 100644 --- a/modules/frontend/src/lib/resource/json/array.svelte +++ b/modules/frontend/src/lib/resource/json/array.svelte @@ -6,8 +6,8 @@ export let values: (FhirObject | FhirPrimitive)[]; -{'[\n'}{#each values as value}{'[\n'}{#each values as value, index}{/each}{' '.repeat(indent)}{']'} + />{index < values.length - 1 ? ',\n' : '\n'}{/each}{' '.repeat(indent)}{']'} diff --git a/modules/frontend/src/lib/resource/json/value.svelte b/modules/frontend/src/lib/resource/json/value.svelte index 4885b7179..b1045c12a 100644 --- a/modules/frontend/src/lib/resource/json/value.svelte +++ b/modules/frontend/src/lib/resource/json/value.svelte @@ -12,4 +12,4 @@ {#if isPrimitive(value.type)}{insideArray ? ' '.repeat(indent) : ''}{:else}{/if}{insideArray ? '\n' : ''} + />{:else}{/if} diff --git a/modules/frontend/src/routes/__admin/+layout.svelte b/modules/frontend/src/routes/__admin/+layout.svelte index 4983c01b7..0a597e09e 100644 --- a/modules/frontend/src/routes/__admin/+layout.svelte +++ b/modules/frontend/src/routes/__admin/+layout.svelte @@ -1,6 +1,8 @@
@@ -17,6 +19,13 @@ id="/__admin/jobs" label="Jobs" /> + {#if data.features.find((f) => f.key === 'cql-expression-cache')?.enabled} + + {/if}
diff --git a/modules/frontend/src/routes/__admin/+page.ts b/modules/frontend/src/routes/__admin/+layout.ts similarity index 85% rename from modules/frontend/src/routes/__admin/+page.ts rename to modules/frontend/src/routes/__admin/+layout.ts index 6858ab2b8..d5ccce9ec 100644 --- a/modules/frontend/src/routes/__admin/+page.ts +++ b/modules/frontend/src/routes/__admin/+layout.ts @@ -1,4 +1,4 @@ -import type { PageLoad } from './$types'; +import type { LayoutLoad } from './$types'; import { base } from '$app/paths'; import { error, type NumericRange } from '@sveltejs/kit'; @@ -10,6 +10,7 @@ export interface Setting { } export interface Feature { + key: string; name: string; toggle: string; enabled: boolean; @@ -20,7 +21,7 @@ export interface Data { features: Feature[]; } -export const load: PageLoad = async ({ fetch }) => { +export const load: LayoutLoad = async ({ fetch }) => { const res = await fetch(`${base}/__admin`, { headers: { Accept: 'application/json' } }); diff --git a/modules/frontend/src/routes/__admin/+page.svelte b/modules/frontend/src/routes/__admin/+page.svelte index 7d2958fdb..45e0f4560 100644 --- a/modules/frontend/src/routes/__admin/+page.svelte +++ b/modules/frontend/src/routes/__admin/+page.svelte @@ -54,7 +54,7 @@ {#each data.features as feature} - + {/each} diff --git a/modules/frontend/src/routes/__admin/cql/+page.svelte b/modules/frontend/src/routes/__admin/cql/+page.svelte new file mode 100644 index 000000000..c94694bb5 --- /dev/null +++ b/modules/frontend/src/routes/__admin/cql/+page.svelte @@ -0,0 +1,45 @@ + + + + CQL - Admin - Blaze + + +
+ +
+

Bloom filters

+

+ Used to improve CQL performance. Only the newest 100 are shown. +

+
+ + + + + + + + + {#each data.bloomFilters as bloomFilter} + + {/each} +
THash# PatientsMem Size
+
diff --git a/modules/frontend/src/routes/__admin/cql/+page.ts b/modules/frontend/src/routes/__admin/cql/+page.ts new file mode 100644 index 000000000..34a81a34b --- /dev/null +++ b/modules/frontend/src/routes/__admin/cql/+page.ts @@ -0,0 +1,29 @@ +import { base } from '$app/paths'; +import { error, type NumericRange } from '@sveltejs/kit'; + +export interface BloomFilter { + hash: string; + t: number; + patientCount: number; + exprForm: string; + memSize: number; +} + +export interface Data { + bloomFilters: BloomFilter[]; +} + +export async function load({ fetch }): Promise { + const res = await fetch(`${base}/__admin/cql/bloom-filters`, { + headers: { Accept: 'application/json' } + }); + + if (!res.ok) { + error(res.status as NumericRange<400, 599>, { + short: undefined, + message: `An error happened while loading CQL Bloom filters. Please try again later.` + }); + } + + return { bloomFilters: await res.json() }; +} diff --git a/modules/frontend/src/routes/__admin/cql/bloom-filter-row.svelte b/modules/frontend/src/routes/__admin/cql/bloom-filter-row.svelte new file mode 100644 index 000000000..6276446ac --- /dev/null +++ b/modules/frontend/src/routes/__admin/cql/bloom-filter-row.svelte @@ -0,0 +1,21 @@ + + + + {bloomFilter.t} + {bloomFilter.hash.substring(0, 8)} + {bloomFilter.patientCount} + {prettyBytes(bloomFilter.memSize, { binary: true, maximumFractionDigits: 1 })} + diff --git a/modules/frontend/src/routes/__admin/cql/bloom-filters/+server.ts b/modules/frontend/src/routes/__admin/cql/bloom-filters/+server.ts new file mode 100644 index 000000000..b9ea59870 --- /dev/null +++ b/modules/frontend/src/routes/__admin/cql/bloom-filters/+server.ts @@ -0,0 +1,9 @@ +import type { RequestHandler } from './$types'; +import { base } from '$app/paths'; + +export const GET: RequestHandler = async ({ fetch }) => { + const res = await fetch(`${base}/__admin/cql/bloom-filters`, { + headers: { Accept: 'application/json' } + }); + return new Response(await res.blob(), res); +}; diff --git a/modules/frontend/src/routes/__admin/feature-row.svelte b/modules/frontend/src/routes/__admin/feature-row.svelte index ecbe107eb..1a5b9503a 100644 --- a/modules/frontend/src/routes/__admin/feature-row.svelte +++ b/modules/frontend/src/routes/__admin/feature-row.svelte @@ -1,15 +1,15 @@ - {name} - {toggle} + {feature.name} + {feature.toggle} {enabled}{feature.enabled} diff --git a/modules/interaction/src/blaze/interaction/util.clj b/modules/interaction/src/blaze/interaction/util.clj index edbc76cdc..028936cb7 100644 --- a/modules/interaction/src/blaze/interaction/util.clj +++ b/modules/interaction/src/blaze/interaction/util.clj @@ -117,7 +117,7 @@ (let [meta (into {} (keep (fn [[k v]] (when (and v (not (#{:versionId :lastUpdated} k))) [k v]))) meta)] (if (empty? meta) (dissoc resource :meta) - (assoc resource :meta (type/map->Meta meta))))) + (assoc resource :meta (type/meta meta))))) (defn keep? "Determines whether `tx-op` is a keep operator." diff --git a/modules/interaction/test/blaze/interaction/transaction/bundle_test.clj b/modules/interaction/test/blaze/interaction/transaction/bundle_test.clj index 99c516cb6..ed80037ed 100644 --- a/modules/interaction/test/blaze/interaction/transaction/bundle_test.clj +++ b/modules/interaction/test/blaze/interaction/transaction/bundle_test.clj @@ -26,8 +26,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-220129" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"POST" @@ -41,8 +41,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-220200" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"POST" @@ -59,8 +59,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-220200" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"POST" @@ -75,8 +75,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-220200" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"POST" @@ -91,8 +91,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-214728" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"PUT" @@ -106,8 +106,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-214728" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"PUT" @@ -124,8 +124,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "id-214728" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH})} + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH})} :request {:fhir/type :fhir.Bundle.entry/request :method #fhir/code"PUT" @@ -158,8 +158,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "0" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH}) + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH}) :gender #fhir/code"male"} :request {:fhir/type :fhir.Bundle.entry/request diff --git a/modules/interaction/test/blaze/interaction/transaction_test.clj b/modules/interaction/test/blaze/interaction/transaction_test.clj index d5e95057d..3128b0eae 100644 --- a/modules/interaction/test/blaze/interaction/transaction_test.clj +++ b/modules/interaction/test/blaze/interaction/transaction_test.clj @@ -438,8 +438,8 @@ [{:fhir/type :fhir.Bundle/entry :resource {:fhir/type :fhir/Patient :id "0" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH}) + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH}) :gender #fhir/code"female"} :request {:fhir/type :fhir.Bundle.entry/request diff --git a/modules/interaction/test/blaze/interaction/update_test.clj b/modules/interaction/test/blaze/interaction/update_test.clj index 93604cf37..da4c5c535 100644 --- a/modules/interaction/test/blaze/interaction/update_test.clj +++ b/modules/interaction/test/blaze/interaction/update_test.clj @@ -499,8 +499,8 @@ ::reitit/match patient-match :headers {"if-match" if-match} :body {:fhir/type :fhir/Patient :id "0" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH}) + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH}) :birthDate #fhir/date"2020"}})] (testing "Returns 200" @@ -541,8 +541,8 @@ ::reitit/match patient-match :headers {"if-match" if-match} :body {:fhir/type :fhir/Patient :id "0" - :meta (type/map->Meta {:versionId #fhir/id"1" - :lastUpdated Instant/EPOCH}) + :meta (type/meta {:versionId #fhir/id"1" + :lastUpdated Instant/EPOCH}) :birthDate #fhir/date"2020"}})] (testing "Returns 200" diff --git a/modules/jepsen/src/blaze/jepsen/register.clj b/modules/jepsen/src/blaze/jepsen/register.clj index 0346092a4..e2b9341a0 100644 --- a/modules/jepsen/src/blaze/jepsen/register.clj +++ b/modules/jepsen/src/blaze/jepsen/register.clj @@ -61,7 +61,7 @@ @(-> (fhir-client/update base-uri {:fhir/type :fhir/Observation :id "0" - :subject (type/map->Reference {:reference (str "Patient/" (random-uuid))})} + :subject (type/reference {:reference (str "Patient/" (random-uuid))})} context) (ac/exceptionally (constantly nil)))) diff --git a/modules/job-async-interaction/src/blaze/job/async_interaction.clj b/modules/job-async-interaction/src/blaze/job/async_interaction.clj index fad07ce7f..cb41d08e1 100644 --- a/modules/job-async-interaction/src/blaze/job/async_interaction.clj +++ b/modules/job-async-interaction/src/blaze/job/async_interaction.clj @@ -38,9 +38,9 @@ :input [(u/request-bundle-input (str "Bundle/" bundle-id)) {:fhir/type :fhir.Task/input - :type (type/map->CodeableConcept + :type (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri u/parameter-uri) :code #fhir/code"t"})]}) :value (type/unsignedInt t)}]}) @@ -91,7 +91,7 @@ (response-bundle context entries))))))) (defn add-response-bundle-reference [job response-bundle-id] - (->> (type/map->Reference {:reference (str "Bundle/" response-bundle-id)}) + (->> (type/reference {:reference (str "Bundle/" response-bundle-id)}) (job-util/add-output job output-uri "bundle"))) (defn- add-processing-duration [job start] diff --git a/modules/job-async-interaction/src/blaze/job/async_interaction/util.clj b/modules/job-async-interaction/src/blaze/job/async_interaction/util.clj index c3685b07d..3364960e2 100644 --- a/modules/job-async-interaction/src/blaze/job/async_interaction/util.clj +++ b/modules/job-async-interaction/src/blaze/job/async_interaction/util.clj @@ -15,15 +15,15 @@ (defn request-bundle-input [reference] {:fhir/type :fhir.Task/input - :type (type/map->CodeableConcept + :type (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri parameter-uri) :code #fhir/code"bundle"})]}) - :value (type/map->Reference {:reference reference})}) + :value (type/reference {:reference reference})}) (defn processing-duration [start] - (type/map->Quantity + (type/quantity {:value (type/decimal (BigDecimal/valueOf (- (System/currentTimeMillis) start) 3)) :unit #fhir/string"s" :system #fhir/uri"http://unitsofmeasure.org" diff --git a/modules/job-async-interaction/test/blaze/job/async_interaction_test.clj b/modules/job-async-interaction/test/blaze/job/async_interaction_test.clj index 1fa36dfc7..ea5d2bed8 100644 --- a/modules/job-async-interaction/test/blaze/job/async_interaction_test.clj +++ b/modules/job-async-interaction/test/blaze/job/async_interaction_test.clj @@ -130,6 +130,7 @@ :kv-store (ig/ref :blaze.db.main/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.main/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} :blaze.db.admin/node @@ -140,6 +141,7 @@ :kv-store (ig/ref :blaze.db.admin/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.admin/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} [::tx-log/local :blaze.db.main/tx-log] @@ -230,6 +232,8 @@ :blaze.db/search-param-registry {:structure-definition-repo structure-definition-repo} + :blaze/scheduler {} + :blaze.test/fixed-clock {} :blaze.test/fixed-rng-fn {}}) diff --git a/modules/job-re-index/test/blaze/job/re_index_test.clj b/modules/job-re-index/test/blaze/job/re_index_test.clj index dd2b40f26..289fab436 100644 --- a/modules/job-re-index/test/blaze/job/re_index_test.clj +++ b/modules/job-re-index/test/blaze/job/re_index_test.clj @@ -85,6 +85,7 @@ :kv-store (ig/ref :blaze.db.main/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.main/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} :blaze.db.admin/node @@ -95,6 +96,7 @@ :kv-store (ig/ref :blaze.db.admin/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.admin/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} [::tx-log/local :blaze.db.main/tx-log] @@ -185,6 +187,8 @@ :blaze.db/search-param-registry {:structure-definition-repo structure-definition-repo} + :blaze/scheduler {} + :blaze.test/fixed-clock {} :blaze.test/offset-clock diff --git a/modules/job-scheduler/src/blaze/job_scheduler.clj b/modules/job-scheduler/src/blaze/job_scheduler.clj index 61591fc5a..f21d313c9 100644 --- a/modules/job-scheduler/src/blaze/job_scheduler.clj +++ b/modules/job-scheduler/src/blaze/job_scheduler.clj @@ -143,7 +143,7 @@ (comp type/integer inc type/value)) (defn- job-number-identifier [job-number] - (type/map->Identifier + (type/identifier {:use #fhir/code"official" :system (type/uri job-util/job-number-url) :value (str job-number)})) diff --git a/modules/job-scheduler/test/blaze/job_scheduler_test.clj b/modules/job-scheduler/test/blaze/job_scheduler_test.clj index ec0c37ac8..f49c6ec42 100644 --- a/modules/job-scheduler/test/blaze/job_scheduler_test.clj +++ b/modules/job-scheduler/test/blaze/job_scheduler_test.clj @@ -125,6 +125,7 @@ :kv-store (ig/ref :blaze.db.main/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.main/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} :blaze.db.admin/node @@ -135,6 +136,7 @@ :kv-store (ig/ref :blaze.db.admin/index-kv-store) :resource-indexer (ig/ref :blaze.db.node.admin/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} [::tx-log/local :blaze.db.main/tx-log] @@ -225,6 +227,8 @@ :blaze.db/search-param-registry {:structure-definition-repo structure-definition-repo} + :blaze/scheduler {} + :blaze.test/fixed-clock {} ::incrementing-rng-fn {}}) @@ -282,9 +286,9 @@ (is (s/valid? :blaze/job-scheduler job-scheduler))))) (defn- job-type [type] - (type/map->CodeableConcept + (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri job-util/type-url) :code (type/code type)})]})) @@ -304,7 +308,7 @@ [#fhir/Coding {:system #fhir/uri"https://samply.github.io/blaze/fhir/CodeSystem/AsyncInteractionJobParameter" :code #fhir/code"bundle"}]} - :value (type/map->Reference {:reference (str "Bundle/" bundle-id)})}]}) + :value (type/reference {:reference (str "Bundle/" bundle-id)})}]}) (defn bundle [id] {:fhir/type :fhir/Bundle @@ -412,7 +416,7 @@ :fhir/type := :fhir/Task job-util/job-number := "1" jtu/combined-status := :ready - bundle-input := (type/map->Reference {:reference (str "Bundle/" bundle-id)}))) + bundle-input := (type/reference {:reference (str "Bundle/" bundle-id)}))) (testing "the bundle is created" (given @(d/pull node (d/resource-handle (d/db node) "Bundle" bundle-id)) diff --git a/modules/job-util/src/blaze/job/util.clj b/modules/job-util/src/blaze/job/util.clj index 64890ea4a..bc02d30b0 100644 --- a/modules/job-util/src/blaze/job/util.clj +++ b/modules/job-util/src/blaze/job/util.clj @@ -22,16 +22,16 @@ "https://samply.github.io/blaze/fhir/CodeSystem/JobOutput") (defn- mk-status-reason [reason] - (type/map->CodeableConcept + (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri status-reason-url) :code (type/code reason)})]})) (defn- mk-sub-status [system-url code] - (type/map->CodeableConcept + (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri system-url) :code (type/code code)})]})) @@ -111,9 +111,9 @@ (defn task-output [system code value] {:fhir/type :fhir.Task/output - :type (type/map->CodeableConcept + :type (type/codeable-concept {:coding - [(type/map->Coding + [(type/coding {:system (type/uri system) :code (type/code code)})]}) :value value}) diff --git a/modules/job-util/test/blaze/job/util_test.clj b/modules/job-util/test/blaze/job/util_test.clj index edc4dc955..3e0cd3010 100644 --- a/modules/job-util/test/blaze/job/util_test.clj +++ b/modules/job-util/test/blaze/job/util_test.clj @@ -26,9 +26,9 @@ (ig/init {:blaze.fhir/structure-definition-repo {}}))) (defn- codeable-concept [system code] - (type/map->CodeableConcept + (type/codeable-concept {:coding - [(type/map->Coding {:system (type/uri system) :code (type/code code)})]})) + [(type/coding {:system (type/uri system) :code (type/code code)})]})) (deftest job-number-test (is (= (job-util/job-number diff --git a/modules/kv/src/blaze/db/kv.clj b/modules/kv/src/blaze/db/kv.clj index 0bc62da4d..685c45afa 100644 --- a/modules/kv/src/blaze/db/kv.clj +++ b/modules/kv/src/blaze/db/kv.clj @@ -183,3 +183,10 @@ Writes are atomic. Blocks." [store entries] (p/-write store entries)) + +(defn estimate-num-keys + "Returns the estimated number of keys in `column-family` of `store`. + + Returns an anomaly if the column-family was not found." + [store column-family] + (p/-estimate-num-keys store column-family)) diff --git a/modules/kv/src/blaze/db/kv/mem.clj b/modules/kv/src/blaze/db/kv/mem.clj index bfab9fd71..2f8eb4dd2 100644 --- a/modules/kv/src/blaze/db/kv/mem.clj +++ b/modules/kv/src/blaze/db/kv/mem.clj @@ -110,14 +110,17 @@ (set! closed? true))) (defn- column-family-not-found-msg [column-family] - (format "column family `%s` not found" (name column-family))) + (format "Column family `%s` not found." (name column-family))) + +(defn- column-family-not-found-anom [column-family] + (ba/not-found (column-family-not-found-msg column-family))) (deftype MemKvSnapshot [db] p/KvSnapshot (-new-iterator [_ column-family] (if-let [db (get db column-family)] (->MemKvIterator db (atom {:rest (seq db)}) false) - (throw-anom (ba/not-found (column-family-not-found-msg column-family))))) + (throw-anom (column-family-not-found-anom column-family)))) (-snapshot-get [_ column-family k] (some-> (get-in db [column-family k]) (copy))) @@ -127,7 +130,7 @@ (defn- assoc-copy [m column-family k v] (when (nil? m) - (throw-anom (ba/not-found (column-family-not-found-msg column-family)))) + (throw-anom (column-family-not-found-anom column-family))) (assoc m (copy k) (copy v))) (defn- put-entries [db entries] @@ -172,7 +175,12 @@ (-write [_ entries] (swap! db write-entries entries) - nil)) + nil) + + (-estimate-num-keys [_ column-family] + (if-let [m (get @db column-family)] + (count m) + (column-family-not-found-anom column-family)))) (def ^:private bytes-cmp (reify Comparator diff --git a/modules/kv/src/blaze/db/kv/protocols.clj b/modules/kv/src/blaze/db/kv/protocols.clj index 1919f7d4e..b924d8151 100644 --- a/modules/kv/src/blaze/db/kv/protocols.clj +++ b/modules/kv/src/blaze/db/kv/protocols.clj @@ -43,4 +43,6 @@ (-delete [store entries]) - (-write [store entries])) + (-write [store entries]) + + (-estimate-num-keys [store column-family])) diff --git a/modules/kv/src/blaze/db/kv_spec.clj b/modules/kv/src/blaze/db/kv_spec.clj index 367513dd7..381b5eea6 100644 --- a/modules/kv/src/blaze/db/kv_spec.clj +++ b/modules/kv/src/blaze/db/kv_spec.clj @@ -4,7 +4,8 @@ [blaze.coll.spec :as cs] [blaze.db.kv :as kv] [blaze.db.kv.spec] - [clojure.spec.alpha :as s])) + [clojure.spec.alpha :as s] + [cognitect.anomalies :as anom])) (s/fdef kv/valid? :args (s/cat :iter ::kv/iterator) @@ -79,3 +80,7 @@ (s/fdef kv/write! :args (s/cat :kv-store :blaze.db/kv-store :entries (cs/coll-of ::kv/write-entry))) + +(s/fdef kv/estimate-num-keys + :args (s/cat :kv-store :blaze.db/kv-store :column-family simple-keyword?) + :ret (s/or :estimate-num-keys nat-int? :anomaly ::anom/anomaly)) diff --git a/modules/kv/test/blaze/db/kv/mem_test.clj b/modules/kv/test/blaze/db/kv/mem_test.clj index 3887f5182..1749ee173 100644 --- a/modules/kv/test/blaze/db/kv/mem_test.clj +++ b/modules/kv/test/blaze/db/kv/mem_test.clj @@ -15,13 +15,14 @@ [clojure.test :as test :refer [deftest is testing]] [cognitect.anomalies :as anom] [integrant.core :as ig] + [juxt.iota :refer [given]] [taoensso.timbre :as log]) (:import [java.lang AutoCloseable])) (set! *warn-on-reflection* true) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -576,5 +577,18 @@ (kv/write! kv-store [[:delete :default (ba 0x00)]]) (is (nil? (kv/get kv-store :default (ba 0x00))))))) +(deftest estimate-num-keys-test + (with-system [{kv-store ::kv/mem} config] + (is (zero? (kv/estimate-num-keys kv-store :default))) + + (given (kv/estimate-num-keys kv-store :foo) + ::anom/category := ::anom/not-found + ::anom/message := "Column family `foo` not found.")) + + (with-system-data [{kv-store ::kv/mem} config] + [[:default (ba 0x00) (ba 0x10)]] + + (is (= 1 (kv/estimate-num-keys kv-store :default))))) + (deftest init-component-test (is (kv/store? (ig/init-key ::kv/mem {})))) diff --git a/modules/module-base/src/blaze/util.clj b/modules/module-base/src/blaze/util.clj index a90ac9274..adda60379 100644 --- a/modules/module-base/src/blaze/util.clj +++ b/modules/module-base/src/blaze/util.clj @@ -16,7 +16,7 @@ [x] (if (or (nil? x) (sequential? x)) x [x])) -(defn strip-leading-slash - "Strips a possible leading slash from `s`." +(defn strip-leading-slashes + "Strips all possible leading slashes from `s`." [s] - (if (str/starts-with? s "/") (subs s 1) s)) + (if (str/starts-with? s "/") (recur (subs s 1)) s)) diff --git a/modules/module-base/test/blaze/util_test.clj b/modules/module-base/test/blaze/util_test.clj index 8c6f823fd..0bf68b41b 100644 --- a/modules/module-base/test/blaze/util_test.clj +++ b/modules/module-base/test/blaze/util_test.clj @@ -27,6 +27,6 @@ (is (= [1] (u/to-seq [1]))))) (deftest strip-leading-slash-test - (satisfies-prop 1000 + (satisfies-prop 10000 (prop/for-all [s gen/string] - (not (str/starts-with? (u/strip-leading-slash s) "/"))))) + (not (str/starts-with? (u/strip-leading-slashes s) "/"))))) diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj index f643a6405..d64f0bd70 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj @@ -5,6 +5,8 @@ [blaze.async.comp :as ac] [blaze.coll.core :as coll] [blaze.db.api :as d] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.spec] [blaze.executors :as ex] [blaze.fhir.operation.evaluate-measure.measure :as measure] [blaze.fhir.operation.evaluate-measure.measure.spec] @@ -121,6 +123,7 @@ (defmethod m/pre-init-spec ::handler [_] (s/keys :req-un [:blaze.db/node ::executor :blaze/clock :blaze/rng-fn] + :opt [::expr/cache] :opt-un [::timeout :blaze/context-path])) (defmethod ig/init-key ::handler [_ context] diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/cql.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/cql.clj index 277a32ce3..d9db991b4 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/cql.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/cql.clj @@ -5,8 +5,8 @@ [blaze.coll.core :as coll] [blaze.db.api :as d] [blaze.elm.compiler :as c] - [blaze.elm.compiler.external-data :as ed] [blaze.elm.expression :as expr] + [blaze.elm.resource :as cr] [blaze.elm.util :as elm-util] [blaze.fhir.spec :as fhir-spec] [taoensso.timbre :as log])) @@ -77,7 +77,7 @@ [{:keys [db] :as context} {:keys [name expression]} subject-handles] (with-open [db (d/new-batch-db db)] (transduce - (comp (map (partial ed/mk-resource db)) + (comp (map (partial cr/mk-resource db)) (result-xf (assoc context :db db) name expression) (halt-when ba/anomaly?)) ((:reduce-op context) db) subject-handles))) diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj index a112d92b1..abbf9062b 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj @@ -5,9 +5,11 @@ [blaze.coll.core :as coll] [blaze.cql-translator :as cql-translator] [blaze.db.api :as d] + [blaze.elm.compiler :as c] [blaze.elm.compiler-spec] - [blaze.elm.compiler.external-data :as ed] [blaze.elm.compiler.library :as library] + [blaze.elm.expression :as-alias expr] + [blaze.elm.resource :as cr] [blaze.fhir.operation.evaluate-measure.cql :as cql] [blaze.fhir.operation.evaluate-measure.measure.group :as group] [blaze.fhir.operation.evaluate-measure.measure.population :as pop] @@ -127,6 +129,9 @@ :fhir/issue "value" :fhir.issue/expression "Measure.library")))) +(defn- remove-unused-defs [expression-defs measure] + (into {} (filter (comp (u/expression-names measure) key)) expression-defs)) + (defn- compile-primary-library "Compiles the CQL code from the first library resource which is referenced from `measure`. @@ -320,21 +325,36 @@ (type/extension {:url "https://samply.github.io/blaze/fhir/StructureDefinition/eval-duration" :value - (type/map->Quantity + (type/quantity {:code #fhir/code"s" :system #fhir/uri"http://unitsofmeasure.org" :unit #fhir/string"s" :value (bigdec duration)})})) +(defn- bloom-filter-ratio + "Creates an extension with a number of available Bloom filters over the total + number of requested Bloom filters." + [bloom-filters] + (type/extension + {:url "https://samply.github.io/blaze/fhir/StructureDefinition/bloom-filter-ratio" + :value + (type/ratio + {:numerator + (type/quantity {:value (bigdec (count (coll/eduction (remove ba/anomaly?) bloom-filters)))}) + :denominator + (type/quantity {:value (bigdec (count bloom-filters))})})})) + (defn- local-ref [handle] (str (name (fhir-spec/fhir-type handle)) "/" (:id handle))) (defn- measure-report - [{:keys [now report-type subject-handle] :as context} measure + [{:keys [now report-type subject-handle bloom-filters] :as context} measure {[start end] :period} [{:keys [result]} duration]] (cond-> {:fhir/type :fhir/MeasureReport - :extension [(eval-duration duration)] + :extension + [(eval-duration duration) + (bloom-filter-ratio bloom-filters)] :status #fhir/code"complete" :type (case report-type @@ -344,12 +364,12 @@ :measure (type/canonical (canonical context measure)) :date now :period - (type/map->Period + (type/period {:start (type/dateTime (str start)) :end (type/dateTime (str end))})} subject-handle - (assoc :subject (type/map->Reference {:reference (local-ref subject-handle)})) + (assoc :subject (type/reference {:reference (local-ref subject-handle)})) (seq result) (assoc :group result))) @@ -385,6 +405,18 @@ (format "Timeout of %d millis eclipsed while evaluating." (.toMillis ^Duration timeout))) +(defn- attach-cache* [cache context [name {:keys [expression] :as expr}]] + [name (c/form expression)] + (let [[expression bloom-filters] (c/attach-cache expression cache)] + (-> (assoc-in context [:expression-defs name] (assoc expr :expression expression)) + (update :bloom-filters into bloom-filters)))) + +(defn- attach-cache [{:keys [expression-defs] :as context} cache] + (reduce + (partial attach-cache* cache) + (assoc context :bloom-filters []) + expression-defs)) + (defn- eval-unfiltered-xf [context] (comp (filter (comp #{"Unfiltered"} :context val)) (map @@ -408,6 +440,7 @@ (defn- enhance-context [{:keys [clock db timeout] :blaze/keys [cancelled?] + ::expr/keys [cache] :or {timeout (time/hours 1)} :as context} measure {:keys [report-type subject-ref]}] @@ -432,10 +465,16 @@ ::luid/generator (luid-generator context)) function-defs (assoc :function-defs function-defs))] - (when-ok [expression-defs (eval-unfiltered context expression-defs)] - (cond-> (assoc context :expression-defs expression-defs) + (when-ok [expression-defs (eval-unfiltered context expression-defs) + expression-defs (library/resolve-all-refs expression-defs)] + (cond-> (assoc + context + :expression-defs + (-> (remove-unused-defs expression-defs measure) + (library/resolve-param-refs parameter-default-values))) + cache (attach-cache cache) subject-handle - (assoc :subject-handle (ed/mk-resource db subject-handle))))))))) + (assoc :subject-handle (cr/mk-resource db subject-handle))))))))) (defn evaluate-measure "Evaluates `measure` inside `context` with `params`. diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/spec.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/spec.clj index c2d00a904..6969de4bb 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/spec.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/spec.clj @@ -1,7 +1,7 @@ (ns blaze.fhir.operation.evaluate-measure.measure.spec (:require [blaze.db.spec] - [blaze.elm.compiler.external-data :as ed] + [blaze.elm.resource :as cr] [blaze.fhir.operation.evaluate-measure.measure :as-alias measure] [blaze.fhir.spec.spec] [clojure.spec.alpha :as s])) @@ -14,10 +14,10 @@ :local-ref :blaze.fhir/literal-ref-tuple)) (s/def ::measure/population-handle - ed/resource?) + cr/resource?) (s/def ::measure/subject-handle - ed/resource?) + cr/resource?) (s/def ::measure/handle (s/keys :req-un [::measure/population-handle ::measure/subject-handle])) diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/util.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/util.clj index 1c4b98de9..840b6ac84 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/util.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure/util.clj @@ -1,6 +1,7 @@ (ns blaze.fhir.operation.evaluate-measure.measure.util (:require [blaze.anomaly :as ba] + [blaze.coll.core :as coll] [blaze.fhir.spec.type :as type] [blaze.luid :as luid])) @@ -30,10 +31,10 @@ expression))) (defn list-reference [list-id] - (type/map->Reference {:reference (str "List/" list-id)})) + (type/reference {:reference (str "List/" list-id)})) (defn- resource-reference [{:keys [id] :as resource}] - (type/map->Reference {:reference (str (name (type/type resource)) "/" id)})) + (type/reference {:reference (str (name (type/type resource)) "/" id)})) (defn population-tx-ops [list-id handles] [[:create @@ -66,3 +67,25 @@ (-> result (ba/map (partial merge-result* ret)) (ba/exceptionally reduced))) + +(defn- expression-name-of-expression [{:keys [language expression]}] + (when (#{"text/cql" "text/cql-identifier"} (type/value language)) + (type/value expression))) + +(defn- expression-name-of-population [{:keys [criteria]}] + (expression-name-of-expression criteria)) + +(defn- expression-names-of-stratifier [{:keys [criteria component]}] + (if criteria + (some-> (expression-name-of-expression criteria) vector) + (coll/eduction (keep expression-name-of-population) component))) + +(defn- expression-names-of-group [{:keys [population stratifier]}] + (-> (into [] (keep expression-name-of-population) population) + (into (mapcat expression-names-of-stratifier) stratifier))) + +(defn expression-names + "Returns a set of alle CQL expression names found in `measure`." + {:arglists '([measure])} + [{:keys [group]}] + (into #{} (mapcat expression-names-of-group) group)) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj index 47ef824c1..b2694f7c0 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj @@ -1,10 +1,10 @@ (ns blaze.fhir.operation.evaluate-measure.cql.spec (:require [blaze.elm.compiler :as-alias c] - [blaze.elm.compiler.external-data :as ed] [blaze.elm.compiler.library.spec] [blaze.elm.expression :as-alias expr] [blaze.elm.expression.spec] + [blaze.elm.resource :as cr] [blaze.fhir.operation.evaluate-measure :as-alias evaluate-measure] [blaze.fhir.operation.evaluate-measure.cql :as-alias cql] [blaze.fhir.operation.evaluate-measure.spec] @@ -35,10 +35,10 @@ :opt-un [::cql/population-basis]))) (s/def ::cql/subject-handle - ed/resource?) + cr/resource?) (s/def ::cql/population-handle - ed/resource?) + cr/resource?) (s/def ::cql/handle (s/keys :req-un [::cql/subject-handle ::cql/population-handle])) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_spec.clj index 7704316c1..e2497f718 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_spec.clj @@ -2,9 +2,8 @@ (:require [blaze.async.comp :as ac] [blaze.elm.compiler :as-alias c] - [blaze.elm.compiler.external-data :as ed] - [blaze.elm.compiler.external-data-spec] [blaze.elm.compiler.spec] + [blaze.elm.resource :as cr] [blaze.fhir.operation.evaluate-measure.cql :as cql] [blaze.fhir.operation.evaluate-measure.cql.spec] [blaze.fhir.spec] @@ -13,7 +12,7 @@ (s/fdef cql/evaluate-expression-1 :args (s/cat :context ::cql/context - :subject (s/nilable ed/resource?) + :subject (s/nilable cr/resource?) :name string? :expression ::c/expression) :ret (s/or :result any? :anomaly ::anom/anomaly)) @@ -25,7 +24,7 @@ (s/fdef cql/evaluate-individual-expression :args (s/cat :context ::cql/evaluate-expression-context - :subject ed/resource? + :subject cr/resource? :name string?) :ret ac/completable-future?) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj index 65f31265d..7fdd823e7 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj @@ -18,6 +18,7 @@ [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [deftest is testing]] [cognitect.anomalies :as anom] + [integrant.core :as ig] [juxt.iota :refer [given]] [taoensso.timbre :as log]) (:import @@ -90,17 +91,25 @@ (fn [_ _ _] (throw (Exception. ^String msg)))) (defn- context - [{:blaze.db/keys [node] :blaze.test/keys [fixed-clock executor]} library] + [{:blaze.db/keys [node] + ::expr/keys [cache] + :blaze.test/keys [fixed-clock executor]} + library] (let [{:keys [expression-defs function-defs]} (compile-library node library)] {:db (d/db node) :now (now fixed-clock) + ::expr/cache cache :interrupted? (constantly nil) :expression-defs expression-defs :function-defs function-defs :executor executor})) (def ^:private config - (assoc mem-node-config :blaze.test/executor {})) + (assoc mem-node-config + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} + :blaze.test/executor {})) (def ^:private conj-reduce-op (fn [_db] conj)) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_spec.clj index b8c6f6379..0290e8865 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_spec.clj @@ -8,3 +8,7 @@ (s/fdef u/expression-name :args (s/cat :population-path-fn fn? :criteria (s/nilable :fhir/Expression)) :ret (s/or :expression-name string? :anomaly ::anom/anomaly)) + +(s/fdef u/expression-names + :args (s/cat :measure :fhir/Measure) + :ret (s/coll-of string?)) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_test.clj index 2619183f7..33339fffd 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/util_test.clj @@ -5,7 +5,7 @@ [blaze.fhir.operation.evaluate-measure.measure.util-spec] [blaze.test-util :as tu :refer [satisfies-prop]] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest testing]] + [clojure.test :as test :refer [are deftest testing]] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] [cognitect.anomalies :as anom] @@ -56,3 +56,43 @@ {:fhir/type :fhir/Expression :language #fhir/code"text/cql" :expression expression})))))) + +(defn- cql-expression [expr] + {:fhir/type :fhir/Expression + :language #fhir/code"text/cql-identifier" + :expression expr}) + +(deftest cql-definition-names-test + (are [measure names] (= names (u/expression-names measure)) + {:fhir/type :fhir/Measure :id "0" + :url #fhir/uri"measure-155502" + :library [#fhir/canonical"0"] + :group + [{:fhir/type :fhir.Measure/group + :population + [{:fhir/type :fhir.Measure.group/population + :criteria (cql-expression "InInitialPopulation")}] + :stratifier + [{:fhir/type :fhir.Measure.group/stratifier + :criteria (cql-expression "Gender")}]}]} + #{"InInitialPopulation" + "Gender"} + + {:fhir/type :fhir/Measure :id "0" + :url #fhir/uri"measure-155502" + :library [#fhir/canonical"0"] + :group + [{:fhir/type :fhir.Measure/group + :population + [{:fhir/type :fhir.Measure.group/population + :criteria (cql-expression "InInitialPopulation")}] + :stratifier + [{:fhir/type :fhir.Measure.group/stratifier + :component + [{:fhir/type :fhir.Measure.group.stratifier/component + :criteria (cql-expression "AgeClass")} + {:fhir/type :fhir.Measure.group.stratifier/component + :criteria (cql-expression "Gender")}]}]}]} + #{"InInitialPopulation" + "AgeClass" + "Gender"})) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj index 0527ab9a6..a9b170017 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj @@ -3,6 +3,7 @@ [blaze.async.comp :as ac] [blaze.cql-translator-spec] [blaze.db.spec] + [blaze.elm.expression :as-alias expr] [blaze.elm.expression.spec] [blaze.fhir.operation.evaluate-measure :as-alias evaluate-measure] [blaze.fhir.operation.evaluate-measure.cql-spec] @@ -18,7 +19,7 @@ (s/def ::context (s/keys :req [:blaze/base-url ::reitit/router] - :opt [:blaze/cancelled?] + :opt [:blaze/cancelled? ::expr/cache] :req-un [:blaze/clock :blaze/rng-fn :blaze.db/db ::evaluate-measure/executor] :opt-un [::evaluate-measure/timeout])) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj index 134227b6c..96ba87da2 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj @@ -1,8 +1,11 @@ (ns blaze.fhir.operation.evaluate-measure.measure-test (:require - [blaze.anomaly :as ba] + [blaze.anomaly :as ba :refer [when-ok]] [blaze.db.api :as d] [blaze.db.api-stub :refer [mem-node-config with-system-data]] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as ec] + [blaze.elm.expression.cache.bloom-filter :as-alias bloom-filter] [blaze.fhir.operation.evaluate-measure.measure :as measure] [blaze.fhir.operation.evaluate-measure.measure-spec] [blaze.fhir.operation.evaluate-measure.measure.group-spec] @@ -19,6 +22,7 @@ [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [deftest is testing]] [cognitect.anomalies :as anom] + [integrant.core :as ig] [java-time.api :as time] [juxt.iota :refer [given]] [reitit.core :as reitit] @@ -60,7 +64,7 @@ :id "1" :url #fhir/uri"0" :content - [(type/map->Attachment + [(type/attachment {:contentType #fhir/code"text/cql" :data (type/->Base64Binary (b64-encode query))})]} :request @@ -75,6 +79,9 @@ (def ^:private config (assoc mem-node-config + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} :blaze.test/fixed-rng-fn {} :blaze.test/executor {})) @@ -82,22 +89,30 @@ ([name] (evaluate name "population")) ([name report-type] + (evaluate name report-type false)) + ([name report-type twice-for-caching] (with-system-data [{:blaze.db/keys [node] + ::expr/keys [cache] :blaze.test/keys [fixed-clock fixed-rng-fn executor]} config] [(tx-ops (:entry (read-data name)))] (let [db (d/db node) context {:clock fixed-clock :rng-fn fixed-rng-fn :db db - :blaze/base-url "" ::reitit/router router + ::expr/cache cache :blaze/base-url "" ::reitit/router router :executor executor} measure @(d/pull node (d/resource-handle db "Measure" "0")) - period [#system/date "2000" #system/date "2020"]] - (try - @(measure/evaluate-measure context measure - {:period period :report-type report-type}) - (catch Exception e - (ex-data (ex-cause e)))))))) + period [#system/date "2000" #system/date "2020"] + params {:period period :report-type report-type} + eval #(try + @(measure/evaluate-measure context measure params) + (catch Exception e + (ex-data (ex-cause e))))] + (if twice-for-caching + (when-ok [_ (eval)] + (Thread/sleep 200) + (eval)) + (eval)))))) (defn- first-population [result] (if (::anom/category result) @@ -172,6 +187,35 @@ define InInitialPopulation: [Encounter]") +(def library-exists-condition + "library Retrieve + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + context Patient + + define InInitialPopulation: + exists [Condition]") + +(def library-medication + "library Retrieve + using FHIR version '4.0.0' + include FHIRHelpers version '4.0.0' + + codesystem atc: 'http://fhir.de/CodeSystem/dimdi/atc' + + context Unfiltered + + define \"Temozolomid Refs\": + [Medication: Code 'L01AX03' from atc] M return 'Medication/' + M.id + + context Patient + + define InInitialPopulation: + Patient.gender = 'female' and + exists from [MedicationStatement] M + where M.medication.reference in \"Temozolomid Refs\"") + (def library-patient-encounter "library Retrieve using FHIR version '4.0.0' @@ -685,6 +729,122 @@ ::anom/category := ::anom/incorrect ::anom/message := "Subject with type `Patient` and id `0` was not found.")))))) +(def ^:private ^:const bloom-filter-ratio-url + "https://samply.github.io/blaze/fhir/StructureDefinition/bloom-filter-ratio") + +(defn- bloom-filter-ratio [extensions] + (some #(when (= bloom-filter-ratio-url (:url %)) %) extensions)) + +(defn- patient-condition-tx-ops [id] + (cond-> [[:put {:fhir/type :fhir/Patient :id (str id)}]] + (even? id) + (conj [:put {:fhir/type :fhir/Condition :id (str id) + :subject (type/reference {:reference (str "Patient/" id)})}]))) + +(defn- patient-medication-tx-ops [id] + (cond-> [[:put {:fhir/type :fhir/Patient :id (str id) + :gender (if (even? id) #fhir/code"female" #fhir/code"male")}]] + (zero? (rem id 4)) + (conj [:put {:fhir/type :fhir/MedicationStatement :id (str id) + :medication #fhir/Reference{:reference "Medication/0"} + :subject (type/reference {:reference (str "Patient/" id)})}]))) + +(deftest evaluate-measure-cache-test + (testing "Condition" + (with-system-data + [{:blaze.db/keys [node] + ::expr/keys [cache] + :blaze.test/keys [fixed-clock fixed-rng-fn executor]} config] + [(into [] (mapcat patient-condition-tx-ops) (range 2000)) + [[:put {:fhir/type :fhir/Library :id "0" :url #fhir/uri"0" + :content [(library-content library-exists-condition)]}]]] + + (let [db (d/db node) + context {:clock fixed-clock :rng-fn fixed-rng-fn :db db + ::expr/cache cache + :executor executor :blaze/base-url "" ::reitit/router router} + measure {:fhir/type :fhir/Measure :id "0" + :library [#fhir/canonical"0"] + :group + [{:fhir/type :fhir.Measure/group + :population + [{:fhir/type :fhir.Measure.group/population + :code (population-concept "initial-population") + :criteria (cql-expression "InInitialPopulation")}]}]}] + + (testing "without bloom filter because it's not available yet" + (let [params {:period [#system/date "2000" #system/date "2100"] + :report-type "population"}] + (given (:resource @(measure/evaluate-measure context measure params)) + :fhir/type := :fhir/MeasureReport + [:extension bloom-filter-ratio :value :numerator :value] := 0M + [:extension bloom-filter-ratio :value :denominator :value] := 1M + [:group 0 :population 0 :count] := 1000))) + + (Thread/sleep 1000) + + (testing "with bloom filter" + (let [params {:period [#system/date "2000" #system/date "2100"] + :report-type "population"}] + (given (:resource @(measure/evaluate-measure context measure params)) + :fhir/type := :fhir/MeasureReport + [:extension bloom-filter-ratio :value :numerator :value] := 1M + [:extension bloom-filter-ratio :value :denominator :value] := 1M + [:group 0 :population 0 :count] := 1000)) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/expr-form] := "(exists (retrieve \"Condition\"))"))))) + + (testing "Medication" + (log/set-min-level! :info) + (with-system-data + [{:blaze.db/keys [node] + ::expr/keys [cache] + :blaze.test/keys [fixed-clock fixed-rng-fn executor]} config] + [[[:put {:fhir/type :fhir/Medication :id "0" + :code #fhir/CodeableConcept{:coding [#fhir/Coding{:system #fhir/uri"http://fhir.de/CodeSystem/dimdi/atc" :code #fhir/code"L01AX03"}]}}]] + (into [] (mapcat patient-medication-tx-ops) (range 2000)) + [[:put {:fhir/type :fhir/Library :id "0" :url #fhir/uri"0" + :content [(library-content library-medication)]}]]] + + (let [db (d/db node) + context {:clock fixed-clock :rng-fn fixed-rng-fn :db db + ::expr/cache cache + :executor executor :blaze/base-url "" ::reitit/router router} + measure {:fhir/type :fhir/Measure :id "0" + :library [#fhir/canonical"0"] + :group + [{:fhir/type :fhir.Measure/group + :population + [{:fhir/type :fhir.Measure.group/population + :code (population-concept "initial-population") + :criteria (cql-expression "InInitialPopulation")}]}]}] + + (testing "without bloom filter because it's not available yet" + (let [params {:period [#system/date "2000" #system/date "2100"] + :report-type "population"}] + (given (:resource @(measure/evaluate-measure context measure params)) + :fhir/type := :fhir/MeasureReport + [:extension bloom-filter-ratio :value :numerator :value] := 0M + [:extension bloom-filter-ratio :value :denominator :value] := 1M + [:group 0 :population 0 :count] := 500))) + + (Thread/sleep 1000) + + (testing "with bloom filter" + (let [params {:period [#system/date "2000" #system/date "2100"] + :report-type "population"}] + (given (:resource @(measure/evaluate-measure context measure params)) + :fhir/type := :fhir/MeasureReport + [:extension bloom-filter-ratio :value :numerator :value] := 1M + [:extension bloom-filter-ratio :value :denominator :value] := 1M + [:group 0 :population 0 :count] := 500)) + + (given (into [] (ec/list-by-t cache)) + count := 1 + [0 ::bloom-filter/expr-form] := "(exists (eduction-query (comp (filter (fn [M] (contains [\"Medication/0\"] (call \"ToString\" (:reference (:medication M)))))) distinct) (retrieve \"MedicationStatement\")))")))))) + (defmacro testing-query [name count] `(testing ~name (is (= ~count (:count (first-population (evaluate ~name))))))) @@ -766,6 +926,12 @@ (testing-query "q53-population-basis-boolean" 2) + (testing "q57-mii-specimen-reference" + (given (evaluate "q57-mii-specimen-reference" "population" true) + [:resource :extension bloom-filter-ratio :value :numerator :value] := 6M + [:resource :extension bloom-filter-ratio :value :denominator :value] := 6M + [first-population :count] := 1)) + (let [result (evaluate "q1" "subject-list")] (testing "MeasureReport is valid" (is (s/valid? :blaze/resource (:resource result)))) @@ -973,7 +1139,7 @@ (given (first-stratifier-strata (evaluate "q52-sort-with-missing-values")) count := 1 - [0 :value :text] := "Condition[id = 0, t = 1]" + [0 :value :text] := "Condition[id = 0, t = 1, last-change-t = 1]" [0 :population 0 :count] := #fhir/integer 1) (given (first-stratifier-strata (evaluate "q54-stratifier-condition-code")) @@ -1020,6 +1186,12 @@ [1 :component 1 :extension 0 :value :code] := #fhir/code"cm" [1 :population 0 :count] := #fhir/integer 1)) +(deftest queries-with-errors-test + (testing "overly large non-parsable query" + (given (evaluate "q58-overly-large-nonparsable-query") + ::anom/category := ::anom/fault + ::anom/message := "Error while parsing the ELM representation of a CQL library: Could not convert library to JSON using JAXB serializer."))) + (comment (log/set-level! :debug) - (evaluate "q53-population-basis-boolean")) + (evaluate "q58-overly-large-nonparsable-query")) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q50-specimen-condition-reference.cql b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q50-specimen-condition-reference.cql index c49d57a8f..04a88ae55 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q50-specimen-condition-reference.cql +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q50-specimen-condition-reference.cql @@ -12,4 +12,4 @@ context Patient define InInitialPopulation: exists [Specimen: "Serum Specimen"] S with [Condition: "Diabetes mellitus"] C - such that (S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value as Reference).reference = 'Condition/' + C.id + such that (S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value as Reference).reference = 'Condition/' + C.id diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.cql b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.cql new file mode 100644 index 000000000..371a8d694 --- /dev/null +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.cql @@ -0,0 +1,40 @@ +library "q57-mii-specimen-reference" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' +codesystem snomed: 'http://snomed.info/sct' + +context Patient + +define "Criterion 1": + Patient.gender = 'female' + +define "Diagnose E13.9": + [Condition: Code 'E13.9' from icd10] union + [Condition: Code 'E13.91' from icd10] union + [Condition: Code 'E13.90' from icd10] + +define "Criterion 2": + exists (from [Specimen: Code '119364003' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) or + exists (from [Specimen: Code '258590006' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) or + exists (from [Specimen: Code '866034009' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) or + exists (from [Specimen: Code '866035005' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) or + exists (from [Specimen: Code '442427000' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) or + exists (from [Specimen: Code '737089009' from snomed] S + with "Diagnose E13.9" C + such that S.extension.where(url='https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose').first().value.as(Reference).reference = 'Condition/' + C.id) + +define InInitialPopulation: + "Criterion 1" and + "Criterion 2" diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.json b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.json new file mode 100644 index 000000000..1d88bb188 --- /dev/null +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q57-mii-specimen-reference.json @@ -0,0 +1,139 @@ +{ + "resourceType": "Bundle", + "type": "transaction", + "entry": [ + { + "resource": { + "resourceType": "Patient", + "id": "0", + "gender": "female" + }, + "request": { + "method": "PUT", + "url": "Patient/0" + } + }, + { + "resource": { + "resourceType": "Patient", + "id": "1", + "gender": "female" + }, + "request": { + "method": "PUT", + "url": "Patient/1" + } + }, + { + "resource": { + "resourceType": "Patient", + "id": "2" + }, + "request": { + "method": "PUT", + "url": "Patient/2" + } + }, + { + "resource": { + "resourceType": "Condition", + "id": "0", + "code": { + "coding": [ + { + "system": "http://fhir.de/CodeSystem/bfarm/icd-10-gm", + "code": "E13.91" + } + ] + }, + "subject": { + "reference": "Patient/0" + } + }, + "request": { + "method": "PUT", + "url": "Condition/0" + } + }, + { + "resource": { + "resourceType": "Specimen", + "id": "0", + "extension": [ + { + "url": "https://www.medizininformatik-initiative.de/fhir/ext/modul-biobank/StructureDefinition/Diagnose", + "valueReference": { + "reference": "Condition/0" + } + } + ], + "type": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "866035005" + } + ] + }, + "subject": { + "reference": "Patient/0" + } + }, + "request": { + "method": "PUT", + "url": "Specimen/0" + } + }, + { + "resource": { + "resourceType": "Measure", + "id": "0", + "url": "0", + "status": "active", + "subjectCodeableConcept": { + "coding": [ + { + "system": "http://hl7.org/fhir/resource-types", + "code": "Patient" + } + ] + }, + "library": [ + "0" + ], + "scoring": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-scoring", + "code": "cohort" + } + ] + }, + "group": [ + { + "population": [ + { + "code": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-population", + "code": "initial-population" + } + ] + }, + "criteria": { + "language": "text/cql-identifier", + "expression": "InInitialPopulation" + } + } + ] + } + ] + }, + "request": { + "method": "PUT", + "url": "Measure/0" + } + } + ] +} diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.cql b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.cql new file mode 100644 index 000000000..63caffc93 --- /dev/null +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.cql @@ -0,0 +1,509 @@ +library "q58-overly-large-nonparsable-query" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '0' from sct] or + exists [Condition: Code '1' from sct] or + exists [Condition: Code '2' from sct] or + exists [Condition: Code '3' from sct] or + exists [Condition: Code '4' from sct] or + exists [Condition: Code '5' from sct] or + exists [Condition: Code '6' from sct] or + exists [Condition: Code '7' from sct] or + exists [Condition: Code '8' from sct] or + exists [Condition: Code '9' from sct] or + exists [Condition: Code '10' from sct] or + exists [Condition: Code '11' from sct] or + exists [Condition: Code '12' from sct] or + exists [Condition: Code '13' from sct] or + exists [Condition: Code '14' from sct] or + exists [Condition: Code '15' from sct] or + exists [Condition: Code '16' from sct] or + exists [Condition: Code '17' from sct] or + exists [Condition: Code '18' from sct] or + exists [Condition: Code '19' from sct] or + exists [Condition: Code '20' from sct] or + exists [Condition: Code '21' from sct] or + exists [Condition: Code '22' from sct] or + exists [Condition: Code '23' from sct] or + exists [Condition: Code '24' from sct] or + exists [Condition: Code '25' from sct] or + exists [Condition: Code '26' from sct] or + exists [Condition: Code '27' from sct] or + exists [Condition: Code '28' from sct] or + exists [Condition: Code '29' from sct] or + exists [Condition: Code '30' from sct] or + exists [Condition: Code '31' from sct] or + exists [Condition: Code '32' from sct] or + exists [Condition: Code '33' from sct] or + exists [Condition: Code '34' from sct] or + exists [Condition: Code '35' from sct] or + exists [Condition: Code '36' from sct] or + exists [Condition: Code '37' from sct] or + exists [Condition: Code '38' from sct] or + exists [Condition: Code '39' from sct] or + exists [Condition: Code '40' from sct] or + exists [Condition: Code '41' from sct] or + exists [Condition: Code '42' from sct] or + exists [Condition: Code '43' from sct] or + exists [Condition: Code '44' from sct] or + exists [Condition: Code '45' from sct] or + exists [Condition: Code '46' from sct] or + exists [Condition: Code '47' from sct] or + exists [Condition: Code '48' from sct] or + exists [Condition: Code '49' from sct] or + exists [Condition: Code '50' from sct] or + exists [Condition: Code '51' from sct] or + exists [Condition: Code '52' from sct] or + exists [Condition: Code '53' from sct] or + exists [Condition: Code '54' from sct] or + exists [Condition: Code '55' from sct] or + exists [Condition: Code '56' from sct] or + exists [Condition: Code '57' from sct] or + exists [Condition: Code '58' from sct] or + exists [Condition: Code '59' from sct] or + exists [Condition: Code '60' from sct] or + exists [Condition: Code '61' from sct] or + exists [Condition: Code '62' from sct] or + exists [Condition: Code '63' from sct] or + exists [Condition: Code '64' from sct] or + exists [Condition: Code '65' from sct] or + exists [Condition: Code '66' from sct] or + exists [Condition: Code '67' from sct] or + exists [Condition: Code '68' from sct] or + exists [Condition: Code '69' from sct] or + exists [Condition: Code '70' from sct] or + exists [Condition: Code '71' from sct] or + exists [Condition: Code '72' from sct] or + exists [Condition: Code '73' from sct] or + exists [Condition: Code '74' from sct] or + exists [Condition: Code '75' from sct] or + exists [Condition: Code '76' from sct] or + exists [Condition: Code '77' from sct] or + exists [Condition: Code '78' from sct] or + exists [Condition: Code '79' from sct] or + exists [Condition: Code '80' from sct] or + exists [Condition: Code '81' from sct] or + exists [Condition: Code '82' from sct] or + exists [Condition: Code '83' from sct] or + exists [Condition: Code '84' from sct] or + exists [Condition: Code '85' from sct] or + exists [Condition: Code '86' from sct] or + exists [Condition: Code '87' from sct] or + exists [Condition: Code '88' from sct] or + exists [Condition: Code '89' from sct] or + exists [Condition: Code '90' from sct] or + exists [Condition: Code '91' from sct] or + exists [Condition: Code '92' from sct] or + exists [Condition: Code '93' from sct] or + exists [Condition: Code '94' from sct] or + exists [Condition: Code '95' from sct] or + exists [Condition: Code '96' from sct] or + exists [Condition: Code '97' from sct] or + exists [Condition: Code '98' from sct] or + exists [Condition: Code '99' from sct] or + exists [Condition: Code '100' from sct] or + exists [Condition: Code '101' from sct] or + exists [Condition: Code '102' from sct] or + exists [Condition: Code '103' from sct] or + exists [Condition: Code '104' from sct] or + exists [Condition: Code '105' from sct] or + exists [Condition: Code '106' from sct] or + exists [Condition: Code '107' from sct] or + exists [Condition: Code '108' from sct] or + exists [Condition: Code '109' from sct] or + exists [Condition: Code '110' from sct] or + exists [Condition: Code '111' from sct] or + exists [Condition: Code '112' from sct] or + exists [Condition: Code '113' from sct] or + exists [Condition: Code '114' from sct] or + exists [Condition: Code '115' from sct] or + exists [Condition: Code '116' from sct] or + exists [Condition: Code '117' from sct] or + exists [Condition: Code '118' from sct] or + exists [Condition: Code '119' from sct] or + exists [Condition: Code '120' from sct] or + exists [Condition: Code '121' from sct] or + exists [Condition: Code '122' from sct] or + exists [Condition: Code '123' from sct] or + exists [Condition: Code '124' from sct] or + exists [Condition: Code '125' from sct] or + exists [Condition: Code '126' from sct] or + exists [Condition: Code '127' from sct] or + exists [Condition: Code '128' from sct] or + exists [Condition: Code '129' from sct] or + exists [Condition: Code '130' from sct] or + exists [Condition: Code '131' from sct] or + exists [Condition: Code '132' from sct] or + exists [Condition: Code '133' from sct] or + exists [Condition: Code '134' from sct] or + exists [Condition: Code '135' from sct] or + exists [Condition: Code '136' from sct] or + exists [Condition: Code '137' from sct] or + exists [Condition: Code '138' from sct] or + exists [Condition: Code '139' from sct] or + exists [Condition: Code '140' from sct] or + exists [Condition: Code '141' from sct] or + exists [Condition: Code '142' from sct] or + exists [Condition: Code '143' from sct] or + exists [Condition: Code '144' from sct] or + exists [Condition: Code '145' from sct] or + exists [Condition: Code '146' from sct] or + exists [Condition: Code '147' from sct] or + exists [Condition: Code '148' from sct] or + exists [Condition: Code '149' from sct] or + exists [Condition: Code '150' from sct] or + exists [Condition: Code '151' from sct] or + exists [Condition: Code '152' from sct] or + exists [Condition: Code '153' from sct] or + exists [Condition: Code '154' from sct] or + exists [Condition: Code '155' from sct] or + exists [Condition: Code '156' from sct] or + exists [Condition: Code '157' from sct] or + exists [Condition: Code '158' from sct] or + exists [Condition: Code '159' from sct] or + exists [Condition: Code '160' from sct] or + exists [Condition: Code '161' from sct] or + exists [Condition: Code '162' from sct] or + exists [Condition: Code '163' from sct] or + exists [Condition: Code '164' from sct] or + exists [Condition: Code '165' from sct] or + exists [Condition: Code '166' from sct] or + exists [Condition: Code '167' from sct] or + exists [Condition: Code '168' from sct] or + exists [Condition: Code '169' from sct] or + exists [Condition: Code '170' from sct] or + exists [Condition: Code '171' from sct] or + exists [Condition: Code '172' from sct] or + exists [Condition: Code '173' from sct] or + exists [Condition: Code '174' from sct] or + exists [Condition: Code '175' from sct] or + exists [Condition: Code '176' from sct] or + exists [Condition: Code '177' from sct] or + exists [Condition: Code '178' from sct] or + exists [Condition: Code '179' from sct] or + exists [Condition: Code '180' from sct] or + exists [Condition: Code '181' from sct] or + exists [Condition: Code '182' from sct] or + exists [Condition: Code '183' from sct] or + exists [Condition: Code '184' from sct] or + exists [Condition: Code '185' from sct] or + exists [Condition: Code '186' from sct] or + exists [Condition: Code '187' from sct] or + exists [Condition: Code '188' from sct] or + exists [Condition: Code '189' from sct] or + exists [Condition: Code '190' from sct] or + exists [Condition: Code '191' from sct] or + exists [Condition: Code '192' from sct] or + exists [Condition: Code '193' from sct] or + exists [Condition: Code '194' from sct] or + exists [Condition: Code '195' from sct] or + exists [Condition: Code '196' from sct] or + exists [Condition: Code '197' from sct] or + exists [Condition: Code '198' from sct] or + exists [Condition: Code '199' from sct] or + exists [Condition: Code '200' from sct] or + exists [Condition: Code '201' from sct] or + exists [Condition: Code '202' from sct] or + exists [Condition: Code '203' from sct] or + exists [Condition: Code '204' from sct] or + exists [Condition: Code '205' from sct] or + exists [Condition: Code '206' from sct] or + exists [Condition: Code '207' from sct] or + exists [Condition: Code '208' from sct] or + exists [Condition: Code '209' from sct] or + exists [Condition: Code '210' from sct] or + exists [Condition: Code '211' from sct] or + exists [Condition: Code '212' from sct] or + exists [Condition: Code '213' from sct] or + exists [Condition: Code '214' from sct] or + exists [Condition: Code '215' from sct] or + exists [Condition: Code '216' from sct] or + exists [Condition: Code '217' from sct] or + exists [Condition: Code '218' from sct] or + exists [Condition: Code '219' from sct] or + exists [Condition: Code '220' from sct] or + exists [Condition: Code '221' from sct] or + exists [Condition: Code '222' from sct] or + exists [Condition: Code '223' from sct] or + exists [Condition: Code '224' from sct] or + exists [Condition: Code '225' from sct] or + exists [Condition: Code '226' from sct] or + exists [Condition: Code '227' from sct] or + exists [Condition: Code '228' from sct] or + exists [Condition: Code '229' from sct] or + exists [Condition: Code '230' from sct] or + exists [Condition: Code '231' from sct] or + exists [Condition: Code '232' from sct] or + exists [Condition: Code '233' from sct] or + exists [Condition: Code '234' from sct] or + exists [Condition: Code '235' from sct] or + exists [Condition: Code '236' from sct] or + exists [Condition: Code '237' from sct] or + exists [Condition: Code '238' from sct] or + exists [Condition: Code '239' from sct] or + exists [Condition: Code '240' from sct] or + exists [Condition: Code '241' from sct] or + exists [Condition: Code '242' from sct] or + exists [Condition: Code '243' from sct] or + exists [Condition: Code '244' from sct] or + exists [Condition: Code '245' from sct] or + exists [Condition: Code '246' from sct] or + exists [Condition: Code '247' from sct] or + exists [Condition: Code '248' from sct] or + exists [Condition: Code '249' from sct] or + exists [Condition: Code '250' from sct] or + exists [Condition: Code '251' from sct] or + exists [Condition: Code '252' from sct] or + exists [Condition: Code '253' from sct] or + exists [Condition: Code '254' from sct] or + exists [Condition: Code '255' from sct] or + exists [Condition: Code '256' from sct] or + exists [Condition: Code '257' from sct] or + exists [Condition: Code '258' from sct] or + exists [Condition: Code '259' from sct] or + exists [Condition: Code '260' from sct] or + exists [Condition: Code '261' from sct] or + exists [Condition: Code '262' from sct] or + exists [Condition: Code '263' from sct] or + exists [Condition: Code '264' from sct] or + exists [Condition: Code '265' from sct] or + exists [Condition: Code '266' from sct] or + exists [Condition: Code '267' from sct] or + exists [Condition: Code '268' from sct] or + exists [Condition: Code '269' from sct] or + exists [Condition: Code '270' from sct] or + exists [Condition: Code '271' from sct] or + exists [Condition: Code '272' from sct] or + exists [Condition: Code '273' from sct] or + exists [Condition: Code '274' from sct] or + exists [Condition: Code '275' from sct] or + exists [Condition: Code '276' from sct] or + exists [Condition: Code '277' from sct] or + exists [Condition: Code '278' from sct] or + exists [Condition: Code '279' from sct] or + exists [Condition: Code '280' from sct] or + exists [Condition: Code '281' from sct] or + exists [Condition: Code '282' from sct] or + exists [Condition: Code '283' from sct] or + exists [Condition: Code '284' from sct] or + exists [Condition: Code '285' from sct] or + exists [Condition: Code '286' from sct] or + exists [Condition: Code '287' from sct] or + exists [Condition: Code '288' from sct] or + exists [Condition: Code '289' from sct] or + exists [Condition: Code '290' from sct] or + exists [Condition: Code '291' from sct] or + exists [Condition: Code '292' from sct] or + exists [Condition: Code '293' from sct] or + exists [Condition: Code '294' from sct] or + exists [Condition: Code '295' from sct] or + exists [Condition: Code '296' from sct] or + exists [Condition: Code '297' from sct] or + exists [Condition: Code '298' from sct] or + exists [Condition: Code '299' from sct] or + exists [Condition: Code '300' from sct] or + exists [Condition: Code '301' from sct] or + exists [Condition: Code '302' from sct] or + exists [Condition: Code '303' from sct] or + exists [Condition: Code '304' from sct] or + exists [Condition: Code '305' from sct] or + exists [Condition: Code '306' from sct] or + exists [Condition: Code '307' from sct] or + exists [Condition: Code '308' from sct] or + exists [Condition: Code '309' from sct] or + exists [Condition: Code '310' from sct] or + exists [Condition: Code '311' from sct] or + exists [Condition: Code '312' from sct] or + exists [Condition: Code '313' from sct] or + exists [Condition: Code '314' from sct] or + exists [Condition: Code '315' from sct] or + exists [Condition: Code '316' from sct] or + exists [Condition: Code '317' from sct] or + exists [Condition: Code '318' from sct] or + exists [Condition: Code '319' from sct] or + exists [Condition: Code '320' from sct] or + exists [Condition: Code '321' from sct] or + exists [Condition: Code '322' from sct] or + exists [Condition: Code '323' from sct] or + exists [Condition: Code '324' from sct] or + exists [Condition: Code '325' from sct] or + exists [Condition: Code '326' from sct] or + exists [Condition: Code '327' from sct] or + exists [Condition: Code '328' from sct] or + exists [Condition: Code '329' from sct] or + exists [Condition: Code '330' from sct] or + exists [Condition: Code '331' from sct] or + exists [Condition: Code '332' from sct] or + exists [Condition: Code '333' from sct] or + exists [Condition: Code '334' from sct] or + exists [Condition: Code '335' from sct] or + exists [Condition: Code '336' from sct] or + exists [Condition: Code '337' from sct] or + exists [Condition: Code '338' from sct] or + exists [Condition: Code '339' from sct] or + exists [Condition: Code '340' from sct] or + exists [Condition: Code '341' from sct] or + exists [Condition: Code '342' from sct] or + exists [Condition: Code '343' from sct] or + exists [Condition: Code '344' from sct] or + exists [Condition: Code '345' from sct] or + exists [Condition: Code '346' from sct] or + exists [Condition: Code '347' from sct] or + exists [Condition: Code '348' from sct] or + exists [Condition: Code '349' from sct] or + exists [Condition: Code '350' from sct] or + exists [Condition: Code '351' from sct] or + exists [Condition: Code '352' from sct] or + exists [Condition: Code '353' from sct] or + exists [Condition: Code '354' from sct] or + exists [Condition: Code '355' from sct] or + exists [Condition: Code '356' from sct] or + exists [Condition: Code '357' from sct] or + exists [Condition: Code '358' from sct] or + exists [Condition: Code '359' from sct] or + exists [Condition: Code '360' from sct] or + exists [Condition: Code '361' from sct] or + exists [Condition: Code '362' from sct] or + exists [Condition: Code '363' from sct] or + exists [Condition: Code '364' from sct] or + exists [Condition: Code '365' from sct] or + exists [Condition: Code '366' from sct] or + exists [Condition: Code '367' from sct] or + exists [Condition: Code '368' from sct] or + exists [Condition: Code '369' from sct] or + exists [Condition: Code '370' from sct] or + exists [Condition: Code '371' from sct] or + exists [Condition: Code '372' from sct] or + exists [Condition: Code '373' from sct] or + exists [Condition: Code '374' from sct] or + exists [Condition: Code '375' from sct] or + exists [Condition: Code '376' from sct] or + exists [Condition: Code '377' from sct] or + exists [Condition: Code '378' from sct] or + exists [Condition: Code '379' from sct] or + exists [Condition: Code '380' from sct] or + exists [Condition: Code '381' from sct] or + exists [Condition: Code '382' from sct] or + exists [Condition: Code '383' from sct] or + exists [Condition: Code '384' from sct] or + exists [Condition: Code '385' from sct] or + exists [Condition: Code '386' from sct] or + exists [Condition: Code '387' from sct] or + exists [Condition: Code '388' from sct] or + exists [Condition: Code '389' from sct] or + exists [Condition: Code '390' from sct] or + exists [Condition: Code '391' from sct] or + exists [Condition: Code '392' from sct] or + exists [Condition: Code '393' from sct] or + exists [Condition: Code '394' from sct] or + exists [Condition: Code '395' from sct] or + exists [Condition: Code '396' from sct] or + exists [Condition: Code '397' from sct] or + exists [Condition: Code '398' from sct] or + exists [Condition: Code '399' from sct] or + exists [Condition: Code '400' from sct] or + exists [Condition: Code '401' from sct] or + exists [Condition: Code '402' from sct] or + exists [Condition: Code '403' from sct] or + exists [Condition: Code '404' from sct] or + exists [Condition: Code '405' from sct] or + exists [Condition: Code '406' from sct] or + exists [Condition: Code '407' from sct] or + exists [Condition: Code '408' from sct] or + exists [Condition: Code '409' from sct] or + exists [Condition: Code '410' from sct] or + exists [Condition: Code '411' from sct] or + exists [Condition: Code '412' from sct] or + exists [Condition: Code '413' from sct] or + exists [Condition: Code '414' from sct] or + exists [Condition: Code '415' from sct] or + exists [Condition: Code '416' from sct] or + exists [Condition: Code '417' from sct] or + exists [Condition: Code '418' from sct] or + exists [Condition: Code '419' from sct] or + exists [Condition: Code '420' from sct] or + exists [Condition: Code '421' from sct] or + exists [Condition: Code '422' from sct] or + exists [Condition: Code '423' from sct] or + exists [Condition: Code '424' from sct] or + exists [Condition: Code '425' from sct] or + exists [Condition: Code '426' from sct] or + exists [Condition: Code '427' from sct] or + exists [Condition: Code '428' from sct] or + exists [Condition: Code '429' from sct] or + exists [Condition: Code '430' from sct] or + exists [Condition: Code '431' from sct] or + exists [Condition: Code '432' from sct] or + exists [Condition: Code '433' from sct] or + exists [Condition: Code '434' from sct] or + exists [Condition: Code '435' from sct] or + exists [Condition: Code '436' from sct] or + exists [Condition: Code '437' from sct] or + exists [Condition: Code '438' from sct] or + exists [Condition: Code '439' from sct] or + exists [Condition: Code '440' from sct] or + exists [Condition: Code '441' from sct] or + exists [Condition: Code '442' from sct] or + exists [Condition: Code '443' from sct] or + exists [Condition: Code '444' from sct] or + exists [Condition: Code '445' from sct] or + exists [Condition: Code '446' from sct] or + exists [Condition: Code '447' from sct] or + exists [Condition: Code '448' from sct] or + exists [Condition: Code '449' from sct] or + exists [Condition: Code '450' from sct] or + exists [Condition: Code '451' from sct] or + exists [Condition: Code '452' from sct] or + exists [Condition: Code '453' from sct] or + exists [Condition: Code '454' from sct] or + exists [Condition: Code '455' from sct] or + exists [Condition: Code '456' from sct] or + exists [Condition: Code '457' from sct] or + exists [Condition: Code '458' from sct] or + exists [Condition: Code '459' from sct] or + exists [Condition: Code '460' from sct] or + exists [Condition: Code '461' from sct] or + exists [Condition: Code '462' from sct] or + exists [Condition: Code '463' from sct] or + exists [Condition: Code '464' from sct] or + exists [Condition: Code '465' from sct] or + exists [Condition: Code '466' from sct] or + exists [Condition: Code '467' from sct] or + exists [Condition: Code '468' from sct] or + exists [Condition: Code '469' from sct] or + exists [Condition: Code '470' from sct] or + exists [Condition: Code '471' from sct] or + exists [Condition: Code '472' from sct] or + exists [Condition: Code '473' from sct] or + exists [Condition: Code '474' from sct] or + exists [Condition: Code '475' from sct] or + exists [Condition: Code '476' from sct] or + exists [Condition: Code '477' from sct] or + exists [Condition: Code '478' from sct] or + exists [Condition: Code '479' from sct] or + exists [Condition: Code '480' from sct] or + exists [Condition: Code '481' from sct] or + exists [Condition: Code '482' from sct] or + exists [Condition: Code '483' from sct] or + exists [Condition: Code '484' from sct] or + exists [Condition: Code '485' from sct] or + exists [Condition: Code '486' from sct] or + exists [Condition: Code '487' from sct] or + exists [Condition: Code '488' from sct] or + exists [Condition: Code '489' from sct] or + exists [Condition: Code '490' from sct] or + exists [Condition: Code '491' from sct] or + exists [Condition: Code '492' from sct] or + exists [Condition: Code '493' from sct] or + exists [Condition: Code '494' from sct] or + exists [Condition: Code '495' from sct] or + exists [Condition: Code '496' from sct] or + exists [Condition: Code '497' from sct] or + exists [Condition: Code '498' from sct] or + exists [Condition: Code '499' from sct] diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.json b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.json new file mode 100644 index 000000000..4ed75a355 --- /dev/null +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/q58-overly-large-nonparsable-query.json @@ -0,0 +1,57 @@ +{ + "resourceType": "Bundle", + "type": "transaction", + "entry": [ + { + "resource": { + "resourceType": "Measure", + "id": "0", + "url": "0", + "status": "active", + "subjectCodeableConcept": { + "coding": [ + { + "system": "http://hl7.org/fhir/resource-types", + "code": "Patient" + } + ] + }, + "library": [ + "0" + ], + "scoring": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-scoring", + "code": "cohort" + } + ] + }, + "group": [ + { + "population": [ + { + "code": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-population", + "code": "initial-population" + } + ] + }, + "criteria": { + "language": "text/cql-identifier", + "expression": "InInitialPopulation" + } + } + ] + } + ] + }, + "request": { + "method": "PUT", + "url": "Measure/0" + } + } + ] +} diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/test_util.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/test_util.clj index cbabea32e..f3d5d76b0 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/test_util.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/test_util.clj @@ -2,8 +2,8 @@ (:require [blaze.async.comp :as ac] [blaze.db.api :as d] - [blaze.elm.compiler.external-data :as ed] - [blaze.elm.compiler.external-data-spec] + [blaze.elm.resource :as cr] + [blaze.elm.resource-spec] [blaze.handler.util :as handler-util])) (defn wrap-error [handler] @@ -15,7 +15,7 @@ {:population-handle subject-handle :subject-handle subject-handle}) (defn handle-mapper [db] - (comp (ed/resource-mapper db) (map handle))) + (comp (cr/resource-mapper db) (map handle))) (defn resource [db type id] - (ed/mk-resource db (d/resource-handle db type id))) + (cr/mk-resource db (d/resource-handle db type id))) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj index f013bb48e..4299c42f0 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj @@ -5,6 +5,7 @@ [blaze.db.api-stub :as api-stub :refer [with-system-data]] [blaze.db.node :refer [node?]] [blaze.db.resource-store :as rs] + [blaze.elm.expression :as-alias expr] [blaze.executors :as ex] [blaze.fhir.operation.evaluate-measure :as evaluate-measure] [blaze.fhir.operation.evaluate-measure.test-util :refer [wrap-error]] @@ -140,6 +141,7 @@ api-stub/mem-node-config ::evaluate-measure/handler {:node (ig/ref :blaze.db/node) + ::expr/cache (ig/ref ::expr/cache) :executor (ig/ref :blaze.test/executor) :clock (ig/ref :blaze.test/fixed-clock) :rng-fn (ig/ref :blaze.test/fixed-rng-fn)} @@ -147,6 +149,9 @@ {:node (ig/ref :blaze.db/node) :clock (ig/ref :blaze.test/fixed-clock) :rng-fn (ig/ref :blaze.test/fixed-rng-fn)} + ::expr/cache + {:node (ig/ref :blaze.db/node) + :executor (ig/ref :blaze.test/executor)} :blaze.test/executor {} :blaze.test/fixed-rng-fn {})) diff --git a/modules/rest-util/deps.edn b/modules/rest-util/deps.edn index 3dc860155..2e5dded83 100644 --- a/modules/rest-util/deps.edn +++ b/modules/rest-util/deps.edn @@ -23,7 +23,7 @@ :exclusions [com.cognitect/transit-clj]} ring/ring-core - {:mvn/version "1.12.1" + {:mvn/version "1.12.2" :exclusions [org.apache.commons/commons-fileupload2-core crypto-equality/crypto-equality diff --git a/modules/rest-util/src/blaze/handler/fhir/util.clj b/modules/rest-util/src/blaze/handler/fhir/util.clj index 0d8024fc5..e6c7312f4 100644 --- a/modules/rest-util/src/blaze/handler/fhir/util.clj +++ b/modules/rest-util/src/blaze/handler/fhir/util.clj @@ -201,7 +201,7 @@ if-match :ifMatch if-none-exist :ifNoneExist} :request :keys [resource]}] - (let [url (-> url type/value u/strip-leading-slash) + (let [url (-> url type/value u/strip-leading-slashes) [url query-string] (str/split url #"\?") method (keyword (str/lower-case (type/value method))) return-preference (or return-preference @@ -415,7 +415,7 @@ {:arglists '([idx entry])} [idx {:keys [request resource] :as entry}] (let [method (some-> request :method type/value) - [url] (some-> request :url type/value u/strip-leading-slash (str/split #"\?")) + [url] (some-> request :url type/value u/strip-leading-slashes (str/split #"\?")) {:keys [type id kind]} (some-> url match-url)] (cond (nil? request) diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb.clj index 26e0e05d1..78129ed90 100644 --- a/modules/rocksdb/src/blaze/db/kv/rocksdb.clj +++ b/modules/rocksdb/src/blaze/db/kv/rocksdb.clj @@ -206,6 +206,9 @@ (impl/write-wb! cfhs wb entries) (.write db write-opts wb))) + (-estimate-num-keys [store column-family] + (p/-long-property store column-family "rocksdb.estimate-num-keys")) + p/Rocks (-path [_] path) diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj index 9ed3d0757..5479801e7 100644 --- a/modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj +++ b/modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj @@ -203,13 +203,13 @@ (def ^:private bloom-filter-full-positive-total (counter-metric "blaze_rocksdb_bloom_filter_full_positive_total" - "Number of times bloom FullFilter has not avoided the reads." + "Number of times Bloom FullFilter has not avoided the reads." TickerType/BLOOM_FILTER_FULL_POSITIVE)) (def ^:private bloom-filter-full-true-positive-total (counter-metric "blaze_rocksdb_bloom_filter_full_true_positive_total" - "Number of times bloom FullFilter has not avoided the reads and data actually exist." + "Number of times Bloom FullFilter has not avoided the reads and data actually exist." TickerType/BLOOM_FILTER_FULL_TRUE_POSITIVE)) (def ^:private blocks-compressed-total diff --git a/modules/rocksdb/test/blaze/db/kv/rocksdb_test.clj b/modules/rocksdb/test/blaze/db/kv/rocksdb_test.clj index 9ceb6a4ec..19079339a 100644 --- a/modules/rocksdb/test/blaze/db/kv/rocksdb_test.clj +++ b/modules/rocksdb/test/blaze/db/kv/rocksdb_test.clj @@ -27,7 +27,7 @@ (set! *warn-on-reflection* true) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -624,6 +624,19 @@ (kv/write! db [[:delete :default (ba 0x00)]]) (is (nil? (kv/get db :default (ba 0x00))))))) +(deftest estimate-num-keys-test + (with-system [{db ::kv/rocksdb} (config (new-temp-dir!))] + (is (zero? (kv/estimate-num-keys db :default))) + + (given (kv/estimate-num-keys db :foo) + ::anom/category := ::anom/not-found + ::anom/message := "Column family `foo` not found.")) + + (with-system-data [{db ::kv/rocksdb} (config (new-temp-dir!))] + [[:default (ba 0x00) (ba 0x10)]] + + (is (= 1 (kv/estimate-num-keys db :default))))) + (deftest path-test (with-system [{db ::kv/rocksdb} (config (new-temp-dir!))] (is (string? (rocksdb/path db))))) diff --git a/modules/scheduler/src/blaze/scheduler.clj b/modules/scheduler/src/blaze/scheduler.clj index 00cc904f8..78beac8a9 100644 --- a/modules/scheduler/src/blaze/scheduler.clj +++ b/modules/scheduler/src/blaze/scheduler.clj @@ -11,6 +11,13 @@ (set! *warn-on-reflection* true) +(defn submit + "Submits the function `f` to be called later. + + Returns a future that can be used in `cancel`." + [scheduler f] + (p/-submit scheduler f)) + (defn schedule-at-fixed-rate "Schedules the function `f` to be called at a rate of `period` with an `initial-delay`. @@ -24,6 +31,9 @@ (extend-protocol p/Scheduler ScheduledExecutorService + (-submit [scheduler f] + (.submit scheduler ^Runnable f)) + (-schedule-at-fixed-rate [scheduler f initial-delay period] (.scheduleAtFixedRate scheduler diff --git a/modules/scheduler/src/blaze/scheduler_spec.clj b/modules/scheduler/src/blaze/scheduler_spec.clj index 9d87fdd47..e0d7eb908 100644 --- a/modules/scheduler/src/blaze/scheduler_spec.clj +++ b/modules/scheduler/src/blaze/scheduler_spec.clj @@ -5,6 +5,10 @@ [clojure.spec.alpha :as s] [java-time.api :as time])) +(s/fdef sched/submit + :args (s/cat :scheduler :blaze/scheduler :f fn?) + :ret :blaze.scheduler/future) + (s/fdef sched/schedule-at-fixed-rate :args (s/cat :scheduler :blaze/scheduler :f fn? :initial-delay time/duration? :period time/duration?) diff --git a/modules/server/deps.edn b/modules/server/deps.edn index fb5d28bef..4cdfd48aa 100644 --- a/modules/server/deps.edn +++ b/modules/server/deps.edn @@ -6,7 +6,7 @@ {:local/root "../module-base"} ring/ring-core - {:mvn/version "1.12.1" + {:mvn/version "1.12.2" :exclusions [org.apache.commons/commons-fileupload2-core crypto-equality/crypto-equality diff --git a/modules/thread-pool-executor-collector/test/blaze/thread_pool_executor_collector_test.clj b/modules/thread-pool-executor-collector/test/blaze/thread_pool_executor_collector_test.clj index b3de602c8..d9ef51831 100644 --- a/modules/thread-pool-executor-collector/test/blaze/thread_pool_executor_collector_test.clj +++ b/modules/thread-pool-executor-collector/test/blaze/thread_pool_executor_collector_test.clj @@ -17,7 +17,7 @@ (set! *warn-on-reflection* true) (st/instrument) -(log/set-level! :trace) +(log/set-min-level! :trace) (test/use-fixtures :each tu/fixture) @@ -86,7 +86,7 @@ [6 :samples 0 :value] := 0.0)) (testing "one active thread" - (ex/execute! pool #(Thread/sleep 1000)) + (ex/execute! pool #(Thread/sleep 2000)) (given (metrics/collect collector) [0 :name] := "thread_pool_executor_active_count" [0 :samples 0 :value] := 1.0 diff --git a/profiling/blaze/profiling.clj b/profiling/blaze/profiling.clj index bb48315e4..49682c842 100644 --- a/profiling/blaze/profiling.clj +++ b/profiling/blaze/profiling.clj @@ -2,9 +2,13 @@ "Profiling namespace without test dependencies." (:require [blaze.system :as system] - [blaze.db.cache-collector :as cc] + [blaze.cache-collector :as cc] + [blaze.cache-collector.protocols :as ccp] [blaze.db.kv.rocksdb :as rocksdb] [blaze.db.resource-cache :as resource-cache] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.cache :as ec] + [blaze.system :as system] [clojure.tools.namespace.repl :refer [refresh]] [taoensso.timbre :as log])) @@ -43,16 +47,22 @@ ;; Transaction Cache (comment - (str (cc/-stats (:blaze.db/tx-cache system))) + (str (ccp/-stats (:blaze.db/tx-cache system))) (resource-cache/invalidate-all! (:blaze.db/tx-cache system)) ) ;; Resource Cache (comment - (str (cc/-stats (:blaze.db/resource-cache system))) + (str (ccp/-stats (:blaze.db/resource-cache system))) (resource-cache/invalidate-all! (:blaze.db/resource-cache system)) ) +;; CQL Expression Cache +(comment + (into [] (ec/list-by-t (::expr/cache system))) + (str (ccp/-stats (::expr/cache system))) + ) + ;; DB (comment (str (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats])) @@ -69,6 +79,7 @@ (rocksdb/property index-db :resource-as-of-index "rocksdb.stats") (rocksdb/property index-db :type-as-of-index "rocksdb.stats") (rocksdb/property index-db :system-as-of-index "rocksdb.stats") + (rocksdb/property index-db :patient-last-change-index "rocksdb.stats") (rocksdb/property index-db :type-stats-index "rocksdb.stats") (rocksdb/property index-db :system-stats-index "rocksdb.stats") diff --git a/resources/blaze.edn b/resources/blaze.edn index 2573aa5cf..bd56d7f34 100644 --- a/resources/blaze.edn +++ b/resources/blaze.edn @@ -188,6 +188,11 @@ :blaze.rest-api/async-status-cancel-handler {:job-scheduler #blaze/ref :blaze/job-scheduler} + ;; + ;; CQL Evaluation Engine + ;; + :blaze.cql/retrieve-total {} + ;; ;; FHIR Operation Evaluate Measure ;; @@ -276,6 +281,7 @@ :kv-store #blaze/ref :blaze.db.main/index-kv-store :resource-indexer #blaze/ref :blaze.db.node.main/resource-indexer :search-param-registry #blaze/ref :blaze.db/search-param-registry + :scheduler #blaze/ref :blaze/scheduler :enforce-referential-integrity #blaze/cfg ["ENFORCE_REFERENTIAL_INTEGRITY" boolean? true]} [:blaze.db.node/indexer-executor :blaze.db.node.main/indexer-executor] {} @@ -291,7 +297,8 @@ :resource-store #blaze/ref :blaze.db/resource-cache :kv-store #blaze/ref :blaze.db.admin/index-kv-store :resource-indexer #blaze/ref :blaze.db.node.admin/resource-indexer - :search-param-registry #blaze/ref :blaze.db/search-param-registry} + :search-param-registry #blaze/ref :blaze.db/search-param-registry + :scheduler #blaze/ref :blaze/scheduler} [:blaze.db.node/indexer-executor :blaze.db.node.admin/indexer-executor] {} @@ -327,7 +334,7 @@ :blaze.db.node.tx-indexer/duration-seconds {} - :blaze.db/cache-collector + :blaze/cache-collector {:caches {"tx-cache" #blaze/ref :blaze.db.main/tx-cache "resource-cache" #blaze/ref :blaze.db/resource-cache}} @@ -403,7 +410,10 @@ :blaze.job/re-index {:main-node #blaze/ref :blaze.db.main/node :admin-node #blaze/ref :blaze.db.admin/node - :clock #blaze/ref :blaze/clock}} + :clock #blaze/ref :blaze/clock + :extra-bundle-file #blaze/cfg ["DB_SEARCH_PARAM_BUNDLE" string?]} + + :blaze/scheduler {}} :storage {:in-memory @@ -427,8 +437,11 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-last-change-index nil :type-stats-index nil - :system-stats-index nil}} + :system-stats-index nil + :cql-bloom-filter nil + :cql-bloom-filter-by-t nil}} ;; ;; Admin In-Memory, Volatile Index Store @@ -449,8 +462,11 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-last-change-index nil :type-stats-index nil - :system-stats-index nil}} + :system-stats-index nil + :cql-bloom-filter nil + :cql-bloom-filter-by-t nil}} ;; ;; Main Local Transaction Log for Single Node Deployments @@ -614,6 +630,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-last-change-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -624,7 +646,22 @@ {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 :target-file-size-base-in-mb 2 - :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + + :cql-bloom-filter + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :enable-blob-files? true + :min-blob-size 512} + + :cql-bloom-filter-by-t + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :reverse-comparator? true}}} [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store.main/stats] {} @@ -699,6 +736,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-last-change-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -709,7 +752,22 @@ {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 :target-file-size-base-in-mb 2 - :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + + :cql-bloom-filter + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :enable-blob-files? true + :min-blob-size 512} + + :cql-bloom-filter-by-t + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :reverse-comparator? true}}} ;; ;; Main Local Transaction Log for Single Node Deployments @@ -926,6 +984,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-last-change-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -936,7 +1000,22 @@ {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 :target-file-size-base-in-mb 2 - :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + + :cql-bloom-filter + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :enable-blob-files? true + :min-blob-size 512} + + :cql-bloom-filter-by-t + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :reverse-comparator? true}}} [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store.main/stats] {} @@ -1011,6 +1090,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-last-change-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -1021,7 +1106,22 @@ {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 :target-file-size-base-in-mb 2 - :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + + :cql-bloom-filter + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :enable-blob-files? true + :min-blob-size 512} + + :cql-bloom-filter-by-t + {:write-buffer-size-in-mb 2 + :max-bytes-for-level-base-in-mb 8 + :target-file-size-base-in-mb 2 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] + :reverse-comparator? true}}} :blaze.db.kv.rocksdb/block-cache {:size-in-mb #blaze/cfg ["DB_BLOCK_CACHE_SIZE" int? 128]} @@ -1155,6 +1255,33 @@ [:blaze/http-client :blaze.openid-auth/http-client] {:trust-store #blaze/cfg ["OPENID_CLIENT_TRUST_STORE" string?] - :trust-store-pass #blaze/cfg ["OPENID_CLIENT_TRUST_STORE_PASS" string?]} + :trust-store-pass #blaze/cfg ["OPENID_CLIENT_TRUST_STORE_PASS" string?]}}} - :blaze/scheduler {}}}]} + {:key :cql-expression-cache + :name "CQL Expression Cache" + :toggle "CQL_EXPR_CACHE_SIZE" + :config + {:blaze.fhir.operation.evaluate-measure/handler + {:blaze.elm.expression/cache #blaze/ref :blaze.elm.expression/cache} + + :blaze.elm.expression/cache + {:node #blaze/ref :blaze.db.main/node + :max-size-in-mb #blaze/cfg ["CQL_EXPR_CACHE_SIZE" nat-int?] + :refresh #blaze/cfg ["CQL_EXPR_CACHE_REFRESH" java-time.api/duration? "PT24H"] + :executor #blaze/ref :blaze.elm.expression.cache/executor} + + :blaze.elm.expression.cache/executor + {:num-threads #blaze/cfg ["CQL_EXPR_CACHE_THREADS" pos-int? 4]} + + :blaze.elm.expression.cache/bloom-filter-creation-duration-seconds {} + :blaze.elm.expression.cache/bloom-filter-useful-total {} + :blaze.elm.expression.cache/bloom-filter-not-useful-total {} + :blaze.elm.expression.cache/bloom-filter-false-positive-total {} + :blaze.elm.expression.cache/bloom-filter-bytes {} + + :blaze/cache-collector + {:caches + {"cql-expr-cache" #blaze/ref :blaze.elm.expression/cache}} + + :blaze/admin-api + {:blaze.elm.expression/cache #blaze/ref :blaze.elm.expression/cache}}}]} diff --git a/src/blaze/system.clj b/src/blaze/system.clj index b232a3a23..d1da5f6b4 100644 --- a/src/blaze/system.clj +++ b/src/blaze/system.clj @@ -92,9 +92,9 @@ (log/info "Loaded the following namespaces:" (str/join ", " loaded-ns)))) (def ^:private root-config - {:blaze/version "0.27.1" + {:blaze/version "0.28.0" - :blaze/release-date "2024-06-14" + :blaze/release-date "2024-06-28" :blaze/clock {} @@ -163,9 +163,10 @@ (-> (assoc-in config [:base-config :blaze.db/storage] (keyword key)) (update :base-config (partial merge-with merge) (get storage (keyword key)))))) -(defn- conj-feature [config {:keys [name toggle]} enabled?] +(defn- conj-feature [config {:keys [key name toggle]} enabled?] (update-in config [:blaze/admin-api :features] (fnil conj []) - {:name name :toggle toggle :enabled enabled?})) + {:key (clojure.core/name key) :name name :toggle toggle + :enabled enabled?})) (defn- merge-features "Merges feature config portions of enabled features into `base-config`." @@ -185,7 +186,7 @@ (defn init! [{level "LOG_LEVEL" :or {level "info"} :as env}] (log/info "Set log level to:" (str/lower-case level)) - (log/set-level! (keyword (str/lower-case level))) + (log/set-min-level! (keyword (str/lower-case level))) (let [config (-> (read-blaze-edn) (merge-storage env) (merge-features env))