From ffc787f9b4161b6916c86d0ef32b9adb5c8b6dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karolis=20Vy=C4=8Dius?= Date: Thu, 18 Jul 2024 08:08:52 +0300 Subject: [PATCH 1/3] Add boundaries.sqlite and checksums to release --- .github/workflows/publish.yml | 39 +++++++++++++++++++++++++++++++---- Dockerfile | 2 +- create-database.sh | 16 +++++++------- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e883c8b..9a11a37 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,12 +11,12 @@ on: permissions: { } jobs: - publish-api-docker-image: - name: Publish API docker image + publish-docker-image: + name: Publish docker image runs-on: ubuntu-latest timeout-minutes: 20 permissions: - contents: read + contents: write packages: write id-token: write attestations: write @@ -42,4 +42,35 @@ jobs: with: subject-name: ghcr.io/govlt/national-boundaries-api subject-digest: ${{ steps.docker.outputs.digest }} - push-to-registry: true \ No newline at end of file + push-to-registry: true + + - name: Extract files from Docker image + uses: shrink/actions-docker-extract@v3 + id: extract + with: + image: ghcr.io/govlt/national-boundaries-api@${{ steps.docker.outputs.digest }} + path: /opt/app + destination: app + + - name: Upload boundaries.sqlite artifact + uses: actions/upload-artifact@v4 + with: + name: boundaries.sqlite + path: app/boundaries.sqlite + if-no-files-found: error + + - name: Upload data-source-checksums.txt artifact + uses: actions/upload-artifact@v4 + with: + name: data-source-checksums.txt + path: app/data-source-checksums.txt + if-no-files-found: error + + - name: Upload artifacts to GitHub release + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + app/boundaries.sqlite + app/data-source-checksums.txt + diff --git a/Dockerfile b/Dockerfile index d81e6db..14c0074 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY src/ src/ COPY --chmod=555 entrypoint.sh /opt/app/entrypoint.sh -COPY --from=database-builder --chmod=444 /opt/database/boundaries.sqlite /opt/database/data-sources/checksums.txt ./ +COPY --from=database-builder --chmod=444 /opt/database/boundaries.sqlite /opt/database/data-sources/data-source-checksums.txt ./ ENTRYPOINT ["/opt/app/entrypoint.sh"] diff --git a/create-database.sh b/create-database.sh index ccb45a0..a5fc42c 100755 --- a/create-database.sh +++ b/create-database.sh @@ -22,14 +22,14 @@ mkdir -p data-sources echo "Importing counties data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/counties.json https://www.registrucentras.lt/aduomenys/?byla=adr_gra_apskritys.json -calculate_md5 data-sources/counties.json >> data-sources/checksums.txt +calculate_md5 data-sources/counties.json >> data-sources/data-source-checksums.txt ogr2ogr -f SQLite boundaries.sqlite data-sources/counties.json -dsco SPATIALITE=YES -lco FID=feature_id -lco GEOMETRY_NAME=geom \ -sql "SELECT FID AS feature_id, CAST(APS_KODAS AS integer(8)) AS code, APS_PAV as name, APS_PLOTAS as area_ha, APS_R AS created_at FROM counties" ogrinfo -sql "CREATE UNIQUE INDEX counties_code ON counties(code)" boundaries.sqlite echo "Importing municipalities data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/municipalities.json https://www.registrucentras.lt/aduomenys/?byla=adr_gra_savivaldybes.json -calculate_md5 data-sources/municipalities.json >> data-sources/checksums.txt +calculate_md5 data-sources/municipalities.json >> data-sources/data-source-checksums.txt ogr2ogr -append -f SQLite boundaries.sqlite data-sources/municipalities.json -lco FID=feature_id -lco GEOMETRY_NAME=geom \ -sql "SELECT FID AS feature_id, CAST(SAV_KODAS AS integer(8)) AS code, SAV_PAV as name, SAV_PLOTAS as area_ha, CAST(APS_KODAS AS integer(8)) as county_code, SAV_R AS created_at FROM municipalities" ogrinfo -sql "CREATE UNIQUE INDEX municipalities_code ON municipalities(code)" boundaries.sqlite @@ -37,7 +37,7 @@ ogrinfo -sql "CREATE INDEX municipalities_county_code ON municipalities(county_c echo "Importing elderships data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/elderships.json https://www.registrucentras.lt/aduomenys/?byla=adr_gra_seniunijos.json -calculate_md5 data-sources/elderships.json >> data-sources/checksums.txt +calculate_md5 data-sources/elderships.json >> data-sources/data-source-checksums.txt ogr2ogr -append -f SQLite boundaries.sqlite data-sources/elderships.json -lco FID=feature_id -lco GEOMETRY_NAME=geom \ -sql "SELECT FID AS feature_id, CAST(SEN_KODAS AS integer(8)) AS code, SEN_PAV as name, SEN_PLOTAS as area_ha, CAST(SAV_KODAS AS integer(8)) AS municipality_code, SEN_R AS created_at FROM elderships" ogrinfo -sql "CREATE UNIQUE INDEX elderships_code ON elderships(code)" boundaries.sqlite @@ -45,7 +45,7 @@ ogrinfo -sql "CREATE INDEX elderships_municipality_code_and_name ON elderships(m echo "Importing residential areas data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/residential_areas.json https://www.registrucentras.lt/aduomenys/?byla=adr_gra_gyvenamosios_vietoves.json -calculate_md5 data-sources/residential_areas.json >> data-sources/checksums.txt +calculate_md5 data-sources/residential_areas.json >> data-sources/data-source-checksums.txt ogr2ogr -append -f SQLite boundaries.sqlite data-sources/residential_areas.json -lco FID=feature_id -lco GEOMETRY_NAME=geom \ -sql "SELECT FID AS feature_id, GYV_KODAS AS code, GYV_PAV as name, PLOTAS as area_ha, CAST(SAV_KODAS AS integer(8)) AS municipality_code, GYV_R as created_at FROM residential_areas" ogrinfo -sql "CREATE UNIQUE INDEX residential_areas_code ON residential_areas(code)" boundaries.sqlite @@ -53,7 +53,7 @@ ogrinfo -sql "CREATE INDEX residential_municipality_code_and_name ON residential echo "Importing streets data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/streets.json https://www.registrucentras.lt/aduomenys/?byla=adr_gra_gatves.json -calculate_md5 data-sources/streets.json >> data-sources/checksums.txt +calculate_md5 data-sources/streets.json >> data-sources/data-source-checksums.txt ogr2ogr -append -f SQLite boundaries.sqlite data-sources/streets.json -lco FID=feature_id -lco GEOMETRY_NAME=geom \ -sql "SELECT FID AS feature_id, GAT_KODAS AS code, GAT_PAV as name, GAT_PAV_PI AS full_name, GAT_ILGIS as length_m, GYV_KODAS AS residential_area_code, GTV_R AS created_at FROM streets" ogrinfo -sql "CREATE UNIQUE INDEX streets_code ON streets(code)" boundaries.sqlite @@ -62,7 +62,7 @@ ogrinfo -sql "CREATE INDEX streets_residential_area_code_and_name ON streets(res echo "Importing addresses data into SQLite" curl -f -L --max-redirs 5 --retry 3 -o data-sources/addresses-information.psv https://www.registrucentras.lt/aduomenys/?byla=adr_stat_lr.csv -calculate_md5 data-sources/addresses-information.psv >> data-sources/checksums.txt +calculate_md5 data-sources/addresses-information.psv >> data-sources/data-source-checksums.txt ogr2ogr -f GPKG data-sources/addresses.gpkg data-sources/addresses-information.psv -nln info @@ -74,7 +74,7 @@ echo "Importing address points for each municipality" curl -sf "https://www.registrucentras.lt/aduomenys/?byla=adr_savivaldybes.csv" | csvcut -d "|" -c "SAV_KODAS" | tail -n +2 | while read -r code; do echo "Converting https://www.registrucentras.lt/aduomenys/?byla=adr_gra_$code.json" curl -f -L --max-redirs 5 --retry 3 -o "data-sources/addresses-$code.json" "https://www.registrucentras.lt/aduomenys/?byla=adr_gra_$code.json" - calculate_md5 "data-sources/addresses-$code.json" >> data-sources/checksums.txt + calculate_md5 "data-sources/addresses-$code.json" >> data-sources/data-source-checksums.txt ogr2ogr -append -f GPKG data-sources/addresses.gpkg "data-sources/addresses-$code.json" -nln points done @@ -89,7 +89,7 @@ ogrinfo -sql "CREATE INDEX addresses_street_code ON addresses(street_code, plot_ echo "Importing rooms" curl -f -L --max-redirs 5 --retry 3 -o data-sources/rooms.psv https://www.registrucentras.lt/aduomenys/?byla=adr_pat_lr.csv -calculate_md5 data-sources/rooms.psv >> data-sources/checksums.txt +calculate_md5 data-sources/rooms.psv >> data-sources/data-source-checksums.txt ogr2ogr -append -f SQLite boundaries.sqlite data-sources/rooms.psv -lco FID=code \ -sql "SELECT CAST(PAT_KODAS AS integer(8)) as code, CAST(AOB_KODAS AS integer(8)) AS address_code, PATALPOS_NR AS room_number, PAT_NUO AS created_at FROM rooms" ogrinfo -sql "CREATE INDEX rooms_address_code ON rooms(address_code, room_number)" boundaries.sqlite From 5824fa645c02265b2328ded80149559f2f24e453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karolis=20Vy=C4=8Dius?= Date: Thu, 18 Jul 2024 08:22:05 +0300 Subject: [PATCH 2/3] Debug --- .github/workflows/publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a11a37..75cd48a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -52,6 +52,10 @@ jobs: path: /opt/app destination: app + - run: ls -lah + + - run: ls -lah app + - name: Upload boundaries.sqlite artifact uses: actions/upload-artifact@v4 with: From c8554b694625c6b3909eaabcd205eb1e0e4e66f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karolis=20Vy=C4=8Dius?= Date: Thu, 18 Jul 2024 08:59:10 +0300 Subject: [PATCH 3/3] Fix extract path --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 75cd48a..9043728 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: id: extract with: image: ghcr.io/govlt/national-boundaries-api@${{ steps.docker.outputs.digest }} - path: /opt/app + path: /opt/app/. destination: app - run: ls -lah