From 407f140f10450b817793275f7b10e9a6023d3900 Mon Sep 17 00:00:00 2001
From: James McKinney <26463+jpmckinney@users.noreply.github.com>
Date: Wed, 1 May 2024 01:47:44 -0400
Subject: [PATCH] kingfisher-collect(incremental): Add transaction to replace
 _result table

---
 salt/kingfisher/collect/files/bi/cron.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/salt/kingfisher/collect/files/bi/cron.sh b/salt/kingfisher/collect/files/bi/cron.sh
index fb669bd9..afe89c0f 100644
--- a/salt/kingfisher/collect/files/bi/cron.sh
+++ b/salt/kingfisher/collect/files/bi/cron.sh
@@ -13,7 +13,7 @@ env SENTRY_DSN={{ SENTRY_DSN }} .ve/bin/scrapy crawl \
     --logfile="{{ userdir }}/logs/{{ crawl.spider }}-$(date +%F).log"
 
 # shellcheck disable=all
-{%- if crawl.powerbi %}
+{%- if 'powerbi' in crawl and crawl.powerbi %}
 psql -U kingfisher_collect -h localhost -t -c 'SELECT data FROM {{ crawl.spider }}' -o {{ scratchdir }}/{{ crawl.spider }}.jsonl
 
 ocdscardinal prepare \
@@ -32,6 +32,9 @@ ocdscardinal indicators \
 {{ userdir }}/bin/manage.py json-to-csv {{ scratchdir }}/{{ crawl.spider }}.json {{ scratchdir }}/{{ crawl.spider }}.csv
 
 psql postgresql://kingfisher_collect@localhost:5432/kingfisher_collect \
+    -c "BEGIN" \
+    -c "DELETE FROM {{ crawl.spider }}_result" \
     -c "\copy {{ crawl.spider }}_result (ocid, subject, code, result, buyer_id, procuring_entity_id, tenderer_id, created_at) FROM stdin DELIMITER ',' CSV HEADER" \
+    -c "END" \
     < {{scratchdir}}/{{crawl.spider}}.csv
 {%- endif %}