chore: base updates

zimmerman-team · Oct 30, 2023 · b6b6b16 · b6b6b16
1 parent 2371316
commit b6b6b16
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 2 deletions.
diff --git a/direct_indexing/data_sources/currency_monthlyaverage.json b/direct_indexing/data_sources/currency_monthlyaverage.json
diff --git a/direct_indexing/metadata/dataset.py b/direct_indexing/metadata/dataset.py
@@ -91,7 +91,6 @@ def prepare_update(dataset_metadata):
     old_datasets = [d for d in dataset_metadata if d['id'] in existing_datasets]
     changed_datasets = [
         d for d in old_datasets if d['resources'][0]['hash'] != existing_datasets[d['id']]['hash']
-        and existing_datasets[d['id']]['filetype'] == 'activity'
     ]  # Skip organisation files for incremental updates
     updated_datasets = new_datasets + changed_datasets
     updated_datasets_bools = [False for _ in new_datasets] + [True for _ in changed_datasets]

diff --git a/direct_indexing/processing/dataset.py b/direct_indexing/processing/dataset.py
@@ -3,6 +3,7 @@
 import os
 import xml.etree.ElementTree as ET
 
+from datetime import datetime
 from django.conf import settings
 from pysolr import Solr
 from xmljson import badgerfish as bf
@@ -64,6 +65,7 @@ def fun(dataset, update=False):
                                                          dataset_metadata)
     # Add an indexing status to the dataset metadata.
     dataset['iati_cloud_indexed'] = indexed
+    dataset['iati_cloud_indexed_datetime'] = str(datetime.now())
 
     # Index the dataset metadata
     logging.info('-- Save the dataset metadata')

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,6 +6,8 @@ services:
     image: postgres:latest
     ports:
       - 5432:5432
+    env_file:
+      - .env
     volumes:
       - db_data:/var/lib/postgresql/data
     environment:
@@ -26,6 +28,8 @@ services:
     ports:
       - 5672:5672
       - 15672:15672
+    env_file:
+      - .env
     volumes:
       - rabbitmq_data:/var/lib/rabbitmq
     healthcheck:
@@ -39,6 +43,8 @@ services:
     container_name: mongo
     restart: always
     image: mongo:latest
+    env_file:
+      - .env
     environment:
       - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME}
       - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD}
@@ -54,6 +60,8 @@ services:
     image: bitnami/solr:9.1.1
     ports:
       - 8983:8983
+    env_file:
+      - .env
     environment:
       # core setup
       - SOLR_CORES=activity,budget,dataset,organisation,publisher,result,transaction
@@ -72,6 +80,8 @@ services:
     restart: always
     build: .
     image: iaticloud/main
+    env_file:
+      - .env
     command: python manage.py runserver 0.0.0.0:8000
     volumes:
       # Mount the local project directory to the container's /app directory
@@ -90,6 +100,8 @@ services:
     image: iaticloud/main
     volumes:
       - ./:/app
+    env_file:
+      - .env
     command: celery -A iaticloud worker -l INFO
     depends_on:
       - rabbitmq
@@ -101,6 +113,8 @@ services:
     image: iaticloud/main
     volumes:
       - ./:/app
+    env_file:
+      - .env
     command: celery -A iaticloud worker -l INFO -n revoke@%%h -Q revoke_queue
     depends_on:
       - rabbitmq
@@ -112,6 +126,8 @@ services:
     image: iaticloud/main
     volumes:
       - ./:/app
+    env_file:
+      - .env
     command: celery -A iaticloud beat -l INFO
     depends_on:
       - celeryworker
@@ -124,6 +140,8 @@ services:
     image: iaticloud/main
     volumes:
       - ./:/app
+    env_file:
+      - .env
     command: celery -A iaticloud flower -l INFO --port=5555 --basic_auth=${CELERYFLOWER_USER}:${CELERYFLOWER_PASSWORD}
     depends_on:
       - celeryworker
@@ -138,6 +156,8 @@ services:
   nginx:
     container_name: nginx
     image: iaticloud/nginx
+    env_file:
+      - .env
     build: 
       context: ./services/nginx
       args: