Merge pull request #31 from nasa/harmony-1247

HARMONY-1247: Write the error file to s3 if metadata dir is s3

harmony/aws.py

@@ -5,8 +5,11 @@
 This module relies on the harmony.util.config and its environment variables to be
 set for correct operation. See that module and the project README for details.
 """
+from urllib.parse import urlparse
+from os import environ
 import boto3
 from botocore.config import Config
+from harmony import util
 
 
 def is_s3(url: str) -> bool:
@@ -43,6 +46,25 @@ def aws_parameters(use_localstack, localstack_host, region):
         }
 
 
+def write_s3(url, txt):
+    """
+    Writes text to the given  s3 url.
+
+    Parameters
+    ----------
+    url: The s3 file url.
+    txt: The file contents.
+    """
+    parsed = urlparse(url)
+    config = util.config(validate=environ.get('ENV') != 'test')
+    service_params = aws_parameters(
+            config.use_localstack, config.localstack_host, config.aws_default_region)
+    bucket = parsed.netloc
+    key = parsed.path[1:]
+    s3 = boto3.resource("s3", **service_params)
+    s3.Object(bucket, key).put(Body=txt)
+
+
 def _get_aws_client(config, service, user_agent=None):
     """
     Returns a boto3 client for accessing the provided service.  Accesses the service in us-west-2

harmony/cli.py

@@ -19,6 +19,7 @@
 from harmony.util import (receive_messages, delete_message, change_message_visibility,
                           config, create_decrypter)
 from harmony.version import get_version
+from harmony.aws import is_s3, write_s3
 
 
 def setup_cli(parser):
@@ -143,8 +144,13 @@ def _write_error(metadata_dir, message, category='Unknown'):
     category : string
         The error category to write
     """
-    with open(path.join(metadata_dir, 'error.json'), 'w') as file:
-        json.dump({'error': message, 'category': category}, file)
+    error_data = {'error': message, 'category': category}
+    if is_s3(metadata_dir):
+        json_str = json.dumps(error_data)
+        write_s3(f'{metadata_dir}error.json', json_str)
+    else:
+        with open(path.join(metadata_dir, 'error.json'), 'w') as file:
+            json.dump(error_data, file)
 
 
 def _build_adapter(AdapterClass, message_string, sources_path, data_location, config):
@@ -203,12 +209,15 @@ def _invoke(adapter, metadata_dir):
     """
     try:
         logging.info(f'Invoking adapter with harmony-service-lib-py version {get_version()}')
-        makedirs(metadata_dir, exist_ok=True)
+        is_s3_metadata_dir = is_s3(metadata_dir)
+        if not is_s3_metadata_dir:
+            makedirs(metadata_dir, exist_ok=True)
         (out_message, out_catalog) = adapter.invoke()
         out_catalog.normalize_and_save(metadata_dir, CatalogType.SELF_CONTAINED)
 
-        with open(path.join(metadata_dir, 'message.json'), 'w') as file:
-            json.dump(out_message.output_data, file)
+        if not is_s3_metadata_dir:
+            with open(path.join(metadata_dir, 'message.json'), 'w') as file:
+                json.dump(out_message.output_data, file)
     except HarmonyException as err:
         logging.error(err, exc_info=1)
         _write_error(metadata_dir, err.message, err.category)

harmony/s3_stac_io.py

@@ -47,14 +47,8 @@ def write(uri, txt):
     uri: The STAC file uri.
     txt: The STAC contents.
     """
-    config = util.config(validate=environ.get('ENV') != 'test')
-    service_params = aws.aws_parameters(
-        config.use_localstack, config.localstack_host, config.aws_default_region)
     parsed = urlparse(uri)
     if parsed.scheme == 's3':
-        bucket = parsed.netloc
-        key = parsed.path[1:]
-        s3 = boto3.resource("s3", **service_params)
-        s3.Object(bucket, key).put(Body=txt)
+        aws.write_s3(uri, txt)
     else:
         STAC_IO.default_write_text_method(uri, txt)