Skip to content
This repository has been archived by the owner on Sep 23, 2024. It is now read-only.

Commit

Permalink
Add log message indicating which stream failed to load (#59)
Browse files Browse the repository at this point in the history
  • Loading branch information
Limess authored Jun 9, 2020
1 parent a1416e7 commit a7fed8b
Showing 1 changed file with 12 additions and 9 deletions.
21 changes: 12 additions & 9 deletions target_redshift/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,17 +345,20 @@ def flush_streams(


def load_stream_batch(stream, records_to_load, row_count, db_sync, delete_rows=False, compression=None, slices=None, temp_dir=None):
# Load into redshift
if row_count[stream] > 0:
flush_records(stream, records_to_load, row_count[stream], db_sync, compression, slices, temp_dir)
# Load into redshift
try:
if row_count[stream] > 0:
flush_records(stream, records_to_load, row_count[stream], db_sync, compression, slices, temp_dir)

# Delete soft-deleted, flagged rows - where _sdc_deleted at is not null
if delete_rows:
db_sync.delete_rows(stream)

# reset row count for the current stream
row_count[stream] = 0
# Delete soft-deleted, flagged rows - where _sdc_deleted at is not null
if delete_rows:
db_sync.delete_rows(stream)

# reset row count for the current stream
row_count[stream] = 0
except Exception as e:
LOGGER.exception("Failed to load stream %s to Redshift", stream)
raise e

def chunk_iterable(iterable, size):
"""Yield successive n-sized chunks from iterable. The last chunk is not padded"""
Expand Down

0 comments on commit a7fed8b

Please sign in to comment.