Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Add as_posix to s3 push file paths in deployment step
Browse files Browse the repository at this point in the history
Add as_posix test

Completed tests
  • Loading branch information
markbruning committed Sep 18, 2023
1 parent bde96a7 commit d2d080a
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 1 deletion.
4 changes: 3 additions & 1 deletion prefect_aws/deployments/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def push_to_s3(
continue
elif not local_file_path.is_dir():
remote_file_path = Path(folder) / local_file_path.relative_to(local_path)
client.upload_file(str(local_file_path), bucket, str(remote_file_path))
client.upload_file(
str(local_file_path), bucket, str(remote_file_path.as_posix())
)

return {
"bucket": bucket,
Expand Down
40 changes: 40 additions & 0 deletions tests/deploments/test_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,24 @@ def tmp_files(tmp_path: Path):
return tmp_path


@pytest.fixture
def tmp_files_win(tmp_path: Path):
files = [
"testfile1.txt",
"testfile2.txt",
"testfile3.txt",
"testdir1\\testfile4.txt",
"testdir2\\testfile5.txt",
]

for file in files:
filepath = tmp_path / file
filepath.parent.mkdir(parents=True, exist_ok=True)
filepath.write_text("Sample text")

return tmp_path


@pytest.fixture
def mock_aws_credentials(monkeypatch):
# Set mock environment variables for AWS credentials
Expand Down Expand Up @@ -77,6 +95,28 @@ def test_push_to_s3(s3_setup, tmp_files, mock_aws_credentials):
assert set(object_keys) == set(expected_keys)


def test_push_to_s3_as_posix(s3_setup, tmp_files_win, mock_aws_credentials):
s3, bucket_name = s3_setup
folder = "my-project"

os.chdir(tmp_files_win)

push_to_s3(bucket_name, folder)

s3_objects = s3.list_objects_v2(Bucket=bucket_name)
object_keys = [item["Key"] for item in s3_objects["Contents"]]

expected_keys = [
f"{folder}/testfile1.txt",
f"{folder}/testfile2.txt",
f"{folder}/testfile3.txt",
f"{folder}/testdir1/testfile4.txt",
f"{folder}/testdir2/testfile5.txt",
]

assert set(object_keys) == set(expected_keys)


def test_pull_from_s3(s3_setup, tmp_path, mock_aws_credentials):
s3, bucket_name = s3_setup
folder = "my-project"
Expand Down

0 comments on commit d2d080a

Please sign in to comment.