Skip to content

Commit

Permalink
Techdebt: Replace sure with regular assertions in Kinesis (#6579)
Browse files Browse the repository at this point in the history
  • Loading branch information
bblommers authored Aug 1, 2023
1 parent 58a981a commit 6843eb4
Show file tree
Hide file tree
Showing 8 changed files with 340 additions and 350 deletions.
241 changes: 120 additions & 121 deletions tests/test_kinesis/test_kinesis.py

Large diffs are not rendered by default.

221 changes: 112 additions & 109 deletions tests/test_kinesis/test_kinesis_boto3.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
from .test_kinesis import get_stream_arn

import sure # noqa # pylint: disable=unused-import


@mock_kinesis
def test_describe_stream_limit_parameter():
Expand All @@ -17,26 +15,26 @@ def test_describe_stream_limit_parameter():
client.create_stream(StreamName=stream_name, ShardCount=5)

without_filter = client.describe_stream(StreamName=stream_name)["StreamDescription"]
without_filter["Shards"].should.have.length_of(5)
without_filter["HasMoreShards"].should.equal(False)
assert len(without_filter["Shards"]) == 5
assert without_filter["HasMoreShards"] is False

with_filter = client.describe_stream(StreamName=stream_name, Limit=2)[
"StreamDescription"
]
with_filter["Shards"].should.have.length_of(2)
with_filter["HasMoreShards"].should.equal(True)
assert len(with_filter["Shards"]) == 2
assert with_filter["HasMoreShards"] is True

with_filter = client.describe_stream(StreamName=stream_name, Limit=5)[
"StreamDescription"
]
with_filter["Shards"].should.have.length_of(5)
with_filter["HasMoreShards"].should.equal(False)
assert len(with_filter["Shards"]) == 5
assert with_filter["HasMoreShards"] is False

with_filter = client.describe_stream(StreamName=stream_name, Limit=6)[
"StreamDescription"
]
with_filter["Shards"].should.have.length_of(5)
with_filter["HasMoreShards"].should.equal(False)
assert len(with_filter["Shards"]) == 5
assert with_filter["HasMoreShards"] is False


@mock_kinesis
Expand All @@ -53,23 +51,24 @@ def test_list_shards():
)

shard_list = conn.list_shards(StreamName=stream_name)["Shards"]
shard_list.should.have.length_of(2)
assert len(shard_list) == 2
# Verify IDs
[s["ShardId"] for s in shard_list].should.equal(
["shardId-000000000000", "shardId-000000000001"]
)
assert [s["ShardId"] for s in shard_list] == [
"shardId-000000000000",
"shardId-000000000001",
]
# Verify hash range
for shard in shard_list:
shard.should.have.key("HashKeyRange")
shard["HashKeyRange"].should.have.key("StartingHashKey")
shard["HashKeyRange"].should.have.key("EndingHashKey")
shard_list[0]["HashKeyRange"]["EndingHashKey"].should.equal(
str(int(shard_list[1]["HashKeyRange"]["StartingHashKey"]) - 1)
assert "HashKeyRange" in shard
assert "StartingHashKey" in shard["HashKeyRange"]
assert "EndingHashKey" in shard["HashKeyRange"]
assert shard_list[0]["HashKeyRange"]["EndingHashKey"] == str(
int(shard_list[1]["HashKeyRange"]["StartingHashKey"]) - 1
)
# Verify sequence numbers
for shard in shard_list:
shard.should.have.key("SequenceNumberRange")
shard["SequenceNumberRange"].should.have.key("StartingSequenceNumber")
assert "SequenceNumberRange" in shard
assert "StartingSequenceNumber" in shard["SequenceNumberRange"]


@mock_kinesis
Expand All @@ -80,46 +79,43 @@ def test_list_shards_paging():

# Get shard 1-10
shard_list = client.list_shards(StreamName=stream_name)
shard_list["Shards"].should.have.length_of(10)
shard_list.should_not.have.key("NextToken")
assert len(shard_list["Shards"]) == 10
assert "NextToken" not in shard_list

# Get shard 1-4
resp = client.list_shards(StreamName=stream_name, MaxResults=4)
resp["Shards"].should.have.length_of(4)
[s["ShardId"] for s in resp["Shards"]].should.equal(
[
"shardId-000000000000",
"shardId-000000000001",
"shardId-000000000002",
"shardId-000000000003",
]
)
resp.should.have.key("NextToken")
assert len(resp["Shards"]) == 4
assert [s["ShardId"] for s in resp["Shards"]] == [
"shardId-000000000000",
"shardId-000000000001",
"shardId-000000000002",
"shardId-000000000003",
]
assert "NextToken" in resp

# Get shard 4-8
resp = client.list_shards(
StreamName=stream_name, MaxResults=4, NextToken=str(resp["NextToken"])
)
resp["Shards"].should.have.length_of(4)
[s["ShardId"] for s in resp["Shards"]].should.equal(
[
"shardId-000000000004",
"shardId-000000000005",
"shardId-000000000006",
"shardId-000000000007",
]
)
resp.should.have.key("NextToken")
assert len(resp["Shards"]) == 4
assert [s["ShardId"] for s in resp["Shards"]] == [
"shardId-000000000004",
"shardId-000000000005",
"shardId-000000000006",
"shardId-000000000007",
]
assert "NextToken" in resp

# Get shard 8-10
resp = client.list_shards(
StreamName=stream_name, MaxResults=4, NextToken=str(resp["NextToken"])
)
resp["Shards"].should.have.length_of(2)
[s["ShardId"] for s in resp["Shards"]].should.equal(
["shardId-000000000008", "shardId-000000000009"]
)
resp.should_not.have.key("NextToken")
assert len(resp["Shards"]) == 2
assert [s["ShardId"] for s in resp["Shards"]] == [
"shardId-000000000008",
"shardId-000000000009",
]
assert "NextToken" not in resp


@mock_kinesis
Expand All @@ -129,26 +125,26 @@ def test_create_shard():

resp = client.describe_stream(StreamName="my-stream")
desc = resp["StreamDescription"]
desc.should.have.key("StreamName").equal("my-stream")
desc.should.have.key("StreamARN").equal(
f"arn:aws:kinesis:us-west-2:{ACCOUNT_ID}:stream/my-stream"
assert desc["StreamName"] == "my-stream"
assert (
desc["StreamARN"] == f"arn:aws:kinesis:us-west-2:{ACCOUNT_ID}:stream/my-stream"
)
desc.should.have.key("Shards").length_of(2)
desc.should.have.key("StreamStatus").equals("ACTIVE")
desc.should.have.key("HasMoreShards").equals(False)
desc.should.have.key("RetentionPeriodHours").equals(24)
desc.should.have.key("StreamCreationTimestamp")
desc.should.have.key("EnhancedMonitoring").should.equal([{"ShardLevelMetrics": []}])
desc.should.have.key("EncryptionType").should.equal("NONE")
assert len(desc["Shards"]) == 2
assert desc["StreamStatus"] == "ACTIVE"
assert desc["HasMoreShards"] is False
assert desc["RetentionPeriodHours"] == 24
assert "StreamCreationTimestamp" in desc
assert desc["EnhancedMonitoring"] == [{"ShardLevelMetrics": []}]
assert desc["EncryptionType"] == "NONE"

shards = desc["Shards"]
shards[0].should.have.key("ShardId").equal("shardId-000000000000")
shards[0].should.have.key("HashKeyRange")
shards[0]["HashKeyRange"].should.have.key("StartingHashKey").equals("0")
shards[0]["HashKeyRange"].should.have.key("EndingHashKey")
shards[0].should.have.key("SequenceNumberRange")
shards[0]["SequenceNumberRange"].should.have.key("StartingSequenceNumber")
shards[0]["SequenceNumberRange"].shouldnt.have.key("EndingSequenceNumber")
assert shards[0]["ShardId"] == "shardId-000000000000"
assert "HashKeyRange" in shards[0]
assert shards[0]["HashKeyRange"]["StartingHashKey"] == "0"
assert "EndingHashKey" in shards[0]["HashKeyRange"]
assert "SequenceNumberRange" in shards[0]
assert "StartingSequenceNumber" in shards[0]["SequenceNumberRange"]
assert "EndingSequenceNumber" not in shards[0]["SequenceNumberRange"]


@mock_kinesis
Expand All @@ -163,9 +159,10 @@ def test_split_shard_with_invalid_name():
NewStartingHashKey="170141183460469231731687303715884105728",
)
err = exc.value.response["Error"]
err["Code"].should.equal("ValidationException")
err["Message"].should.equal(
"1 validation error detected: Value '?' at 'shardToSplit' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== "1 validation error detected: Value '?' at 'shardToSplit' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"
)


Expand All @@ -181,9 +178,10 @@ def test_split_shard_with_unknown_name():
NewStartingHashKey="170141183460469231731687303715884105728",
)
err = exc.value.response["Error"]
err["Code"].should.equal("ResourceNotFoundException")
err["Message"].should.equal(
"Could not find shard unknown in stream my-stream under account 123456789012."
assert err["Code"] == "ResourceNotFoundException"
assert (
err["Message"]
== "Could not find shard unknown in stream my-stream under account 123456789012."
)


Expand All @@ -199,9 +197,10 @@ def test_split_shard_invalid_hashkey():
NewStartingHashKey="sth",
)
err = exc.value.response["Error"]
err["Code"].should.equal("ValidationException")
err["Message"].should.equal(
"1 validation error detected: Value 'sth' at 'newStartingHashKey' failed to satisfy constraint: Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})"
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== "1 validation error detected: Value 'sth' at 'newStartingHashKey' failed to satisfy constraint: Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})"
)


Expand All @@ -217,9 +216,10 @@ def test_split_shard_hashkey_out_of_bounds():
NewStartingHashKey="170141183460469231731687303715884000000",
)
err = exc.value.response["Error"]
err["Code"].should.equal("InvalidArgumentException")
err["Message"].should.equal(
f"NewStartingHashKey 170141183460469231731687303715884000000 used in SplitShard() on shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} is not both greater than one plus the shard's StartingHashKey 170141183460469231731687303715884105728 and less than the shard's EndingHashKey 340282366920938463463374607431768211455."
assert err["Code"] == "InvalidArgumentException"
assert (
err["Message"]
== f"NewStartingHashKey 170141183460469231731687303715884000000 used in SplitShard() on shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} is not both greater than one plus the shard's StartingHashKey 170141183460469231731687303715884105728 and less than the shard's EndingHashKey 340282366920938463463374607431768211455."
)


Expand Down Expand Up @@ -248,32 +248,34 @@ def test_split_shard():

resp = client.describe_stream(StreamName=stream_name)["StreamDescription"]
shards = resp["Shards"]
shards.should.have.length_of(4)
shards[0].should.have.key("ShardId").equals("shardId-000000000000")
shards[0].should.have.key("HashKeyRange")
shards[0].shouldnt.have.key("ParentShardId")

shards[1].should.have.key("ShardId").equals("shardId-000000000001")
shards[1].shouldnt.have.key("ParentShardId")
shards[1].should.have.key("HashKeyRange")
shards[1]["HashKeyRange"].should.have.key("StartingHashKey").equals(
original_shards[1]["HashKeyRange"]["StartingHashKey"]
assert len(shards) == 4
assert shards[0]["ShardId"] == "shardId-000000000000"
assert "HashKeyRange" in shards[0]
assert "ParentShardId" not in shards[0]

assert shards[1]["ShardId"] == "shardId-000000000001"
assert "ParentShardId" not in shards[1]
assert "HashKeyRange" in shards[1]
assert (
shards[1]["HashKeyRange"]["StartingHashKey"]
== original_shards[1]["HashKeyRange"]["StartingHashKey"]
)
shards[1]["HashKeyRange"].should.have.key("EndingHashKey").equals(
original_shards[1]["HashKeyRange"]["EndingHashKey"]
assert (
shards[1]["HashKeyRange"]["EndingHashKey"]
== original_shards[1]["HashKeyRange"]["EndingHashKey"]
)
shards[1]["SequenceNumberRange"].should.have.key("StartingSequenceNumber")
shards[1]["SequenceNumberRange"].should.have.key("EndingSequenceNumber")
assert "StartingSequenceNumber" in shards[1]["SequenceNumberRange"]
assert "EndingSequenceNumber" in shards[1]["SequenceNumberRange"]

shards[2].should.have.key("ShardId").equals("shardId-000000000002")
shards[2].should.have.key("ParentShardId").equals(shards[1]["ShardId"])
shards[2]["SequenceNumberRange"].should.have.key("StartingSequenceNumber")
shards[2]["SequenceNumberRange"].shouldnt.have.key("EndingSequenceNumber")
assert shards[2]["ShardId"] == "shardId-000000000002"
assert shards[2]["ParentShardId"] == shards[1]["ShardId"]
assert "StartingSequenceNumber" in shards[2]["SequenceNumberRange"]
assert "EndingSequenceNumber" not in shards[2]["SequenceNumberRange"]

shards[3].should.have.key("ShardId").equals("shardId-000000000003")
shards[3].should.have.key("ParentShardId").equals(shards[1]["ShardId"])
shards[3]["SequenceNumberRange"].should.have.key("StartingSequenceNumber")
shards[3]["SequenceNumberRange"].shouldnt.have.key("EndingSequenceNumber")
assert shards[3]["ShardId"] == "shardId-000000000003"
assert shards[3]["ParentShardId"] == shards[1]["ShardId"]
assert "StartingSequenceNumber" in shards[3]["SequenceNumberRange"]
assert "EndingSequenceNumber" not in shards[3]["SequenceNumberRange"]


@mock_kinesis
Expand All @@ -295,9 +297,10 @@ def test_split_shard_that_was_split_before():
NewStartingHashKey="170141183460469231731687303715884105829",
)
err = exc.value.response["Error"]
err["Code"].should.equal("InvalidArgumentException")
err["Message"].should.equal(
f"Shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} has already been merged or split, and thus is not eligible for merging or splitting."
assert err["Code"] == "InvalidArgumentException"
assert (
err["Message"]
== f"Shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} has already been merged or split, and thus is not eligible for merging or splitting."
)


Expand All @@ -322,22 +325,22 @@ def test_update_shard_count(initial, target, expected_total):
StreamName="my-stream", TargetShardCount=target, ScalingType="UNIFORM_SCALING"
)

resp.should.have.key("StreamName").equals("my-stream")
resp.should.have.key("CurrentShardCount").equals(initial)
resp.should.have.key("TargetShardCount").equals(target)
assert resp["StreamName"] == "my-stream"
assert resp["CurrentShardCount"] == initial
assert resp["TargetShardCount"] == target

stream = client.describe_stream(StreamName="my-stream")["StreamDescription"]
stream["StreamStatus"].should.equal("ACTIVE")
stream["Shards"].should.have.length_of(expected_total)
assert stream["StreamStatus"] == "ACTIVE"
assert len(stream["Shards"]) == expected_total

active_shards = [
shard
for shard in stream["Shards"]
if "EndingSequenceNumber" not in shard["SequenceNumberRange"]
]
active_shards.should.have.length_of(target)
assert len(active_shards) == target

resp = client.describe_stream_summary(StreamName="my-stream")
stream = resp["StreamDescriptionSummary"]

stream["OpenShardCount"].should.equal(target)
assert stream["OpenShardCount"] == target
Loading

0 comments on commit 6843eb4

Please sign in to comment.