Skip to content

Commit

Permalink
no more dtype checks in read and write. This shall be done in seperat…
Browse files Browse the repository at this point in the history
…e calls using testing.assert_table_has_dtypes().
  • Loading branch information
relleums committed May 13, 2024
1 parent eaa7a06 commit 1192bf4
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 23 deletions.
18 changes: 4 additions & 14 deletions sparse_numeric_table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ def make_rectangular_DataFrame(table):
# ============


def write(path, table, dtypes=None):
def write(path, table):
"""
Writes the table to path.
Expand All @@ -334,15 +334,7 @@ def write(path, table, dtypes=None):
table : dict of recarrays
The sparse table.
dtypes : dict (default: None)
The dtypes of the table. If provided it is asserted that the
table written has the provided dtypes.
"""

if dtypes:
testing.assert_table_has_dtypes(table=table, dtypes=dtypes)

with sequential_tar.open(name=path + ".tmp", mode="w") as tar:
for level_key in table:
assert IDX in table[level_key].dtype.names
Expand All @@ -364,7 +356,7 @@ def _split_level_column_dtype(path):
return level_key, column_key, dtype_key


def read(path, dtypes=None, dynamic=True):
def read(path, dynamic=True):
"""
Returns table which is read from path.
Expand Down Expand Up @@ -394,9 +386,6 @@ def read(path, dtypes=None, dynamic=True):
else:
out[level_key] = level_recarray

if dtypes:
testing.assert_table_has_dtypes(table=out, dtypes=dtypes)

return out


Expand All @@ -414,7 +403,8 @@ def concatenate_files(list_of_table_paths, dtypes):

with tempfile.TemporaryDirectory(prefix="sparse_table_concatenate") as tmp:
for part_table_path in list_of_table_paths:
part_table = read(path=part_table_path, dtypes=dtypes)
part_table = read(path=part_table_path)
testing.assert_table_has_dtypes(table=part_table, dtypes=dtypes)
for level_key in dtypes:
with open(os.path.join(tmp, level_key), "ab") as fa:
fa.write(part_table[level_key].tobytes())
Expand Down
34 changes: 26 additions & 8 deletions sparse_numeric_table/tests/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ def test_from_records():

path = os.path.join(tmp, "{:06d}.tar".format(j))
job_result_paths.append(path)
snt.write(path=path, table=table, dtypes=dtypes)
snt.testing.assert_table_has_dtypes(table=table, dtypes=dtypes)
snt.write(path=path, table=table)

# reduce
# ------
Expand All @@ -72,8 +73,14 @@ def test_write_read_full_table():
my_table_dtypes = snt.testing.make_example_table_dtypes()
with tempfile.TemporaryDirectory(prefix="test_sparse_table") as tmp:
path = os.path.join(tmp, "my_table.tar")
snt.write(path=path, table=my_table, dtypes=my_table_dtypes)
my_table_back = snt.read(path=path, dtypes=my_table_dtypes)
snt.testing.assert_table_has_dtypes(
table=my_table, dtypes=my_table_dtypes
)
snt.write(path=path, table=my_table)
my_table_back = snt.read(path=path)
snt.testing.assert_table_has_dtypes(
table=my_table_back, dtypes=my_table_dtypes
)
snt.testing.assert_tables_are_equal(my_table, my_table_back)

# no dtypes
Expand All @@ -93,8 +100,14 @@ def test_write_read_empty_table():
empty_table_dtypes = snt.testing.make_example_table_dtypes()
with tempfile.TemporaryDirectory(prefix="test_sparse_table") as tmp:
path = os.path.join(tmp, "my_empty_table.tar")
snt.write(path=path, table=empty_table, dtypes=empty_table_dtypes)
my_table_back = snt.read(path=path, dtypes=empty_table_dtypes)
snt.testing.assert_table_has_dtypes(
table=empty_table, dtypes=empty_table_dtypes
)
snt.write(path=path, table=empty_table)
my_table_back = snt.read(path=path)
snt.testing.assert_table_has_dtypes(
table=my_table_back, dtypes=empty_table_dtypes
)
snt.testing.assert_tables_are_equal(empty_table, my_table_back)

# no dtypes
Expand Down Expand Up @@ -241,10 +254,13 @@ def test_concatenate_several_tables():
)
table_i_dtypes = snt.testing.make_example_table_dtypes()
paths.append(os.path.join(tmp, "{:06d}.tar".format(i)))
snt.testing.assert_table_has_dtypes(
table=table_i,
dtypes=table_i_dtypes,
)
snt.write(
path=paths[-1],
table=table_i,
dtypes=table_i_dtypes,
)
output_path = os.path.join(tmp, "full.tar")
full_table = snt.concatenate_files(
Expand Down Expand Up @@ -316,6 +332,8 @@ def test_only_index_in_level():

with tempfile.TemporaryDirectory(prefix="test_sparse_table") as tmp:
path = os.path.join(tmp, "table_with_index_only_level.tar")
snt.write(path=path, table=table, dtypes=dtypes)
table_back = snt.read(path=path, dtypes=dtypes)
snt.testing.assert_table_has_dtypes(table=table, dtypes=dtypes)
snt.write(path=path, table=table)
table_back = snt.read(path=path)
snt.testing.assert_table_has_dtypes(table=table_back, dtypes=dtypes)
snt.testing.assert_tables_are_equal(table, table_back)
2 changes: 1 addition & 1 deletion sparse_numeric_table/version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.2.3"
__version__ = "0.2.4"

0 comments on commit 1192bf4

Please sign in to comment.