From 4f56e3020703dde225d06a4abd17a7a22ef4ed4f Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Thu, 5 Sep 2024 13:46:27 +0200 Subject: [PATCH] file: use 64bit indexing for block size This patch changes the block size and block compression functions to use 64bit indexing. From reading the specs, I came to the conclusion that the block size table can be larger than 4Gi entries. (to be exact, I believe that the worst case block table is: UINT64_MAX (max file size) / 4096 (minimal block size) While we don't support files that large yet, we lay the foundation to support them in the future. --- include/sqsh_file.h | 33 +++++++++++++++++++++++++++-- libsqsh/include/sqsh_file_private.h | 2 +- libsqsh/src/file/file.c | 16 ++++++++++++++ libsqsh/src/file/file_iterator.c | 22 +++++++++---------- tools/src/stat.c | 8 +++---- 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/include/sqsh_file.h b/include/sqsh_file.h index d21554cb..ccb430b3 100644 --- a/include/sqsh_file.h +++ b/include/sqsh_file.h @@ -472,9 +472,25 @@ uint64_t sqsh_file_block_count2(const struct SqshFile *context); * * @return the size of the block with the index. */ -uint32_t sqsh_file_block_size(const struct SqshFile *context, uint32_t index); +uint32_t sqsh_file_block_size2(const struct SqshFile *context, uint64_t index); /** + * @deprecated Since 1.6.0. Use sqsh_file_block_size2() instead. + * @memberof SqshFile + * @brief Getter the size of a block of the file content. This is only + * internally used and will be used while retrieving the file content. + * + * @param[in] context The file context. + * @param index The index of the block. + * + * @return the size of the block with the index. + */ +__attribute__((deprecated("Since 1.6.0. Use sqsh_file_block_size2() instead."))) +uint32_t +sqsh_file_block_size(const struct SqshFile *context, uint32_t index); + +/** + * @deprecated Since 1.6.0. Use sqsh_file_block_is_compressed2() instead. * @memberof SqshFile * @brief Checks whether a certain block is compressed. * @@ -483,9 +499,22 @@ uint32_t sqsh_file_block_size(const struct SqshFile *context, uint32_t index); * * @return true if the block is compressed, false otherwise. */ -bool +__attribute__((deprecated( + "Since 1.6.0. Use sqsh_file_block_is_compressed2() instead."))) bool sqsh_file_block_is_compressed(const struct SqshFile *context, uint32_t index); +/** + * @memberof SqshFile + * @brief Checks whether a certain block is compressed. + * + * @param[in] context The file context. + * @param index The index of the block. + * + * @return true if the block is compressed, false otherwise. + */ +bool +sqsh_file_block_is_compressed2(const struct SqshFile *context, uint64_t index); + /** * @memberof SqshFile * @brief retrieve the fragment block index. This is only internally used diff --git a/libsqsh/include/sqsh_file_private.h b/libsqsh/include/sqsh_file_private.h index 95a772ff..5bdd9a3e 100644 --- a/libsqsh/include/sqsh_file_private.h +++ b/libsqsh/include/sqsh_file_private.h @@ -147,7 +147,7 @@ struct SqshFileIterator { struct SqshFragmentView fragment_view; size_t sparse_size; size_t block_size; - uint32_t block_index; + uint64_t block_index; const uint8_t *data; size_t size; }; diff --git a/libsqsh/src/file/file.c b/libsqsh/src/file/file.c index 6baa9eac..7d6804e6 100644 --- a/libsqsh/src/file/file.c +++ b/libsqsh/src/file/file.c @@ -309,6 +309,14 @@ sqsh_file_block_count(const struct SqshFile *context) { return (uint32_t)block_count; } +uint32_t +sqsh_file_block_size2(const struct SqshFile *context, uint64_t index) { + const uint32_t size_info = + context->impl->block_size_info(get_inode(context), index); + + return sqsh_datablock_size(size_info); +} + uint32_t sqsh_file_block_size(const struct SqshFile *context, uint32_t index) { const uint32_t size_info = @@ -317,6 +325,14 @@ sqsh_file_block_size(const struct SqshFile *context, uint32_t index) { return sqsh_datablock_size(size_info); } +bool +sqsh_file_block_is_compressed2(const struct SqshFile *context, uint64_t index) { + const uint32_t size_info = + context->impl->block_size_info(get_inode(context), index); + + return sqsh_datablock_is_compressed(size_info); +} + bool sqsh_file_block_is_compressed(const struct SqshFile *context, uint32_t index) { const uint32_t size_info = diff --git a/libsqsh/src/file/file_iterator.c b/libsqsh/src/file/file_iterator.c index 89e477b2..ad2f92c2 100644 --- a/libsqsh/src/file/file_iterator.c +++ b/libsqsh/src/file/file_iterator.c @@ -135,9 +135,8 @@ map_block_compressed( iterator->compression_manager; const struct SqshFile *file = iterator->file; struct SqshExtractView *extract_view = &iterator->extract_view; - const uint32_t block_index = iterator->block_index; - const sqsh_index_t data_block_size = - sqsh_file_block_size(file, block_index); + const uint64_t block_index = iterator->block_index; + const uint32_t data_block_size = sqsh_file_block_size2(file, block_index); rv = sqsh__map_reader_advance( &iterator->map_reader, next_offset, data_block_size); @@ -168,7 +167,7 @@ map_block_uncompressed( size_t desired_size) { int rv = 0; const struct SqshFile *file = iterator->file; - uint32_t block_index = iterator->block_index; + uint64_t block_index = iterator->block_index; struct SqshMapReader *reader = &iterator->map_reader; const uint64_t block_count = sqsh_file_block_count2(file); size_t outer_size = 0; @@ -176,14 +175,14 @@ map_block_uncompressed( for (; iterator->sparse_size == 0 && block_index < block_count; block_index++) { - if (sqsh_file_block_is_compressed(file, block_index)) { + if (sqsh_file_block_is_compressed2(file, block_index)) { break; } if (outer_size >= desired_size) { break; } const uint32_t data_block_size = - sqsh_file_block_size(file, block_index); + sqsh_file_block_size2(file, block_index); /* Set the sparse size only if we are not at the last block. */ if (block_index + 1 != block_count) { iterator->sparse_size = iterator->block_size - data_block_size; @@ -241,11 +240,12 @@ map_block(struct SqshFileIterator *iterator, size_t desired_size) { int rv = 0; const struct SqshFile *file = iterator->file; - const uint32_t block_index = iterator->block_index; + const uint64_t block_index = iterator->block_index; const size_t block_size = iterator->block_size; - const bool is_compressed = sqsh_file_block_is_compressed(file, block_index); + const bool is_compressed = + sqsh_file_block_is_compressed2(file, block_index); const uint64_t file_size = sqsh_file_size(file); - const size_t data_block_size = sqsh_file_block_size(file, block_index); + const size_t data_block_size = sqsh_file_block_size2(file, block_index); const sqsh_index_t next_offset = sqsh__map_reader_size(&iterator->map_reader); @@ -374,10 +374,10 @@ sqsh_file_iterator_skip2( } sqsh_index_t reader_forward = 0; - uint32_t block_index = iterator->block_index; + uint64_t block_index = iterator->block_index; const uint64_t block_count = sqsh_file_block_count2(iterator->file); for (sqsh_index_t i = 0; i < skip_index && block_index < block_count; i++) { - reader_forward += sqsh_file_block_size(iterator->file, block_index); + reader_forward += sqsh_file_block_size2(iterator->file, block_index); block_index += 1; } rv = sqsh__map_reader_advance(&iterator->map_reader, reader_forward, 0); diff --git a/tools/src/stat.c b/tools/src/stat.c index 9523e1f7..3b7c39b2 100644 --- a/tools/src/stat.c +++ b/tools/src/stat.c @@ -294,11 +294,11 @@ stat_file(struct SqshArchive *archive, const char *path) { } printf(" number of blocks: %" PRIu64 "\n", sqsh_file_block_count2(file)); - for (uint32_t i = 0; i < sqsh_file_block_count2(file); i++) { - bool is_compressed = sqsh_file_block_is_compressed(file, i); - uint32_t size = sqsh_file_block_size(file, i); + for (uint64_t i = 0; i < sqsh_file_block_count2(file); i++) { + bool is_compressed = sqsh_file_block_is_compressed2(file, i); + uint32_t size = sqsh_file_block_size2(file, i); - printf(" % 9i - %i (compressed: %s)\n", i, size, + printf(" % 9" PRIi64 " - %i (compressed: %s)\n", i, size, is_compressed ? "yes" : "no"); } break;