From e66fe05e2f81fa6e0755031f40cf7cc091326f3b Mon Sep 17 00:00:00 2001 From: aerosoul Date: Thu, 23 Jan 2020 10:59:26 -0600 Subject: [PATCH 1/5] Add PS3 FF compression support --- src/IW4/Zone.cpp | 16 ++++-- src/ZoneUtils/Zone/ZoneBuffer.cpp | 85 +++++++++++++++++++++++++------ src/ZoneUtils/Zone/ZoneBuffer.hpp | 6 +-- 3 files changed, 85 insertions(+), 22 deletions(-) diff --git a/src/IW4/Zone.cpp b/src/IW4/Zone.cpp index e898383..2b3c08a 100644 --- a/src/IW4/Zone.cpp +++ b/src/IW4/Zone.cpp @@ -175,7 +175,8 @@ namespace ZoneTool // write zone header auto mem_ptr = buf->at>(); - buf->write_stream(&mem, 4, target_ == zone_target::pc ? 10 : 8); // write correct amount of streams, skip last 2 streams for console + + buf->write_stream(&mem, 4, target_ == zone_target::pc ? 10 : target_ == zone_target::ps3 ? 9 : 8); // write correct amount of streams, skip last 2 streams for console std::uintptr_t pad = 0xFFFFFFFF; std::uintptr_t zero = 0; @@ -273,11 +274,11 @@ namespace ZoneTool buf->pop_stream(); // update zone header - zone->size = buf->size() - header_size; + zone->size = buf->size() - (target_ == zone_target::pc ? 0x28 : target_ == zone_target::ps3 ? 0x24 : 0x20); zone->externalsize = 0; // Update stream data - for (int i = 0; i < (target_ != zone_target::pc ? 6 : 8); i++) + for (int i = 0; i < (target_ == zone_target::pc ? 8 : target_ == zone_target::ps3 ? 7 : 6); i++) { zone->streams[i] = buf->stream_offset(i); } @@ -296,11 +297,18 @@ namespace ZoneTool } } + if (this->get_target() == zone_target::ps3) + { + auto zoneSize = ((buf->size() + (0x10000 - 1)) & ~(0x10000 - 1)); + std::vector pad(zoneSize - buf->size(), 0); + buf->write(pad.data(), zoneSize - buf->size()); + } + // Dump zone to disk (DEBUGGING PURPOSES) buf->save("debug\\" + this->name_ + ".zone"); // Compress buffer - auto buf_compressed = buf->compress_zlib(); + auto buf_compressed = buf->compress_zlib(this->get_target() == zone_target::ps3 ? true : false); // Generate FF header auto header = this->m_zonemem->Alloc(); diff --git a/src/ZoneUtils/Zone/ZoneBuffer.cpp b/src/ZoneUtils/Zone/ZoneBuffer.cpp index 57e5ca6..65eb07a 100644 --- a/src/ZoneUtils/Zone/ZoneBuffer.cpp +++ b/src/ZoneUtils/Zone/ZoneBuffer.cpp @@ -257,31 +257,86 @@ namespace ZoneTool this->image_pak()->save(filename); } - std::vector ZoneBuffer::compress_zlib(const std::uint8_t* data, const std::size_t data_size) + std::vector ZoneBuffer::compress_zlib(const std::uint8_t* data, const std::size_t data_size, bool compress_blocks) { auto compressBound = [](unsigned long sourceLen) { return static_cast((ceil(sourceLen * 1.001)) + 12); }; - // calculate buffer size needed for current zone - auto size = compressBound(data_size); + if (compress_blocks == false) + { + // calculate buffer size needed for current zone + auto size = compressBound(data_size); - // alloc array for compressed data - std::vector compressed; - compressed.resize(size); + // alloc array for compressed data + std::vector compressed; + compressed.resize(size); - // compress buffer - auto status = compress2(compressed.data(), &size, data, data_size, ZLIB_BEST_COMPRESSION); - compressed.resize(size); + // compress buffer + auto status = compress2(compressed.data(), &size, data, data_size, ZLIB_BEST_COMPRESSION); + compressed.resize(size); - // return compressed buffer - return compressed; + // return compressed buffer + return compressed; + } + else + { + // data should be 0x10000 byte aligned + const auto block_size = 0x10000; + auto bound_size = compressBound(block_size); + auto num_blocks = data_size / block_size; + + std::vector> blocks; + blocks.resize(num_blocks); + + auto data_ptr = data; + for (auto& block : blocks) + { + // allocate for compressed data + block.resize(bound_size); + + // compress block buffer + unsigned long size; + auto status = compress2(block.data(), &size, data_ptr, block_size, ZLIB_BEST_COMPRESSION); + if (size >= block_size) + { + // discard compressed data and just store uncompressed data + block.resize(block_size + 2); + + // 0 block size is uncompressed + block[0] = 0; + block[1] = 0; + memcpy(block.data() + 2, data_ptr, block_size); + } + else + { + block.resize(size); + + // overwrite zlib header with block size + size -= 2; + block[0] = (size & 0xff00) >> 8; + block[1] = size & 0xff; + } + + // go to next block + data_ptr += block_size; + } + + std::vector compressed; + for (auto& block : blocks) + { + compressed.insert(compressed.end(), block.begin(), block.end()); + } + + + return compressed; + } } - std::vector ZoneBuffer::compress_zlib(const std::vector& data) + std::vector ZoneBuffer::compress_zlib(const std::vector& data, bool compress_blocks) { - return ZoneBuffer::compress_zlib(data.data(), data.size()); + return ZoneBuffer::compress_zlib(data.data(), data.size(), compress_blocks); } std::vector ZoneBuffer::compress_zstd() @@ -307,9 +362,9 @@ namespace ZoneTool return compressed; } - std::vector ZoneBuffer::compress_zlib() + std::vector ZoneBuffer::compress_zlib(bool compress_blocks) { - return ZoneBuffer::compress_zlib(this->m_buf.data(), this->m_pos); + return ZoneBuffer::compress_zlib(this->m_buf.data(), this->m_pos, compress_blocks); } void GenerateKeys(XZoneKey* key) diff --git a/src/ZoneUtils/Zone/ZoneBuffer.hpp b/src/ZoneUtils/Zone/ZoneBuffer.hpp index 0a4013d..8786c94 100644 --- a/src/ZoneUtils/Zone/ZoneBuffer.hpp +++ b/src/ZoneUtils/Zone/ZoneBuffer.hpp @@ -349,11 +349,11 @@ namespace ZoneTool void save(const std::string& filename); void save_image_pak(const std::string& filename); - static std::vector compress_zlib(const std::uint8_t* data, const std::size_t size); - static std::vector compress_zlib(const std::vector& data); + static std::vector compress_zlib(const std::uint8_t* data, const std::size_t size, bool compress_blocks = false); + static std::vector compress_zlib(const std::vector& data, bool compress_blocks = false); std::vector compress_zstd(); - std::vector compress_zlib(); + std::vector compress_zlib(bool compress_blocks = false); void encrypt(); }; } From d53683f71bfda51c119a142c321fe32f4844ceb8 Mon Sep 17 00:00:00 2001 From: a-c130 <62227313+a-c130@users.noreply.github.com> Date: Sun, 12 Jul 2020 19:36:57 -0400 Subject: [PATCH 2/5] Fix Spelling Error --- src/ZoneTool/ZoneTool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ZoneTool/ZoneTool.cpp b/src/ZoneTool/ZoneTool.cpp index e936ec4..178c683 100644 --- a/src/ZoneTool/ZoneTool.cpp +++ b/src/ZoneTool/ZoneTool.cpp @@ -238,7 +238,7 @@ namespace ZoneTool { if (row->fields_[0] == "localize"s && row->numOfFields_ >= 3) { - ZONETOOL_INFO("Adding lcoalized string to zone..."); + ZONETOOL_INFO("Adding localized string to zone..."); struct LocalizeStruct { From 97a6df5bbdae3755be9a11d8fcbdfcf98dc2eb46 Mon Sep 17 00:00:00 2001 From: Pilphe <49091240+Pilphe@users.noreply.github.com> Date: Tue, 25 Aug 2020 00:15:10 +0200 Subject: [PATCH 3/5] Fix zone csv not being created when dumping more than 1 zone --- src/IW4/IW4.cpp | 7 ++++++- src/IW5/Patches/AssetHandler.cpp | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/IW4/IW4.cpp b/src/IW4/IW4.cpp index b6d2918..152ff80 100644 --- a/src/IW4/IW4.cpp +++ b/src/IW4/IW4.cpp @@ -193,11 +193,12 @@ char**>(0x00799278)[type]); auto fastfile = static_cast((char*)(*(DWORD*)0x112A680 + 4)); // generate CSV for fastfile + static FILE* csvFile = nullptr; + if (isVerifying || isDumping) { FileSystem::SetFastFile(fastfile); - static FILE* csvFile; // open csv file for dumping if (!csvFile) @@ -247,6 +248,10 @@ char**>(0x00799278)[type]); // clear referenced assets array because we are done dumping referencedAssets.clear(); + // clear csv file static variable for next dumps + FileSystem::FileClose(csvFile); + csvFile = nullptr; + // mark dumping as complete to exit the process if it has been started using the command line if (currentDumpingZone == fastfile) { diff --git a/src/IW5/Patches/AssetHandler.cpp b/src/IW5/Patches/AssetHandler.cpp index 7e30bb1..b98ae0f 100644 --- a/src/IW5/Patches/AssetHandler.cpp +++ b/src/IW5/Patches/AssetHandler.cpp @@ -124,7 +124,7 @@ namespace ZoneTool { FileSystem::SetFastFile(fastfile); - static FILE* csvFile; + static FILE* csvFile = nullptr; // open csv file for dumping if (!csvFile) @@ -167,6 +167,7 @@ namespace ZoneTool referencedAssets.clear(); FileSystem::FileClose(csvFile); + csvFile = nullptr; is_dumping_complete = true; } From f7b0c5a22ea35eb4c09df3b89a0c1d81a6ffa4bf Mon Sep 17 00:00:00 2001 From: Pilphe <49091240+Pilphe@users.noreply.github.com> Date: Tue, 25 Aug 2020 00:24:50 +0200 Subject: [PATCH 4/5] Fix uncompressed rawfiles not being dumped --- src/IW4/Assets/RawFile.cpp | 6 +++++- src/IW5/Assets/RawFile.cpp | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/IW4/Assets/RawFile.cpp b/src/IW4/Assets/RawFile.cpp index 1acc653..4770f7d 100644 --- a/src/IW4/Assets/RawFile.cpp +++ b/src/IW4/Assets/RawFile.cpp @@ -122,7 +122,7 @@ namespace ZoneTool if (fp) { - if (asset->compressedLen) + if (asset->compressedLen > 0) { std::vector uncompressed; uncompressed.resize(asset->len); @@ -132,6 +132,10 @@ namespace ZoneTool fwrite(uncompressed.data(), uncompressed.size(), 1, fp); } + else if (asset->len > 0) + { + fwrite(asset->buffer, asset->len, 1, fp); + } } FileSystem::FileClose(fp); diff --git a/src/IW5/Assets/RawFile.cpp b/src/IW5/Assets/RawFile.cpp index 4ece700..a1b1f46 100644 --- a/src/IW5/Assets/RawFile.cpp +++ b/src/IW5/Assets/RawFile.cpp @@ -110,7 +110,7 @@ namespace ZoneTool if (fp) { - if (asset->compressedLen) + if (asset->compressedLen > 0) { std::vector uncompressed; uncompressed.resize(asset->len); @@ -120,6 +120,10 @@ namespace ZoneTool fwrite(uncompressed.data(), uncompressed.size(), 1, fp); } + else if (asset->len > 0) + { + fwrite(asset->buffer, asset->len, 1, fp); + } } FileSystem::FileClose(fp); From 9443368c0b2f10df04bd4da40d35cdc3143f64a9 Mon Sep 17 00:00:00 2001 From: RektInator <7003455+RektInator@users.noreply.github.com> Date: Sun, 29 Nov 2020 13:11:12 +0100 Subject: [PATCH 5/5] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 50591fd..a471946 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ ![stars](https://img.shields.io/github/stars/ZoneTool/zonetool.svg) ![GitHub forks](https://img.shields.io/github/forks/ZoneTool/zonetool) ![GitHub issues](https://img.shields.io/github/issues/ZoneTool/zonetool) -[![discord](https://discordapp.com/api/guilds/290238678352134145/widget.png)](https://discord.gg/a6JM2Tv)

Plutonium # zonetool @@ -125,6 +124,9 @@ Special thanks to the following people: * homura * Sofika +## Discord +Join our discord server at https://discord.gg/a6JM2Tv or https://discord.gg/plutonium + ## Donate If you like my work, feel free to contribute!