From a8b1ee87a1c952c58b546af8a16ca6444c8a1b2c Mon Sep 17 00:00:00 2001 From: Montague <6136789+MontagueM@users.noreply.github.com> Date: Tue, 15 Jun 2021 11:10:30 +0100 Subject: [PATCH] Added api-like shader extraction --- api.cpp | 276 ++++++++++++++++++++++++++++++++++++++++++++++++++++ api.h | 5 +- dynamic.cpp | 7 +- helpers.cpp | 6 ++ helpers.h | 3 +- main.cpp | 27 +++-- texture.cpp | 1 - 7 files changed, 313 insertions(+), 12 deletions(-) diff --git a/api.cpp b/api.cpp index e5ac6de..31a67f5 100644 --- a/api.cpp +++ b/api.cpp @@ -1,5 +1,49 @@ #include "api.h" +std::unordered_map channelNames = +{ + {662199250, "ArmorPlate"}, + {1367384683, "ArmorSuit"}, + {218592586, "ArmorCloth"}, + {1667433279, "Weapon1"}, + {1667433278, "Weapon2"}, + {1667433277, "Weapon3"}, + {3073305669, "ShipUpper"}, + {3073305668, "ShipDecals"}, + {3073305671, "ShipLower"}, + {1971582085, "SparrowUpper"}, + {1971582084, "SparrowEngine"}, + {1971582087, "SparrowLower"}, + {373026848, "GhostMain"}, + {373026849, "GhostHighlights"}, + {373026850, "GhostDecals"}, +}; + +std::vector dataNames = +{ + "Detail Diffuse Transform", + "Detail Normal Transform", + "\"spec_aa_transform\"", + "Primary Color", + "\"primary_emissive_tint_color_and_intensity_bias\"", + "\"primary_material_params\"", + "\"primary_material_advanced_params\"", + "Primary Roughness Remap", + "Primary Wear Color", + "Primary Wear Remap", + "Primary Worn Roughness Remap", + "\"primary_worn_material_parameters\"", + "Secondary Color", + "\"secondary_emissive_tint_color_and_intensity_bias\"", + "\"secondary_material_params\"", + "\"secondary_material_advanced_params\"", + "Secondary Roughness Remap", + "Secondary Wear Color", + "Secondary Wear Remap", + "Secondary Worn Roughness Remap", + "\"secondary_worn_material_parameters\"", +}; + std::vector getAPIModelHashes(uint32_t apiHash, std::string packagesPath, std::unordered_map hash64Table, bool& bSingle) { std::vector modelHashes; @@ -69,6 +113,238 @@ uint32_t getArtArrangementHash(uint32_t apiHash, std::string packagesPath) } +void getAPIShader(uint32_t apiHash, std::string outputPath, std::string packagesPath, std::unordered_map hash64Table) +{ + File* dataTable = new File("26FCDD80", packagesPath); + dataTable->getData(); + + uint32_t tableOffset = 0x30; + uint32_t tableCount; + uint32_t val; + uint32_t val2; + std::unordered_map defaultChannelDyeMap; + std::unordered_map customChannelDyeMap; + memcpy((char*)&tableCount, dataTable->data + 8, 4); + for (int i = tableOffset; i < tableOffset + tableCount * 0x20; i += 0x20) + { + memcpy((char*)&val, dataTable->data + i, 4); + if (val == apiHash) + { + memcpy((char*)&val, dataTable->data + i + 0x10, 4); + File dataFile = File(uint32ToHexStr(val), packagesPath); + dataFile.getData(); + memcpy((char*)&val, dataFile.data + 0x88, 4); + val += 0x88; + memcpy((char*)&val2, dataFile.data + val - 4, 4); + if (val2 != 2155901815) + { + printf("Given shader is not valid!\n"); + return; + } + uint32_t defaultDyeTableCount; + uint32_t defaultDyeTableOffset; + uint32_t customDyeTableCount; + uint32_t customDyeTableOffset; + memcpy((char*)&defaultDyeTableCount, dataFile.data + val + 0x28, 4); + memcpy((char*)&defaultDyeTableOffset, dataFile.data + val + 0x30, 4); + defaultDyeTableOffset += 0x10 + val + 0x30; + memcpy((char*)&customDyeTableCount, dataFile.data + val + 0x38, 4); + memcpy((char*)&customDyeTableOffset, dataFile.data + val + 0x40, 4); + customDyeTableOffset += 0x10 + val + 0x40; + uint16_t channelIndex; + uint16_t dyeIndex; + for (int j = defaultDyeTableOffset; j < defaultDyeTableOffset + defaultDyeTableCount * 4; j += 4) + { + memcpy((char*)&channelIndex, dataFile.data + j, 2); + memcpy((char*)&dyeIndex, dataFile.data + j + 2, 2); + defaultChannelDyeMap[channelIndex] = dyeIndex; + } + for (int j = customDyeTableOffset; j < customDyeTableOffset + customDyeTableCount * 4; j += 4) + { + memcpy((char*)&channelIndex, dataFile.data + j, 2); + memcpy((char*)&dyeIndex, dataFile.data + j + 2, 2); + customChannelDyeMap[channelIndex] = dyeIndex; + } + } + } + if (defaultChannelDyeMap.size() == 0) return; + + File* channelTable = new File("C92FCF80", packagesPath); + channelTable->getData(); + File* dyeManifestTable = new File("A77AD080", packagesPath); + dyeManifestTable->getData(); + File* dyeFileTable = new File("BDB2C180", packagesPath); + dyeFileTable->getData(); + uint32_t channelHash; + uint32_t dyeManifestHash; + std::string dyeFileHash; + File* finalDyeFile = nullptr; + std::string channelName; + // For each pair, find the channel hash to pair it with a name + find the dye file + + std::unordered_map>> defaultDyes; + std::unordered_map>> customDyes; + + std::unordered_map> defaultTextures; + std::unordered_map> customTextures; + + for (int q = 0; q < 2; q++) + { + auto map = customChannelDyeMap; + if (q == 0) auto map = defaultChannelDyeMap; + + for (auto& it : map) + { + finalDyeFile = nullptr; + + // Get channel name + memcpy((char*)&channelHash, channelTable->data + 0x30 + 4 * it.first, 4); + channelName = channelNames[channelHash]; + + // Get dye file + memcpy((char*)&dyeManifestHash, dyeManifestTable->data + 0x30 + 8 * it.second + 4, 4); + tableOffset = 0x40; + memcpy((char*)&tableCount, dyeFileTable->data + 8, 4); + for (int i = tableOffset; i < tableOffset + tableCount * 0x18; i += 0x18) + { + memcpy((char*)&val, dyeFileTable->data + i, 4); + if (val == dyeManifestHash) + { + memcpy((char*)&val, dyeFileTable->data + i + 0xC, 4); + if (val == 1) + { + uint32_t dyeFileID; + memcpy((char*)&dyeFileID, dyeFileTable->data + i + 0x8, 4); + dyeFileHash = uint32ToHexStr(dyeFileID); + } + else + { + // H64 + uint64_t h64; + memcpy((char*)&h64, dyeFileTable->data + i + 0x10, 8); + dyeFileHash = getHash64(h64, hash64Table); + } + + File file = File(dyeFileHash, packagesPath); + size_t fileSize = file.getData(); + if (fileSize != 0x18) break; + uint32_t dyeFileID; + memcpy((char*)&dyeFileID, file.data + 0x8, 4); + dyeFileHash = uint32ToHexStr(dyeFileID); + file = File(dyeFileHash, packagesPath); + fileSize = file.getData(); + if (fileSize < 0x10) break; + memcpy((char*)&dyeFileID, file.data + 0xC, 4); + dyeFileHash = uint32ToHexStr(dyeFileID); + finalDyeFile = new File(dyeFileHash, packagesPath); + finalDyeFile->getData(); + break; + } + } + // Get dye textures + memcpy((char*)&tableCount, finalDyeFile->data + 0x48, 4); + memcpy((char*)&tableOffset, finalDyeFile->data + 0x50, 4); + tableOffset += 0x60; + std::string texHash; + std::string diffuseName; + std::string normalName; + for (int i = tableOffset; i < tableOffset + tableCount * 0x18; i += 0x18) + { + // H64 + uint64_t h64; + memcpy((char*)&h64, finalDyeFile->data + i + 0x10, 8); + texHash = getHash64(h64, hash64Table); + uint32_t texID; + memcpy((char*)&texID, finalDyeFile->data + i, 4); + + // Save texture + Texture tex = Texture(texHash, packagesPath); + std::string addString = "dif"; + if (texID % 2 != 0) + { + addString = "norm"; + normalName = channelName + "_" + std::to_string(texID) + "_" + texHash + "_" + addString; + } + else diffuseName = channelName + "_" + std::to_string(texID) + "_" + texHash + "_" + addString; + tex.tex2Other(outputPath + "/" + channelName + "_" + std::to_string(texID) + "_" + texHash + "_" + addString + ".dds", "png"); + } + // Get dye data + if (finalDyeFile == nullptr) continue; + std::unordered_map> dyeData; + std::unordered_map texData; + memcpy((char*)&tableCount, finalDyeFile->data + 0x90, 4); + memcpy((char*)&tableOffset, finalDyeFile->data + 0x98, 4); + tableOffset += 0xA8; + float_t fval; + int c = 0; + for (int i = tableOffset; i < tableOffset + tableCount * 0x10; i += 0x10) + { + std::vector data; + for (int j = 0; j < 4; j++) + { + memcpy((char*)&fval, finalDyeFile->data + i + j * 4, 4); + data.push_back(fval); + } + dyeData[dataNames[c]] = data; + c++; + } + + + if (q == 0) + { + defaultDyes[channelName] = dyeData; + texData["Diffuse"] = diffuseName; + texData["Normal"] = normalName; + defaultTextures[channelName] = texData; + } + else + { + customDyes[channelName] = dyeData; + texData["Diffuse"] = diffuseName; + texData["Normal"] = normalName; + customTextures[channelName] = texData; + } + } + } + + for (int q = 0; q < 2; q++) + { + if (q == 0) writeShader(defaultDyes, defaultTextures, false, outputPath); + else writeShader(customDyes, customTextures, true, outputPath); + } +} + +void writeShader(std::unordered_map>> dyes, std::unordered_map> textures, bool bCustom, std::string outputPath) +{ + std::string stringFactoryShader = ""; + if (!bCustom) stringFactoryShader += "Default dyes:\n"; + else stringFactoryShader += "Custom dyes:\n"; + for (auto& it : dyes) + { + stringFactoryShader += " " + it.first + ":\n"; + if (it.first.find("Cloth", 0) != std::string::npos) stringFactoryShader += " Is cloth: True\n"; + else stringFactoryShader += " Is cloth: False\n"; + stringFactoryShader += " Properties:\n"; + for (auto& it2 : it.second) + { + if (it.first.find("Diffuse", 0) != std::string::npos) break; + std::string floatString = "["; + for (auto& flt : it2.second) floatString += std::to_string(flt) + ", "; + stringFactoryShader += " " + it2.first + ": " + floatString.substr(0, floatString.size()-2) + "]\n"; + } + stringFactoryShader += " Diffuse: " + textures[it.first]["Diffuse"] + "\n"; + stringFactoryShader += " Normal: " + textures[it.first]["Normal"] + "\n"; + } + stringFactoryShader += "\n"; + + FILE* shaderFile; + std::string path = outputPath + "/shader.txt"; + fopen_s(&shaderFile, path.c_str(), "w"); + fwrite(stringFactoryShader.c_str(), stringFactoryShader.size(), 1, shaderFile); + fclose(shaderFile); +} + + std::vector getAPISingleHashes(uint32_t mHash, uint32_t fHash, std::string packagesPath, std::unordered_map hash64Table) { std::vector h64Files = { "", "" }; diff --git a/api.h b/api.h index 61d5f82..ca339a4 100644 --- a/api.h +++ b/api.h @@ -1,9 +1,12 @@ #pragma once #include "helpers.h" +#include "texture.h" #include std::vector getAPIModelHashes(uint32_t apiHash, std::string packagesPath, std::unordered_map hash64Table, bool& bSingle); std::vector getAPISingleHashes(uint32_t mHash, uint32_t fHash, std::string packagesPath, std::unordered_map hash64Table); std::vector getAPIMultiHashes(uint32_t tableOffset, File* modelTable, std::string packagesPath, std::unordered_map hash64Table); std::vector getHashesFromH64s(std::vector h64Files, std::string packagesPath, std::unordered_map hash64Table); -uint32_t getArtArrangementHash(uint32_t apiHash, std::string packagesPath); \ No newline at end of file +uint32_t getArtArrangementHash(uint32_t apiHash, std::string packagesPath); +void getAPIShader(uint32_t apiHash, std::string outputPath, std::string packagesPath, std::unordered_map hash64Table); +void writeShader(std::unordered_map>> dyes, std::unordered_map> textures, bool bCustom, std::string outputPath); diff --git a/dynamic.cpp b/dynamic.cpp index c18a614..c7d88df 100644 --- a/dynamic.cpp +++ b/dynamic.cpp @@ -133,9 +133,14 @@ void Dynamic::getDyn3Files() if (off + 572 - 4 >= fileSize) { printf("\nDynamic has no mesh data (C), skipping..."); - return; + continue; } memcpy((char*)&off, dyn2->data + off + 572, 4); + if (off < 2155872256) + { + printf("\nDynamic has no mesh data (D), skipping..."); + continue; + } File* dyn3 = new File(uint32ToHexStr(off), packagesPath); if (std::find(existingDyn3s.begin(), existingDyn3s.end(), dyn3->hash) != existingDyn3s.end() || dyn3->getData() == 0) diff --git a/helpers.cpp b/helpers.cpp index 37a6e89..22d30a0 100644 --- a/helpers.cpp +++ b/helpers.cpp @@ -32,6 +32,12 @@ std::string getPkgID(std::string hash) return pkgID; } +uint16_t getPkgID(uint32_t hash) +{ + uint16_t pkgID = floor((hash - 0x80800000) / 8192); + return pkgID; +} + std::string getHash64(uint64_t hash64, std::unordered_map hash64Table) { std::string h64 = ""; diff --git a/helpers.h b/helpers.h index 6f0fdc1..1e04d56 100644 --- a/helpers.h +++ b/helpers.h @@ -126,4 +126,5 @@ class DynamicMesh : public Mesh std::string getReferenceFromHash(std::string hash, std::string pkgsPath); std::string getHash64(uint64_t hash64, std::unordered_map hash64Table); -std::string getPkgID(std::string hash); \ No newline at end of file +std::string getPkgID(std::string hash); +uint16_t getPkgID(uint32_t hash); \ No newline at end of file diff --git a/main.cpp b/main.cpp index 0f23ef7..5c93ba5 100644 --- a/main.cpp +++ b/main.cpp @@ -26,10 +26,10 @@ int main(int argc, char** argv) sarge.setArgument("i", "inputhash", "hash of Dynamic Model Header 1", true); sarge.setArgument("t", "textures", "enables textures", false); sarge.setArgument("b", "batch", "batch with pkg ID", true); - sarge.setArgument("h", "help", "help shows arguments", false); sarge.setArgument("a", "api", "api hash", true); sarge.setArgument("s", "skeloverride", "skeleton override", true); sarge.setArgument("c", "cbuffer", "enable cbuffer extraction", false); + sarge.setArgument("h", "shader", "shader hash", true); sarge.setDescription("Destiny 2 dynamic model extractor by Monteven."); sarge.setUsage("MontevenDynamicExtractor"); @@ -40,12 +40,6 @@ int main(int argc, char** argv) return 1; } - if (sarge.exists("help")) - { - show_usage(); - return 0; - } - std::string pkgsPath; std::string outputPath; std::string fileName; @@ -56,16 +50,20 @@ int main(int argc, char** argv) int skeletonOverride = -1; std::string batchPkg; std::string apiHashStr = ""; + std::string shaderHashStr = ""; uint32_t apiHash = 0; + uint32_t shaderHash = 0; sarge.getFlag("pkgspath", pkgsPath); sarge.getFlag("outputpath", outputPath); sarge.getFlag("filename", fileName); sarge.getFlag("inputhash", modelHash); sarge.getFlag("api", apiHashStr); + sarge.getFlag("shader", shaderHashStr); sarge.getFlag("skeloverride", skeletonOverrideStr); if (skeletonOverrideStr != "") skeletonOverride = std::stol(skeletonOverrideStr); if (apiHashStr != "") apiHash = std::stoul(apiHashStr); + if (shaderHashStr != "") shaderHash = std::stoul(shaderHashStr); bTextures = sarge.exists("textures"); bCBuffer = sarge.exists("cbuffer"); sarge.getFlag("batch", batchPkg); @@ -74,12 +72,14 @@ int main(int argc, char** argv) { if (apiHashStr != "") fileName = apiHashStr; + if (shaderHashStr != "") + fileName = shaderHashStr; else fileName = modelHash; } // Checking params are valid - if (pkgsPath == "" || outputPath == "" || (modelHash == "" && batchPkg == "" && apiHash == 0)) + if (pkgsPath == "" || outputPath == "" || (modelHash == "" && batchPkg == "" && apiHash == 0 && shaderHash == 0)) { std::cerr << "Invalid parameters, potentially backslashes in paths or paths not given.\n"; show_usage(); @@ -116,6 +116,17 @@ int main(int argc, char** argv) saveH64Table(hash64Table); } + if (shaderHash != 0) + { + printf("Shader flag found, getting shader data...\n"); + std::string savePath = outputPath + "/" + std::to_string(shaderHash); + std::filesystem::create_directories(savePath); + getAPIShader(shaderHash, savePath, pkgsPath, hash64Table); + + printf("Shader rip done!"); + return 0; + } + if (apiHash != 0) { printf("API flag found, getting api models...\n"); diff --git a/texture.cpp b/texture.cpp index 4ed88c8..0f4d304 100644 --- a/texture.cpp +++ b/texture.cpp @@ -274,7 +274,6 @@ std::string getCBufferFromOffset(unsigned char* data, int offset, int count, uin else if (cbType == 2155872400 || offset == 0) { std::string allFloat4 = "static float4 cb" + name + '[' + std::to_string(count) + "] = \n{\n"; - allFloat4.reserve(count); float_t val; for (int i = 0; i < count; i++) {