diff --git a/src/acts/actscli.hpp b/src/acts/actscli.hpp index 1898cb9..8eb1b94 100644 --- a/src/acts/actscli.hpp +++ b/src/acts/actscli.hpp @@ -18,6 +18,7 @@ namespace actscli { bool noIWHash{}; bool markHash{}; const char* dumpHashmap{}; + bool dumpHashmapUnknown{ false }; const char* wniFiles{}; bool installDirHashes{ true }; bool show0Hash{ false }; diff --git a/src/acts/hashutils.cpp b/src/acts/hashutils.cpp index 82801db..8e07e90 100644 --- a/src/acts/hashutils.cpp +++ b/src/acts/hashutils.cpp @@ -10,6 +10,7 @@ namespace { std::set g_extracted{}; const char* hashPrefix{}; bool g_saveExtracted = false; + bool g_saveExtractedUnk = false; bool show0 = false; bool markHash = false; bool heavyHashes = false; @@ -126,8 +127,9 @@ namespace hashutils { } } - void SaveExtracted(bool value) { + void SaveExtracted(bool value, bool unk) { g_saveExtracted = value; + g_saveExtractedUnk = unk; g_extracted.clear(); } @@ -155,7 +157,7 @@ namespace hashutils { out.close(); // clear and unset extract - SaveExtracted(false); + SaveExtracted(false, false); LOG_TRACE("End write extracted into {}", file); } @@ -336,10 +338,12 @@ namespace hashutils { } return true; } + const auto res = g_hashMap.find(hash & hashutils::MASK63); if (g_saveExtracted) { - g_extracted.emplace(hash); + if (g_saveExtractedUnk || res != g_hashMap.end()) { + g_extracted.emplace(hash); + } } - const auto res = g_hashMap.find(hash & hashutils::MASK63); if (res == g_hashMap.end()) { snprintf(out, outSize, heavyHashes ? "%s_%016llX" : "%s_%llx", type, hash); return false; diff --git a/src/acts/hashutils.hpp b/src/acts/hashutils.hpp index 1b1a216..8c5c6f4 100644 --- a/src/acts/hashutils.hpp +++ b/src/acts/hashutils.hpp @@ -34,8 +34,9 @@ namespace hashutils { /* * Save the extract hashes for a future use with WriteExtracted * @param value save extracted + * @param unk unk values */ - void SaveExtracted(bool value); + void SaveExtracted(bool value, bool unk); /* * Save the extract hashes * @param file file to read diff --git a/src/acts/main.cpp b/src/acts/main.cpp index 6d4d4f9..b7ca1e3 100644 --- a/src/acts/main.cpp +++ b/src/acts/main.cpp @@ -157,6 +157,9 @@ namespace { return false; } opt.dumpHashmap = argv[++i]; + } + else if (!_strcmpi("--extracted-unk", arg)) { + opt.dumpHashmapUnknown = true; } else if (!strcmp("-p", arg) || !_strcmpi("--pack", arg)) { if (i + 1 == argc) { @@ -242,6 +245,7 @@ namespace { LOG_INFO(" --log-path [p] : Set the log path(s), semicolon separated"); LOG_INFO(" -d --debug : Enable debug mode"); LOG_INFO(" -x --extracted [f] : Write the extracted hashes into a file after the process"); + LOG_INFO(" --extracted-unk : with -x Extract the unknown values"); LOG_INFO(" -t --no-title : Hide ACTS title at start"); LOG_INFO(" -p --pack [f] : Load ACTS pack file"); LOG_INFO(" -P --profiler [f] : Save profiler file after tool usage"); diff --git a/src/acts/tools/compatibility/scobalula_wnigen.cpp b/src/acts/tools/compatibility/scobalula_wnigen.cpp index bb7d46a..8d5a94d 100644 --- a/src/acts/tools/compatibility/scobalula_wnigen.cpp +++ b/src/acts/tools/compatibility/scobalula_wnigen.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "actscli.hpp" #include "compatibility/scobalula_wni.hpp" @@ -26,7 +27,7 @@ namespace { return DT_INVALID; } - int wni_r(Process& proc, int argc, const char* argv[]) { + int wni_r(int argc, const char* argv[]) { if (argc < 5) { return tool::BAD_USAGE; } @@ -60,16 +61,175 @@ namespace { return tool::OK; } - int wni_gen(Process& proc, int argc, const char* argv[]) { - if (argc < 4) { + enum Algorithms : uint64_t { + ALG_FNV = 1, + ALG_FNV_IW_RES = 1 << 2, + ALG_DVAR = 1 << 3, + ALG_FNV32 = 1 << 4, + ALG_SCR_T7 = 1 << 5, + ALG_SCR_T89 = 1 << 6, + ALG_SCR_JUP = 1 << 7, + ALG_SCR_T10 = 1 << 8, + ALG_SCR_T10_SP = 1 << 9, + + ALG_ALL = static_cast(-1), + }; + + int wni_gen_csv(int argc, const char* argv[]) { + if (tool::NotEnoughParam(argc, 2)) { return tool::BAD_USAGE; } + std::filesystem::path csv{ argv[2] }; + std::filesystem::path out{ argv[3] }; + LOG_DEBUG("Reading HASH CSV {}", csv.string()); + std::string buffer{}; - auto* in = argv[2]; - auto* out = argv[3]; - bool canon{}; - if (argc > 4) { - canon = !_strcmpi(argv[4], "true"); + if (!utils::ReadFile(csv, buffer)) { + LOG_WARNING("Can't read hash csv {}", csv.string()); + return tool::BASIC_ERROR; + } + + + rapidcsv::Document doc{}; + + std::stringstream stream{ buffer }; + + doc.Load(stream, rapidcsv::LabelParams(-1, -1), rapidcsv::SeparatorParams(',')); + + if (doc.GetColumnCount() < 2) { + LOG_WARNING("Can't read hash csv {}: invalid file", csv.string()); + return tool::BASIC_ERROR; + } + + uint32_t count{}; + + std::vector rawdata{}; + for (size_t i = 0; i < doc.GetRowCount(); i++) { + const std::string hash = doc.GetCell(0, i); + const std::string value = doc.GetCell(1, i); + + uint64_t hashVal; + try { + hashVal = std::strtoull(hash.c_str(), nullptr, 16); + } + catch (std::runtime_error& e) { + LOG_WARNING("Error when reading {}: invalid line {}: {}", csv.string(), i, e.what()); + continue; + } + + utils::WriteValue(rawdata, hashVal); + utils::WriteString(rawdata, value.c_str()); + count++; + } + + + std::ofstream os{ out, std::ios::binary }; + utils::CloseEnd osce{ os }; + + if (!os) { + LOG_ERROR("Can't open output {}", out.string()); + return tool::BASIC_ERROR; + } + + if (rawdata.size() >= LZ4_MAX_INPUT_SIZE) { + LOG_ERROR("File too big."); + return tool::BASIC_ERROR; + } + + int bound = LZ4_compressBound((int)rawdata.size()); + + auto comp = std::make_unique(bound); + + LOG_DEBUG("Compressing..."); + + int compressedSize = LZ4_compress_default((const char*)&rawdata[0], &comp[0], (int)rawdata.size(), bound); + if (compressedSize <= 0) { + LOG_ERROR("Failed to compress, abort."); + return tool::BASIC_ERROR; + } + + // magic + auto magic = compatibility::scobalula::wni::WNI_MAGIC; + os.write(reinterpret_cast(&magic), sizeof(magic)); + // version + auto version = compatibility::scobalula::wni::WNI_VERSION; + os.write(reinterpret_cast(&version), sizeof(version)); + + uint32_t tmp; + + // entries + tmp = count; + LOG_INFO("Entries: {}", tmp); + os.write(reinterpret_cast(&tmp), sizeof(tmp)); + // compressedSize + tmp = compressedSize; + LOG_INFO("Compressed Size: {}", tmp); + os.write(reinterpret_cast(&tmp), sizeof(tmp)); + // decompressedSize + tmp = (uint32_t)rawdata.size(); + LOG_INFO("Decompressed Size: {}", tmp); + os.write(reinterpret_cast(&tmp), sizeof(tmp)); + // buffer + os.write(reinterpret_cast(&comp[0]), compressedSize + 1); + + LOG_INFO("Created into {}", out.string()); + + return tool::OK; + } + + + int wni_gen(int argc, const char* argv[]) { + if (tool::NotEnoughParam(argc, 2)) { + return tool::BAD_USAGE; + } + + const char* in = argv[2]; + const char* out = argv[3]; + uint64_t alg{}; + if (4 == argc) { + alg |= ALG_ALL; + } + else { + for (size_t i = 4; i < argc; i++) { + switch (hash::Hash64(argv[i])) { + case hash::Hash64("fnv"): + alg |= ALG_FNV; + break; + case hash::Hash64("fnv32"): + alg |= ALG_FNV32; + break; + case hash::Hash64("iwres"): + alg |= ALG_FNV_IW_RES; + break; + case hash::Hash64("t7"): + alg |= ALG_SCR_T7; + break; + case hash::Hash64("t8"): + case hash::Hash64("t9"): + case hash::Hash64("t89"): + alg |= ALG_SCR_T7; + break; + case hash::Hash64("t10"): + alg |= ALG_SCR_T10; + break; + case hash::Hash64("t10sp"): + alg |= ALG_SCR_T10; + break; + case hash::Hash64("jup"): + alg |= ALG_SCR_JUP; + break; + case hash::Hash64("dvar"): + alg |= ALG_DVAR; + break; + case hash::Hash64("all"): + alg |= ALG_ALL; + break; + default: + LOG_ERROR("Invalid algorithm {}", argv[i]); + return tool::BAD_USAGE; + } + + } } std::ifstream is{ in }; @@ -92,33 +252,53 @@ namespace { std::string line; uint32_t count{}; - + std::vector rawdata{}; LOG_DEBUG("Loading strings..."); while (is.good() && std::getline(is, line)) { - if (canon) { + if (alg & ALG_SCR_T89) { utils::WriteValue(rawdata, (uint64_t)hash::HashT89Scr(line.c_str())); utils::WriteString(rawdata, line.c_str()); } - if (useTreyarch) { + + if (alg & ALG_FNV) { utils::WriteValue(rawdata, hash::Hash64(line.c_str())); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_SCR_T7) { utils::WriteValue(rawdata, hash::HashT7(line.c_str())); utils::WriteString(rawdata, line.c_str()); } - if (useIW) { + + if (alg & ALG_FNV_IW_RES) { utils::WriteValue(rawdata, hash::HashIWRes(line.c_str())); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_SCR_JUP) { utils::WriteValue(rawdata, hash::HashJupScr(line.c_str())); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_FNV32) { utils::WriteValue(rawdata, hash::Hash64(line.c_str(), 0x811C9DC5, 0x1000193) & 0xFFFFFFFF); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_DVAR) { utils::WriteValue(rawdata, hash::HashIWDVar(line.c_str())); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_SCR_T10) { utils::WriteValue(rawdata, hash::HashT10Scr(line.c_str())); utils::WriteString(rawdata, line.c_str()); + } + + if (alg & ALG_SCR_T10_SP) { utils::WriteValue(rawdata, hash::HashT10ScrSP(line.c_str())); utils::WriteString(rawdata, line.c_str()); } @@ -179,6 +359,7 @@ namespace { return tool::OK; } - ADD_TOOL(wni_r, "compatibility", " [input] [output] [type=csv,txt]", "Read WNI file/dir", nullptr, wni_r); - ADD_TOOL(wni_gen, "compatibility", " [input] [output] [canon=false]", "Gen WNI file", nullptr, wni_gen); + ADD_TOOL(wni_r, "compatibility", " [input] [output] [type=csv,txt]", "Read WNI file/dir", wni_r); + ADD_TOOL(wni_gen_csv, "compatibility", " [input] [output]", "Gen WNI file from csv", wni_gen_csv); + ADD_TOOL(wni_gen, "compatibility", " [input] [output] [algorithms=all]+", "Gen WNI file with algo", wni_gen); } \ No newline at end of file diff --git a/src/acts/tools/cw/poolt9.cpp b/src/acts/tools/cw/poolt9.cpp index ccaad9d..4760421 100644 --- a/src/acts/tools/cw/poolt9.cpp +++ b/src/acts/tools/cw/poolt9.cpp @@ -945,7 +945,9 @@ int cw::pool::pooltool(Process& proc, int argc, const char* argv[]) { return tool::OK; } - hashutils::SaveExtracted(opt.m_dump_hashmap != NULL); + if (opt.m_dump_hashmap) { + hashutils::SaveExtracted(true, false); + } hashutils::ReadDefaultFile(); std::error_code ec; diff --git a/src/acts/tools/gsc.cpp b/src/acts/tools/gsc.cpp index 18e2f0e..9a8e2f5 100644 --- a/src/acts/tools/gsc.cpp +++ b/src/acts/tools/gsc.cpp @@ -3529,9 +3529,9 @@ int tool::gsc::gscinfo(Process& proc, int argc, const char* argv[]) { const char* globalHM = actscli::options().dumpHashmap; - if (!globalHM) { + if (!globalHM && gdctx.opt.m_dump_hashmap != nullptr) { // keep the option for backward compatibility - hashutils::SaveExtracted(gdctx.opt.m_dump_hashmap != nullptr); + hashutils::SaveExtracted(true, false); } bool computed{}; int ret{ tool::OK }; diff --git a/src/acts/tools/pool.cpp b/src/acts/tools/pool.cpp index 3ecdd1f..1cb560f 100644 --- a/src/acts/tools/pool.cpp +++ b/src/acts/tools/pool.cpp @@ -1367,7 +1367,9 @@ int tool::pool::pooltool(Process& proc, int argc, const char* argv[]) { return tool::OK; } - hashutils::SaveExtracted(opt.m_dump_hashmap != NULL); + if (opt.m_dump_hashmap) { + hashutils::SaveExtracted(true, false); + } std::error_code ec; std::filesystem::create_directories(opt.m_output, ec); diff --git a/src/acts/tools/sp23/gscinfo.cpp b/src/acts/tools/sp23/gscinfo.cpp index 034fb18..5a5fc6f 100644 --- a/src/acts/tools/sp23/gscinfo.cpp +++ b/src/acts/tools/sp23/gscinfo.cpp @@ -638,7 +638,7 @@ namespace { } hashutils::ReadDefaultFile(); - hashutils::SaveExtracted(true); + hashutils::SaveExtracted(true, true); if (opt.m_outputDir) { diff --git a/src/acts/tools/tools.cpp b/src/acts/tools/tools.cpp index ac3ca25..d42acf2 100644 --- a/src/acts/tools/tools.cpp +++ b/src/acts/tools/tools.cpp @@ -308,7 +308,9 @@ namespace tool { } } - hashutils::SaveExtracted(opt.dumpHashmap != nullptr); + if (opt.dumpHashmap) { + hashutils::SaveExtracted(true, opt.dumpHashmapUnknown); + } const clock_t beginTime = clock();