produce wni using csv and better hash extract

src/acts/actscli.hpp

@@ -18,6 +18,7 @@ namespace actscli {
 		bool noIWHash{};
 		bool markHash{};
 		const char* dumpHashmap{};
+		bool dumpHashmapUnknown{ false };
 		const char* wniFiles{};
 		bool installDirHashes{ true };
 		bool show0Hash{ false };

src/acts/hashutils.cpp

@@ -10,6 +10,7 @@ namespace {
 	std::set<uint64_t> g_extracted{};
 	const char* hashPrefix{};
 	bool g_saveExtracted = false;
+	bool g_saveExtractedUnk = false;
 	bool show0 = false;
 	bool markHash = false;
 	bool heavyHashes = false;
@@ -126,8 +127,9 @@ namespace hashutils {
 		}
 	}
 
-	void SaveExtracted(bool value) {
+	void SaveExtracted(bool value, bool unk) {
 		g_saveExtracted = value;
+		g_saveExtractedUnk = unk;
 		g_extracted.clear();
 	}
 
@@ -155,7 +157,7 @@ namespace hashutils {
 
 		out.close();
 		// clear and unset extract
-		SaveExtracted(false);
+		SaveExtracted(false, false);
 		LOG_TRACE("End write extracted into {}", file);
 	}
 
@@ -336,10 +338,12 @@ namespace hashutils {
 			}
 			return true;
 		}
+		const auto res = g_hashMap.find(hash & hashutils::MASK63);
 		if (g_saveExtracted) {
-			g_extracted.emplace(hash);
+			if (g_saveExtractedUnk || res != g_hashMap.end()) {
+				g_extracted.emplace(hash);
+			}
 		}
-		const auto res = g_hashMap.find(hash & hashutils::MASK63);
 		if (res == g_hashMap.end()) {
 			snprintf(out, outSize, heavyHashes ? "%s_%016llX" : "%s_%llx", type, hash);
 			return false;

src/acts/hashutils.hpp

@@ -34,8 +34,9 @@ namespace hashutils {
 	/*
 	 * Save the extract hashes for a future use with WriteExtracted
 	 * @param value save extracted
+	 * @param unk unk values
 	 */
-	void SaveExtracted(bool value);
+	void SaveExtracted(bool value, bool unk);
 	/*
 	 * Save the extract hashes
 	 * @param file file to read

src/acts/main.cpp

@@ -157,6 +157,9 @@ namespace {
 					return false;
 				}
 				opt.dumpHashmap = argv[++i];
+				}
+			else if (!_strcmpi("--extracted-unk", arg)) {
+				opt.dumpHashmapUnknown = true;
 			}
 			else if (!strcmp("-p", arg) || !_strcmpi("--pack", arg)) {
 				if (i + 1 == argc) {
@@ -242,6 +245,7 @@ namespace {
 		LOG_INFO(" --log-path [p]     : Set the log path(s), semicolon separated");
 		LOG_INFO(" -d --debug         : Enable debug mode");
 		LOG_INFO(" -x --extracted [f] : Write the extracted hashes into a file after the process");
+		LOG_INFO(" --extracted-unk    : with -x Extract the unknown values");
 		LOG_INFO(" -t --no-title      : Hide ACTS title at start");
 		LOG_INFO(" -p --pack [f]      : Load ACTS pack file");
 		LOG_INFO(" -P --profiler [f]  : Save profiler file after tool usage");

src/acts/tools/compatibility/scobalula_wnigen.cpp

@@ -1,5 +1,6 @@
 #include <includes.hpp>
 #include <lz4.h>
+#include <rapidcsv.h>
 #include "actscli.hpp"
 #include "compatibility/scobalula_wni.hpp"
 
@@ -26,7 +27,7 @@ namespace {
 		return DT_INVALID;
 	}
 
-	int wni_r(Process& proc, int argc, const char* argv[]) {
+	int wni_r(int argc, const char* argv[]) {
 		if (argc < 5) {
 			return tool::BAD_USAGE;
 		}
@@ -60,16 +61,175 @@ namespace {
 		return tool::OK;
 	}
 
-	int wni_gen(Process& proc, int argc, const char* argv[]) {
-		if (argc < 4) {
+	enum Algorithms : uint64_t {
+		ALG_FNV = 1,
+		ALG_FNV_IW_RES = 1 << 2,
+		ALG_DVAR = 1 << 3,
+		ALG_FNV32 = 1 << 4,
+		ALG_SCR_T7 = 1 << 5,
+		ALG_SCR_T89 = 1 << 6,
+		ALG_SCR_JUP = 1 << 7,
+		ALG_SCR_T10 = 1 << 8,
+		ALG_SCR_T10_SP = 1 << 9,
+
+		ALG_ALL = static_cast<Algorithms>(-1),
+	};
+
+	int wni_gen_csv(int argc, const char* argv[]) {
+		if (tool::NotEnoughParam(argc, 2)) {
 			return tool::BAD_USAGE;
 		}
+		std::filesystem::path csv{ argv[2] };
+		std::filesystem::path out{ argv[3] };
+		LOG_DEBUG("Reading HASH CSV {}", csv.string());
+		std::string buffer{};
 
-		auto* in = argv[2];
-		auto* out = argv[3];
-		bool canon{};
-		if (argc > 4) {
-			canon = !_strcmpi(argv[4], "true");
+		if (!utils::ReadFile(csv, buffer)) {
+			LOG_WARNING("Can't read hash csv {}", csv.string());
+			return tool::BASIC_ERROR;
+		}
+
+
+		rapidcsv::Document doc{};
+
+		std::stringstream stream{ buffer };
+
+		doc.Load(stream, rapidcsv::LabelParams(-1, -1), rapidcsv::SeparatorParams(','));
+
+		if (doc.GetColumnCount() < 2) {
+			LOG_WARNING("Can't read hash csv {}: invalid file", csv.string());
+			return tool::BASIC_ERROR;
+		}
+
+		uint32_t count{};
+
+		std::vector<byte> rawdata{};
+		for (size_t i = 0; i < doc.GetRowCount(); i++) {
+			const std::string hash = doc.GetCell<std::string>(0, i);
+			const std::string value = doc.GetCell<std::string>(1, i);
+
+			uint64_t hashVal;
+			try {
+				hashVal = std::strtoull(hash.c_str(), nullptr, 16);
+			}
+			catch (std::runtime_error& e) {
+				LOG_WARNING("Error when reading {}: invalid line {}: {}", csv.string(), i, e.what());
+				continue;
+			}
+
+			utils::WriteValue(rawdata, hashVal);
+			utils::WriteString(rawdata, value.c_str());
+			count++;
+		}
+
+
+		std::ofstream os{ out, std::ios::binary };
+		utils::CloseEnd osce{ os };
+
+		if (!os) {
+			LOG_ERROR("Can't open output {}", out.string());
+			return tool::BASIC_ERROR;
+		}
+
+		if (rawdata.size() >= LZ4_MAX_INPUT_SIZE) {
+			LOG_ERROR("File too big.");
+			return tool::BASIC_ERROR;
+		}
+
+		int bound = LZ4_compressBound((int)rawdata.size());
+
+		auto comp = std::make_unique<char[]>(bound);
+
+		LOG_DEBUG("Compressing...");
+
+		int compressedSize = LZ4_compress_default((const char*)&rawdata[0], &comp[0], (int)rawdata.size(), bound);
+		if (compressedSize <= 0) {
+			LOG_ERROR("Failed to compress, abort.");
+			return tool::BASIC_ERROR;
+		}
+
+		// magic
+		auto magic = compatibility::scobalula::wni::WNI_MAGIC;
+		os.write(reinterpret_cast<const char*>(&magic), sizeof(magic));
+		// version
+		auto version = compatibility::scobalula::wni::WNI_VERSION;
+		os.write(reinterpret_cast<const char*>(&version), sizeof(version));
+
+		uint32_t tmp;
+
+		// entries
+		tmp = count;
+		LOG_INFO("Entries: {}", tmp);
+		os.write(reinterpret_cast<const char*>(&tmp), sizeof(tmp));
+		// compressedSize
+		tmp = compressedSize;
+		LOG_INFO("Compressed Size: {}", tmp);
+		os.write(reinterpret_cast<const char*>(&tmp), sizeof(tmp));
+		// decompressedSize
+		tmp = (uint32_t)rawdata.size();
+		LOG_INFO("Decompressed Size: {}", tmp);
+		os.write(reinterpret_cast<const char*>(&tmp), sizeof(tmp));
+		// buffer
+		os.write(reinterpret_cast<const char*>(&comp[0]), compressedSize + 1);
+
+		LOG_INFO("Created into {}", out.string());
+
+		return tool::OK;
+	}
+
+
+	int wni_gen(int argc, const char* argv[]) {
+		if (tool::NotEnoughParam(argc, 2)) {
+			return tool::BAD_USAGE;
+		}
+
+		const char* in = argv[2];
+		const char* out = argv[3];
+		uint64_t alg{};
+		if (4 == argc) {
+			alg |= ALG_ALL;
+		}
+		else {
+			for (size_t i = 4; i < argc; i++) {
+				switch (hash::Hash64(argv[i])) {
+				case hash::Hash64("fnv"): 
+					alg |= ALG_FNV; 
+					break;
+				case hash::Hash64("fnv32"): 
+					alg |= ALG_FNV32; 
+					break;
+				case hash::Hash64("iwres"): 
+					alg |= ALG_FNV_IW_RES; 
+					break;
+				case hash::Hash64("t7"): 
+					alg |= ALG_SCR_T7; 
+					break;
+				case hash::Hash64("t8"):
+				case hash::Hash64("t9"):
+				case hash::Hash64("t89"):
+					alg |= ALG_SCR_T7; 
+					break;
+				case hash::Hash64("t10"):
+					alg |= ALG_SCR_T10;
+					break;
+				case hash::Hash64("t10sp"):
+					alg |= ALG_SCR_T10;
+					break;
+				case hash::Hash64("jup"):
+					alg |= ALG_SCR_JUP; 
+					break;
+				case hash::Hash64("dvar"):
+					alg |= ALG_DVAR;
+					break;
+				case hash::Hash64("all"):
+					alg |= ALG_ALL;
+					break;
+				default: 
+					LOG_ERROR("Invalid algorithm {}", argv[i]);
+					return tool::BAD_USAGE;
+				}
+
+			}
 		}
 
 		std::ifstream is{ in };
@@ -92,33 +252,53 @@ namespace {
 
 		std::string line;
 		uint32_t count{};
-		
+
 		std::vector<byte> rawdata{};
 
 		LOG_DEBUG("Loading strings...");
 
 		while (is.good() && std::getline(is, line)) {
-			if (canon) {
+			if (alg & ALG_SCR_T89) {
 				utils::WriteValue(rawdata, (uint64_t)hash::HashT89Scr(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
 			}
-			if (useTreyarch) {
+
+			if (alg & ALG_FNV) {
 				utils::WriteValue(rawdata, hash::Hash64(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_SCR_T7) {
 				utils::WriteValue(rawdata, hash::HashT7(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
 			}
-			if (useIW) {
+
+			if (alg & ALG_FNV_IW_RES) {
 				utils::WriteValue(rawdata, hash::HashIWRes(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_SCR_JUP) {
 				utils::WriteValue(rawdata, hash::HashJupScr(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_FNV32) {
 				utils::WriteValue(rawdata, hash::Hash64(line.c_str(), 0x811C9DC5, 0x1000193) & 0xFFFFFFFF);
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_DVAR) {
 				utils::WriteValue(rawdata, hash::HashIWDVar(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_SCR_T10) {
 				utils::WriteValue(rawdata, hash::HashT10Scr(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
+			}
+
+			if (alg & ALG_SCR_T10_SP) {
 				utils::WriteValue(rawdata, hash::HashT10ScrSP(line.c_str()));
 				utils::WriteString(rawdata, line.c_str());
 			}
@@ -179,6 +359,7 @@ namespace {
 		return tool::OK;
 	}
 
-	ADD_TOOL(wni_r, "compatibility", " [input] [output] [type=csv,txt]", "Read WNI file/dir", nullptr, wni_r);
-	ADD_TOOL(wni_gen, "compatibility", " [input] [output] [canon=false]", "Gen WNI file", nullptr, wni_gen);
+	ADD_TOOL(wni_r, "compatibility", " [input] [output] [type=csv,txt]", "Read WNI file/dir", wni_r);
+	ADD_TOOL(wni_gen_csv, "compatibility", " [input] [output]", "Gen WNI file from csv", wni_gen_csv);
+	ADD_TOOL(wni_gen, "compatibility", " [input] [output] [algorithms=all]+", "Gen WNI file with algo", wni_gen);
 }

src/acts/tools/cw/poolt9.cpp

@@ -945,7 +945,9 @@ int cw::pool::pooltool(Process& proc, int argc, const char* argv[]) {
         return tool::OK;
     }
 
-    hashutils::SaveExtracted(opt.m_dump_hashmap != NULL);
+    if (opt.m_dump_hashmap) {
+        hashutils::SaveExtracted(true, false);
+    }
     hashutils::ReadDefaultFile();
 
     std::error_code ec;

src/acts/tools/gsc.cpp

@@ -3529,9 +3529,9 @@ int tool::gsc::gscinfo(Process& proc, int argc, const char* argv[]) {
 
 
     const char* globalHM = actscli::options().dumpHashmap;
-    if (!globalHM) {
+    if (!globalHM && gdctx.opt.m_dump_hashmap != nullptr) {
         // keep the option for backward compatibility
-        hashutils::SaveExtracted(gdctx.opt.m_dump_hashmap != nullptr);
+        hashutils::SaveExtracted(true, false);
     }
     bool computed{};
     int ret{ tool::OK };

src/acts/tools/pool.cpp

@@ -1367,7 +1367,9 @@ int tool::pool::pooltool(Process& proc, int argc, const char* argv[]) {
         return tool::OK;
     }
 
-    hashutils::SaveExtracted(opt.m_dump_hashmap != NULL);
+    if (opt.m_dump_hashmap) {
+        hashutils::SaveExtracted(true, false);
+    }
 
     std::error_code ec;
     std::filesystem::create_directories(opt.m_output, ec);

src/acts/tools/sp23/gscinfo.cpp

@@ -638,7 +638,7 @@ namespace {
 		}
 
 		hashutils::ReadDefaultFile();
-		hashutils::SaveExtracted(true);
+		hashutils::SaveExtracted(true, true);
 
 
 		if (opt.m_outputDir) {