Add functionality to extract THOR files

README.md

@@ -1,16 +1,17 @@
 # zextractor
 
-An utility tool to extract data from Gravity's GRF/GPF file formats as used by Ragnarok Online.
+An utility tool to extract data from Gravity's GRF/GPF file formats as used by Ragnarok Online as well as Aeomin's THOR patcher files.
 
 ## Usage
 `./zextractor -h`
 ```
-A tool to extract files from one or multiple Gravity Resource Files (GRF)
+A tool to extract files from one or multiple Gravity Resource/Patch Files (GRF,GPF) or THOR files
 -c     --configFile Specific config file to use instead of the default. Default: zextractor.conf
               --grf List of GRF filenames to load. Can contain multiple comma separated values. E.g. rdata.grf,data.grf. In that case rdata.grf would be loaded first. Default: 
+             --thor THOR filename to load. Can only extract one THOR file at a time. Default: 
    --keepLettercase Use original filenames. Whether the extracted files should have the same lower/uppercase as in the GRF. Default: false
       --outputAscii Output ascii filenames. If set to true, the output filenames will use the raw ascii filenames instead of the converted korean utf encoding. Default: false
-   --printFiletable Creates a <grf filenames>_filetable.txt that contains the filetable information. Default: false
+   --printFiletable Creates a <filenames>__filetable.txt that contains the filetable information. Default: false
            --outdir Directory to place the extracted files into. Default: output
       --filtersfile Filename of the filters text file to use. Default: filters.txt
           --filters Comma separated filters. Default: 
@@ -28,8 +29,10 @@ A tool to extract files from one or multiple Gravity Resource Files (GRF)
 `./zextractor --grf=rdata.grf,data.grf --filters=data\\texture\\*,data\\sprite\\*`
 - **Extract the single file `data/model/prontera/oven.rsm` from `data.grf`.**  
 `./zextractor --grf=data.grf --filters=data\\model\\prontera\\oven.rsm`
-- **Do not extract anything, but print the filetable of `data.grf` to `data.grf_filetable.txt`.**  
+- **Do not extract anything, but print the filetable of `data.grf` to `data.grf__filetable.txt`.**  
 `./zextractor --grf=data.grf --extract=false --printFiletable`
+- **Extract all files of `my-patch.thor`.**  
+`./zextractor --thor=my-patch.thor`
 
 ## Filters
 If wanting to select multiple very specific files and or directories a filters file can be provided.

dub.json

@@ -9,7 +9,8 @@
     "targetPath": "bin",
     "dependencies": {
         "zconfig": "~>1.0.0",
-        "zgrf": "~>1.1.0"
+        "zgrf": "~>1.1.0",
+        "zthor": "~>1.0.1"
     },
     "subPackages": [
         "./configgenerator/"

source/app.d

@@ -1,10 +1,11 @@
-import std.getopt;
-import logger;
-import zgrf : VirtualGRF, GRF;
 import config;
+import logger;
+import std.getopt;
 import zconfig : initializeConfig, getConfigArguments;
+import zgrf : VirtualGRF, GRF, GRFFiletable;
+import zthor : THOR, THORFiletable;
 
-enum usage = "A tool to extract files from one or multiple Gravity Resource Files (GRF)";
+enum usage = "A tool to extract files from one or multiple Gravity Resource/Patch Files (GRF,GPF) or THOR files";
 
 int main(string[] args)
 {
@@ -36,9 +37,9 @@ int main(string[] args)
         return 0;
     }
 
-    if (config.grf.length == 0)
+    if (config.grf.length == 0 && config.thor.length == 0)
     {
-        log_fatal("No GRFs to load.");
+        log_fatal("No files to load.");
         return 1;
     }
 
@@ -72,10 +73,10 @@ int main(string[] args)
         loadGRF!VirtualGRF(grf, config, filters);
         if (config.extract)
         {
-            extractFiles(grf.files, config);
+            extractFiles!GRFFiletable(grf.files, config);
         }
     }
-    else
+    else if (config.grf.length == 1)
     {
         import zgrf : close;
 
@@ -94,7 +95,31 @@ int main(string[] args)
         loadGRF!GRF(grf, config, filters);
         if (config.extract)
         {
-            extractFiles(grf.files, config);
+            extractFiles!GRFFiletable(grf.files, config);
+        }
+    }
+
+    if (config.thor.length > 0)
+    {
+        import zthor : close;
+
+        THOR thor;
+        scope (exit)
+            thor.close();
+
+        try
+        {
+            thor = THOR(config.thor);
+        }
+        catch (Exception e)
+        {
+            logf_fatal("Couldn't open THOR file. Message: %s", e.msg);
+            return 1;
+        }
+        loadTHOR(thor, config, filters);
+        if (config.extract)
+        {
+            extractFiles!THORFiletable(thor.files, config);
         }
     }
 
@@ -151,40 +176,78 @@ wstring[] parseFilters(const string filtersfile)
 ref T loadGRF(T)(return ref T grf, const Config conf, const(wstring)[] filters)
         if (is(T == VirtualGRF) || is(T == GRF))
 {
+    import core.time : MonoTime;
     import zgrf : parse;
 
+    log_info("Parsing GRFs and loading filetables...");
+    auto startTime = MonoTime.currTime;
+
     grf.parse(filters);
 
+    auto elapsed = MonoTime.currTime - startTime;
+    logf_info("Finished loading filetables with a total of %d files after %s", grf.files.length, elapsed);
+
     if (conf.printFiletable)
     {
+        import std.path : baseName;
+
         string filename;
         static if (is(T == VirtualGRF))
         {
             foreach (const ref g; grf.grfs)
             {
-                filename ~= g.filename;
+                filename ~= baseName(g.filename);
             }
         }
         else static if (is(T == GRF))
         {
-            filename = grf.filename;
+            filename = baseName(grf.filename);
         }
-        filename ~= "_filetable.txt";
+        filename ~= "__filetable.txt";
 
         logf_info("Writing filetable to %s", filename);
-        printFiles(filename, grf.files);
+        printGRFFiles(filename, grf.files);
     }
 
     return grf;
 }
 
-import zgrf : GRFFiletable;
+ref THOR loadTHOR(return ref THOR thor, const Config conf, const(wstring)[] filters)
+{
+    import core.time : MonoTime;
+    import zthor : parse;
+
+    log_info("Parsing THOR and loading filetable...");
+    auto startTime = MonoTime.currTime;
+
+    thor.parse(filters);
+
+    auto elapsed = MonoTime.currTime - startTime;
+    logf_info("Finished loading filetable with %d files after %s", thor.files.length, elapsed);
+
+    if (conf.printFiletable)
+    {
+        import std.path : baseName;
+
+        string filename = baseName(thor.filename) ~ "__filetable.txt";
+
+        logf_info("Writing filetable to %s", filename);
+        printTHORFiles(filename, thor.files);
+    }
+    return thor;
+}
+
 
-void extractFiles(ref GRFFiletable files, const Config conf)
+void extractFiles(T)(ref T files, const Config conf)
+    if (is(T == GRFFiletable) || is(T == THORFiletable))
 {
+    import core.time : MonoTime;
     import std.file : exists, isDir, mkdirRecurse, FileException;
     import std.path : dirSeparator, dirName, buildPath;
     import zgrf : GRFFile;
+    import zthor : THORFile;
+
+    log_info("Extracting files...");
 
     if (!exists(conf.outdir))
     {
@@ -205,6 +268,13 @@ void extractFiles(ref GRFFiletable files, const Config conf)
         return;
     }
 
+    auto startTime = MonoTime.currTime;
+    scope (exit)
+    {
+        auto elapsed = MonoTime.currTime - startTime;
+        logf_info("Finished extracting after %s", elapsed);
+    }
+
     ulong index = 1;
 
     foreach (ref file; files)
@@ -273,9 +343,30 @@ void extractFiles(ref GRFFiletable files, const Config conf)
         {
             import std.stdio : File;
             import std.typecons : No;
-            import zgrf : getFileData;
 
-            scope data = getFileData(*file.grf, file, No.useCache);
+            static if (is(T == GRFFiletable))
+            {
+                import zgrf : getFileData;
+
+                scope data = getFileData(*file.grf, file, No.useCache);
+            }
+            else static if (is(T == THORFiletable))
+            {
+                import zthor : FileFlags;
+
+                scope ubyte[] data;
+
+                if (!(file.flags & FileFlags.remove))
+                {
+                    import zthor : getFileData;
+
+                    data = getFileData(*file.thor, file, No.useCache);
+                }
+                else if (conf.verbose)
+                {
+                    logf_info("%s has 'remove' flag. Nothing to extract.", file.name);
+                }
+            }
 
             auto f = File(buildPath(utf8path, base), "w+");
             f.rawWrite(data);
@@ -318,7 +409,7 @@ string asciiToLower(inout string text) nothrow
     return cast(immutable(char)[]) wasteOfMemory;
 }
 
-void printFiles(const string filename, const ref GRFFiletable files)
+void printGRFFiles(const string filename, const ref GRFFiletable files)
 {
     import std.stdio : File;
 
@@ -368,3 +459,52 @@ void printFiles(const string filename, const ref GRFFiletable files)
         f.write(app.data);
     }
 }
+
+void printTHORFiles(const string filename, const ref THORFiletable files)
+{
+    import std.stdio : File;
+
+    auto f = File(filename, "w+");
+    scope (exit)
+        f.close();
+
+    import zthor : THORFile;
+
+    foreach (const ref file; files)
+    {
+        import std.array : appender;
+        import std.format : formattedWrite;
+
+        auto app = appender!wstring;
+
+        app.put("============\n");
+        app.put("Filename (ASCII): ");
+        foreach (const b; file.rawName)
+        {
+            if (b == 0)
+                break;
+
+            app.put(cast(wchar) b);
+        }
+        app.put("\n");
+
+        app.formattedWrite("Filename: %s\n", file.name);
+        app.formattedWrite("Hash (CRC32): %X\n", file.hash);
+        app.formattedWrite("Filesize: %s bytes\n", file.size);
+        app.formattedWrite("Filesize (compressed): %s bytes\n", file.compressed_size);
+        app.formattedWrite("Flags: %d", file.flags);
+        import zthor : FileFlags;
+
+        if (file.flags == 0)
+            app.put(" ADD");
+        if (file.flags & FileFlags.remove)
+            app.put(" REMOVE");
+        app.put("\n");
+        app.formattedWrite("Offset: %d\n", file.offset);
+        app.formattedWrite("THOR: %s\n", file.thor.filename);
+
+        f.write(app.data);
+    }
+}
+
+

source/config.d

@@ -11,13 +11,15 @@ struct Config
         @Desc("List of GRF filenames to load. Can contain multiple comma separated values. " ~
                 "E.g. rdata.grf,data.grf. In that case rdata.grf would be loaded first.")
         string[] grf = [];
+        @Desc("THOR filename to load. Can only extract one THOR file at a time.")
+        string thor;
         @Desc("Use original filenames. Whether the extracted files should have the same" ~
                 " lower/uppercase as in the GRF.")
         bool keepLettercase = false;
         @Desc("Output ascii filenames. If set to true, the output filenames will use the raw " ~
                 "ascii filenames instead of the converted korean utf encoding.")
         bool outputAscii = false;
-        @Desc("Creates a <grf filenames>_filetable.txt that contains the filetable information.")
+        @Desc("Creates a <filenames>__filetable.txt that contains the filetable information.")
         bool printFiletable = false;
         @Desc("Directory to place the extracted files into.")
         string outdir = "output";

zextractor.example.conf

@@ -4,6 +4,10 @@
 ; Default value: 
 ;grf=
 
+; THOR filename to load. Can only extract one THOR file at a time.
+; Default value: 
+;thor=
+
 ; Use original filenames. Whether the extracted files should have the same
 ; lower/uppercase as in the GRF.
 ; Default value: false
@@ -14,7 +18,7 @@
 ; Default value: false
 ;outputAscii=false
 
-; Creates a <grf filenames>_filetable.txt that contains the filetable
+; Creates a <filenames>_<container>_filetable.txt that contains the filetable
 ; information.
 ; Default value: false
 ;printFiletable=false