Merge pull request #3 from KostromDan/dev

2.2.5

.flake8

@@ -2,7 +2,8 @@
 exclude =
     MDGUi,
     .venv,
-    result
+    result,
+    tmp
 ignore =
     E501
     W504

.gitignore

@@ -166,3 +166,5 @@ cython_debug/
 /result/*
 /tmp/*
 /logs/*
+/decompiler/data/*
+*.zip

MDG.py

@@ -1,14 +1,16 @@
 import faulthandler
+import multiprocessing
 import sys
 
+from MDGUi.generate_ui import generate_ui
 from PySide6.QtWidgets import QApplication
 
-from MDGUi.generate_ui import generate_ui
 from MDGUtil import PathUtils
 from MDGUtil.MDGLogger import MDGLogger
 
 if __name__ == '__main__':
     faulthandler.enable()
+    multiprocessing.freeze_support()
 
     if not PathUtils.check_pyinstaller_env():
         generate_ui()

MDGLogic/AbstractDeobfDecompMainThread.py

@@ -4,15 +4,14 @@
 
 from MDGLogic.AbstractMDGThread import AbstractMDGThread
 from MDGLogic.DecompilationThread import DecompilationThread
-from MDGLogic.DeobfuscationThread import DeobfuscationThread
 
 
 class AbstractDeobfDecompMainThread(AbstractMDGThread):
     failed_mod_signal = Signal(str)
 
     def __init__(self, widgets: dict[str, dict[str, Any] | list[str]]) -> None:
         super().__init__(widgets)
-        self.threads: list[Optional[DecompilationThread, DeobfuscationThread]] = []
+        self.threads: list[Optional[DecompilationThread]] = []
 
     def terminate(self) -> None:
         for thread in self.threads:

MDGLogic/AbstractMDGThread.py

@@ -6,7 +6,7 @@
 
 class AbstractMDGThread(QThread):
     progress = Signal(int, str)
-    critical_signal = Signal(str, str)
+    critical_signal = Signal(str, str, str)
     progress_bar = Signal(int)
 
     def __init__(self, serialized_widgets: dict[str, dict[str, Any] | list[str]]) -> None:

MDGLogic/CopyThread.py

@@ -1,3 +1,5 @@
+import hashlib
+import json
 import logging
 import os
 import shutil
@@ -19,81 +21,131 @@ def run(self) -> None:
 
         FileUtils.create_folder(PathUtils.TMP_MODS_PATH)
 
-        jar_in_jar_mods_list = []
+        mods_list = list(filter(lambda x: x.endswith('.jar'), os.listdir(mods_path)))
+        mods_count = len(mods_list)
+
+        for i, mod in enumerate(mods_list):
+            mod_path = os.path.join(mods_path, mod)
+            self.progress.emit(int((i / mods_count) * 100), f'Copying {mod}')
+            logging.info(f'Starting copying {mod}')
+            shutil.copy(mod_path, PathUtils.TMP_MODS_PATH)
+            try:
+                os.rename(os.path.join(PathUtils.TMP_MODS_PATH, mod),
+                          os.path.join(PathUtils.TMP_MODS_PATH, FileUtils.remove_unsupported_symbols(mod)))
+            except FileExistsError:
+                pass
+            logging.info(f'Finished copying {mod}')
+
+        self.progress.emit(100, 'Finished copying mods.')
+        logging.info(f'Found and copied from mods folder files with .jar extension: {mods_count}')
+
         if self.serialized_widgets['jar_in_jar_check_box']['isChecked']:
+            self.progress.emit(100, 'Analysing mods for jar in jar.')
             logging.info('Started analysing mods for jar in jar.')
-            for mod in os.listdir(mods_path):
-                if not mod.endswith('.jar'):
-                    continue
-                FileUtils.extract_jars_from_jar(os.path.join(mods_path, mod), PathUtils.TMP_MODS_PATH)
-            jar_in_jar_mods_list += os.listdir(PathUtils.TMP_MODS_PATH)
+            for mod in os.listdir(PathUtils.TMP_MODS_PATH):
+                FileUtils.extract_jars_from_jar(os.path.join(PathUtils.TMP_MODS_PATH, mod), PathUtils.TMP_MODS_PATH)
             logging.info('Finished analysing mods for jar in jar. '
-                         f'Found and extracted: {len(jar_in_jar_mods_list)}.')
+                         f'Found and extracted: {len(os.listdir(PathUtils.TMP_MODS_PATH)) - mods_count}.')
 
-        mods_list = list(filter(lambda x: x.endswith('.jar'), os.listdir(mods_path)))
-        mods_list += jar_in_jar_mods_list  # add jar in jar
-        mods_list_without_space = [mod.replace(' ', '_') for mod in mods_list]
+        mods_list = os.listdir(PathUtils.TMP_MODS_PATH)
         mods_count = len(mods_list)
-        logging.info(f'Found in mods folder files with .jar extension: {mods_count}')
-        FileUtils.create_folder(PathUtils.TMP_MODS_PATH)
+
+        logging.info(f'Total mods count including jar in jar: {mods_count}')
+
+        mod_hashes = dict()
+        for mod_name in mods_list:
+            mod_path = os.path.join(PathUtils.TMP_MODS_PATH, mod_name)
+            mod_hash = hashlib.sha256(open(mod_path, 'rb').read()).hexdigest()
+            mod_hashes[mod_name.removesuffix('.jar')] = mod_hash
+        with open(PathUtils.TMP_MODS_HASHES_PATH, 'w') as f:
+            f.write(json.dumps(mod_hashes))
+
+        if not os.path.exists(PathUtils.DEOBFUSCATED_CACHE_PATH):
+            FileUtils.remove_folder(PathUtils.DEOBFUSCATED_MODS_PATH)
+        try:  # remove mods deobfuscation of which was interrupted
+            with open(PathUtils.DEOBFUSCATED_CACHE_PATH, 'r') as f:
+                cache = json.loads(f.read())
+            for mod in os.listdir(PathUtils.DEOBFUSCATED_MODS_PATH):
+                mod_path = os.path.join(PathUtils.DEOBFUSCATED_MODS_PATH, mod)
+                if (mod.removesuffix('.jar').removesuffix('_mapped_official') not in cache and
+                        not os.path.basename(mod_path).endswith('.json')):
+                    os.remove(mod_path)
+                    logging.info(f'Found {mod} in deobfuscated mods.'
+                                 f"But it's not in cache. Removing. "
+                                 f'Maybe deobfuscation of it was interrupted.')
+        except FileNotFoundError:
+            pass
+
         use_cached_decomp = []
         use_cached_deobf = []
         if cache_enabled:
-            try:  # remove mods from decompiled_mods which not in current mods_list
+            if not os.path.exists(PathUtils.DECOMPILED_CACHE_PATH):
+                FileUtils.remove_folder(PathUtils.DECOMPILED_MODS_PATH)
+            elif os.path.exists(PathUtils.DECOMPILED_MODS_PATH):
+                with open(PathUtils.DECOMPILED_CACHE_PATH, 'r') as f:
+                    cache = json.loads(f.read())
                 for mod in os.listdir(PathUtils.DECOMPILED_MODS_PATH):
-                    path_to_mod = os.path.join(PathUtils.DECOMPILED_MODS_PATH, mod)
-                    if os.path.isfile(path_to_mod):
-                        continue
-                    mod_name = mod.removesuffix('_mapped_official') + '.jar'
+                    mod_path = os.path.join(PathUtils.DECOMPILED_MODS_PATH, mod)
+                    mod_name_without_jar = mod.removesuffix('.jar').removesuffix('_mapped_official')
+                    mod_name_with_jar = mod_name_without_jar + '.jar'
                     decompiled_with_deobf = '_mapped_official' in mod
                     deobf_enabled = self.serialized_widgets['deobf_check_box']['isChecked']
-                    if (mod_name in mods_list_without_space and ((decompiled_with_deobf and deobf_enabled) or
-                                                                 (not decompiled_with_deobf and not deobf_enabled))):
-                        use_cached_decomp.append(mod_name)
-                        logging.info(f'Found {mod_name} in decompiled_mods. Coping skipped.')
+                    if mod_name_without_jar.endswith('.json'):
+                        continue
+                    if mod_name_without_jar not in cache:
+                        shutil.rmtree(mod_path)
+                        logging.info(f'Found {mod} in decompiled mods.'
+                                     f"But it's not in cache. Removing. "
+                                     f'Maybe decompilation of it was interrupted.')
+                        continue
+                    if (mod_name_without_jar in mod_hashes and
+                            cache[mod_name_without_jar] != mod_hashes[mod_name_without_jar]):
+                        shutil.rmtree(mod_path)
+                        logging.info(f'Found {mod} in decompiled mods.'
+                                     f'But hash is not same with mod from current mod list. Removing. '
+                                     f"Maybe mod changed without changing it's name.")
+                        continue
+                    if (mod_name_with_jar in mods_list and ((decompiled_with_deobf and deobf_enabled) or
+                                                            (not decompiled_with_deobf and not deobf_enabled))):
+                        use_cached_decomp.append(mod_name_with_jar)
                     else:
-                        shutil.rmtree(path_to_mod)
-                        logging.info(f'Found {mod_name} in decompiled_mods.'
+                        shutil.rmtree(mod_path)
+                        logging.info(f'Found {mod_name_with_jar} in decompiled_mods. '
                                      f"Current mod list doesn't include it. Removing.")
-            except FileNotFoundError:
-                pass
-            # try:  # remove mods from deobfuscated_mods which in decompiled_mods
-            #     for mod in os.listdir(PathUtils.DEOBFUSCATED_MODS_PATH):
-            #         mod_name = mod.removesuffix('.jar').removesuffix('_mapped_official') + '.jar'
-            #         if mod_name in use_cached:
-            #             os.remove(os.path.join(os.path.join(PathUtils.DEOBFUSCATED_MODS_PATH, mod)))
-            #             logging.info(f'Removed {mod_name} from deobuscated mods, '
-            #                          f'since it already in decompilated mods cache.')
-            # except FileNotFoundError:
-            #     pass
-            try:  # remove mods from deobfuscated_mods which not in current mods_list
+
+            if not os.path.exists(PathUtils.DEOBFUSCATED_CACHE_PATH):
+                FileUtils.remove_folder(PathUtils.DEOBFUSCATED_MODS_PATH)
+            elif os.path.exists(PathUtils.DEOBFUSCATED_MODS_PATH):
+                with open(PathUtils.DEOBFUSCATED_CACHE_PATH, 'r') as f:
+                    cache = json.loads(f.read())
                 for mod in os.listdir(PathUtils.DEOBFUSCATED_MODS_PATH):
-                    mod_name = mod.removesuffix('.jar').removesuffix('_mapped_official') + '.jar'
-                    if mod_name in mods_list_without_space:
-                        use_cached_deobf.append(mod_name)
-                        logging.info(f'Found {mod_name} in deobfuscated mods. Coping skipped.')
+                    mod_path = os.path.join(PathUtils.DEOBFUSCATED_MODS_PATH, mod)
+                    mod_name_without_jar = mod.removesuffix('.jar').removesuffix('_mapped_official')
+                    mod_name_with_jar = mod_name_without_jar + '.jar'
+                    if mod_name_without_jar.endswith('.json'):
+                        continue
+                    if mod_name_without_jar not in cache:
+                        os.remove(mod_path)
+                        logging.info(f'Found {mod} in deobfuscated mods.'
+                                     f"But it's not in cache. Removing. "
+                                     f'Maybe deobfuscation of it was interrupted.')
+                        continue
+                    if (mod_name_without_jar in mod_hashes and
+                            cache[mod_name_without_jar] != mod_hashes[mod_name_without_jar]):
+                        os.remove(mod_path)
+                        logging.info(f'Found {mod} in deobfuscated mods.'
+                                     f'But hash is not same with mod from current mod list. Removing. '
+                                     f"Maybe mod changed without changing it's name")
+                        continue
+                    if mod_name_with_jar in mods_list:
+                        use_cached_deobf.append(mod_name_with_jar)
                     else:
-                        os.remove(os.path.join(PathUtils.DEOBFUSCATED_MODS_PATH, mod))
-                        logging.info(f'Found {mod_name} in deobfuscated mods.'
+                        os.remove(mod_path)
+                        logging.info(f'Found {mod_name_with_jar} in deobfuscated mods. '
                                      f"Current mod list doesn't include it. Removing.")
-            except FileNotFoundError:
-                pass
-        self.use_cached_signal.emit(use_cached_decomp)
 
-        for mod in jar_in_jar_mods_list:
-            if mod in use_cached_deobf:
-                os.remove(os.path.join(PathUtils.TMP_MODS_PATH, mod))
+        for mod_name in use_cached_deobf:
+            logging.info(f'Found {mod_name} in cache. Removing from tmp folder.')
+            os.remove(os.path.join(PathUtils.TMP_MODS_PATH, mod_name))
 
-        for i, mod in enumerate(mods_list):
-            if mod.replace(' ', '_') in use_cached_deobf or mod in jar_in_jar_mods_list:
-                continue
-            mod_path = os.path.join(mods_path, mod)
-            self.progress.emit(int((i / mods_count) * 100), f'Copying {mod}')
-            logging.info(f'Starting copying {mod}')
-            shutil.copy(mod_path, PathUtils.TMP_MODS_PATH)
-            if ' ' in mod:
-                os.rename(os.path.join(PathUtils.TMP_MODS_PATH, mod),
-                          os.path.join(PathUtils.TMP_MODS_PATH, mod.replace(' ', '_')))
-            logging.info(f'Finished copying {mod}')
-
-        self.progress.emit(100, 'Finished copying mods.')
+        self.use_cached_signal.emit(use_cached_decomp)

MDGLogic/CriticalMBThread.py

@@ -4,11 +4,12 @@
 
 
 class CriticalMBThread(AbstractMDGThread):
-    def __init__(self, title: str, text: str) -> None:
+    def __init__(self, title: str, text: str, widget_name: str) -> None:
         self.title = title
         self.text = text
+        self.widget_name = widget_name
         super().__init__(None)
 
     def run(self) -> None:
         time.sleep(0.1)
-        self.critical_signal.emit(self.title, self.text)
+        self.critical_signal.emit(self.title, self.text, self.widget_name)

MDGLogic/DecompilationMainThread.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import time
@@ -10,17 +9,13 @@
 
 
 def get_mods_iter(use_cached: list) -> Iterator[os.PathLike]:
-    try:
+    if os.path.exists(PathUtils.TMP_MODS_PATH):
         for mod in os.listdir(PathUtils.TMP_MODS_PATH):
             yield os.path.join(PathUtils.TMP_MODS_PATH, mod)
-    except FileNotFoundError:
-        pass
-    try:
+    if os.path.exists(PathUtils.DEOBFUSCATED_MODS_PATH):
         for mod in os.listdir(PathUtils.DEOBFUSCATED_MODS_PATH):
-            if mod.removesuffix('_mapped_official.jar') + '.jar' not in use_cached:
+            if mod.removesuffix('_mapped_official.jar') + '.jar' not in use_cached and not mod.endswith('.json'):
                 yield os.path.join(PathUtils.DEOBFUSCATED_MODS_PATH, mod)
-    except FileNotFoundError:
-        pass
 
 
 class DecompilationMainThread(AbstractDeobfDecompMainThread):
@@ -42,12 +37,13 @@ def run(self) -> None:
 
         FileUtils.create_folder(PathUtils.DECOMPILED_MODS_PATH)
 
-        with open(os.path.join(PathUtils.DECOMPILED_MODS_PATH, 'cache.json'), 'w') as f:
-            cache = []
-            for mod in os.listdir(PathUtils.DECOMPILED_MODS_PATH):
-                if os.path.isdir(os.path.join(PathUtils.DECOMPILED_MODS_PATH, mod)):
-                    cache.append(mod)
-            f.write(json.dumps(cache))
+        try:
+            os.remove(PathUtils.DECOMPILED_CACHE_PATH)
+        except FileNotFoundError:
+            pass
+        for mod in os.listdir(PathUtils.DECOMPILED_MODS_PATH):
+            if os.path.isdir(os.path.join(PathUtils.DECOMPILED_MODS_PATH, mod)):
+                FileUtils.append_cache(PathUtils.DECOMPILED_CACHE_PATH, mod, FileUtils.get_original_mod_hash(mod))
 
         while processed_mods_count < mods_to_decomp_count:
             if len(self.threads) < allocated_threads_count and started_mods_count < mods_to_decomp_count:

MDGLogic/DecompilationThread.py

@@ -1,4 +1,3 @@
-import json
 import os.path
 import subprocess
 import threading
@@ -16,10 +15,16 @@ class DecompilationThread(AbstractDeobfDecompThread):
 
     def run(self) -> None:
         decomp_cmd = self.serialized_widgets['decomp_cmd_line_edit']['text']
+        mod_name = os.path.basename(self.mod_path)
         result_folder = os.path.join(PathUtils.DECOMPILED_MODS_PATH,
-                                     os.path.basename(self.mod_path.removesuffix('.jar')))
+                                     mod_name.removesuffix('.jar'))
         FileUtils.create_folder(result_folder)
-        decomp_cmd_formatted = decomp_cmd.format(path_to_jar=self.mod_path, out_path=result_folder)
+
+        java_home = self.serialized_widgets['decompiler_java_home_line_edit']['text']
+        decomp_cmd_formatted = (PathUtils.format_decompiler_command(decomp_cmd,
+                                                                    java_home,
+                                                                    self.mod_path,
+                                                                    result_folder))
         self.cmd = subprocess.Popen(decomp_cmd_formatted, shell=True)
         while self.cmd.poll() is None:
             time.sleep(0.1)
@@ -35,9 +40,6 @@ def run(self) -> None:
         if os.listdir(result_folder) or list(Path(result_folder).rglob('*.java')):
             self.success = True
             with self.cache_lock:
-                cache_path = os.path.join(PathUtils.DECOMPILED_MODS_PATH, 'cache.json')
-                with open(cache_path, 'r') as f:
-                    cache = json.loads(f.read())
-                cache.append(os.path.basename(result_folder))
-                with open(cache_path, 'w') as f:
-                    f.write(json.dumps(cache))
+                FileUtils.append_cache(PathUtils.DECOMPILED_CACHE_PATH,
+                                       mod_name,
+                                       FileUtils.get_original_mod_hash(mod_name))