diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..1137b8b --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,41 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Pytest and flake8 linting + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + poetry lock + poetry install --with dev + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + poetry run python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + poetry run python -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + poetry run python -m pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a278402..07dce49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,28 @@ - # See https://pre-commit.com for more information - # See https://pre-commit.com/hooks.html for more hooks - repos: - - repo: https://github.com/ambv/black - rev: 24.4.2 - hooks: - - id: black - language_version: python3.12 - - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 - hooks: - - id: flake8 - language_version: python3.12 - - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.10.0' # Use the sha / tag you want to point at - hooks: - - id: mypy - language_version: python3.12 - additional_dependencies: [types-PyYAML==6.0.12.*] \ No newline at end of file +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/ambv/black + rev: 24.10.0 + hooks: + - id: black + language_version: python3.12 + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + language_version: python3.12 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.13.0" # Use the sha / tag you want to point at + hooks: + - id: mypy + language_version: python3.12 + additional_dependencies: [types-PyYAML==6.0.12.*] + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.3 + hooks: + - id: gitleaks \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c9ab5d5 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Stronghold Crusader Dashboard + +## What is this project about? + +Stronghold Crusader Dashboard is a python based CLI and dashboard app designed to track and visualize Stronghold Crusader Extreme statistics. + +[Stronghold Crusader Extreme](https://fireflyworlds.com/games/strongholdcrusader/) is a real time strategy game made by Firefly Studios. +While the game does feature a stats screen after each match, it does miss more modern features like a graph of stats over time and some deeper statistics. This project aims to provide easily explorable stat tracking for your Stronghold Crusader Extreme matches. + +## Project Inspiration + +This project is inspired by the constant work and videos from [Udwin](https://www.youtube.com/@Udwin), [Krarilotus](https://www.youtube.com/@Krarilotus), [Xander10alpha](https://www.youtube.com/@Xander10alpha) and many more. +Most of the work figuring out the memory adresses was already done in other [projects](https://github.com/patel-nikhil/SHCLiveStatReader) by [patel-nikhil](https://github.com/patel-nikhil) and others. + +## Disclaimer + +Stronghold Crusader is the legal property of Firefly Studios. This project has no affiliation and is not endorsed or supported by Firefly Studios. This content provided in this project does not modify the executable or other game files in any way. diff --git a/db/README.MD b/db/README.MD new file mode 100644 index 0000000..5724f80 --- /dev/null +++ b/db/README.MD @@ -0,0 +1,4 @@ +# db folder + +This folder will contain the db.sqlite file. +The db.sqlite file will be automatically be created at module initilization. diff --git a/img/Image253.png b/img/Image253.png new file mode 100644 index 0000000..25cf03c Binary files /dev/null and b/img/Image253.png differ diff --git a/img/Image254.png b/img/Image254.png new file mode 100644 index 0000000..ee1c21d Binary files /dev/null and b/img/Image254.png differ diff --git a/img/Image255.png b/img/Image255.png new file mode 100644 index 0000000..38ddf58 Binary files /dev/null and b/img/Image255.png differ diff --git a/img/Image264.png b/img/Image264.png new file mode 100644 index 0000000..ba6e295 Binary files /dev/null and b/img/Image264.png differ diff --git a/img/Image570.png b/img/Image570.png new file mode 100644 index 0000000..87c06cc Binary files /dev/null and b/img/Image570.png differ diff --git a/img/Image620.png b/img/Image620.png new file mode 100644 index 0000000..cb81495 Binary files /dev/null and b/img/Image620.png differ diff --git a/img/Image622.png b/img/Image622.png new file mode 100644 index 0000000..3cceb22 Binary files /dev/null and b/img/Image622.png differ diff --git a/img/Image624.png b/img/Image624.png new file mode 100644 index 0000000..4be0b94 Binary files /dev/null and b/img/Image624.png differ diff --git a/img/Image628.png b/img/Image628.png new file mode 100644 index 0000000..f23fcd5 Binary files /dev/null and b/img/Image628.png differ diff --git a/img/Image632.png b/img/Image632.png new file mode 100644 index 0000000..0cd3400 Binary files /dev/null and b/img/Image632.png differ diff --git a/img/Image634.png b/img/Image634.png new file mode 100644 index 0000000..7675dc3 Binary files /dev/null and b/img/Image634.png differ diff --git a/img/Image636.png b/img/Image636.png new file mode 100644 index 0000000..afb4aa6 Binary files /dev/null and b/img/Image636.png differ diff --git a/img/Image638.png b/img/Image638.png new file mode 100644 index 0000000..c510465 Binary files /dev/null and b/img/Image638.png differ diff --git a/img/Image640.png b/img/Image640.png new file mode 100644 index 0000000..7ca44eb Binary files /dev/null and b/img/Image640.png differ diff --git a/img/Image642.png b/img/Image642.png new file mode 100644 index 0000000..56697a1 Binary files /dev/null and b/img/Image642.png differ diff --git a/img/Image644.png b/img/Image644.png new file mode 100644 index 0000000..8d12b4e Binary files /dev/null and b/img/Image644.png differ diff --git a/img/Image646.png b/img/Image646.png new file mode 100644 index 0000000..a4b040c Binary files /dev/null and b/img/Image646.png differ diff --git a/img/Image648.png b/img/Image648.png new file mode 100644 index 0000000..069aea3 Binary files /dev/null and b/img/Image648.png differ diff --git a/img/Image650.png b/img/Image650.png new file mode 100644 index 0000000..72af2cb Binary files /dev/null and b/img/Image650.png differ diff --git a/img/Image652.png b/img/Image652.png new file mode 100644 index 0000000..d90ddf6 Binary files /dev/null and b/img/Image652.png differ diff --git a/img/Image654.png b/img/Image654.png new file mode 100644 index 0000000..e6ca559 Binary files /dev/null and b/img/Image654.png differ diff --git a/img/Image656.png b/img/Image656.png new file mode 100644 index 0000000..614285d Binary files /dev/null and b/img/Image656.png differ diff --git a/img/Image658.png b/img/Image658.png new file mode 100644 index 0000000..3cf1153 Binary files /dev/null and b/img/Image658.png differ diff --git a/img/Image660.png b/img/Image660.png new file mode 100644 index 0000000..d5149ff Binary files /dev/null and b/img/Image660.png differ diff --git a/img/Image662.png b/img/Image662.png new file mode 100644 index 0000000..ed2f595 Binary files /dev/null and b/img/Image662.png differ diff --git a/img/Image664.png b/img/Image664.png new file mode 100644 index 0000000..7625fb7 Binary files /dev/null and b/img/Image664.png differ diff --git a/img/Image730.png b/img/Image730.png new file mode 100644 index 0000000..6870d6b Binary files /dev/null and b/img/Image730.png differ diff --git a/img/ST03_Woodcutters_Hut.png b/img/ST03_Woodcutters_Hut.png new file mode 100644 index 0000000..7c94cdd Binary files /dev/null and b/img/ST03_Woodcutters_Hut.png differ diff --git a/img/ST05_Iron_Mine.png b/img/ST05_Iron_Mine.png new file mode 100644 index 0000000..a0e3fa4 Binary files /dev/null and b/img/ST05_Iron_Mine.png differ diff --git a/img/ST06_Pitch_Digger.png b/img/ST06_Pitch_Digger.png new file mode 100644 index 0000000..b1d8687 Binary files /dev/null and b/img/ST06_Pitch_Digger.png differ diff --git a/img/ST20_Quarry.png b/img/ST20_Quarry.png new file mode 100644 index 0000000..50c9cc4 Binary files /dev/null and b/img/ST20_Quarry.png differ diff --git a/img/ST26_Tradepost.png b/img/ST26_Tradepost.png new file mode 100644 index 0000000..e951c06 Binary files /dev/null and b/img/ST26_Tradepost.png differ diff --git a/img/armys1.png b/img/armys1.png new file mode 100644 index 0000000..21bc419 Binary files /dev/null and b/img/armys1.png differ diff --git a/img/armys10.png b/img/armys10.png new file mode 100644 index 0000000..e672ab7 Binary files /dev/null and b/img/armys10.png differ diff --git a/img/armys11.png b/img/armys11.png new file mode 100644 index 0000000..2647a30 Binary files /dev/null and b/img/armys11.png differ diff --git a/img/armys12.png b/img/armys12.png new file mode 100644 index 0000000..0d97c0f Binary files /dev/null and b/img/armys12.png differ diff --git a/img/armys13.png b/img/armys13.png new file mode 100644 index 0000000..6b36284 Binary files /dev/null and b/img/armys13.png differ diff --git a/img/armys14.png b/img/armys14.png new file mode 100644 index 0000000..f31cb09 Binary files /dev/null and b/img/armys14.png differ diff --git a/img/armys15.png b/img/armys15.png new file mode 100644 index 0000000..d8846f6 Binary files /dev/null and b/img/armys15.png differ diff --git a/img/armys16.png b/img/armys16.png new file mode 100644 index 0000000..fa87e59 Binary files /dev/null and b/img/armys16.png differ diff --git a/img/armys17.png b/img/armys17.png new file mode 100644 index 0000000..e92993e Binary files /dev/null and b/img/armys17.png differ diff --git a/img/armys18.png b/img/armys18.png new file mode 100644 index 0000000..0eb3721 Binary files /dev/null and b/img/armys18.png differ diff --git a/img/armys19.png b/img/armys19.png new file mode 100644 index 0000000..9021821 Binary files /dev/null and b/img/armys19.png differ diff --git a/img/armys2.png b/img/armys2.png new file mode 100644 index 0000000..621b625 Binary files /dev/null and b/img/armys2.png differ diff --git a/img/armys20.png b/img/armys20.png new file mode 100644 index 0000000..1049493 Binary files /dev/null and b/img/armys20.png differ diff --git a/img/armys21.png b/img/armys21.png new file mode 100644 index 0000000..704b1fc Binary files /dev/null and b/img/armys21.png differ diff --git a/img/armys22.png b/img/armys22.png new file mode 100644 index 0000000..00f494d Binary files /dev/null and b/img/armys22.png differ diff --git a/img/armys23.png b/img/armys23.png new file mode 100644 index 0000000..68392c3 Binary files /dev/null and b/img/armys23.png differ diff --git a/img/armys24.png b/img/armys24.png new file mode 100644 index 0000000..fbe38ad Binary files /dev/null and b/img/armys24.png differ diff --git a/img/armys25.png b/img/armys25.png new file mode 100644 index 0000000..9a2d3a8 Binary files /dev/null and b/img/armys25.png differ diff --git a/img/armys26.png b/img/armys26.png new file mode 100644 index 0000000..d3d0db5 Binary files /dev/null and b/img/armys26.png differ diff --git a/img/armys3.png b/img/armys3.png new file mode 100644 index 0000000..1559f47 Binary files /dev/null and b/img/armys3.png differ diff --git a/img/armys4.png b/img/armys4.png new file mode 100644 index 0000000..ac16866 Binary files /dev/null and b/img/armys4.png differ diff --git a/img/armys5.png b/img/armys5.png new file mode 100644 index 0000000..ef86bb6 Binary files /dev/null and b/img/armys5.png differ diff --git a/img/armys6.png b/img/armys6.png new file mode 100644 index 0000000..30a7794 Binary files /dev/null and b/img/armys6.png differ diff --git a/img/armys7.png b/img/armys7.png new file mode 100644 index 0000000..90df421 Binary files /dev/null and b/img/armys7.png differ diff --git a/img/armys8.png b/img/armys8.png new file mode 100644 index 0000000..086473f Binary files /dev/null and b/img/armys8.png differ diff --git a/img/armys9.png b/img/armys9.png new file mode 100644 index 0000000..74e1305 Binary files /dev/null and b/img/armys9.png differ diff --git a/img/contents07_town_buildings.png b/img/contents07_town_buildings.png new file mode 100644 index 0000000..d48a1b0 Binary files /dev/null and b/img/contents07_town_buildings.png differ diff --git a/img/null_sketch.png b/img/null_sketch.png new file mode 100644 index 0000000..81bd7e5 Binary files /dev/null and b/img/null_sketch.png differ diff --git a/memory/building.yaml b/memory/building.yaml new file mode 100644 index 0000000..a12d994 --- /dev/null +++ b/memory/building.yaml @@ -0,0 +1,11 @@ +address: 0x00F98DB2 +type: integer +category: building +offsets: + owneroffset: 0x4 + workersneededoffset: 0xC6 + workersworkingoffset: 0xC8 + workersmissingoffset: 0xCA + snoozedoffset: 0x1C4 + offset: 0x32C +total: 0x00F989A0 diff --git a/memory/greatestlord.yaml b/memory/greatestlord.yaml new file mode 100644 index 0000000..6c60a99 --- /dev/null +++ b/memory/greatestlord.yaml @@ -0,0 +1,90 @@ +Map: + MapName: + address: 0x24B649C + type: string + MapStartYear: + address: 0x24BA938 + type: integer + MapStartMonth: + address: 0x24BA93C + type: byte + MapEndYear: + address: 0x24BA940 + type: integer + MapEndMonth: + address: 0x24BA944 + type: byte + AdvantageSetting: + address: 0x24B7F4C + type: byte +Player: + Active: + address: 0x24BA557 + type: boolean + category: other + offset: 0x1 + Name: + address: 0x24BA286 + type: string + category: other + offset: 0x5A + Team: + address: 0x24BAAB6 + type: byte + category: other + offset: 0x1 + Gold: + address: 0x24BA564 + type: integer + category: other + offset: 0x4 + Units: + address: 0x24BA888 + type: integer + category: military + offset: 0x4 + WeightedTroopsKilled: + address: 0x24BA840 + type: integer + category: military + offset: 0x4 + WeightedBuildingsDestroyed: + address: 0x24BA864 + type: integer + category: military + offset: 0x4 + LordKills: + address: 0x24BA7EA + type: byte + category: military + offset: 0x1 + Food: + address: 0x24BA730 + type: integer + category: resource + offset: 0x4 + Wood: + address: 0x24BA79C + type: integer + category: resource + offset: 0x4 + Stone: + address: 0x24BA778 + type: integer + category: resource + offset: 0x4 + Weapons: + address: 0x24BA7F8 + type: integer + category: resource + offset: 0x4 + GoodsSent: + address: 0x24BA8D0 + type: integer + category: resource + offset: 0x4 + GoodsReceived: + address: 0x24BA8AC + type: integer + category: resource + offset: 0x4 diff --git a/memory/lord.yaml b/memory/lord.yaml new file mode 100644 index 0000000..e1b0100 --- /dev/null +++ b/memory/lord.yaml @@ -0,0 +1,178 @@ +map_offsets: + offset: 0x0 + memory: + - address: 0x24b649c + stat_offsets: + - name: map_name + offset: 0x0 + type: string +lord_basic_offsets: + offset: 0x1 + memory: + - address: 0x24ba557 + stat_offsets: + - name: active + offset: 0x0 + type: boolean + - name: team + offset: 0x55f + type: byte +lord_global_offsets: + offset: 0x4 + memory: + - address: 0x11f5ab0 + stat_offsets: + - name: num_eco_buildings + offset: 0x0 + type: int + - name: num_total_buildings + offset: 0x9dccc + type: int + - address: 0x24ba564 + stat_offsets: + - name: total_gold + offset: 0x0 + type: int + - name: weighted_troops_killed + offset: 0x2dc + type: int + - name: weighted_buildings_destroyed + offset: 0x300 + type: int + - name: goods_received + offset: 0x348 + type: int + - name: goods_sent + offset: 0x36c + type: int +lord_name_offsets: + offset: 0x5a + memory: + - address: 0x24ba286 + stat_offsets: + - name: name + offset: 0x0 + type: string +lord_stat_offsets: + offset: 0x39f4 + memory: + - address: 0x11f24a0 + stat_offsets: + - name: housing + offset: 0x0 + type: word + - name: units + offset: 0x344 + type: int + - name: siege_engines + offset: 0x348 + type: int + - name: popularity + offset: 0x3d0 + type: word + - name: wood + offset: 0x464 + type: int + - name: hops + offset: 0x468 + type: int + - name: stone + offset: 0x46c + type: int + - name: iron + offset: 0x474 + type: int + - name: pitch + offset: 0x478 + type: int + - name: wheat + offset: 0x480 + type: int + - name: bread + offset: 0x484 + type: int + - name: cheese + offset: 0x488 + type: int + - name: meat + offset: 0x48c + type: int + - name: apples + offset: 0x490 + type: int + - name: ale + offset: 0x494 + type: int + - name: gold + offset: 0x498 + type: int + - name: flour + offset: 0x49c + type: int + - name: bows + offset: 0x4a0 + type: int + - name: crossbows + offset: 0x4a4 + type: int + - name: spears + offset: 0x4a8 + type: int + - name: pikes + offset: 0x4ac + type: int + - name: maces + offset: 0x4b0 + type: int + - name: swords + offset: 0x4b4 + type: int + - name: leather_armor + offset: 0x4b8 + type: int + - name: metal_armor + offset: 0x4bc + type: int + - address: 0x11f44c8 + stat_offsets: + - name: total_food + offset: 0x0 + type: int + - name: population + offset: 0xe4 + type: word + - name: taxes + offset: 0xec + type: int + - name: weighted_losses + offset: 0x204 + type: int + - name: ale_coverage + offset: 0x208 + type: int + - name: num_inns + offset: 0x234 + type: int + # - name: working_woodcutters + # offset: 0xa80 + # type: int + - name: num_bakery + offset: 0xaf4 + type: int + - address: 0x11f5d14 + stat_offsets: + - name: weighted_units + offset: 0x0 + type: int + - name: num_farms + offset: 0x40 + type: int + - name: num_iron_mines + offset: 0xa4 + type: int + - name: num_pitchrigs + offset: 0xa8 + type: int + - name: num_quarries + offset: 0xac + type: int diff --git a/memory/names.yaml b/memory/names.yaml new file mode 100644 index 0000000..ae726e1 --- /dev/null +++ b/memory/names.yaml @@ -0,0 +1,315 @@ +Buildings: + 1: hovel + 2: house + 3: woodcuttershut + 4: oxtether + 5: ironmine + 6: pitchrig + 7: huntershut + 8: mercenarypost + 9: barracks + 10: stockpile + 11: armory + 12: fletcher + 13: blacksmith + 14: poleturner + 15: armourer + 16: tanner + 17: bakery + 18: brewery + 19: granary + 20: quarry + 21: quarrypile + 22: inn + 23: apothecary + 24: engineerguild + 25: tunnelerguild + 26: marketplace + 27: well + 28: oilsmelter + 29: siege_tent + 30: wheat_farm + 31: hop_farm + 32: apple_farm + 33: dairy_farm + 34: mill + 35: stables + 36: chapel + 37: church + 38: cathedral + 39: unused + 40: keep_one + 41: keep_two + 42: keep_three + 43: keep_four + 44: keep_five + 45: large_gatehouse + 46: small_gatehouse + 47: main_wood + 48: postern_gate + 49: drawbridge + 50: tunnel + 51: camp_fire + 52: signpost + 53: parade_ground + 54: s_fballista + 55: campground + 56: parade_ground + 57: parade_ground + 58: parade_ground + 59: parade_ground + 60: gatehouse + 61: tower + 62: gallows + 63: stocks + 64: witch_hoist + 65: maypole + 66: garden + 67: killingpit + 68: pitchditch + 69: unused + 70: waterpot + 71: keepdoor_left + 72: keepdoor_right + 73: keepdoor + 74: tower_one + 75: tower_two + 76: tower_three + 77: tower_four + 78: tower_five + 79: unused + 80: s_catapult + 81: s_trebuchet + 82: s_batteringram + 83: s_siegetower + 84: s_shield + 85: unused + 86: s_mangonel + 87: s_balista + 88: unused + 89: unused + 90: unused + 91: cesspit + 92: burningstake + 93: gibbet + 94: dungeon + 95: stretchingrack + 96: rack_flogging + 97: choppingblock + 98: dunkingstool + 99: dogcage + 100: statue + 101: shrine + 102: bee_hive + 103: dancingbear + 104: pond + 105: bear cave + 106: outpost + 107: outpost +Units: + 1: artifacts + 2: artifacts + 3: artifacts + 4: artifacts + 5: artifacts + 6: artifacts + 7: artifacts + 8: artifacts + 9: artifacts + 10: artifacts + 11: artifacts + 12: artifacts + 13: artifacts + 14: artifacts + 15: artifacts + 16: artifacts + 17: peasant + 18: europ archer + 19: woodcutter + 20: fletcher walking + 21: ox + 22: black humanoid mess + 23: fleshy something + 24: black humanoid mess + 25: stone mason quarry worker + 26: windmillboy inside + 27: fletcher working + 28: artifacts + 29: palm tree falling + 30: palm tree falling + 31: birch tree falling + 32: quarry ox leader + 33: farm worker + 34: ladder from ladderman + 35: ladder from ladderman + 36: baker + 37: windmillboy outside + 38: artifacts + 39: spearman + 40: pikeman + 41: crossbowman + 42: swordsman + 43: maceman + 44: knight horse + 45: artifacts + 46: artifacts + 47: some worker + 48: artifacts + 49: artifacts + 50: brewery stirring shadow + 51: brewery stirring + 52: artifacts + 53: brewery woman walking + 54: artifacts + 55: artifacts + 56: artifacts + 57: something inside + 58: swordsmith inside + 59: armourer outside + 60: artifacts + 61: iron mine worker + 62: catapult firing + 63: cow + 64: poleturner inside + 65: pitchrig worker inside + 66: baker shoving bread + 67: wood log being cut + 68: tanner inside + 69: tanner inside + 70: worker and tree + 71: worker and tree + 72: worker and tree + 73: pitchworker walking + 74: poleturner with pike + 75: tanner with leatherarmour + 76: different flags + 77: horse + 78: merchant walking + 79: some shadow + 80: some shadow + 81: unknown person walking and laying down + 82: siege tent + 83: mangonel + 84: trebuchet loading + 85: glitchy worker + 86: engineer (with shovel) + 87: artifacts + 88: hunter / ... / wheel of cheese + 89: hunter (with animal) + 90: skinned animal (by hunter) + 91: deer + 92: lion + 93: rabbit + 94: camel + 95: dog + 96: priest + 97: farm tree stages + something + 98: horse in stable + 99: castle lady + 100: europ lord + 101: jester + 102: armourer (carrying iron) + 103: armourer hammering + 104: shields being stacked + 105: shadow of tunneler + 106: tunneler + 107: artifacts + 108: monk shadow + 109: monk + 110: knight riding + 111: engineer filling pitch shadows + 112: engineer filling pitch shadows + 113: engineer filling pitch + 114: maypole idle + 115: maypole + 116: dancing bear + 117: dancing bear + 118: dancing bear + 119: dancing bear + 120: tower ballista firing + 121: shield walking + 122: part of something + 123: part of battering ram + 124: parts of siege tower + 125: chicken shadow + 126: chicken + 127: mother with child (hovel) + 128: child (male) + 129: child (female) + 130: tunneler building tunnel + 131: jester juggling + 132: fireeater + 133: healer + 134: cow landing + 135: cow landing 2 + 136: artifacts + 137: artifacts + 138: artifacts + 139: artifacts + 140: artifacts + 141: artifacts + 142: artifacts + 143: some shadow + 144: person of 143 + 145: person / fire arrow + 146: innkeeper rolling beer + 147: armourer idle + 148: artifacts + 149: artifacts + 150: artifacts + 151: artifacts + 152: artifacts / chicken + 153: artifacts / chicken + 154: chicken + 155: artifacts / merchant in shop + 156: artifacts + 157: shadow of 159 + 158: shadow of 160 + 159: part of something + 160: part of something + 161: part of something + 162: healer mixing herbs + 163: artifacts + 164: artifacts + 165: artifacts + 166: artifacts + 167: bird + 168: bird + 169: people in inn + 170: people in inn + 171: people in inn + 172: artifacts + 173: burning stake + 174: burning stake + 175: burning stake + 176: water boy + 177: water boy shadow + 178: water boy + 179: water boy + 180: water boy + 181: water boy + 182: water boy + 183: shadows + 184: shadows + 185: something + 186: something + 187: something + 188: something + 189: flag / part of arab archer + 190: arab archer + 191: slave + 192: slinger + 193: assassin + 194: archer riding + 195: arab swordsman + 196: fire thrower + 197: fire ballista engineers / firing + 198: horse shadow + 199: horse archer + 200: part of dog and flag + 201: part of dog and flag + 202: dog shadow + 203: part of dog + 204: part of arab lord + 205: arab lord + diff --git a/memory/player.yaml b/memory/player.yaml new file mode 100644 index 0000000..473bf29 --- /dev/null +++ b/memory/player.yaml @@ -0,0 +1,245 @@ +Active: + address: 0x24BA557 + type: boolean + category: other + offset: 0x1 +Name: + address: 0x24BA286 + type: string + category: other + offset: 0x5A +Team: + address: 0x24BAAB6 + type: byte + category: other + offset: 0x1 +MapName: + address: 0x24B649C + type: string + category: other + offset: 0 +TotalGold: + address: 0x24BA564 + type: integer + category: other + offset: 0x4 +Gold: + address: 0x11F2938 + type: integer + category: other + offset: 0x39F4 +TaxesSetting: + address: 0x11F45B4 + type: integer + category: other + offset: 0x39F4 +Units: + address: 0x11F27E4 + type: integer + category: military + offset: 0x39F4 +WeightedUnits: + address: 0x11F5D14 + type: integer + category: military + offset: 0x39F4 +SiegeEngineCount: + address: 0x11F27E8 + type: integer + category: military + offset: 0x39F4 +CurrentEconomyBuildings: + address: 0x11F5AB0 + type: integer + category: military + offset: 0x4 +CurrentTotalBuildings: + address: 0x129377C + type: integer + category: military + offset: 0x4 +FarmCount: + address: 0x11F5D54 + type: integer + category: other + offset: 0x39F4 +InnCount: + address: 0x11F46FC + type: integer + category: other + offset: 0x39F4 +PitchRigCount: + address: 0x11F5DBC + type: integer + category: other + offset: 0x39F4 +IronMineCount: + address: 0x11F5DB8 + type: integer + category: other + offset: 0x39F4 +StoneQuarryCount: + address: 0x11F5DC0 + type: integer + category: other + offset: 0x39F4 +BakeryCount: + address: 0x11F4FBC + type: integer + category: other + offset: 0x39F4 +WorkingWoodcutterCount: + address: 0x11F4F48 + type: integer + category: other + offset: 0x39F4 +TotalFood: + address: 0x11f44C8 + type: integer + category: other + offset: 0x39F4 +Popularity: + address: 0x11F2870 + type: word + category: other + offset: 0x39F4 +Population: + address: 0x11F45AC + type: word + category: other + offset: 0x39F4 +Housing: + address: 0x11F24A0 + type: word + category: other + offset: 0x39F4 +Wood: + address: 0x11F2904 + type: integer + category: resource + offset: 0x39F4 +Stone: + address: 0x11F290C + type: integer + category: resource + offset: 0x39F4 +Iron: + address: 0x11F2914 + type: integer + category: resource + offset: 0x39F4 +Pitch: + address: 0x11F2918 + type: integer + category: resource + offset: 0x39F4 +Apples: + address: 0x11F2930 + type: integer + category: resource + offset: 0x39F4 +Meat: + address: 0x11F292C + type: integer + category: resource + offset: 0x39F4 +Cheese: + address: 0x11F2928 + type: integer + category: resource + offset: 0x39F4 +Bread: + address: 0x11F2924 + type: integer + category: resource + offset: 0x39F4 +MetalArmor: + address: 0x11F295C + type: integer + category: resource + offset: 0x39F4 +LeatherArmor: + address: 0x11F2958 + type: integer + category: resource + offset: 0x39F4 +Swords: + address: 0x11F2954 + type: integer + category: resource + offset: 0x39F4 +Maces: + address: 0x11F2950 + type: integer + category: resource + offset: 0x39F4 +Pikes: + address: 0x11F294C + type: integer + category: resource + offset: 0x39F4 +Spears: + address: 0x11F2948 + type: integer + category: resource + offset: 0x39F4 +Crossbows: + address: 0x11F2944 + type: integer + category: resource + offset: 0x39F4 +Bows: + address: 0x11F2940 + type: integer + category: resource + offset: 0x39F4 +Flour: + address: 0x11F293C + type: integer + category: resource + offset: 0x39F4 +Hops: + address: 0x11F2908 + type: integer + category: resource + offset: 0x39F4 +Ale: + address: 0x11F2934 + type: integer + category: resource + offset: 0x39F4 +Wheat: + address: 0x11F2920 + type: integer + category: resource + offset: 0x39F4 +WeightedLosses: + address: 0x11F46CC + type: integer + category: military + offset: 0x39F4 +WeightedTroopsKilled: + address: 0x24BA840 + type: integer + category: military + offset: 0x4 +WeightedBuildingsDestroyed: + address: 0x24BA864 + type: integer + category: military + offset: 0x4 +GoodsSent: + address: 0x24BA8D0 + type: integer + category: resource + offset: 0x4 +GoodsReceived: + address: 0x24BA8AC + type: integer + category: resource + offset: 0x4 +AleCoveragePercent: + address: 0x011F46D0 + type: integer + category: popularity + offset: 0x39F4 diff --git a/memory/unit.yaml b/memory/unit.yaml new file mode 100644 index 0000000..24101db --- /dev/null +++ b/memory/unit.yaml @@ -0,0 +1,35 @@ +address: 0x145D4D8 #0x145D048 #0x148E598 +type: integer +category: unit +offsets: + coloroffset: 0x4 #{0:white,1:red,2:orange,3:yellow,4:blue,5:black,6:purple,7:cyan,8:green, >9: game} + moving: 0x24 + selected: 0x28 + hp_bar_percent: 0x2C #{0-10: 0-100%, 19: walls up,20: walls down, 21: compass_north, 22: compass_west, 23: compass_south, + # 24: compass_east, 25: compass_west, 26: compass_south, 27: compass_east,28: compass_north, + # 29: zoom_in, 30: zoom_out} + owneroffset: 0x74 + cur_hp: 0x3BC + max_hp: 0x3C0 + x_coord_cam: 0x410 + y_coord_cam: 0x414 + offset: 0x490 + unknown: + - 0x0C + - 0x10 + - 0x14 + - 0x20 # y_texture offset ? + - 0x38 # non-settable + - 0x3C + - 0x2C4 + - 0x2D0 + - 0x3C4 + - 0x3C8 + - 0x3E0 + - 0x3F0 + - 0x424 + - 0x42C + - 0x430 + - 0x438 +total: 0x145CA2C + \ No newline at end of file diff --git a/memory/weights.yaml b/memory/weights.yaml new file mode 100644 index 0000000..935af66 --- /dev/null +++ b/memory/weights.yaml @@ -0,0 +1,130 @@ +Buildings: + "1": 1 + "2": 1 + "3": 1 + "4": 1 + "5": 3 + "6": 3 + "7": 1 + "8": 3 + "9": 3 + "10": 0 + "11": 10 + "12": 5 + "13": 5 + "14": 5 + "15": 5 + "16": 5 + "17": 3 + "18": 3 + "19": 10 + "20": 3 + "21": 0 + "22": 3 + "23": 3 + "24": 5 + "25": 5 + "26": 3 + "27": 3 + "28": 3 + "29": 0 + "30": 1 + "31": 1 + "32": 1 + "33": 1 + "34": 3 + "35": 3 + "36": 3 + "37": 3 + "38": 3 + "39": 0 + "40": 0 + "41": 0 + "42": 0 + "43": 0 + "44": 0 + "45": 5 + "46": 5 + "47": 5 + "48": 5 + "49": 3 + "50": 0 + "51": 0 + "52": 0 + "53": 0 + "54": 0 + "55": 0 + "56": 0 + "57": 0 + "58": 0 + "59": 0 + "60": 5 + "61": 5 + "62": 1 + "63": 1 + "64": 1 + "65": 1 + "66": 1 + "67": 1 + "68": 1 + "69": 0 + "70": 3 + "71": 0 + "72": 0 + "73": 0 + "74": 5 + "75": 5 + "76": 5 + "77": 5 + "78": 5 + "79": 0 + "80": 0 + "81": 0 + "82": 0 + "83": 0 + "84": 0 + "85": 0 + "86": 0 + "87": 0 + "88": 0 + "89": 0 + "90": 0 + "91": 1 + "92": 1 + "93": 1 + "94": 1 + "95": 1 + "96": 1 + "97": 1 + "98": 1 + "99": 1 + "100": 1 + "101": 1 + "102": 1 + "103": 1 + "104": 1 + "105": 1 + "106": 20 + "107": 20 + "108": 0 +Resources: + Wood: 1 + Stone: 7 + Iron: 23 + Pitch: 15 + Apples: 4 + Meat: 4 + Cheese: 4 + Bread: 4 + MetalArmor: 30 + LeatherArmor: 12 + Swords: 30 + Maces: 30 + Pikes: 18 + Spears: 10 + Crossbows: 30 + Bows: 15 + Flour: 10 + Hops: 8 + Ale: 10 + Wheat: 8 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..bae8b2a --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,17 @@ +"""This module initializes the database and contains the other submodules.""" + +import pathlib + +import sqlalchemy as sa + +from .setup_logging import NonErrorFilter, setup_logging # noqa: F401 + +db_file = pathlib.Path.cwd() / "db" / "db.sqlite" +if not db_file.exists(): + db_file.touch() +engine = sa.create_engine("sqlite:///db/db.sqlite") + +setup_logging() + +PROCESS_NAME = "Stronghold_Crusader_Extreme.exe" +SHC_COLORS = ["#ef0008", "#d67300", "#e7c600", "#0084e7", "#6b6b6b", "#9c21b5", "#31a5bd", "#18bd10"] diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 0000000..54841c5 --- /dev/null +++ b/src/app/__init__.py @@ -0,0 +1 @@ +"""This module contains the layout and logic of the dash web app.""" diff --git a/src/app/__main__.py b/src/app/__main__.py new file mode 100644 index 0000000..024e8ae --- /dev/null +++ b/src/app/__main__.py @@ -0,0 +1,10 @@ +"""Serve the dash web app locally.""" + +import sys + +from .app import init_dash_app + +if __name__ == "__main__": + check_per_s = int(sys.argv[1]) + app = init_dash_app(1 / check_per_s) + app.run_server(port=8050, debug=True) diff --git a/src/app/app.py b/src/app/app.py new file mode 100644 index 0000000..1751631 --- /dev/null +++ b/src/app/app.py @@ -0,0 +1,80 @@ +"""This script defines the dash app and its template layout.""" + +import dash +import dash_bootstrap_components as dbc +import diskcache +import numpy as np +from dash import dcc, html +from dash.long_callback import DiskcacheLongCallbackManager + +from . import data_callbacks, graph_callbacks, ui_callbacks # noqa: F401 + + +def init_dash_app(read_interval: float = 10) -> dash.Dash: + cache = diskcache.Cache("./cache") + long_callback_manager = DiskcacheLongCallbackManager(cache) + app = dash.Dash( + __name__, + use_pages=True, + external_stylesheets=[dbc.themes.MATERIA, "assets/style.css"], + long_callback_manager=long_callback_manager, + update_title="", + ) + nav_link_style = { + "margin": "1em 1em", + "textAlign": "center", + "padding": "0.5em 2em", + } + + navbar = dbc.Navbar( + [ + # Use row and col to control vertical alignment of logo / brand + dbc.Nav( + [ + dbc.NavLink("Game", href="/", active="exact", style=nav_link_style), + dbc.NavLink( + "Dashboard", + href="/dashboard", + active="exact", + style=nav_link_style, + ), + dbc.NavLink( + "Settings", + href="/settings", + active="exact", + style=nav_link_style, + ), + ], + vertical=False, + pills=True, + ) + ], + style={ + "padding-left": "10em", + "padding-bottom": "3em", + "border": "none", + "width": "auto", + "box-shadow": "none", + "background-color": "none", + }, + ) + + app.layout = dbc.Container( + children=[ + navbar, + dbc.Row(html.Div(dash.page_container)), + dcc.Interval(id="game_read", interval=np.round(1000 * read_interval)), + dcc.Interval(id="1_min", interval=1000 * 60), + dcc.Interval(id="10_min", interval=1000 * 10 * 60), + dcc.Store("data_store", storage_type="session"), + dcc.Store("cards_store_train", storage_type="session"), + dcc.Store("cards_store_game", storage_type="session"), + dcc.Store("settings_store", storage_type="session"), + dcc.Store("game_store", storage_type="session"), + dcc.Store("col_store", storage_type="session"), + dcc.Store("lord_store", storage_type="session"), + ], + className="dbc", + fluid=True, + ) + return app diff --git a/src/app/assets/style.css b/src/app/assets/style.css new file mode 100644 index 0000000..1e1903b --- /dev/null +++ b/src/app/assets/style.css @@ -0,0 +1,19 @@ +/*Color definitions*/ +:root { + --tag-red: 238, 95, 91; + --tag-cyan: 91, 192, 222; + --tag-blue: 0, 123, 255; + --tag-pink: 232, 62, 140; + --tag-yellow: 248, 244, 6; + --tag-green: 98, 196, 98; + --tag-orange: 249, 118, 11; + --tag-indigo: 102, 16, 242; + --tag-teal: 32, 201, 151; + --link-active-bg: 0, 123, 255; + --link-active: 255, 255, 255; + --tag-alpha: 0.15; + --bg-light: #43474d; + --btn-bg: 91, 192, 222; +} + +/*Global settings*/ \ No newline at end of file diff --git a/src/app/data_callbacks.py b/src/app/data_callbacks.py new file mode 100644 index 0000000..5d3230c --- /dev/null +++ b/src/app/data_callbacks.py @@ -0,0 +1,65 @@ +"""This script contains the callbacks used in the apps data collection.""" + +import logging + +import numpy as np +import pandas as pd +from dash import Input, Output, State, callback +from dash.exceptions import PreventUpdate + +from src import PROCESS_NAME +from src.stats.building import Building +from src.stats.lord import Lord +from src.stats.read_data import read_config +from src.stats.state_machine import StateMachine +from src.stats.unit import Unit + +logger = logging.getLogger(__name__) +build_config = read_config("building") +b = Building.from_dict(build_config) +lord_config = read_config("lord") +lord = Lord.from_dict(lord_config) +unit_config = read_config("unit") +unit = Unit.from_dict(unit_config) +sm = StateMachine() + + +@callback(Output("game_store", "data"), Input("game_read", "n_intervals"), State("game_store", "data")) +def read_data_from_memory(n_intervals: int, data: list | None) -> list: + data = data or [] + game_data = pd.DataFrame(data) + state = sm.update_state(PROCESS_NAME) + if state == "game": + lord_glob_df = lord.get_lord_global_stats() + lord_det_df = lord.get_lord_detailed_stats() + buildings_df = b.calculate_all_stats() + unit_df = unit.calculate_units() + cur_tick_df = ( + pd.concat([lord_glob_df, lord_det_df], axis=1) + .merge(buildings_df, how="left", on="p_ID") + .merge(unit_df, how="left", on="p_ID") + ) + cur_tick_df["time"] = n_intervals + game_data = pd.concat( + [game_data, cur_tick_df], + ignore_index=True, + ) + return game_data.to_dict("records") + elif state == "lobby": + game_data = pd.DataFrame() + return game_data.to_dict("records") + + +@callback(Output("lord_store", "data"), Input("game_read", "n_intervals"), State("lord_store", "data")) +def save_lord_names(n_intervals: int, data: list | None) -> list: + if any((s > 0 for s in np.array(data).shape)): + array = np.array(data) + else: + array = np.array([]) + state = sm.update_state(PROCESS_NAME) + if state != "stats": + lord.get_active_lords() + lord.get_lord_names() + if not np.array_equal(lord.lord_names, array): + return lord.lord_names.tolist() + raise PreventUpdate() diff --git a/src/app/graph_callbacks.py b/src/app/graph_callbacks.py new file mode 100644 index 0000000..ac2b793 --- /dev/null +++ b/src/app/graph_callbacks.py @@ -0,0 +1,55 @@ +"""This script contains the callbacks used in the apps graph generation.""" + +import logging + +import pandas as pd +import plotly.graph_objects as go +from dash import ALL, Input, Output, State, callback, ctx + +from src import SHC_COLORS + +logger = logging.getLogger(__name__) + + +@callback( + Output("stat-display", "figure"), + Output("col_store", "data"), + Input("game_store", "data"), + Input({"type": "col-switch", "index": ALL}, "n_clicks"), + State("col_store", "data"), +) +def update_graph(data: list, _: dict[str, int], last_column: str | None): + column = last_column or "num_buildings" + fig = go.Figure() + logger.info(ctx.triggered_id) + if data and ctx.triggered_id is not None: + if isinstance(ctx.triggered_id, dict): + column = ctx.triggered_id.get("index", "") + df = pd.DataFrame(data) + tolerance = 5 + df = df.sort_values(["p_ID", "time"]) + # Identify outliers where the value is far from both its neighbors + df["is_outlier"] = ( + ((df["num_buildings"].shift(1) - df["num_buildings"]).abs() > tolerance) + & ((df["num_buildings"].shift(-1) - df["num_buildings"]).abs() > tolerance) + & ((df["num_buildings"].shift(1) - df["num_buildings"].shift(-1)).abs() <= tolerance) + & (df["p_ID"].shift(1) == df["p_ID"]) + ) + + # Filter out the outliers + df = df.loc[~df["is_outlier"], :].drop(columns=["is_outlier"]) + for p_id, group in df.groupby("p_ID"): + assert isinstance(p_id, int) + fig.add_trace( + go.Scatter( + x=group["time"], + y=group[column], + mode="lines", + name=f"p_ID {p_id}", + marker_color=SHC_COLORS[p_id - 1], + ) + ) + + # Customize layout (optional) + fig.update_layout(title=f"{column} over time", xaxis_title="Timesteps", yaxis_title=column) + return fig, column diff --git a/src/app/pages/dashboard.py b/src/app/pages/dashboard.py new file mode 100644 index 0000000..bca2d88 --- /dev/null +++ b/src/app/pages/dashboard.py @@ -0,0 +1,76 @@ +"""This script defines the page layout for the game page.""" + +import dash +import dash_bootstrap_components as dbc +from dash import dcc + +dash.register_page( + __name__, + path="/", + name="Live Game Stats", + title="Live Game Stats", + description="Landing page.", +) + +col_list = [ + "num_eco_buildings", + "num_total_buildings", + "total_gold", + "weighted_troops_killed", + "weighted_buildings_destroyed", + "goods_received", + "goods_sent", + "housing", + "units", + "siege_engines", + "popularity", + "wood", + "hops", + "stone", + "iron", + "pitch", + "wheat", + "bread", + "cheese", + "meat", + "apples", + "ale", + "gold", + "flour", + "bows", + "crossbows", + "spears", + "pikes", + "maces", + "swords", + "leather_armor", + "metal_armor", + "total_food", + "population", + "taxes", + "weighted_losses", + "weighted_units", + "num_farms", + "num_iron_mines", + "num_pitchrigs", + "num_quarries", + "num_buildings", + "workers_needed", + "workers_working", + "workers_missing", + "snoozed", +] + +layout = [ + dbc.Row( + [ + dbc.Col(dcc.Dropdown(id="lord-select", multi=True)), + ] + ), + dbc.Row( + [ + dbc.Col([dbc.Button(col, id={"type": "col-switch", "index": col}) for col in col_list]), + ] + ), + dbc.Row(dcc.Graph(id="stat-display")), +] diff --git a/src/app/ui_callbacks.py b/src/app/ui_callbacks.py new file mode 100644 index 0000000..c48b187 --- /dev/null +++ b/src/app/ui_callbacks.py @@ -0,0 +1,13 @@ +"""This script contains the callbacks used in the app ui generation.""" + +from dash import Input, Output, callback +from dash.exceptions import PreventUpdate + + +@callback(Output("lord-select", "options"), Input("lord_store", "data")) +def display_lord_dd(lord_names): + if lord_names: + return [{"label": "No owner", "value": 0}] + [ + {"label": lord_name, "value": i + 1} for i, lord_name in enumerate(lord_names) + ] + raise PreventUpdate() diff --git a/src/parser/__init__.py b/src/parser/__init__.py new file mode 100644 index 0000000..358cc24 --- /dev/null +++ b/src/parser/__init__.py @@ -0,0 +1 @@ +"""This Module contains the data transformation and plotting functions.""" diff --git a/src/parser/building.py b/src/parser/building.py new file mode 100644 index 0000000..69b7eec --- /dev/null +++ b/src/parser/building.py @@ -0,0 +1,136 @@ +"""Script containing code to manage building related tasks and calculations.""" + +import numpy as np +import pandas as pd + +from src import PROCESS_NAME +from src.stats.read_data import D_Types, read_config, read_memory, read_memory_chunk + + +class Building: + """Class to read buildings from memory and compute stats.""" + + def __init__(self, base: int, offsets: dict, total_buildings: int) -> None: + """Initialize the Building class. + + Args: + base (int): base address of the buildings array in memory + offsets (dict): offset to next building in memory + total_buildings (int): address where number of total buildings is stored + """ + self.building_names = read_config("names")["Buildings"] + self.base = base + self.offset = offsets["offset"] + self.owner = offsets["owneroffset"] + self.workers_needed = offsets["workersneededoffset"] + self.workers = offsets["workersworkingoffset"] + self.workers_missing = offsets["workersmissingoffset"] + self.snoozed = offsets["snoozedoffset"] + self.total_buildings = total_buildings + + @staticmethod + def from_dict(config: dict) -> "Building": + """Initialize Building from a dictionary. + + Args: + config (dict): configuration dictionary + + Returns: + Building: instantiated class + """ + return Building(config["address"], config["offsets"], config["total"]) + + def list_buildings(self, player_id: int = 0) -> pd.DataFrame: + """List all buildings present in the game. + + Args: + player_id (int, optional): player id to filter buildings. Defaults to 0 and not filtering. + + Returns: + pd.DataFrame: buildings data + """ + num_buildings = int(read_memory(PROCESS_NAME, self.total_buildings, D_Types.INT)) + offset_list = [0, self.owner, self.workers_needed, self.workers, self.workers_missing, self.snoozed] + buildings_list = read_memory_chunk( + PROCESS_NAME, + self.base, + [i * self.offset + extra_off for i in range(num_buildings) for extra_off in offset_list], + D_Types.WORD, + ) + buildings_array = np.array(buildings_list).reshape((num_buildings, len(offset_list))) + if player_id != 0: + mask = buildings_array[:, 2] == player_id + else: + mask = (buildings_array[:, 2] >= 1) & (buildings_array[:, 2] <= 8) + + filtered_buildings = buildings_array[mask] + + building_names_array = np.vectorize(self.building_names.get)(filtered_buildings[:, 0]) + # address_array = (self.base + np.arange(buildings_array.shape[0]) * self.offset).reshape(-1, 1) + buildings_array = np.column_stack((building_names_array, filtered_buildings)) + return pd.DataFrame( + buildings_array, + columns=[ + "b_name", + "ID", + "owner", + "workers_needed", + "workers_working", + "workers_missing", + "snoozed", + ], + ).astype( + { + "b_name": pd.StringDtype(), + "ID": pd.Int64Dtype(), + "owner": pd.Int32Dtype(), + "workers_needed": pd.Int32Dtype(), + "workers_working": pd.Int32Dtype(), + "workers_missing": pd.Int32Dtype(), + "snoozed": pd.Int32Dtype(), + } + ) + + def calculate_all_stats(self) -> pd.DataFrame: + """Calculate all building and worker related stats into a dataframe. + + Returns: + pd.DataFrame: All building stats. + """ + false_worker_ids = [1, 2, 8, 9, 21, 29] + ground_ids = [53, 55, 56, 57, 58, 59] + keep_ids = [71, 72, 73] + siege_engines = [80, 81, 82, 83, 84, 86, 87] + building_info_df = pd.DataFrame( + columns=["num_buildings", "workers_needed", "workers_working", "workers_missing", "snoozed"] + ) + building_mem_df = self.list_buildings() + building_mem_df = building_mem_df.loc[ + ~(building_mem_df["ID"].isin(ground_ids + keep_ids + siege_engines)), + :, + ] + building_info_df["num_buildings"] = ( + building_mem_df.loc[:, ["owner"]] + .groupby("owner") + .size() + .to_frame() + .rename(columns={"size": "num_buildings"}) + ) + building_info_df["snoozed"] = ( + building_mem_df.loc[building_mem_df["snoozed"] == 1, ["owner"]] + .groupby("owner") + .size() + .to_frame() + .rename(columns={"size": "snoozed"}) + ) + building_info_df["snoozed"] = building_info_df["snoozed"].fillna(0).astype(pd.Int32Dtype()) + building_mem_df = building_mem_df.loc[ + ~(building_mem_df["ID"].isin(false_worker_ids)) & (building_mem_df["snoozed"] == 0), + :, + ] + building_info_df[["workers_needed", "workers_working", "workers_missing"]] = ( + building_mem_df.loc[:, ["owner", "workers_needed", "workers_working", "workers_missing"]] + .groupby("owner") + .sum() + ) + return building_info_df.reset_index(names=["p_ID"]) diff --git a/src/parser/lord.py b/src/parser/lord.py new file mode 100644 index 0000000..5f7bbf0 --- /dev/null +++ b/src/parser/lord.py @@ -0,0 +1,129 @@ +import numpy as np +import pandas as pd + +from src import PROCESS_NAME + +from .read_data import D_Types, read_memory_chunk + + +class Lord: + def __init__(self, map: dict, lord_basic: dict, lord_global: dict, lord_name: dict, lord_stat: dict) -> None: + self.map = map + self.lord_basic = lord_basic["memory"] + self.lord_basic_off = lord_basic["offset"] + self.lord_name = lord_name["memory"] + self.lord_name_off = lord_name["offset"] + self.lord_global = lord_global["memory"] + self.lord_global_off = lord_global["offset"] + self.lord_stat = lord_stat["memory"] + self.lord_stat_off = lord_stat["offset"] + self.active_lords = np.empty(8) + self.num_lords = 8 + self.teams = np.empty(1) + self.lord_names = np.empty(1) + + @staticmethod + def from_dict(config: dict) -> "Lord": + return Lord( + config["map_offsets"], + config["lord_basic_offsets"], + config["lord_global_offsets"], + config["lord_name_offsets"], + config["lord_stat_offsets"], + ) + + def get_active_lords(self) -> None: + lord_basic = self.lord_basic[0] + basic_offsets = [ + i * self.lord_basic_off + extra_off["offset"] for extra_off in lord_basic["stat_offsets"] for i in range(8) + ] + dtypes = [D_Types[extra_off["type"].upper()] for extra_off in lord_basic["stat_offsets"] for i in range(8)] + lord_basic_mem = read_memory_chunk( + PROCESS_NAME, + lord_basic["address"], + basic_offsets, + dtypes, + ) + lord_basic_arr = np.reshape(np.array(lord_basic_mem), (2, 8)) + self.active_lords = lord_basic_arr[0, :] + self.num_lords = np.max([lord_basic_arr[0, :].sum(), (lord_basic_arr[1, :] >= 0).sum()]) + self.teams = lord_basic_arr[1, 0 : self.num_lords] # noqa: E203 + + def get_lord_names(self) -> None: + lord_name = self.lord_name[0] + names_offsets = [ + i * self.lord_name_off + extra_off["offset"] + for extra_off in lord_name["stat_offsets"] + for i in range(self.num_lords) + ] + dtypes = [ + D_Types[extra_off["type"].upper()] + for extra_off in lord_name["stat_offsets"] + for i in range(self.num_lords) + ] + lord_names_mem = read_memory_chunk( + PROCESS_NAME, + lord_name["address"], + names_offsets, + dtypes, + ) + self.lord_names = np.reshape(np.array(lord_names_mem), (self.num_lords, 1)) + + def get_lord_global_stats(self) -> pd.DataFrame: + cols = ["p_ID"] + [ + extra_off["name"] for lord_global in self.lord_global for extra_off in lord_global["stat_offsets"] + ] + total_arr = np.arange(1, self.num_lords + 1).reshape(-1, 1) + for lord_global in self.lord_global: + global_offsets = [ + i * self.lord_global_off + extra_off["offset"] + for extra_off in lord_global["stat_offsets"] + for i in range(self.num_lords) + ] + dtypes = [ + D_Types[extra_off["type"].upper()] + for extra_off in lord_global["stat_offsets"] + for _ in range(self.num_lords) + ] + lord_global_mem = read_memory_chunk( + PROCESS_NAME, + lord_global["address"], + global_offsets, + dtypes, + ) + lord_global_arr = np.reshape( + np.array(lord_global_mem), + ( + len(lord_global["stat_offsets"]), + self.num_lords, + ), + ).T + total_arr = np.concat((total_arr, lord_global_arr), axis=1) + return pd.DataFrame(total_arr, columns=cols) + + def get_lord_detailed_stats(self) -> pd.DataFrame: + cols = [extra_off["name"] for lord_stat in self.lord_stat for extra_off in lord_stat["stat_offsets"]] + total_arr = np.empty((self.num_lords, 0)) + for lord_stat in self.lord_stat: + stat_offsets = [ + i * self.lord_stat_off + extra_off["offset"] + for i in range(self.num_lords) + for extra_off in lord_stat["stat_offsets"] + ] + dtypes = [ + D_Types[extra_off["type"].upper()] + for extra_off in lord_stat["stat_offsets"] + for _ in range(self.num_lords) + ] + lord_stat_mem = read_memory_chunk( + PROCESS_NAME, + lord_stat["address"], + stat_offsets, + dtypes, + ) + lord_stat_arr = np.reshape( + np.array(lord_stat_mem), + (self.num_lords, -1), + ) + total_arr = np.concat((total_arr, lord_stat_arr), axis=1) + return pd.DataFrame(total_arr, columns=cols) diff --git a/src/parser/read_data.py b/src/parser/read_data.py new file mode 100644 index 0000000..b466be9 --- /dev/null +++ b/src/parser/read_data.py @@ -0,0 +1,240 @@ +import ctypes +import logging +import pathlib +from ctypes import Array, c_bool, c_byte, c_char, c_uint16, c_uint32, wintypes +from enum import Enum + +import numpy as np +import psutil +import yaml + +logger = logging.getLogger(__name__) + +# Constants for permissions +PROCESS_ALL_ACCESS = 0x1F0FFF + + +def read_config(filename: str) -> dict: + path = pathlib.Path().cwd() / "memory" / f"{filename}.yaml" + with open(path, "r", encoding="utf8") as file: + if path.suffix == ".yaml": + data = yaml.safe_load(file) + return data + + +# Helper function to find a process by name +def get_process_by_name(name): + for proc in psutil.process_iter(["name"]): + # print(proc.info["name"]) + if proc.info["name"] == name: + return proc + return None + + +class D_Types(Enum): + INT = 0 + STRING = 1 + BYTE = 2 + BOOLEAN = 3 + WORD = 4 + + +d_types: dict[D_Types, c_uint32 | c_byte | c_bool | c_uint16 | Array[c_char]] = { + D_Types.INT: ctypes.c_uint32(), + D_Types.STRING: ctypes.create_string_buffer(256), + D_Types.BYTE: ctypes.c_byte(), + D_Types.BOOLEAN: ctypes.c_bool(), + D_Types.WORD: ctypes.c_uint16(), +} + + +class MemoryReadError(Exception): + """Custom exception for memory read errors.""" + + def __init__(self, message, process_name=None, address=None): + self.message = message + self.process_name = process_name + self.address = address + super().__init__(self._build_message()) + + def _build_message(self): + details = [] + if self.process_name: + details.append(f"Process: '{self.process_name}'") + if self.address is not None: + details.append(f"Address: {hex(self.address) if isinstance(self.address, int) else self.address}") + details.append(self.message) + return "; ".join(details) + + +def slice_ctypes_array(ctypes_array: Array, offset: int, length: int) -> Array: + array_type = ctypes.c_byte * length + return array_type(*ctypes_array[offset : offset + length]) # noqa: E203 + + +def read_memory(process_name: str, address: int, dtype: D_Types) -> int | bool | str: + """Read the memory value from an address within a process. + + Args: + process_name (str): name of the target process + address (int): target address + dtype (D_Types): address value data type + + Raises: + MemoryReadError: Can't find target process. + MemoryReadError: Can't open target process. + ValueError: Invalid data type. + MemoryReadError: Can't read target address. + e: General exception. + + Returns: + int | bool | str: value of memory address + """ + try: + # Locate the process + proc = get_process_by_name(process_name) + if not proc: + raise MemoryReadError("Process not found.", process_name=process_name) + + # Open the process with the necessary access + process_handle = ctypes.windll.kernel32.OpenProcess(PROCESS_ALL_ACCESS, False, proc.pid) + if not process_handle: + raise MemoryReadError("Failed to open process.", process_name=process_name) + + try: + # Prepare a buffer to store the read value + if dtype not in d_types: + raise ValueError(f"Unsupported dtype '{dtype}'. Supported types: {list(d_types.keys())}") + + buffer = d_types[dtype] + bytes_read = wintypes.SIZE() + + # Perform the read + success = ctypes.windll.kernel32.ReadProcessMemory( + process_handle, + ctypes.c_void_p(address), + ctypes.byref(buffer), + ctypes.sizeof(buffer), + ctypes.byref(bytes_read), + ) + + if not success: + raise MemoryReadError("Failed to read memory.", process_name=process_name, address=address) + + # Return the read value + if isinstance(buffer, ctypes.Array): + return buffer.value.decode("utf-8", errors="ignore") + + return buffer.value + + finally: + # Ensure the process handle is always closed + ctypes.windll.kernel32.CloseHandle(process_handle) + + except Exception as e: + raise e + + +def read_memory_chunk( + process_name: str, base_address: int, offsets: list[int], dtype: D_Types | list[D_Types] +) -> list[int | str | bool]: + """Read a chunk of memory at different offsets. + + Args: + process_name (str): name of the target process + base_address (int): target address + offsets (list[int]): offsets from target address to be read + + Raises: + ValueError: Offsets must be nonempty + MemoryReadError: Can't find target process. + MemoryReadError: Can't open target process. + MemoryReadError: Can't read target address. + e: General exception. + + Returns: + list[int]: list of memory values at offsets. + """ + try: + if not offsets: + raise ValueError("Offsets list cannot be empty.") + + if not isinstance(dtype, list): + dtype = [dtype] * len(offsets) + + if len(dtype) != len(offsets): + raise ValueError("The length of dtype list must match the length of offsets.") + + if offsets != sorted(offsets): + offsets, dtype = map(list, zip(*sorted(zip(offsets, dtype)))) + + # Locate the process + proc = get_process_by_name(process_name) + if not proc: + raise MemoryReadError("Process not found.", process_name=process_name) + + # Open the process + process_handle = ctypes.windll.kernel32.OpenProcess(PROCESS_ALL_ACCESS, False, proc.pid) + if not process_handle: + raise MemoryReadError("Failed to open process.", process_name=process_name) + + try: + # Calculate the range of memory to read + type_sizes = { + D_Types.INT: ctypes.sizeof(c_uint32()), + D_Types.STRING: 256, # Assuming a fixed size for strings + D_Types.BYTE: ctypes.sizeof(c_byte()), + D_Types.BOOLEAN: ctypes.sizeof(c_bool()), + D_Types.WORD: ctypes.sizeof(c_uint16()), + } + size = offsets[-1] + type_sizes[dtype[-1]] + + # Read the contiguous memory block + buffer: Array[c_byte] = (ctypes.c_byte * size)() + bytes_read = wintypes.SIZE() + + success = ctypes.windll.kernel32.ReadProcessMemory( + process_handle, + ctypes.c_void_p(base_address), + buffer, + size, + ctypes.byref(bytes_read), + ) + + if not success: + raise MemoryReadError("Failed to read memory.", process_name=process_name, address=base_address) + + # Extract the values dynamically + results = [] + for offset, d in zip(offsets, dtype): + slice = slice_ctypes_array(buffer, offset, type_sizes[d]) + if d == D_Types.INT: + value: int | str | bool = ctypes.c_uint32.from_buffer_copy(slice).value + elif d == D_Types.STRING: + # print(chardet.detect(bytes(np.mod(buffer[offset : offset + type_sizes[d]], 256).tolist()))) + value = ctypes.create_string_buffer( + bytes(np.mod(buffer[offset : offset + type_sizes[d]], 256)) # noqa: E203 + ).value.decode("ISO-8859-1") + elif d == D_Types.BYTE: + value = ctypes.c_byte.from_buffer_copy(slice).value + elif d == D_Types.BOOLEAN: + value = ctypes.c_bool.from_buffer_copy(slice).value + elif d == D_Types.WORD: + value = ctypes.c_uint16.from_buffer_copy(slice).value + else: + raise ValueError( + ( + "ctypes.c_uint32.from_buffer_copy(buffer[offset : offset + type_sizes[d]])" + f".valueUnsupported data type: {d}" + ) + ) + results.append(value) + + return results + + finally: + # Ensure the process handle is always closed + ctypes.windll.kernel32.CloseHandle(process_handle) + + except Exception as e: + raise e diff --git a/src/parser/state_machine.py b/src/parser/state_machine.py new file mode 100644 index 0000000..6d8ebc9 --- /dev/null +++ b/src/parser/state_machine.py @@ -0,0 +1,28 @@ +from .read_data import D_Types, read_memory + + +class StateMachine: + def __init__(self): + self.previous_state = None # Tracks the last state + + def update_state(self, process_name: str) -> str: + # Read current conditions + is_year_zero = read_memory(process_name, 0x24BA938, D_Types.INT) == 0 + in_game = not is_year_zero and read_memory(process_name, 0x1311607, D_Types.STRING) != "shc_back.tgx" + + # Determine the next state + if is_year_zero: + current_state = "lobby" + elif in_game: + current_state = "game" + else: + current_state = "stats" + + # Ensure "stats" can only follow "game" + if current_state == "stats" and self.previous_state != "game": + current_state = "lobby" # Default to lobby or another appropriate state + + # Update the previous state tracker + self.previous_state = current_state + + return current_state diff --git a/src/parser/unit.py b/src/parser/unit.py new file mode 100644 index 0000000..2fe49b4 --- /dev/null +++ b/src/parser/unit.py @@ -0,0 +1,137 @@ +import numpy as np +import pandas as pd + +from src import PROCESS_NAME +from src.stats.read_data import D_Types, read_config, read_memory, read_memory_chunk + + +class MemoryAddress: + def __init__(self, base, offset, val_offset): + self.base = base + self.offset = offset + self.val_offset = val_offset + + def calculate_address(self, multiplier): + return self.base + multiplier * self.offset + self.val_offset + + +class Unit: + def __init__(self, base: int, offsets: dict, total_units: int) -> None: + self.unit_names = read_config("names")["Units"] + self.base = base + self.offset = offsets["offset"] + self.color = offsets["coloroffset"] + self.owner = offsets["owneroffset"] + self.moving = offsets["moving"] + self.selected = offsets["selected"] + self.hp_bar = offsets["hp_bar_percent"] + self.cur_hp = offsets["cur_hp"] + self.max_hp = offsets["max_hp"] + self.x = offsets["x_coord_cam"] + self.y = offsets["y_coord_cam"] + self.unknown = [MemoryAddress(self.base, self.offset, unknown_offset) for unknown_offset in offsets["unknown"]] + + self.total_units = total_units + + @staticmethod + def from_dict(config: dict) -> "Unit": + return Unit(config["address"], config["offsets"], config["total"]) + + def list_units(self, player_id: int | None = None) -> pd.DataFrame: + num_units = int(read_memory(PROCESS_NAME, self.total_units, D_Types.INT)) + offset_list = [ + 0, + self.color, + self.moving, + self.selected, + self.hp_bar, + self.owner, + self.cur_hp, + self.max_hp, + self.x, + self.y, + ] + unit_info = read_memory_chunk( + PROCESS_NAME, + self.base, + [i * self.offset + extra_off for i in range(num_units) for extra_off in offset_list], + D_Types.WORD, + ) + unit_info = np.array(unit_info).reshape((num_units, len(offset_list))) + if player_id is not None: + mask = unit_info[:, 2] == player_id + else: + mask = (unit_info[:, 2] >= 0) & (unit_info[:, 2] <= 8) + + filtered_units = unit_info[mask] + + unit_names_array = np.vectorize(self.unit_names.get)(filtered_units[:, 0]) + address_array = (self.base + np.arange(unit_info.shape[0]) * self.offset)[mask].reshape(-1, 1) + unit_info = np.column_stack((address_array, unit_names_array, filtered_units)) + return pd.DataFrame( + unit_info, + columns=[ + "address", + "unit_name", + "ID", + "color", + "moving", + "selected", + "hp_percent", + "p_ID", + "cur_hp", + "max_hp", + "x", + "y", + ], + ).astype( + { + "address": pd.Int64Dtype(), + "unit_name": pd.StringDtype(), + "ID": pd.Int64Dtype(), + "color": pd.Int64Dtype(), + "moving": pd.Int32Dtype(), + "selected": pd.Int32Dtype(), + "hp_percent": pd.Int32Dtype(), + "p_ID": pd.Int32Dtype(), + "cur_hp": pd.Int64Dtype(), + "max_hp": pd.Int64Dtype(), + "x": pd.Int64Dtype(), + "y": pd.Int64Dtype(), + } + ) + + def list_units_exp(self, player_id: int | None = None) -> pd.DataFrame: + num_units = int(read_memory(PROCESS_NAME, self.total_units, D_Types.INT)) + offset_list = [0, *[obj.val_offset for obj in self.unknown]] + unit_info = read_memory_chunk( + PROCESS_NAME, + self.base, + [i * self.offset + extra_off for i in range(num_units) for extra_off in offset_list], + D_Types.WORD, + ) + unit_info = np.array(unit_info).reshape((num_units, len(offset_list))) + if player_id is not None: + mask = unit_info[:, 2] == player_id + else: + mask = (unit_info[:, 2] >= 0) & (unit_info[:, 2] <= 8) + + filtered_units = unit_info[mask] + + unit_names_array = np.vectorize(self.unit_names.get)(filtered_units[:, 0]) + address_array = (self.base + np.arange(unit_info.shape[0]) * self.offset).reshape(-1, 1) + unit_info = np.column_stack((address_array, unit_names_array, filtered_units)) + return pd.DataFrame( + unit_info, + columns=[ + "address", + "unit_name", + "ID", + *[hex(off.val_offset) for off in self.unknown], + ], + ) + + def calculate_units(self, player_id: int | None = None) -> pd.DataFrame: + units = self.list_units(player_id) + unit_stats_df = units.groupby(["p_ID", "unit_name"]).size().unstack(fill_value=0) + return unit_stats_df diff --git a/src/setup_logging.py b/src/setup_logging.py new file mode 100644 index 0000000..de5ef68 --- /dev/null +++ b/src/setup_logging.py @@ -0,0 +1,26 @@ +import atexit +import logging +import logging.config +import logging.handlers +import pathlib +from typing import override + +import yaml + + +def setup_logging(): + config_file = pathlib.Path.cwd() / "log_config.yaml" + with open(config_file) as f_in: + config = yaml.load(f_in, yaml.SafeLoader) + + logging.config.dictConfig(config) + queue_handler = logging.getHandlerByName("queue_handler") + if queue_handler is not None: + queue_handler.listener.start() + atexit.register(queue_handler.listener.stop) + + +class NonErrorFilter(logging.Filter): + @override + def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: + return record.levelno <= logging.INFO diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29