diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index e0c51a68..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: CI - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Download required file for splat - run: | - curl -o disc/SCUS_971.98 "${{ secrets.FILE_URL }}" - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install requests - - - name: Run build script - run: | - python configure.py - ninja - - - name: Post to frogress endpoint - run: | - curl -X POST https://progress.deco.mp/data/slycooper1/ntsc-u/ \ - -H "Content-Type: application/json" \ - -d '{ - "api_key": "${{ secrets.FROGRESS_API_KEY }}", - "entries": [ - { - "git_hash": "${{ github.sha }}", - "timestamp": "${{ github.event.head_commit.timestamp }}", - "categories": { - "default": { - // metrics - } - } - } - ] - }' diff --git a/.github/workflows/frogress.yml b/.github/workflows/frogress.yml new file mode 100644 index 00000000..b5827c2e --- /dev/null +++ b/.github/workflows/frogress.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: + - main + - frogress + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -U -r requirements.txt + + - name: Setup wine + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install wine32 + + - name: Setup build environment + run: | + sudo apt-get install binutils-mips-linux-gnu + scripts/setup_progd_linux.sh + curl -o disc/SCUS_971.98 "${{ secrets.FILE_URL }}" + + - name: Run build script + run: | + python configure.py + ninja + + - name: Post to frogress endpoint + run: | + python3 scripts/upload_progress.py --frogress_api_key ${{ secrets.FROGRESS_API_KEY }} --git_hash ${{ github.sha }} --timestamp ${{ github.event.head_commit.timestamp }} diff --git a/config/sly1.yaml b/config/sly1.yaml index b7e26efd..b66c0e57 100644 --- a/config/sly1.yaml +++ b/config/sly1.yaml @@ -103,8 +103,7 @@ segments: - [0x3dcf8, asm, P2/break] - - [0x3efd8, asm] # TODO - #- [0x, asm, P2/bsp] + - [0x3efd8, asm, P2/bsp] #- [0x, asm, P2/button] #- [0x, asm, P2/cat] (might not be in release) #- [0x, asm, P2/can] @@ -140,7 +139,7 @@ segments: - [0x5a6e0, asm, P2/eyes] - [0x5aad0, asm, P2/fader] - - [0x5aca0, asm] #? unknown file + - [0x5aca0, asm, P2/unk_f] #? unknown file - [0x5af20, asm, P2/find] - [0x5b8b0, asm, P2/flash] @@ -183,8 +182,7 @@ segments: - [0x86c18, asm, P2/mark] - [0x894c0, asm, P2/mat] - - [0x8b388, asm] # TODO - #- [0x, asm, P2/mecha] + - [0x8b388, asm, P2/mecha] #- [0x, asm, P2/mb] #- [0x, asm, P2/memcard] #- [0x, asm, P2/memory] @@ -205,8 +203,7 @@ segments: - [0x94210, c, P2/prog] - - [0x94598, asm] # TODO - #- [0x, asm, P2/prompt] + - [0x94598, asm, P2/prompt] #- [0x, asm, P2/proxy] #- [0x, asm, P2/puffer] #- [0x, asm, P2/pzo] @@ -311,14 +308,14 @@ segments: #-------------------------------------------------------- # Libs #-------------------------------------------------------- - - [0xF69C4, asm] # TBD #MARK: libs + - [0xF69C4, asm, sce/libs] # #MARK: libs + #- [0xF76F8, asm, sce/rand] + #- [0xF7738, asm] - - [0xF76F8, asm, sce/rand] - - - [0xF7738, asm] #-------------------------------------------------------- - + # Vutext + #-------------------------------------------------------- - [0x112E10, textbin, vutext] #MARK: vutext #-------------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index e972fce0..88a5e6c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ pygfxd crunch64 ninja ninja_syntax +mapfile_parser diff --git a/scripts/upload_progress.py b/scripts/upload_progress.py new file mode 100644 index 00000000..6c521797 --- /dev/null +++ b/scripts/upload_progress.py @@ -0,0 +1,125 @@ +""" +Script for uploading progress to the frogress. + +Adapted from https://github.com/zeldaret/af/blob/aeb01dcb95e8281f89f355604dbeba519ef073d5/tools/progress.py +MIT License: https://opensource.org/license/mit +""" +from pathlib import Path +import argparse +import mapfile_parser + +ASMPATH = Path("asm") +NONMATCHINGS = "nonmatchings" +BASE_URL = "https://progress.deco.mp" +SLUG = "slycooper1" +VERSION = "us" + +def getProgressFromMapFile(mapFile: mapfile_parser.MapFile, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]: + totalStats = mapfile_parser.ProgressStats() + progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict() + + for segment in mapFile: + for file in segment: + if len(file) == 0: + continue + + folder = file.filepath.parts[pathIndex] + + if ".a" in folder: + folder = folder.split('.a')[0] + + if folder in aliases: + folder = aliases[folder] + + if folder not in progressPerFolder: + progressPerFolder[folder] = mapfile_parser.ProgressStats() + + originalFilePath = Path(*file.filepath.parts[pathIndex:]) + + extensionlessFilePath = originalFilePath + while extensionlessFilePath.suffix: + extensionlessFilePath = extensionlessFilePath.with_suffix("") + + fullAsmFile = asmPath / extensionlessFilePath.with_suffix(".s") + + handwrittenAsmFiles = [Path("sce/crt0.o")] + + if originalFilePath in handwrittenAsmFiles: + wholeFileIsUndecomped = False + else: + wholeFileIsUndecomped = fullAsmFile.exists() + + for func in file: + funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s" + + symSize = 0 + if func.size is not None: + symSize = func.size + + if wholeFileIsUndecomped: + totalStats.undecompedSize += symSize + progressPerFolder[folder].undecompedSize += symSize + elif funcAsmPath.exists(): + totalStats.undecompedSize += symSize + progressPerFolder[folder].undecompedSize += symSize + else: + totalStats.decompedSize += symSize + progressPerFolder[folder].decompedSize += symSize + + return totalStats, progressPerFolder + +def getProgress(mapPath: Path) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]: + """ + Gets the progress of the using the mapfile parser. + """ + mapFile = mapfile_parser.MapFile() + mapFile.readMapFile(mapPath) + + for segment in mapFile: + for file in segment: + if len(file) == 0: + continue + + filepathParts = list(file.filepath.parts) + file.filepath = Path(*filepathParts) + + nonMatchingsPath = ASMPATH / NONMATCHINGS + + progress = getProgressFromMapFile(mapFile.filterBySectionType(".text"), ASMPATH, nonMatchingsPath, aliases={"ultralib": "libultra"}) + + return progress + +def main(args: argparse.ArgumentParser) -> None: + """ + Main function, calculates the progress and uploads it to the frogress. + """ + frogress_api_key = args.frogress_api_key + git_hash = args.git_hash + timestamp = args.timestamp + + if not frogress_api_key: + raise ValueError("Missing frogress API key.") + + if not git_hash: + raise ValueError("Missing git hash.") + + if not timestamp: + raise ValueError("Missing timestamp.") + timestamp = int(timestamp) + + mapPath = "out/SCUS_971.98.map" + + codeTotalStats, codeProgressPerFolder = getProgress(mapPath) + codeEntries: dict[str, int] = mapfile_parser.frontends.upload_frogress.getFrogressEntriesFromStats(codeTotalStats, codeProgressPerFolder, verbose=True) + + url = mapfile_parser.utils.generateFrogressEndpointUrl(BASE_URL, SLUG, VERSION) + mapfile_parser.frontends.upload_frogress.uploadEntriesToFrogress(codeEntries, "all", url, apikey=frogress_api_key, verbose=True) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Upload progress to the frogress") + parser.add_argument("--frogress_api_key", help="API key for the frogress") + parser.add_argument("--git_hash", help="Git hash of the current commit") + parser.add_argument("--timestamp", help="Timestamp of the build") + + args = parser.parse_args() + main(args)