From 06463e0294e32ac86d53a8619fd06f36b8cace13 Mon Sep 17 00:00:00 2001 From: Derek Hensley Date: Fri, 16 Feb 2024 09:08:04 -0800 Subject: [PATCH] Build Cleanup (#39) * Some cleanup * first_diff * ci * readme * Move built linker script to build --- .github/workflows/ci.yml | 3 ++ .gitignore | 5 +- Makefile | 107 +++++++++++++++++++++++---------------- README.md | 4 +- baseroms/us/checksum.md5 | 1 + tools/first_diff.py | 4 +- tools/progress.py | 2 +- tools/upload_frogress.py | 2 +- yamls/us/header.yaml | 4 +- yoshisstory.us.md5 | 1 - 10 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 baseroms/us/checksum.md5 delete mode 100644 yoshisstory.us.md5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46adad3..e9062dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - name: Get the dependency run: cp deps_repo/yoshis-story/* . + - name: venv + run: make venv -j $(nproc) VERSION=${{ mactrix.version }} + - name: Setup run: make setup -j $(nproc) VERSION=${{ matrix.version }} diff --git a/.gitignore b/.gitignore index d21f71b..2c5e4a8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,8 @@ build/ linker_scripts/us/auto/ linker_scripts/us/yoshisstory.ld *_auto.ld -yoshisstory.ld -yoshisstory.us.yaml +yoshisstory-*.ld +yoshisstory-*.yaml __pycache__/ .make_options @@ -22,6 +22,7 @@ ctx.c.m2c *.v64 *.plf *.sym +*.venv .vscode/* !.vscode/c_cpp_properties.json diff --git a/Makefile b/Makefile index cbc9f12..7918bf0 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,29 @@ MAKEFLAGS += --no-builtin-rules SHELL = /bin/bash .SHELLFLAGS = -o pipefail -c -#### Defaults #### +# OS Detection +ifeq ($(OS),Windows_NT) + DETECTED_OS = windows + MAKE = make + VENV_BIN_DIR = Scripts +else + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + DETECTED_OS = linux + MAKE = make + VENV_BIN_DIR = bin + endif + ifeq ($(UNAME_S),Darwin) + DETECTED_OS = macos + MAKE = gmake + VENV_BIN_DIR = bin + endif +endif +#### Defaults #### +# Target game version. Currently only the following version is supported: +# us N64 USA (default) +VERSION ?= us # If COMPARE is 1, check the output md5sum after building COMPARE ?= 1 # If NON_MATCHING is 1, define the NON_MATCHING C flag when building @@ -27,25 +48,28 @@ OBJDUMP_BUILD ?= 0 FULL_DISASM ?= 0 # Number of threads to compress with N_THREADS ?= $(shell nproc) - -# Set prefix to mips binutils binaries (mips-linux-gnu-ld => 'mips-linux-gnu-') - Change at your own risk! -# In nearly all cases, not having 'mips-linux-gnu-*' binaries on the PATH is indicative of missing dependencies +# MIPS toolchain prefix MIPS_BINUTILS_PREFIX ?= mips-linux-gnu- +# Python virtual environment +VENV ?= .venv +# Python interpreter +PYTHON ?= $(VENV)/$(VENV_BIN_DIR)/python3 +# Emulator w/ flags +N64_EMULATOR ?= -VERSION ?= us - -BASEROM := baserom.$(VERSION).z64 -TARGET := yoshisstory +BASEROM_DIR := baseroms/$(VERSION) +BASEROM := $(BASEROM_DIR)/baserom.z64 +TARGET := yoshisstory ### Output ### BUILD_DIR := build -ROM := $(BUILD_DIR)/$(TARGET).$(VERSION).z64 -ELF := $(BUILD_DIR)/$(TARGET).$(VERSION).elf -LD_MAP := $(BUILD_DIR)/$(TARGET).$(VERSION).map -LD_SCRIPT := linker_scripts/$(VERSION)/$(TARGET).ld +ROM := $(BUILD_DIR)/$(TARGET)-$(VERSION).z64 +ELF := $(ROM:.z64=.elf) +MAP := $(ROM:.z64=.map) +LDSCRIPT := $(ROM:.z64=.ld) #### Setup #### @@ -64,19 +88,11 @@ ifeq ($(NON_MATCHING),1) COMPARE := 0 endif -MAKE = make CPPFLAGS += -fno-dollars-in-identifiers -P LDFLAGS := --no-check-sections --accept-unknown-input-arch --emit-relocs -UNAME_S := $(shell uname -s) -ifeq ($(OS),Windows_NT) -$(error Native Windows is currently unsupported for building this repository, use WSL instead c:) -else ifeq ($(UNAME_S),Linux) - DETECTED_OS := linux -else ifeq ($(UNAME_S),Darwin) - DETECTED_OS := macos - MAKE := gmake - CPPFLAGS += -xc++ +ifeq ($(DETECTED_OS), macos) + CPPFLAGS += -xc++ endif #### Tools #### @@ -95,13 +111,13 @@ OBJCOPY := $(MIPS_BINUTILS_PREFIX)objcopy OBJDUMP := $(MIPS_BINUTILS_PREFIX)objdump CPP := cpp ICONV := iconv -ASM_PROC := python3 tools/asm-processor/build.py +ASM_PROC := $(PYTHON) tools/asm-processor/build.py CAT := cat ASM_PROC_FLAGS := --input-enc=utf-8 --output-enc=euc-jp --convert-statics=global-with-filename -SPLAT ?= splat split -SPLAT_YAML ?= $(TARGET).$(VERSION).yaml +SPLAT ?= $(PYTHON) -m splat split +SPLAT_YAML ?= $(TARGET)-$(VERSION).yaml @@ -156,12 +172,6 @@ else OBJCOPY_BIN = @: endif -# rom compression flags -COMPFLAGS := --threads $(N_THREADS) -ifeq ($(NON_MATCHING),0) - COMPFLAGS += --matching -endif - SPLAT_FLAGS ?= ifneq ($(FULL_DISASM), 0) SPLAT_FLAGS += --disassemble-all @@ -203,12 +213,12 @@ build/src/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) -- #### Main Targets ### -all: uncompressed +all: rom -uncompressed: $(ROM) +rom: $(ROM) ifneq ($(COMPARE),0) @md5sum $(ROM) - @md5sum -c $(TARGET).$(VERSION).md5 + @md5sum -c $(BASEROM_DIR)/checksum.md5 endif clean: @@ -216,9 +226,14 @@ clean: distclean: clean $(RM) -r $(BUILD_DIR) asm/ assets/ .splat/ - $(RM) -r linker_scripts/$(VERSION)/auto $(LD_SCRIPT) + $(RM) -r linker_scripts/$(VERSION)/auto $(LDSCRIPT) $(MAKE) -C tools distclean +venv: + test -d $(VENV) || python3 -m venv $(VENV) + $(PYTHON) -m pip install -U pip + $(PYTHON) -m pip install -U -r requirements.txt + setup: $(MAKE) -C tools @@ -227,20 +242,26 @@ extract: $(CAT) yamls/$(VERSION)/header.yaml yamls/$(VERSION)/makerom.yaml yamls/$(VERSION)/main.yaml > $(SPLAT_YAML) $(SPLAT) $(SPLAT_FLAGS) $(SPLAT_YAML) -diff-init: uncompressed +diff-init: rom $(RM) -rf expected/ mkdir -p expected/ cp -r $(BUILD_DIR) expected/$(BUILD_DIR) -init: - $(MAKE) distclean +init: distclean + $(MAKE) venv $(MAKE) setup $(MAKE) extract $(MAKE) all $(MAKE) diff-init -.PHONY: all uncompressed clean distclean setup extract diff-init init -.DEFAULT_GOAL := uncompressed +run: $(ROM) +ifeq ($(N64_EMULATOR),) + $(error Emulator path not set. Set N64_EMULATOR in the Makefile, .make_options, or define it as an environment variable) +endif + $(N64_EMULATOR) $< + +.PHONY: all rom clean distclean setup extract diff-init init venv run +.DEFAULT_GOAL := rom # Prevent removing intermediate files .SECONDARY: @@ -252,11 +273,11 @@ $(ROM): $(ELF) # TODO: update rom header checksum # TODO: avoid using auto/undefined -$(ELF): $(LIBULTRA_O) $(O_FILES) $(LD_SCRIPT) $(BUILD_DIR)/linker_scripts/$(VERSION)/hardware_regs.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/undefined_syms.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/pif_syms.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_syms_auto.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_funcs_auto.ld - $(LD) $(LDFLAGS) -T $(LD_SCRIPT) \ +$(ELF): $(LIBULTRA_O) $(O_FILES) $(LDSCRIPT) $(BUILD_DIR)/linker_scripts/$(VERSION)/hardware_regs.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/undefined_syms.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/pif_syms.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_syms_auto.ld $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_funcs_auto.ld + $(LD) $(LDFLAGS) -T $(LDSCRIPT) \ -T $(BUILD_DIR)/linker_scripts/$(VERSION)/hardware_regs.ld -T $(BUILD_DIR)/linker_scripts/$(VERSION)/undefined_syms.ld -T $(BUILD_DIR)/linker_scripts/$(VERSION)/pif_syms.ld \ -T $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_syms_auto.ld -T $(BUILD_DIR)/linker_scripts/$(VERSION)/auto/undefined_funcs_auto.ld \ - -Map $(LD_MAP) -o $@ + -Map $(MAP) -o $@ $(BUILD_DIR)/%.ld: %.ld $(CPP) $(CPPFLAGS) $(BUILD_DEFINES) $(IINC) $< > $@ diff --git a/README.md b/README.md index d6a5e74..1f3506b 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,12 @@ The build process has a few python packages required that are located in `requir To install them simply run in a terminal: ```bash -python3 -m pip install -U -r requirements.txt +make venv ``` #### 3. Prepare a base ROM -Copy your ROM to inside the root of this new project directory, and rename the file of the baserom to reflect the version of ROM you are using. ex: `baserom.us.z64` +Copy your ROM to inside `baseroms/` in the version folder corresponding to your ROM's version, and rename the file of the baserom to `baserom.z64`. ex: `baseroms/us/baserom.z64` #### 4. Make and Build the ROM diff --git a/baseroms/us/checksum.md5 b/baseroms/us/checksum.md5 new file mode 100644 index 0000000..e4a5785 --- /dev/null +++ b/baseroms/us/checksum.md5 @@ -0,0 +1 @@ +586a092e22604840973b82dfaceac77a build/yoshisstory-us.z64 diff --git a/tools/first_diff.py b/tools/first_diff.py index ab1c024..925f96a 100755 --- a/tools/first_diff.py +++ b/tools/first_diff.py @@ -46,8 +46,8 @@ def firstDiffMain(): buildFolder = Path("build") - BUILTROM = buildFolder / f"yoshisstory.{args.version}.z64" - BUILTMAP = buildFolder / f"yoshisstory.{args.version}.map" + BUILTROM = buildFolder / f"yoshisstory-{args.version}.z64" + BUILTMAP = buildFolder / f"yoshisstory-{args.version}.map" EXPECTEDROM = "expected" / BUILTROM EXPECTEDMAP = "expected" / BUILTMAP diff --git a/tools/progress.py b/tools/progress.py index cbd39e2..fc056d3 100755 --- a/tools/progress.py +++ b/tools/progress.py @@ -70,7 +70,7 @@ def progressMain(): args = parser.parse_args() - mapPath = Path("build") / f"yoshisstory.{args.version}.map" + mapPath = Path("build") / f"yoshisstory-{args.version}.map" totalStats, progressPerFolder = getProgress(mapPath, args.version) diff --git a/tools/upload_frogress.py b/tools/upload_frogress.py index 6988013..6d020fd 100755 --- a/tools/upload_frogress.py +++ b/tools/upload_frogress.py @@ -25,7 +25,7 @@ def uploadProgressMain(): version: str = args.version category: str = "code" apikey: str = args.apikey - mapPath = Path("build") / f"yoshisstory.{args.version}.map" + mapPath = Path("build") / f"yoshisstory-{args.version}.map" totalStats, progressPerFolder = progress.getProgress(mapPath, version) diff --git a/yamls/us/header.yaml b/yamls/us/header.yaml index bc21719..a890af1 100644 --- a/yamls/us/header.yaml +++ b/yamls/us/header.yaml @@ -2,8 +2,8 @@ name: Yoshi Story (North America) sha1: b13072fef6c6df48c07d8822c01e5bc59036f6da options: basename: yoshisstory - target_path: baserom.us.z64 - ld_script_path: linker_scripts/us/yoshisstory.ld + target_path: baseroms/us/baserom.z64 + ld_script_path: build/yoshisstory-us.ld base_path: . compiler: IDO find_file_boundaries: True diff --git a/yoshisstory.us.md5 b/yoshisstory.us.md5 deleted file mode 100644 index 6be4564..0000000 --- a/yoshisstory.us.md5 +++ /dev/null @@ -1 +0,0 @@ -586a092e22604840973b82dfaceac77a build/yoshisstory.us.z64