Updates TGS support (shiptest-ss13#2185)

## Changelog

:cl: Dominion, Mark Suckerberg
server: We should now support TGS quick setup
admin: The master controller now actually recovers

/:cl:

<!-- Both :cl:'s are required for the changelog to work! You can put
your name to the right of the first :cl: if you want to overwrite your
GitHub username as author ingame. -->
<!-- You can use multiple of the same prefix (they're only used for the
icon ingame) and delete the unneeded ones. Despite some of the tags,
changelogs should generally represent how a player might be affected by
the changes rather than a summary of the PR's contents. -->

---------

Co-authored-by: John Willard <53777086+JohnFulpWillard@users.noreply.github.com>
Co-authored-by: san7890 <the@san7890.com>
Co-authored-by: Jordan Dominion <Cyberboss@users.noreply.github.com>

.github/workflows/tgs_test.yml

@@ -0,0 +1,63 @@
+name: TGS Test Suite
+on:
+  push:
+    branches:
+    - master
+    - 'gh-readonly-queue/master/**'
+    paths:
+    - '.tgs.yml'
+    - 'dependencies.sh'
+    - 'code/__DEFINES/tgs.config.dm'
+    - 'code/__DEFINES/tgs.dm'
+    - 'code/game/world.dm'
+    - 'code/modules/tgs/**'
+    - 'tools/tgs_scripts/**'
+    - 'tools/tgs_test/**'
+  pull_request:
+    branches:
+    - master
+    paths:
+    - '.tgs.yml'
+    - 'dependencies.sh'
+    - 'code/__DEFINES/tgs.config.dm'
+    - 'code/__DEFINES/tgs.dm'
+    - 'code/game/world.dm'
+    - 'code/modules/tgs/**'
+    - 'tools/tgs_scripts/**'
+    - 'tools/tgs_test/**'
+  merge_group:
+    branches:
+    - master
+env:
+  TGS_API_PORT: 5000
+  PR_NUMBER: ${{ github.event.number }}
+jobs:
+  test_tgs_docker:
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+    name: Test TGS Docker
+    runs-on: ubuntu-22.04
+    concurrency:
+      group: test_tgs_docker-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    services:
+      tgs:
+        image: tgstation/server
+        env:
+          Database__DatabaseType: Sqlite
+          Database__ConnectionString: Data Source=TGS_TGTest.sqlite3;Mode=ReadWriteCreate
+          General__ConfigVersion: 4.1.0
+          General__ApiPort: ${{ env.TGS_API_PORT }}
+          General__SetupWizardMode: Never
+        ports:
+          - 5000:5000 #Can't use env here for some reason
+    steps:
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v2
+        with:
+          dotnet-version: 7.0.x
+
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Test TGS Integration
+        run: dotnet run -c Release --project tools/tgs_test ${{ github.repository }} /tgs_instances/tgstation ${{ env.TGS_API_PORT }} ${{ github.event.head_commit.sha || github.event.pull_request.head.sha }} ${{ secrets.GITHUB_TOKEN }} ${{ env.PR_NUMBER }}

.gitignore

@@ -189,13 +189,9 @@ Temporary Items
 !/config/title_screens/images/exclude
 
 #Linux docker
-/tools/LinuxOneShot/SetupProgram/obj/*
-/tools/LinuxOneShot/SetupProgram/bin/*
-/tools/LinuxOneShot/SetupProgram/.vs
-/tools/LinuxOneShot/Database
-/tools/LinuxOneShot/TGS_Config
-/tools/LinuxOneShot/TGS_Instances
-/tools/LinuxOneShot/TGS_Logs
+/tools/tgs_test/.vs/*
+/tools/tgs_test/bin/*
+/tools/tgs_test/obj/*
 
 # JavaScript tools
 **/node_modules

.tgs.yml

@@ -0,0 +1,25 @@
+# This file is used by TGS (https://github.com/tgstation/tgstation-server) clients to quickly initialize a server instance for the codebase
+# The format isn't documented anywhere but hopefully we never have to change it. If there are questions, contact the TGS maintainer Cyberboss/@Dominion#0444
+version: 1
+# The BYOND version to use (kept in sync with dependencies.sh by the "TGS Test Suite" CI job)
+# Must be interpreted as a string, keep quoted
+byond: "514.1588"
+# Folders to create in "<instance_path>/Configuration/GameStaticFiles/"
+static_files:
+  # Config directory should be static
+  - name: config
+    # This implies the folder should be pre-populated with contents from the repo
+    populate: true
+  # Data directory must be static
+  - name: data
+# String dictionary. The value is the location of the file in the repo to upload to TGS. The key is the name of the file to upload to "<instance_path>/Configuration/EventScripts/"
+# This one is for Linux hosted servers
+linux_scripts:
+  PreCompile.sh: tools/tgs_scripts/PreCompile.sh
+  WatchdogLaunch.sh: tools/tgs_scripts/WatchdogLaunch.sh
+  InstallDeps.sh: tools/tgs_scripts/InstallDeps.sh
+# Same as above for Windows hosted servers
+windows_scripts:
+  PreCompile.bat: tools/tgs_scripts/PreCompile.bat
+# The security level the game should be run at
+security: Trusted

code/__DEFINES/spaceman_dmm.dm

@@ -37,12 +37,6 @@
 /proc/auxtools_expr_stub()
 	CRASH("auxtools not loaded")
 
-/world/proc/enable_debugger()
-	var/dll = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL")
-	if (dll)
-		call(dll, "auxtools_init")()
-		enable_debugging()
-
 /world/Del()
 	var/debug_server = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL")
 	if (debug_server)

code/_debugger.dm

@@ -0,0 +1,13 @@
+//Datum used to init Auxtools debugging as early as possible
+//Datum gets created in master.dm because for whatever reason global code in there gets runs first
+//In case we ever figure out how to manipulate global init order please move the datum creation into this file
+/datum/debugger
+
+/datum/debugger/New()
+		enable_debugger()
+
+/datum/debugger/proc/enable_debugger()
+	var/dll = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL")
+	if (dll)
+		call(dll, "auxtools_init")()
+		enable_debugging()

code/controllers/failsafe.dm

@@ -33,6 +33,22 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 /datum/controller/failsafe/Initialize()
 	set waitfor = 0
 	Failsafe.Loop()
+	if (!Master || defcon == 0) //Master is gone/not responding and Failsafe just exited its loop
+		defcon = 3 //Reset defcon level as its used inside the emergency loop
+		while (defcon > 0)
+			var/recovery_result = emergency_loop()
+			if (recovery_result == 1) //Exit emergency loop and delete self if it was able to recover MC
+				break
+			else if (defcon == 1) //Exit Failsafe if we weren't able to recover the MC in the last stage
+				log_game("FailSafe: Failed to recover MC while in emergency state. Failsafe exiting.")
+				message_admins(span_boldannounce("Failsafe failed criticaly while trying to recreate broken MC. Please manually fix the MC or reboot the server. Failsafe exiting now."))
+				message_admins(span_boldannounce("You can try manually calling these two procs:."))
+				message_admins(span_boldannounce("/proc/recover_all_SS_and_recreate_master: Most stuff should still function but expect instability/runtimes/broken stuff."))
+				message_admins(span_boldannounce("/proc/delete_all_SS_and_recreate_master: Most stuff will be broken but basic stuff like movement and chat should still work."))
+			else if (recovery_result == -1) //Failed to recreate MC
+				defcon--
+			sleep(initial(processing_interval)) //Wait a bit until the next try
+
 	if(!QDELETED(src))
 		qdel(src) //when Loop() returns, we delete ourselves and let the mc recreate us
 
@@ -45,8 +61,8 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 	while(running)
 		lasttick = world.time
 		if(!Master)
-			// Replace the missing Master! This should never, ever happen.
-			new /datum/controller/master()
+			// Break out of the main loop so we go into emergency state
+			break
 		// Only poke it if overrides are not in effect.
 		if(processing_interval > 0)
 			if(Master.processing && Master.iteration)
@@ -106,6 +122,57 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 			defcon = 5
 			sleep(initial(processing_interval))
 
+//Emergency loop used when Master got deleted or the main loop exited while Defcon == 0
+//Loop is driven externally so runtimes only cancel the current recovery attempt
+/datum/controller/failsafe/proc/emergency_loop()
+	//The code in this proc should be kept as simple as possible, anything complicated like to_chat might rely on master existing and runtime
+	//The goal should always be to get a new Master up and running before anything else
+	. = -1
+	switch (defcon) //The lower defcon goes the harder we try to fix the MC
+		if (2 to 3) //Try to normally recreate the MC two times
+			. = Recreate_MC()
+		if (1) //Delete the old MC first so we don't transfer any info, in case that caused any issues
+			del(Master)
+			. = Recreate_MC()
+
+	if (. == 1) //We were able to create a new master
+		master_iteration = 0
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, span_adminnotice("Failsafe recovered MC while in emergency state [defcon_pretty()]"))
+	else
+		log_game("FailSafe: Failsafe in emergency state and was unable to recreate MC while in defcon state [defcon_pretty()].")
+		message_admins(span_boldannounce("Failsafe in emergency state and master down, trying to recreate MC while in defcon level [defcon_pretty()] failed."))
+
+///Recreate all SSs which will still cause data survive due to Recover(), the new Master will then find and take them from global.vars
+/proc/recover_all_SS_and_recreate_master()
+	del(Master)
+	var/list/subsytem_types = subtypesof(/datum/controller/subsystem)
+	sortTim(subsytem_types, /proc/cmp_subsystem_init)
+	for(var/I in subsytem_types)
+		new I
+	. = Recreate_MC()
+	if (. == 1) //We were able to create a new master
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after recovering all subsystems!"))
+	else
+		message_admins(span_boldannounce("Failed to create new MC!"))
+
+///Delete all existing SS to basically start over
+/proc/delete_all_SS_and_recreate_master()
+	del(Master)
+	for(var/global_var in global.vars)
+		if (istype(global.vars[global_var], /datum/controller/subsystem))
+			del(global.vars[global_var])
+	. = Recreate_MC()
+	if (. == 1) //We were able to create a new master
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after deleting and recreating all subsystems!"))
+	else
+		message_admins(span_boldannounce("Failed to create new MC!"))
+
 /datum/controller/failsafe/proc/defcon_pretty()
 	return defcon
 

code/controllers/master.dm

@@ -7,6 +7,11 @@
  *
  **/
 
+//Init the debugger datum first so we can debug Master
+//You might wonder why not just create the debugger datum global in its own file, since its loaded way earlier than this DM file
+//Well for whatever reason then the Master gets created first and then the debugger when doing that
+//So thats why this code lives here now, until someone finds out how Byond inits globals
+GLOBAL_REAL(Debugger, /datum/debugger) = new
 //This is the ABSOLUTE ONLY THING that should init globally like this
 //2019 update: the failsafe,config and Global controllers also do it
 GLOBAL_REAL(Master, /datum/controller/master) = new
@@ -88,15 +93,27 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/list/_subsystems = list()
 	subsystems = _subsystems
 	if (Master != src)
-		if (istype(Master))
+		if (istype(Master)) //If there is an existing MC take over his stuff and delete it
 			Recover()
 			qdel(Master)
+			Master = src
 		else
+			//Code used for first master on game boot or if existing master got deleted
+			Master = src
 			var/list/subsytem_types = subtypesof(/datum/controller/subsystem)
 			sortTim(subsytem_types, /proc/cmp_subsystem_init)
+			//Find any abandoned subsystem from the previous master (if there was any)
+			var/list/existing_subsystems = list()
+			for(var/global_var in global.vars)
+				if (istype(global.vars[global_var], /datum/controller/subsystem))
+					existing_subsystems += global.vars[global_var]
+			//Either init a new SS or if an existing one was found use that
 			for(var/I in subsytem_types)
-				_subsystems += new I
-		Master = src
+				var/datum/controller/subsystem/existing_subsystem = locate(I) in existing_subsystems
+				if (istype(existing_subsystem))
+					_subsystems += existing_subsystem
+				else
+					_subsystems += new I
 
 	if(!GLOB)
 		new /datum/controller/global_vars
@@ -127,7 +144,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/delay = 50 * ++Master.restart_count
 	Master.restart_timeout = world.time + delay
 	Master.restart_clear = world.time + (delay * 2)
-	Master.processing = FALSE //stop ticking this one
+	if (Master) //Can only do this if master hasn't been deleted
+		Master.processing = FALSE //stop ticking this one
 	try
 		new/datum/controller/master()
 	catch
@@ -173,8 +191,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 		current_runlevel = Master.current_runlevel
 		StartProcessing(10)
 	else
-		to_chat(world, "<span class='boldannounce'>The Master Controller is having some issues, we will need to re-initialize EVERYTHING</span>")
-		Initialize(20, TRUE)
+		to_chat(world, span_boldannounce("The Master Controller is having some issues, we will need to re-initialize EVERYTHING"))
+		Initialize(20, TRUE, FALSE)
 
 
 // Please don't stuff random bullshit here,

code/controllers/subsystem.dm

@@ -128,7 +128,8 @@
 	dequeue()
 	can_fire = 0
 	flags |= SS_NO_FIRE
-	Master.subsystems -= src
+	if (Master)
+		Master.subsystems -= src
 	return ..()
 
 //Queue it to run.
@@ -197,9 +198,9 @@
 		queue_next.queue_prev = queue_prev
 	if (queue_prev)
 		queue_prev.queue_next = queue_next
-	if (src == Master.queue_tail)
+	if (Master && (src == Master.queue_tail))
 		Master.queue_tail = queue_prev
-	if (src == Master.queue_head)
+	if (Master && (src == Master.queue_head))
 		Master.queue_head = queue_next
 	queued_time = 0
 	if (state == SS_QUEUED)

code/controllers/subsystem/air.dm

@@ -264,6 +264,19 @@ SUBSYSTEM_DEF(air)
 	*/
 	currentpart = SSAIR_PIPENETS
 
+/datum/controller/subsystem/air/Recover()
+	hotspots = SSair.hotspots
+	networks = SSair.networks
+	rebuild_queue = SSair.rebuild_queue
+	expansion_queue = SSair.expansion_queue
+	atmos_machinery = SSair.atmos_machinery
+	atmos_air_machinery = SSair.atmos_air_machinery
+	pipe_init_dirs_cache = SSair.pipe_init_dirs_cache
+	gas_reactions = SSair.gas_reactions
+	high_pressure_delta = SSair.high_pressure_delta
+	currentrun = SSair.currentrun
+	deferred_airs = SSair.deferred_airs
+	string_mixes = SSair.string_mixes
 
 /**
  * Adds a given machine to the processing system for SSAIR_ATMOSMACHINERY processing.

code/controllers/subsystem/assets.dm

@@ -31,3 +31,7 @@ SUBSYSTEM_DEF(assets)
 	transport.Initialize(cache)
 
 	..()
+
+/datum/controller/subsystem/assets/Recover()
+	cache = SSassets.cache
+	preload = SSassets.preload

code/controllers/subsystem/autotransfer.dm

@@ -15,3 +15,7 @@ SUBSYSTEM_DEF(autotransfer)
 	if (world.time > targettime)
 		SSvote.initiate_vote("transfer",null, FALSE) //WS Edit - Ghost Vote Rework
 		targettime = targettime + CONFIG_GET(number/vote_autotransfer_interval)
+
+/datum/controller/subsystem/autotransfer/Recover()
+	starttime = SSautotransfer.starttime
+	targettime = SSautotransfer.targettime

code/controllers/subsystem/jukeboxes.dm

@@ -43,6 +43,11 @@ SUBSYSTEM_DEF(jukeboxes)
 		M.playsound_local(M, null, 100, channel = youvegotafreejukebox[2], S = song_to_init)
 	return activejukeboxes.len
 
+/datum/controller/subsystem/jukeboxes/Recover()
+	songs = SSjukeboxes.songs
+	activejukeboxes = SSjukeboxes.activejukeboxes
+	freejukeboxchannels = SSjukeboxes.freejukeboxchannels
+
 /datum/controller/subsystem/jukeboxes/proc/removejukebox(IDtoremove)
 	if(islist(activejukeboxes[IDtoremove]))
 		var/jukechannel = activejukeboxes[IDtoremove][2]