Save logging + Fixes

code/__defines/misc.dm

@@ -79,6 +79,8 @@
 #define MAX_NAME_LEN          26
 #define MAX_DESC_LEN          128
 #define MAX_TEXTFILE_LENGTH 128000		// 512GQ file
+//Maximum length of a MEDIUM_TEXT column type in a mariadb database.
+#define MAX_MEDIUM_TEXT_LEN 65535
 
 // Event defines.
 #define EVENT_LEVEL_MUNDANE  1
@@ -300,3 +302,6 @@
 
 //Damage stuff
 #define ITEM_HEALTH_NO_DAMAGE -1
+
+///Formats into a readable string neatly an exception's details.
+#define EXCEPTION_TEXT(E) "'[E.name]' ('[E.type]'): '[E.file]':[E.line][length(E.desc)? ":\n'[E.desc]'" : ""]"

code/__defines/time.dm

@@ -22,3 +22,6 @@
 #define worldtime2stationtime(time) time2text(roundstart_hour HOURS + time, "hh:mm")
 #define round_duration_in_ticks (round_start_time ? world.time - round_start_time : 0)
 #define station_time_in_ticks (roundstart_hour HOURS + round_duration_in_ticks)
+
+///Convert a time value from REALTIMEOFDAY into an amount of seconds
+#define REALTIMEOFDAY2SEC(T) ((REALTIMEOFDAY - T) / (1 SECOND))

code/_macros.dm

@@ -168,6 +168,12 @@
 #define SPAN_PINK(X)     SPAN_CLASS("font_pink",     X)
 #define SPAN_PALEPINK(X) SPAN_CLASS("font_palepink", X)
 #define SPAN_SINISTER(X) SPAN_CLASS("sinister", X)
+
+//Persistence
+#define SPAN_SERIALIZER(X)    SPAN_CLASS("serializer",    X)
+#define SPAN_AUTOSAVE(X)      SPAN_CLASS("autosave",      X)
+#define SPAN_AUTOSAVE_WARN(X) SPAN_CLASS("autosave_warn", X)
+
 // placeholders
 #define SPAN_GOOD(X)     SPAN_GREEN(X)
 #define SPAN_NEUTRAL(X)  SPAN_BLUE(X)

code/controllers/configuration.dm

@@ -848,7 +848,9 @@ var/global/list/gamemode_cache = list()
 					config.show_typing_indicator_for_whispers = TRUE
 
 				else
-					log_misc("Unknown setting in configuration: '[name]'")
+					//Shitty hook to get our extra settings to load until config options code is less dumb
+					if(!load_mod_config(name, value))
+						log_misc("Unknown setting in configuration: '[name]'")
 
 		else if(type == "game_options")
 			if(!value)
@@ -976,7 +978,9 @@ var/global/list/gamemode_cache = list()
 					config.disable_daycycle = TRUE
 
 				else
-					log_misc("Unknown setting in configuration: '[name]'")
+					//Shitty hook to get our extra settings to load until config options code is less dumb
+					if(!load_mod_game_options(name, value))
+						log_misc("Unknown setting in configuration: '[name]'")
 
 	fps = round(fps)
 	if(fps <= 0)
@@ -1020,7 +1024,8 @@ var/global/list/gamemode_cache = list()
 			if ("password")
 				sqlpass = value
 			else
-				log_misc("Unknown setting in configuration: '[name]'")
+				if(!load_mod_dbconfig(name, value))
+					log_misc("Unknown setting in configuration: '[name]'")
 
 /datum/configuration/proc/pick_mode(mode_name)
 	// I wish I didn't have to instance the game modes in order to look up
@@ -1042,3 +1047,15 @@ var/global/list/gamemode_cache = list()
 	var/event_info = safe_file2text(filename, FALSE)
 	if(event_info)
 		custom_event_msg = event_info
+
+///Hook stub for loading modpack specific configs. Just override in modpack.
+/datum/configuration/proc/load_mod_config(var/name, var/value)
+	return
+
+///Hook stub for loading modpack specific game_options. Just override in modpack.
+/datum/configuration/proc/load_mod_game_options(var/name, var/value)
+	return
+
+///Hook stub for loading modpack specific dbconfig. Just override in modpack.
+/datum/configuration/proc/load_mod_dbconfig(var/name, var/value)
+	return

code/controllers/subsystems/air.dm

@@ -441,3 +441,9 @@ Total Unsimulated Turfs: [world.maxx*world.maxy*world.maxz - simulated_turf_coun
 		active_edges -= E
 	if(processing_edges)
 		processing_edges -= E
+
+/datum/controller/subsystem/air/proc/invalidate_all_zones()
+	while(SSair.zones.len)
+		var/zone/zone = SSair.zones[SSair.zones.len]
+		SSair.zones.len--
+		zone.c_invalidate()

code/controllers/subsystems/machines.dm

@@ -215,6 +215,16 @@ if(current_step == this_step || (check_resumed && !resumed)) {\
 	if (istype(SSmachines.power_objects))
 		power_objects = SSmachines.power_objects
 
+///Divides the contents of all pipenets into their individual members.
+/datum/controller/subsystem/machines/proc/temporarily_store_pipenets()
+	if(can_fire)
+		log_warning(("Tried to store pipenets air while the subsystem is running!"))
+		CRASH("Tried to store pipenets air while the subsystem is running!")
+
+	for(var/datum/pipe_network/net in SSmachines.pipenets)
+		for(var/datum/pipeline/line in net.line_members)
+			line.temporarily_store_fluids()
+
 #undef SSMACHINES_PIPENETS
 #undef SSMACHINES_MACHINERY
 #undef SSMACHINES_POWERNETS

code/modules/client/darkmode.css

@@ -96,6 +96,11 @@ h1.alert, h2.alert		{color: #a4bad6;}
 .antagdesc				{color: #ff0033; font-size: 125%}
 .reflex_shoot			{color: #000099; font-style: italic;}
 
+/*Persistence*/
+.autosave				{color: #00ff00; font-weight: bold; font-size: 125%;}
+.autosave_warn			{color: #a13434; font-weight: bold; font-size: 125%;}
+.serializer				{color: #a523a5; font-weight: bold;}
+
 /* General purpose colour classes */
 .font_red               {color: #ff0000;}
 .font_orange            {color: #ff7f00;}

code/modules/client/lightmode.css

@@ -92,6 +92,11 @@ h1.alert, h2.alert		{color: #000000;}
 .antagdesc				{color: #ff0033; font-size: 125%}
 .reflex_shoot			{color: #000099; font-style: italic;}
 
+/*Persistence*/
+.autosave				{color: #104b10; font-weight: bold; font-size: 125%;}
+.autosave_warn			{color: #5f2525; font-weight: bold; font-size: 125%;}
+.serializer				{color: #791979; font-weight: bold;}
+
 /* General purpose colour classes */
 .font_red               {color: #ff0000;}
 .font_orange            {color: #ff7f00;}

config/example/config.txt

@@ -462,3 +462,19 @@ RADIATION_LOWER_LIMIT 0.15
 
 ## Uncomment this to show a typing indicator for people writing whispers.
 #SHOW_TYPING_INDICATOR_FOR_WHISPERS
+
+################################
+## Persistence Stuff
+################################
+
+## Time in minutes between automated world saves. Default is every 120 minutes.
+#AUTOSAVE_INTERVAL 120
+
+## Uptime in hours after which the next autosave will force a server reboot. Default is 12 Hours. Setting to 0 disables it.
+#AUTOSAVE_AUTO_RESTART 12
+
+## !! - Use With Caution - !!
+## Set what kind of errors will be skipped over during saving/loading if encountered. Default is "NONE".
+## This value is meant to be used for rescuing a broken save, or forcing a broken save to work. As it will most likely cause problems if used constantly.
+## The possible values are: "NONE"->No errors allowed at all, "RECOVERABLE"->Only recoverable errors allowed, "ANY"->Any error, even connection error will be ignored.
+#SAVE_ERROR_TOLERANCE NONE

mods/persistence/__defines/logging.dm

@@ -1,3 +1,3 @@
 /proc/report_progress_serializer(var/progress_message)
-	admin_notice("<span class='boldannounce' style='color:\"purple\";'>[progress_message]</span>", R_DEBUG)
-	to_world_log(progress_message)
+	admin_notice(SPAN_SERIALIZER(progress_message), R_DEBUG)
+	to_world_log(SPAN_SERIALIZER(progress_message))

mods/persistence/__defines/serializer.dm

@@ -1,35 +1,65 @@
-/*
-	Serialized type names
-*/
-#define SERIALIZER_TYPE_NULL		"NULL"
-#define SERIALIZER_TYPE_VAR 		"VAR"
-#define SERIALIZER_TYPE_TEXT		"TEXT"
-#define SERIALIZER_TYPE_NUM 		"NUM"
-#define SERIALIZER_TYPE_PATH		"PATH"
-#define SERIALIZER_TYPE_FILE		"FILE"
-#define SERIALIZER_TYPE_WRAPPER 	"WRAP"
-#define SERIALIZER_TYPE_LIST		"LIST"
-#define SERIALIZER_TYPE_LIST_EMPTY	"EMPTY"
-#define SERIALIZER_TYPE_DATUM		"OBJ"
-#define SERIALIZER_TYPE_DATUM_FLAT	"FLAT_OBJ"
-#define SERIALIZER_TYPE_FLAT_REF	"FLAT_REF"
-
-/*
-	SQL table names
-*/
-#define SQLS_TABLE_DATUM 				"thing"
-#define SQLS_TABLE_DATUM_VARS 			"thing_var"
-#define SQLS_TABLE_LIST_ELEM 			"list_element"
-#define SQLS_TABLE_Z_LEVELS 			"z_level"
-#define SQLS_TABLE_AREAS 				"areas"
-#define SQLS_TABLE_LIMBO 				"limbo"
-#define SQLS_TABLE_LIMBO_DATUM			"limbo_thing"
-#define SQLS_TABLE_LIMBO_DATUM_VARS 	"limbo_thing_var"
-#define SQLS_TABLE_LIMBO_LIST_ELEM		"limbo_list_element"
+///A macro for getting the current save DB name
+#define SQLS_SAVE_DATABASE global.sqldb
 
+/////////////////////////////////////////////////////////
+// Serialized type names
+/////////////////////////////////////////////////////////
+
+#define SERIALIZER_TYPE_NULL        "NULL"
+#define SERIALIZER_TYPE_VAR         "VAR"
+#define SERIALIZER_TYPE_TEXT        "TEXT"
+#define SERIALIZER_TYPE_NUM         "NUM"
+#define SERIALIZER_TYPE_PATH        "PATH"
+#define SERIALIZER_TYPE_FILE        "FILE"
+#define SERIALIZER_TYPE_WRAPPER     "WRAP"
+#define SERIALIZER_TYPE_LIST        "LIST"
+#define SERIALIZER_TYPE_LIST_EMPTY  "EMPTY"
+#define SERIALIZER_TYPE_DATUM       "OBJ"
+#define SERIALIZER_TYPE_DATUM_FLAT  "FLAT_OBJ"
+#define SERIALIZER_TYPE_FLAT_REF    "FLAT_REF"
+
+/////////////////////////////////////////////////////////
+// SQL table names
+/////////////////////////////////////////////////////////
+
+#define SQLS_TABLE_DATUM            "thing"
+#define SQLS_TABLE_DATUM_VARS       "thing_var"
+#define SQLS_TABLE_LIST_ELEM        "list_element"
+#define SQLS_TABLE_Z_LEVELS         "z_level"
+#define SQLS_TABLE_AREAS            "areas"
+#define SQLS_TABLE_LIMBO            "limbo"
+#define SQLS_TABLE_LIMBO_DATUM      "limbo_thing"
+#define SQLS_TABLE_LIMBO_DATUM_VARS "limbo_thing_var"
+#define SQLS_TABLE_LIMBO_LIST_ELEM  "limbo_list_element"
+
+/////////////////////////////////////////////////////////
+// SQL Stored Functions Names
+/////////////////////////////////////////////////////////
+
+///Name of the stored function that returns the time we last made a world save from the db.
+#define SQLS_FUNC_GET_LAST_SAVE_TIME     "GetLastWorldSaveTime"
+///Log to the table when a world save begins, returns the current save log id.
+#define SQLS_FUNC_LOG_SAVE_WORLD_START   "LogSaveWorldStart"
+///Log to the table when a limbo/storage save begins, returns the current save log id.
+#define SQLS_FUNC_LOG_SAVE_STORAGE_START "LogSaveStorageStart"
+///Log to the table when any save ends, returns the current save log id.
+#define SQLS_FUNC_LOG_SAVE_END           "LogSaveEnd"
+
+/////////////////////////////////////////////////////////
+// SQL Stored Procedures Names
+/////////////////////////////////////////////////////////
+
+///Delete the current world save from the db, so we can write a newer one. Procedures are executed with CALL, and don't return anything.
+#define SQLS_PROC_CLEAR_WORLD_SAVE       "ClearWorldSave"
+
+/////////////////////////////////////////////////////////
+// SQL Helpers
+/////////////////////////////////////////////////////////
+
+///A helper for executing a sql query and throwing the proper exception with a standardized error message. QUERY must be a variable.
 #define SQLS_EXECUTE_AND_REPORT_ERROR(QUERY, ERRORMSG)\
 	if(!QUERY.Execute()){\
-		var/errormsg = ERRORMSG + " '[QUERY.ErrorMsg()]'"; \
+		var/errormsg = ERRORMSG + " '[QUERY.ErrorMsg()]'" + "\n'[QUERY.sql]'"; \
 		to_world_log(errormsg);\
 		throw new /exception/sql_connection(errormsg, __FILE__, __LINE__); \
 	}

mods/persistence/_persistence.dme

@@ -26,6 +26,10 @@
 #include "controllers\subsystems\mapping.dm"
 #include "controllers\subsystems\mining.dm"
 #include "controllers\subsystems\persistence.dm"
+#include "controllers\subsystems\persistence\persistence_loading.dm"
+#include "controllers\subsystems\persistence\persistence_saving.dm"
+#include "controllers\subsystems\persistence\persistence_stats.dm"
+#include "controllers\subsystems\persistence\persistence_storage.dm"
 #include "controllers\subsystems\skills.dm"
 #include "controllers\subsystems\initialization\chargen.dm"
 #include "controllers\subsystems\initialization\fabrication.dm"
@@ -284,6 +288,7 @@
 #include "modules\world_save\serializers\one_off_serializer.dm"
 #include "modules\world_save\serializers\sql_serializer.dm"
 #include "modules\world_save\serializers\sql_serializer_db.dm"
+#include "modules\world_save\save_testing.dm"
 #include "modules\world_save\wrappers\_late_wrapper.dm"
 #include "modules\world_save\wrappers\_wrapper.dm"
 #include "modules\world_save\wrappers\_wrapper_holder.dm"

mods/persistence/code/__defines/world_save.dm

@@ -23,4 +23,14 @@ else{src.custom_saved |= list(VARNAMES);}
 
 //Helper to place at the end of Initialize of saved objects to make sure they lateinit only if they don't get deleted during init and if they were saved!
 #define LATE_INIT_IF_SAVED \
-if(. != INITIALIZE_HINT_QDEL && src.persistent_id){return INITIALIZE_HINT_LATELOAD;}
+if(. != INITIALIZE_HINT_QDEL && src.persistent_id){return INITIALIZE_HINT_LATELOAD;}
+
+////////////////////////////////////////////
+// Persistence Error Tolerance
+////////////////////////////////////////////
+///Do not tolerate any errors during save/load.
+#define PERSISTENCE_ERROR_TOLERANCE_NONE 0
+///Tolerate only recoverable errors during save/load.
+#define PERSISTENCE_ERROR_TOLERANCE_RECOVERABLE 1
+///Tolerate ANY errors during save/load. This is likely to cause a corrupted save.
+#define PERSISTENCE_ERROR_TOLERANCE_ANY 2

mods/persistence/code/controllers/configuration.dm

@@ -1,3 +1,34 @@
 /datum/configuration
-	var/autosave_interval = 60 MINUTES
-	var/autosave_auto_reset = 12 HOURS
+	var/autosave_interval     = 2 HOURS
+	var/autosave_auto_restart = 12 HOURS
+	var/save_error_tolerance  = PERSISTENCE_ERROR_TOLERANCE_NONE
+
+/datum/configuration/load_mod_config(name, value)
+	. = ..()
+	switch(name)
+		if("autosave_interval")
+			autosave_interval = text2num(value) MINUTES
+			. = TRUE
+		if("autosave_auto_restart")
+			autosave_auto_restart = text2num(value) HOURS
+			. =  TRUE
+		if("save_error_tolerance")
+			value = lowertext(value)
+			switch(value)
+				if("any")
+					save_error_tolerance = PERSISTENCE_ERROR_TOLERANCE_ANY
+				if("recoverable")
+					save_error_tolerance = PERSISTENCE_ERROR_TOLERANCE_RECOVERABLE
+				if("none")
+					save_error_tolerance = PERSISTENCE_ERROR_TOLERANCE_NONE
+				else
+					log_misc("Bad value for '[name]' : '[value]'! (Expected 'any', 'recoverable' or 'none')")
+			. =  TRUE
+
+///Hook to add persistence settings meant to be in the game_options file if any.
+/datum/configuration/load_mod_game_options(name, value)
+	. = ..()
+
+///Hook to add persistence settings meant to be in the dbconfig file if any.
+/datum/configuration/load_mod_dbconfig(name, value)
+	. = ..()