Log an error when endianness change is not compatible with arch (#5945)

Author: wargio

librz/arch/asm.c

@@ -560,6 +560,17 @@ RZ_API ut32 rz_asm_get_endianness(RzAsm *a) {
 	return a->cur->endian;
 }
 
+RZ_API bool rz_asm_support_endianness(RzAsm *a, ut32 endian) {
+	rz_return_val_if_fail(a && a->cur, false);
+
+	if (a->cur->endian == RZ_SYS_ENDIAN_NONE || a->cur->endian == RZ_SYS_ENDIAN_BI) {
+		// always return what the bin or default endianness of the system
+		return true;
+	}
+
+	return endian & a->cur->endian;
+}
+
 RZ_API bool rz_asm_set_big_endian(RzAsm *a, bool b) {
 	rz_return_val_if_fail(a && a->cur, false);
 	a->big_endian = false; // little endian by default

librz/bin/bfile_string.c

@@ -643,7 +643,8 @@ RZ_API RZ_OWN RzPVector /*<RzBinString *>*/ *rz_bin_file_strings(RZ_NONNULL RzBi
 
 	if (bf->o) {
 		const RzBinInfo *binfo = rz_bin_object_get_info(bf->o);
-		prefer_big_endian = binfo ? binfo->big_endian : false;
+		// follow what RzBinInfo says only if the arch is set.
+		prefer_big_endian = binfo && binfo->arch ? binfo->big_endian : false;
 	}
 
 	SharedData shared = {

librz/bin/p/bin_any.c

@@ -8,6 +8,12 @@
 #include <rz_bin.h>
 #include <rz_magic.h>
 
+#if RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_LITTLE
+#define ANY_DEFAULT_ENDIANNESS false
+#else
+#define ANY_DEFAULT_ENDIANNESS true
+#endif
+
 static char *get_filetype(RzBuffer *b) {
 	ut8 buf[4096] = { 0 };
 	char *res = NULL;
@@ -49,13 +55,8 @@ static RzBinInfo *info(RzBinFile *bf) {
 	ret->lang = "";
 	ret->file = rz_str_dup(bf->file);
 	ret->type = get_filetype(bf->buf);
-	ret->has_pi = 0;
-	ret->has_canary = 0;
 	ret->has_retguard = -1;
-	ret->big_endian = 0;
-	ret->has_va = 0;
-	ret->has_nx = 0;
-	ret->dbg_info = 0;
+	ret->big_endian = ANY_DEFAULT_ENDIANNESS;
 	return ret;
 }
 

librz/core/carch.c

@@ -228,9 +228,7 @@ static bool core_arch_default_is_big_endian(RzCore *core) {
 	return false;
 }
 
-RZ_DEPRECATE static void core_update_endianness(RzCore *core) {
-	bool big_endian = core_arch_default_is_big_endian(core);
-
+RZ_DEPRECATE static void core_set_endianness(RzCore *core, bool big_endian) {
 	rz_asm_set_big_endian(core->rasm, big_endian);
 	rz_analysis_set_big_endian(core->analysis, big_endian);
 
@@ -247,6 +245,11 @@ RZ_DEPRECATE static void core_update_endianness(RzCore *core) {
 	}
 }
 
+RZ_DEPRECATE static void core_update_endianness(RzCore *core) {
+	bool big_endian = core_arch_default_is_big_endian(core);
+	core_set_endianness(core, big_endian);
+}
+
 // most of this code is a copy from cconfig
 RZ_DEPRECATE static void core_update_syscall_db(RzCore *core) {
 	if (core->analysis->syscall->db) {
@@ -555,4 +558,30 @@ RZ_DEPRECATE RZ_API bool rz_core_arch_configure(RZ_NONNULL RzCore *core, RZ_NULL
 
 	core_update_config_from_arch(core, is_new_arch);
 	return true;
-}
+}
+
+/**
+ * \brief      Sets the endianness a given RzCore
+ *
+ * \param      core        The core
+ * \param[in]  big_endian  When true, tries to set the endianness to big endian, otherwise little endian
+ *
+ * \return     On success returns true, otherwise false
+ */
+RZ_DEPRECATE RZ_API bool rz_core_set_endianness(RZ_NONNULL RzCore *core, bool big_endian) {
+	rz_return_val_if_fail(core && core->config && core->bin && core->rasm && core->analysis && core->parser && core->dbg && core->egg, false);
+
+	ut32 endianness = big_endian ? RZ_SYS_ENDIAN_BIG : RZ_SYS_ENDIAN_LITTLE;
+
+	if (!rz_asm_support_endianness(core->rasm, endianness)) {
+		const char *arch = rz_core_get_arch(core);
+		const char *want = big_endian ? "big" : "little";
+		const char *supported = (!big_endian) ? "big" : "little";
+		RZ_LOG_ERROR("core: cannot change to %s endian (arch '%s' supports only %s endian)\n", want, arch, supported);
+		return false;
+	}
+
+	core_set_endianness(core, big_endian);
+	core_update_config_b(core, "cfg.bigendian", core->rasm->big_endian);
+	return true;
+}

librz/core/cbin.c

@@ -547,8 +547,6 @@ static void sdb_concat_by_path(Sdb *s, const char *path) {
 
 RZ_API bool rz_core_bin_apply_config(RzCore *r, RzBinFile *binfile) {
 	rz_return_val_if_fail(r && binfile, false);
-	int v;
-	char str[RZ_FLAG_NAME_SIZE];
 	RzBinObject *obj = binfile->o;
 	if (!obj) {
 		return false;
@@ -558,38 +556,44 @@ RZ_API bool rz_core_bin_apply_config(RzCore *r, RzBinFile *binfile) {
 		return false;
 	}
 	rz_config_set(r->config, "file.type", rz_str_get(info->rclass));
-	rz_config_set(r->config, "cfg.bigendian",
-		info->big_endian ? "true" : "false");
-	if (info->lang) {
-		rz_config_set(r->config, "bin.lang", info->lang);
-	}
-	rz_config_set(r->config, "asm.os", info->os);
+	rz_config_set(r->config, "bin.lang", rz_str_get(info->lang));
 	if (info->rclass && !strcmp(info->rclass, "pe")) {
 		rz_config_set(r->config, "analysis.cpp.abi", "msvc");
 	} else {
 		rz_config_set(r->config, "analysis.cpp.abi", "itanium");
 	}
-	rz_config_set(r->config, "asm.arch", info->arch);
+	if (RZ_STR_ISNOTEMPTY(info->os)) {
+		rz_config_set(r->config, "asm.os", info->os);
+	}
+	if (RZ_STR_ISNOTEMPTY(info->arch)) {
+		// update the arch & bits only if really set.
+		rz_config_set(r->config, "asm.arch", info->arch);
+		rz_config_set_i(r->config, "asm.bits", info->bits);
+	}
 	if (RZ_STR_ISNOTEMPTY(info->cpu)) {
+		// update the CPU only if really set.
 		rz_config_set(r->config, "asm.cpu", info->cpu);
 	}
 	if (RZ_STR_ISNOTEMPTY(info->features)) {
+		// update the CPU features only if really set.
 		rz_config_set(r->config, "asm.features", info->features);
 	}
-	rz_config_set(r->config, "analysis.arch", info->arch);
-	snprintf(str, RZ_FLAG_NAME_SIZE, "%i", info->bits);
-	rz_config_set(r->config, "asm.bits", str);
-	rz_config_set(r->config, "asm.debuginfo",
-		(RZ_BIN_DBG_STRIPPED & info->dbg_info) ? "false" : "true");
-	v = rz_analysis_archinfo(r->analysis, RZ_ANALYSIS_ARCHINFO_TEXT_ALIGN);
-	if (v != -1) {
-		rz_config_set_i(r->config, "asm.pcalign", v);
+	if (RZ_STR_ISNOTEMPTY(info->arch)) {
+		// update the endianness only if the arch value is set.
+		ut32 endianness = info->big_endian ? RZ_SYS_ENDIAN_BIG : RZ_SYS_ENDIAN_LITTLE;
+		if (rz_asm_support_endianness(r->rasm, endianness)) {
+			// change only if the current arch supports it.
+			rz_config_set_b(r->config, "cfg.bigendian", info->big_endian);
+		}
 	}
+	rz_config_set_b(r->config, "asm.debuginfo", (RZ_BIN_DBG_STRIPPED & info->dbg_info) ? false : true);
+
 	rz_core_analysis_type_init(r);
 	rz_core_analysis_cc_init(r);
 	if (info->default_cc && rz_analysis_cc_exist(r->analysis, info->default_cc)) {
 		rz_config_set(r->config, "analysis.cc", info->default_cc);
 	}
+
 	char *types_dir = rz_path_system(r->sys_path, RZ_SDB_TYPES);
 	if (!types_dir) {
 		return false;

librz/core/cconfig.c

@@ -13,6 +13,12 @@
 
 #include "core_private.h"
 
+#if RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_LITTLE
+#define CFG_DEFAULT_ENDIANNESS "false"
+#else
+#define CFG_DEFAULT_ENDIANNESS "true"
+#endif
+
 typedef struct config_opt_descr {
 	const char *option;
 	const char *description;
@@ -790,21 +796,7 @@ static bool cb_asmsyntax(void *user, void *data) {
 static bool cb_bigendian(void *user, void *data) {
 	RzCore *core = (RzCore *)user;
 	RzConfigNode *node = (RzConfigNode *)data;
-	// Try to set endian based on preference, restrict by RzAsmPlugin
-	bool isbig = rz_asm_set_big_endian(core->rasm, node->i_value);
-	// Set analysis endianness the same as asm
-	rz_analysis_set_big_endian(core->analysis, isbig);
-	// While analysis sets endianess for TypesDB there might
-	// be cases when it isn't availble for the chosen analysis
-	// plugin but types and printing commands still need the
-	// corresponding endianness. Thus we set these explicitly:
-	rz_type_db_set_endian(core->analysis->typedb, node->i_value);
-	core->print->big_endian = node->i_value;
-	// the big endian should also be assigned to dbg->bp->endian
-	if (core->dbg && core->dbg->bp) {
-		core->dbg->bp->endian = isbig;
-	}
-	return true;
+	return rz_core_set_endianness(core, node->i_value);
 }
 
 static bool cb_cfgdatefmt(void *user, void *data) {
@@ -3197,7 +3189,7 @@ RZ_API int rz_core_config_init(RzCore *core) {
 	SETBPREF("cfg.wseek", "false", "Seek after write");
 	SETICB("cfg.seek.histsize", 63, NULL, "Maximum size of the seek history");
 	SETCB("cfg.seek.silent", "false", NULL, "When true, seek movements are not logged in seek history");
-	SETCB("cfg.bigendian", "false", &cb_bigendian, "Use little (false) or big (true) endianness");
+	SETCB("cfg.bigendian", CFG_DEFAULT_ENDIANNESS, &cb_bigendian, "Use little (false) or big (true) endianness");
 	SETI("cfg.cpuaffinity", 0, "Run on cpuid");
 
 	/* log */

librz/include/rz_asm.h

@@ -178,6 +178,7 @@ RZ_API bool rz_asm_set_arch(RzAsm *a, const char *name, int bits);
 RZ_DEPRECATE RZ_API int rz_asm_set_bits(RzAsm *a, int bits);
 RZ_DEPRECATE RZ_API void rz_asm_set_cpu(RzAsm *a, const char *cpu);
 RZ_API ut32 rz_asm_get_endianness(RzAsm *a);
+RZ_API bool rz_asm_support_endianness(RzAsm *a, ut32 endian);
 RZ_API bool rz_asm_set_big_endian(RzAsm *a, bool big_endian);
 RZ_API bool rz_asm_set_syntax(RzAsm *a, int syntax);
 RZ_API int rz_asm_syntax_from_string(const char *name);

librz/include/rz_core.h

@@ -785,6 +785,7 @@ typedef enum {
 	RZ_CORE_ANALYSIS_EXPERIMENTAL, ///< aaaa
 } RzCoreAnalysisType;
 
+RZ_DEPRECATE RZ_API bool rz_core_set_endianness(RZ_NONNULL RzCore *core, bool big_endian);
 RZ_DEPRECATE RZ_API bool rz_core_arch_configure(RZ_NONNULL RzCore *core, RZ_NULLABLE const char *arch, int bits, RZ_NULLABLE const char *cpu, RZ_NULLABLE const char *os, RZ_NULLABLE const char *platform);
 
 RZ_API RzAnalysisOp *rz_core_analysis_op(RzCore *core, ut64 addr, int mask);

librz/main/rizin.c

@@ -110,6 +110,7 @@ static int main_help(RZ_BORROW RZ_NONNULL RzCore *core, int line) {
 			"-d",          "",          "Debug the executable 'file' or running process 'pid'",
 			"-D",          "backend",   "Enable debug mode (e cfg.debug=true)",
 			"-e",          "k=v",       "Evaluate config var",
+			"-E",          "endian",    "Endianness -E big or -E little",
 			"-f",          "",          "Block size = file size",
 			"-F",          "binplug",   "Force to use that rbin plugin",
 			"-h, -hh",     "",          "Show help message, -hh for long",
@@ -434,6 +435,7 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 	ut64 mapaddr = 0LL;
 	bool quiet = false;
 	int is_gdb = false;
+	ut32 endianness = RZ_SYS_ENDIAN_NONE;
 	const char *s_seek = NULL;
 	bool compute_hashes = true;
 	RzList *cmds = rz_list_new();
@@ -502,7 +504,7 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 	char *debugbackend = rz_str_dup("native");
 
 	RzGetopt opt;
-	rz_getopt_init(&opt, argc, argv, "=012AMCwxfF:H:hm:e:nk:NdqQs:p:b:B:a:Lui:I:l:R:r:c:D:vVSTzuXt");
+	rz_getopt_init(&opt, argc, argv, "=012AMCwxfF:H:hm:E:e:nk:NdqQs:p:b:B:a:Lui:I:l:R:r:c:D:vVSTzuXt");
 	while (argc >= 2 && (c = rz_getopt_next(&opt)) != -1) {
 		switch (c) {
 		case '-':
@@ -581,11 +583,38 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 		case 'e':
 			if (!strcmp(opt.arg, "q")) {
 				rz_core_cmd0(r, "eq");
+			} else if (rz_str_startswith(opt.arg, "asm.arch")) {
+				RZ_LOG_ERROR("use -a argument for setting the arch name\n");
+				ret = 1;
+				goto beach;
+			} else if (rz_str_startswith(opt.arg, "asm.bits")) {
+				RZ_LOG_ERROR("use -b argument for setting the arch bits\n");
+				ret = 1;
+				goto beach;
+			} else if (rz_str_startswith(opt.arg, "asm.os")) {
+				RZ_LOG_ERROR("use -k argument for setting the OS/kern\n");
+				ret = 1;
+				goto beach;
+			} else if (rz_str_startswith(opt.arg, "cfg.bigendian")) {
+				RZ_LOG_ERROR("use -E argument for setting the endianness\n");
+				ret = 1;
+				goto beach;
 			} else {
-				rz_config_eval(r->config, opt.arg);
+				rz_config_eval(r->config, (void *)opt.arg);
 				rz_list_append(evals, (void *)opt.arg);
 			}
 			break;
+		case 'E':
+			if (RZ_STR_EQ(opt.arg, "big")) {
+				endianness = RZ_SYS_ENDIAN_BIG;
+			} else if (RZ_STR_EQ(opt.arg, "little")) {
+				endianness = RZ_SYS_ENDIAN_LITTLE;
+			} else {
+				RZ_LOG_ERROR("Invalid endianness value '%s'\n", opt.arg);
+				ret = 1;
+				goto beach;
+			}
+			break;
 		case 'f':
 			fullfile = true;
 			break;
@@ -956,6 +985,9 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 			if (asmos) {
 				rz_config_set(r->config, "asm.os", asmos);
 			}
+			if (endianness != RZ_SYS_ENDIAN_NONE) {
+				rz_config_set_b(r->config, "cfg.bigendian", endianness == RZ_SYS_ENDIAN_BIG);
+			}
 			// TODO: load rbin thing
 		} else {
 			RZ_LOG_ERROR("Cannot slurp from stdin\n");
@@ -1089,7 +1121,9 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 		if (asmos) {
 			rz_config_set(r->config, "asm.os", asmos);
 		}
-
+		if (endianness != RZ_SYS_ENDIAN_NONE) {
+			rz_config_set_b(r->config, "cfg.bigendian", endianness == RZ_SYS_ENDIAN_BIG);
+		}
 		if (!debug || debug == 2) {
 			const char *dbg_profile = rz_config_get(r->config, "dbg.profile");
 			if (opt.ind == argc && dbg_profile && *dbg_profile) {
@@ -1249,10 +1283,6 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 		if (mapaddr) {
 			rz_core_seek(r, mapaddr, true);
 		}
-		rz_list_foreach (evals, iter, cmdn) {
-			rz_config_eval(r->config, cmdn);
-			rz_cons_flush();
-		}
 		if (asmarch) {
 			rz_config_set(r->config, "asm.arch", asmarch);
 		}
@@ -1262,7 +1292,13 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 		if (asmos) {
 			rz_config_set(r->config, "asm.os", asmos);
 		}
-
+		if (endianness != RZ_SYS_ENDIAN_NONE) {
+			rz_config_set_b(r->config, "cfg.bigendian", endianness == RZ_SYS_ENDIAN_BIG);
+		}
+		rz_list_foreach (evals, iter, cmdn) {
+			rz_config_eval(r->config, cmdn);
+			rz_cons_flush();
+		}
 		debug = r->file && iod && (r->file->fd == iod->fd) && iod->plugin &&
 			(iod->plugin->isdbg || (debug == 2 && !strcmp(iod->plugin->name, "dmp")));
 		if (debug) {
@@ -1336,10 +1372,6 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 	} else {
 		rz_core_block_read(r);
 
-		rz_list_foreach (evals, iter, cmdn) {
-			rz_config_eval(r->config, cmdn);
-			rz_cons_flush();
-		}
 		if (asmarch) {
 			rz_config_set(r->config, "asm.arch", asmarch);
 		}
@@ -1349,6 +1381,13 @@ RZ_API int rz_main_rizin(int argc, const char **argv) {
 		if (asmos) {
 			rz_config_set(r->config, "asm.os", asmos);
 		}
+		if (endianness != RZ_SYS_ENDIAN_NONE) {
+			rz_config_set_b(r->config, "cfg.bigendian", endianness == RZ_SYS_ENDIAN_BIG);
+		}
+		rz_list_foreach (evals, iter, cmdn) {
+			rz_config_eval(r->config, cmdn);
+			rz_cons_flush();
+		}
 	}
 	{
 		char *global_rc = rz_path_system_rc(r->sys_path);