From 830928c6473f7d35ce07da4bb59a84d5c9c1bbab Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 05:44:51 +0000 Subject: [PATCH 01/21] bench: set minimum epoch iterations to improve `pow_hash` stability reverts: - f3002779a09803a9eb639c6bc112b59ffb7d4a0f (logically) --- src/bench/pow_hash.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/bench/pow_hash.cpp b/src/bench/pow_hash.cpp index d5f9c17f9808..79859966ac95 100644 --- a/src/bench/pow_hash.cpp +++ b/src/bench/pow_hash.cpp @@ -28,7 +28,7 @@ inline void Pow_X11(benchmark::Bench& bench, const size_t bytes) { uint256 hash{}; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { hash = HashX11(in.begin(), in.end()); }); } @@ -38,7 +38,7 @@ inline void Pow_Blake512(benchmark::Bench& bench, const size_t bytes) sph_blake512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_blake512_init(&ctx); sph_blake512(&ctx, in.data(), in.size()); sph_blake512_close(&ctx, &hash); @@ -50,7 +50,7 @@ inline void Pow_Bmw512(benchmark::Bench& bench, const size_t bytes) sph_bmw512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_bmw512_init(&ctx); sph_bmw512(&ctx, in.data(), in.size()); sph_bmw512_close(&ctx, &hash); @@ -62,7 +62,7 @@ inline void Pow_Cubehash512(benchmark::Bench& bench, const size_t bytes) sph_cubehash512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_cubehash512_init(&ctx); sph_cubehash512(&ctx, in.data(), in.size()); sph_cubehash512_close(&ctx, &hash); @@ -74,7 +74,7 @@ inline void Pow_Echo512(benchmark::Bench& bench, const size_t bytes) sph_echo512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_echo512_init(&ctx); sph_echo512(&ctx, in.data(), in.size()); sph_echo512_close(&ctx, &hash); @@ -86,7 +86,7 @@ inline void Pow_Groestl512(benchmark::Bench& bench, const size_t bytes) sph_groestl512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_groestl512_init(&ctx); sph_groestl512(&ctx, in.data(), in.size()); sph_groestl512_close(&ctx, &hash); @@ -98,7 +98,7 @@ inline void Pow_Jh512(benchmark::Bench& bench, const size_t bytes) sph_jh512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_jh512_init(&ctx); sph_jh512(&ctx, in.data(), in.size()); sph_jh512_close(&ctx, &hash); @@ -110,7 +110,7 @@ inline void Pow_Keccak512(benchmark::Bench& bench, const size_t bytes) sph_keccak512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_keccak512_init(&ctx); sph_keccak512(&ctx, in.data(), in.size()); sph_keccak512_close(&ctx, &hash); @@ -122,7 +122,7 @@ inline void Pow_Luffa512(benchmark::Bench& bench, const size_t bytes) sph_luffa512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_luffa512_init(&ctx); sph_luffa512(&ctx, in.data(), in.size()); sph_luffa512_close(&ctx, &hash); @@ -134,7 +134,7 @@ inline void Pow_Shavite512(benchmark::Bench& bench, const size_t bytes) sph_shavite512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_shavite512_init(&ctx); sph_shavite512(&ctx, in.data(), in.size()); sph_shavite512_close(&ctx, &hash); @@ -146,7 +146,7 @@ inline void Pow_Simd512(benchmark::Bench& bench, const size_t bytes) sph_simd512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_simd512_init(&ctx); sph_simd512(&ctx, in.data(), in.size()); sph_simd512_close(&ctx, &hash); @@ -158,7 +158,7 @@ inline void Pow_Skein512(benchmark::Bench& bench, const size_t bytes) sph_skein512_context ctx; uint8_t hash[OUTPUT_SIZE]; std::vector in(bytes, 0); - bench.batch(in.size()).unit("byte").run([&] { + bench.minEpochIterations(20).batch(in.size()).unit("byte").run([&] { sph_skein512_init(&ctx); sph_skein512(&ctx, in.data(), in.size()); sph_skein512_close(&ctx, &hash); From c4e7a40d842f52d27e8bafd7d4fd4330a935fe08 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 05:58:58 +0000 Subject: [PATCH 02/21] fix: suppress unaligned memory UBSan warnings if supported by arch --- src/crypto/x11/sph_types.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/crypto/x11/sph_types.h b/src/crypto/x11/sph_types.h index cb0066b6c1ba..6f3b8db480c6 100644 --- a/src/crypto/x11/sph_types.h +++ b/src/crypto/x11/sph_types.h @@ -327,6 +327,19 @@ typedef int_fast64_t sph_s64; #error SPH_UPTR defined, but endianness is not known. #endif +#if defined(__clang__) +#define SPH_NO_SANITIZE(...) __attribute__((no_sanitize(__VA_ARGS__))) +#else +#define SPH_NO_SANITIZE(...) +#endif // __clang__ + +// Suppress unaligned memory access warnings if supported by hardware +#if defined(SPH_UPTR) && SPH_UNALIGNED +#define SPH_UNALIGNED_ATTRIBS SPH_NO_SANITIZE("alignment") +#else +#define SPH_UNALIGNED_ATTRIBS +#endif // SPH_UPTR && SPH_UNALIGNED + #if SPH_I386_GCC && !SPH_NO_ASM /* @@ -469,6 +482,7 @@ sph_dec16le(const void *src) * @param val the 32-bit value to encode */ static SPH_INLINE void +SPH_UNALIGNED_ATTRIBS sph_enc32be(void *dst, sph_u32 val) { #if defined SPH_UPTR @@ -527,6 +541,7 @@ sph_enc32be_aligned(void *dst, sph_u32 val) * @return the decoded value */ static SPH_INLINE sph_u32 +SPH_UNALIGNED_ATTRIBS sph_dec32be(const void *src) { #if defined SPH_UPTR @@ -587,6 +602,7 @@ sph_dec32be_aligned(const void *src) * @param val the 32-bit value to encode */ static SPH_INLINE void +SPH_UNALIGNED_ATTRIBS sph_enc32le(void *dst, sph_u32 val) { #if defined SPH_UPTR @@ -645,6 +661,7 @@ sph_enc32le_aligned(void *dst, sph_u32 val) * @return the decoded value */ static SPH_INLINE sph_u32 +SPH_UNALIGNED_ATTRIBS sph_dec32le(const void *src) { #if defined SPH_UPTR @@ -726,6 +743,7 @@ sph_dec32le_aligned(const void *src) * @param val the 64-bit value to encode */ static SPH_INLINE void +SPH_UNALIGNED_ATTRIBS sph_enc64be(void *dst, sph_u64 val) { #if defined SPH_UPTR @@ -796,6 +814,7 @@ sph_enc64be_aligned(void *dst, sph_u64 val) * @return the decoded value */ static SPH_INLINE sph_u64 +SPH_UNALIGNED_ATTRIBS sph_dec64be(const void *src) { #if defined SPH_UPTR @@ -868,6 +887,7 @@ sph_dec64be_aligned(const void *src) * @param val the 64-bit value to encode */ static SPH_INLINE void +SPH_UNALIGNED_ATTRIBS sph_enc64le(void *dst, sph_u64 val) { #if defined SPH_UPTR @@ -938,6 +958,7 @@ sph_enc64le_aligned(void *dst, sph_u64 val) * @return the decoded value */ static SPH_INLINE sph_u64 +SPH_UNALIGNED_ATTRIBS sph_dec64le(const void *src) { #if defined SPH_UPTR From 7e8607e1cbfd43348f0f29473a78a2e3dc53efcb Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 09:38:42 +0000 Subject: [PATCH 03/21] refactor: remove large footprint Echo512 impl, switch to C++ --- configure.ac | 4 +- src/Makefile.am | 6 +- src/crypto/x11/{echo.c => echo.cpp} | 95 ++--------------------------- src/crypto/x11/sph_echo.h | 16 ++--- test/lint/lint-whitespace.py | 1 + 5 files changed, 15 insertions(+), 107 deletions(-) rename src/crypto/x11/{echo.c => echo.cpp} (78%) diff --git a/configure.ac b/configure.ac index 6bf3a3205718..b85fcc46d003 100644 --- a/configure.ac +++ b/configure.ac @@ -379,7 +379,7 @@ if test "$enable_debug" = "yes"; then AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) else dnl If not debugging, enable more aggressive optimizations for sphlib sources - AX_CHECK_COMPILE_FLAG([-O3], [SPHLIB_CFLAGS="$SPHLIB_CFLAGS -O3"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-O3], [SPHLIB_FLAGS="$SPHLIB_FLAGS -O3"], [], [$CXXFLAG_WERROR]) # We always enable at at least -g1 debug info to support proper stacktraces in crash infos # Stacktraces will be suboptimal due to optimization, but better than nothing. Also, -fno-omit-frame-pointer @@ -1877,7 +1877,7 @@ AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) -AC_SUBST(SPHLIB_CFLAGS) +AC_SUBST(SPHLIB_FLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(CLMUL_CXXFLAGS) diff --git a/src/Makefile.am b/src/Makefile.am index f2246d57c0c0..5b0417573da8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -728,9 +728,9 @@ crypto_libbitcoin_crypto_avx2_la_SOURCES = crypto/sha256_avx2.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above crypto_libbitcoin_crypto_sph_la_LDFLAGS = $(AM_LDFLAGS) -static -crypto_libbitcoin_crypto_sph_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_sph_la_CXXFLAGS = $(AM_CXXFLAGS) $(SPHLIB_FLAGS) $(PIE_FLAGS) -static crypto_libbitcoin_crypto_sph_la_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_sph_la_CFLAGS = $(SPHLIB_CFLAGS) +crypto_libbitcoin_crypto_sph_la_CFLAGS = $(SPHLIB_FLAGS) crypto_libbitcoin_crypto_sph_la_CPPFLAGS += \ -DSPH_SMALL_FOOTPRINT_CUBEHASH=1 \ -DSPH_SMALL_FOOTPRINT_JH=1 @@ -739,7 +739,7 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/blake.c \ crypto/x11/bmw.c \ crypto/x11/cubehash.c \ - crypto/x11/echo.c \ + crypto/x11/echo.cpp \ crypto/x11/groestl.c \ crypto/x11/jh.c \ crypto/x11/keccak.c \ diff --git a/src/crypto/x11/echo.c b/src/crypto/x11/echo.cpp similarity index 78% rename from src/crypto/x11/echo.c rename to src/crypto/x11/echo.cpp index ab531bdca5cf..7de8f20b4a78 100644 --- a/src/crypto/x11/echo.c +++ b/src/crypto/x11/echo.cpp @@ -35,14 +35,6 @@ #include "sph_echo.h" -#ifdef __cplusplus -extern "C"{ -#endif - -#if SPH_SMALL_FOOTPRINT && !defined SPH_SMALL_FOOTPRINT_ECHO -#define SPH_SMALL_FOOTPRINT_ECHO 1 -#endif - /* * We can use a 64-bit implementation only if a 64-bit type is available. */ @@ -69,8 +61,6 @@ extern "C"{ } \ } while (0) -#if SPH_SMALL_FOOTPRINT_ECHO - static void aes_2rounds_all(sph_u64 W[16][2], sph_u32 *pK0, sph_u32 *pK1, sph_u32 *pK2, sph_u32 *pK3) @@ -109,46 +99,6 @@ aes_2rounds_all(sph_u64 W[16][2], aes_2rounds_all(W, &K0, &K1, &K2, &K3); \ } while (0) -#else - -#define AES_2ROUNDS(X) do { \ - sph_u32 X0 = (sph_u32)(X[0]); \ - sph_u32 X1 = (sph_u32)(X[0] >> 32); \ - sph_u32 X2 = (sph_u32)(X[1]); \ - sph_u32 X3 = (sph_u32)(X[1] >> 32); \ - sph_u32 Y0, Y1, Y2, Y3; \ - AES_ROUND_LE(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); \ - AES_ROUND_NOKEY_LE(Y0, Y1, Y2, Y3, X0, X1, X2, X3); \ - X[0] = (sph_u64)X0 | ((sph_u64)X1 << 32); \ - X[1] = (sph_u64)X2 | ((sph_u64)X3 << 32); \ - if ((K0 = T32(K0 + 1)) == 0) { \ - if ((K1 = T32(K1 + 1)) == 0) \ - if ((K2 = T32(K2 + 1)) == 0) \ - K3 = T32(K3 + 1); \ - } \ - } while (0) - -#define BIG_SUB_WORDS do { \ - AES_2ROUNDS(W[ 0]); \ - AES_2ROUNDS(W[ 1]); \ - AES_2ROUNDS(W[ 2]); \ - AES_2ROUNDS(W[ 3]); \ - AES_2ROUNDS(W[ 4]); \ - AES_2ROUNDS(W[ 5]); \ - AES_2ROUNDS(W[ 6]); \ - AES_2ROUNDS(W[ 7]); \ - AES_2ROUNDS(W[ 8]); \ - AES_2ROUNDS(W[ 9]); \ - AES_2ROUNDS(W[10]); \ - AES_2ROUNDS(W[11]); \ - AES_2ROUNDS(W[12]); \ - AES_2ROUNDS(W[13]); \ - AES_2ROUNDS(W[14]); \ - AES_2ROUNDS(W[15]); \ - } while (0) - -#endif - #define SHIFT_ROW1(a, b, c, d) do { \ sph_u64 tmp; \ tmp = W[a][0]; \ @@ -187,8 +137,6 @@ aes_2rounds_all(sph_u64 W[16][2], SHIFT_ROW3(3, 7, 11, 15); \ } while (0) -#if SPH_SMALL_FOOTPRINT_ECHO - static void mix_column(sph_u64 W[16][2], int ia, int ib, int ic, int id) { @@ -217,35 +165,6 @@ mix_column(sph_u64 W[16][2], int ia, int ib, int ic, int id) #define MIX_COLUMN(a, b, c, d) mix_column(W, a, b, c, d) -#else - -#define MIX_COLUMN1(ia, ib, ic, id, n) do { \ - sph_u64 a = W[ia][n]; \ - sph_u64 b = W[ib][n]; \ - sph_u64 c = W[ic][n]; \ - sph_u64 d = W[id][n]; \ - sph_u64 ab = a ^ b; \ - sph_u64 bc = b ^ c; \ - sph_u64 cd = c ^ d; \ - sph_u64 abx = ((ab & C64(0x8080808080808080)) >> 7) * 27U \ - ^ ((ab & C64(0x7F7F7F7F7F7F7F7F)) << 1); \ - sph_u64 bcx = ((bc & C64(0x8080808080808080)) >> 7) * 27U \ - ^ ((bc & C64(0x7F7F7F7F7F7F7F7F)) << 1); \ - sph_u64 cdx = ((cd & C64(0x8080808080808080)) >> 7) * 27U \ - ^ ((cd & C64(0x7F7F7F7F7F7F7F7F)) << 1); \ - W[ia][n] = abx ^ bc ^ d; \ - W[ib][n] = bcx ^ a ^ cd; \ - W[ic][n] = cdx ^ ab ^ d; \ - W[id][n] = abx ^ bcx ^ cdx ^ ab ^ c; \ - } while (0) - -#define MIX_COLUMN(a, b, c, d) do { \ - MIX_COLUMN1(a, b, c, d, 0); \ - MIX_COLUMN1(a, b, c, d, 1); \ - } while (0) - -#endif - #define BIG_MIX_COLUMNS do { \ MIX_COLUMN(0, 1, 2, 3); \ MIX_COLUMN(4, 5, 6, 7); \ @@ -282,7 +201,6 @@ mix_column(sph_u64 W[16][2], int ia, int ib, int ic, int id) FINAL_BIG; \ } while (0) - #define INCR_COUNTER(sc, val) do { \ sc->C0 = T32(sc->C0 + (sph_u32)(val)); \ if (sc->C0 < (sph_u32)(val)) { \ @@ -408,31 +326,28 @@ echo_big_close(sph_echo_big_context *sc, unsigned ub, unsigned n, /* see sph_echo.h */ void -sph_echo512_init(void *cc) +sph_echo512_init(sph_echo512_context *cc) { echo_big_init(cc, 512); } /* see sph_echo.h */ void -sph_echo512(void *cc, const void *data, size_t len) +sph_echo512(sph_echo512_context *cc, const void *data, size_t len) { - echo_big_core(cc, data, len); + echo_big_core(cc, static_cast(data), len); } /* see sph_echo.h */ void -sph_echo512_close(void *cc, void *dst) +sph_echo512_close(sph_echo512_context *cc, void *dst) { echo_big_close(cc, 0, 0, dst, 16); } /* see sph_echo.h */ void -sph_echo512_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst) +sph_echo512_addbits_and_close(sph_echo512_context *cc, unsigned ub, unsigned n, void *dst) { echo_big_close(cc, ub, n, dst, 16); } -#ifdef __cplusplus -} -#endif diff --git a/src/crypto/x11/sph_echo.h b/src/crypto/x11/sph_echo.h index b8403b3c7073..59193a4ae792 100644 --- a/src/crypto/x11/sph_echo.h +++ b/src/crypto/x11/sph_echo.h @@ -36,10 +36,6 @@ #ifndef SPH_ECHO_H__ #define SPH_ECHO_H__ -#ifdef __cplusplus -extern "C"{ -#endif - #include #include "sph_types.h" @@ -80,7 +76,7 @@ typedef sph_echo_big_context sph_echo512_context; * @param cc the ECHO-512 context (pointer to a * sph_echo512_context) */ -void sph_echo512_init(void *cc); +void sph_echo512_init(sph_echo512_context *cc); /** * Process some data bytes. It is acceptable that len is zero @@ -90,7 +86,7 @@ void sph_echo512_init(void *cc); * @param data the input data * @param len the input data length (in bytes) */ -void sph_echo512(void *cc, const void *data, size_t len); +void sph_echo512(sph_echo512_context *cc, const void *data, size_t len); /** * Terminate the current ECHO-512 computation and output the result into @@ -101,7 +97,7 @@ void sph_echo512(void *cc, const void *data, size_t len); * @param cc the ECHO-512 context * @param dst the destination buffer */ -void sph_echo512_close(void *cc, void *dst); +void sph_echo512_close(sph_echo512_context *cc, void *dst); /** * Add a few additional bits (0 to 7) to the current computation, then @@ -117,10 +113,6 @@ void sph_echo512_close(void *cc, void *dst); * @param dst the destination buffer */ void sph_echo512_addbits_and_close( - void *cc, unsigned ub, unsigned n, void *dst); - -#ifdef __cplusplus -} -#endif + sph_echo512_context *cc, unsigned ub, unsigned n, void *dst); #endif diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py index 06b3adf52241..15d009b29c15 100755 --- a/test/lint/lint-whitespace.py +++ b/test/lint/lint-whitespace.py @@ -18,6 +18,7 @@ EXCLUDED_DIRS = ["depends/patches/", "contrib/guix/patches/", + "src/crypto/x11/", "src/leveldb/", "src/crc32c/", "src/secp256k1/", From 386742b5fcaa6154ca851d85912fd37eb39a391b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 15 Sep 2025 22:29:28 +0000 Subject: [PATCH 04/21] refactor: remove large footprint Shavite512 impl, switch to C++ --- src/Makefile.am | 2 +- src/crypto/x11/shavite.c | 967 ----------------------------------- src/crypto/x11/shavite.cpp | 369 +++++++++++++ src/crypto/x11/sph_shavite.h | 16 +- 4 files changed, 374 insertions(+), 980 deletions(-) delete mode 100644 src/crypto/x11/shavite.c create mode 100644 src/crypto/x11/shavite.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 5b0417573da8..5fe041d81bf1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -744,7 +744,7 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/jh.c \ crypto/x11/keccak.c \ crypto/x11/luffa.c \ - crypto/x11/shavite.c \ + crypto/x11/shavite.cpp \ crypto/x11/simd.c \ crypto/x11/skein.c \ crypto/x11/sph_blake.h \ diff --git a/src/crypto/x11/shavite.c b/src/crypto/x11/shavite.c deleted file mode 100644 index d986bbf3212d..000000000000 --- a/src/crypto/x11/shavite.c +++ /dev/null @@ -1,967 +0,0 @@ -/* $Id: shavite.c 227 2010-06-16 17:28:38Z tp $ */ -/* - * SHAvite-3 implementation. - * - * ==========================(LICENSE BEGIN)============================ - * - * Copyright (c) 2007-2010 Projet RNRT SAPHIR - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ===========================(LICENSE END)============================= - * - * @author Thomas Pornin - */ - -#include -#include - -#include "sph_shavite.h" - -#ifdef __cplusplus -extern "C"{ -#endif - -#if SPH_SMALL_FOOTPRINT && !defined SPH_SMALL_FOOTPRINT_SHAVITE -#define SPH_SMALL_FOOTPRINT_SHAVITE 1 -#endif - -#ifdef _MSC_VER -#pragma warning (disable: 4146) -#endif - -#define C32 SPH_C32 - -/* - * As of round 2 of the SHA-3 competition, the published reference - * implementation and test vectors are wrong, because they use - * big-endian AES tables while the internal decoding uses little-endian. - * The code below follows the specification. To turn it into a code - * which follows the reference implementation (the one called "BugFix" - * on the SHAvite-3 web site, published on Nov 23rd, 2009), comment out - * the code below (from the '#define AES_BIG_ENDIAN...' to the definition - * of the AES_ROUND_NOKEY macro) and replace it with the version which - * is commented out afterwards. - */ - -#include "aes_helper.c" - -static const sph_u32 IV512[] = { - C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), - C32(0xD1901A06), C32(0x430AE307), C32(0xB29F5CD1), C32(0xDF07FBFC), - C32(0x8E45D73D), C32(0x681AB538), C32(0xBDE86578), C32(0xDD577E47), - C32(0xE275EADE), C32(0x502D9FCD), C32(0xB9357178), C32(0x022A4B9A) -}; - -#define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ - sph_u32 t0 = (x0); \ - sph_u32 t1 = (x1); \ - sph_u32 t2 = (x2); \ - sph_u32 t3 = (x3); \ - AES_ROUND_NOKEY_LE(t0, t1, t2, t3, x0, x1, x2, x3); \ - } while (0) - -#define KEY_EXPAND_ELT(k0, k1, k2, k3) do { \ - sph_u32 kt; \ - AES_ROUND_NOKEY(k1, k2, k3, k0); \ - kt = (k0); \ - (k0) = (k1); \ - (k1) = (k2); \ - (k2) = (k3); \ - (k3) = kt; \ - } while (0) - -#if SPH_SMALL_FOOTPRINT_SHAVITE - -/* - * This function assumes that "msg" is aligned for 32-bit access. - */ -static void -c512(sph_shavite_big_context *sc, const void *msg) -{ - sph_u32 p0, p1, p2, p3, p4, p5, p6, p7; - sph_u32 p8, p9, pA, pB, pC, pD, pE, pF; - sph_u32 rk[448]; - size_t u; - int r, s; - -#if SPH_LITTLE_ENDIAN - memcpy(rk, msg, 128); -#else - for (u = 0; u < 32; u += 4) { - rk[u + 0] = sph_dec32le_aligned( - (const unsigned char *)msg + (u << 2) + 0); - rk[u + 1] = sph_dec32le_aligned( - (const unsigned char *)msg + (u << 2) + 4); - rk[u + 2] = sph_dec32le_aligned( - (const unsigned char *)msg + (u << 2) + 8); - rk[u + 3] = sph_dec32le_aligned( - (const unsigned char *)msg + (u << 2) + 12); - } -#endif - u = 32; - for (;;) { - for (s = 0; s < 4; s ++) { - sph_u32 x0, x1, x2, x3; - - x0 = rk[u - 31]; - x1 = rk[u - 30]; - x2 = rk[u - 29]; - x3 = rk[u - 32]; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk[u + 0] = x0 ^ rk[u - 4]; - rk[u + 1] = x1 ^ rk[u - 3]; - rk[u + 2] = x2 ^ rk[u - 2]; - rk[u + 3] = x3 ^ rk[u - 1]; - if (u == 32) { - rk[ 32] ^= sc->count0; - rk[ 33] ^= sc->count1; - rk[ 34] ^= sc->count2; - rk[ 35] ^= SPH_T32(~sc->count3); - } else if (u == 440) { - rk[440] ^= sc->count1; - rk[441] ^= sc->count0; - rk[442] ^= sc->count3; - rk[443] ^= SPH_T32(~sc->count2); - } - u += 4; - - x0 = rk[u - 31]; - x1 = rk[u - 30]; - x2 = rk[u - 29]; - x3 = rk[u - 32]; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk[u + 0] = x0 ^ rk[u - 4]; - rk[u + 1] = x1 ^ rk[u - 3]; - rk[u + 2] = x2 ^ rk[u - 2]; - rk[u + 3] = x3 ^ rk[u - 1]; - if (u == 164) { - rk[164] ^= sc->count3; - rk[165] ^= sc->count2; - rk[166] ^= sc->count1; - rk[167] ^= SPH_T32(~sc->count0); - } else if (u == 316) { - rk[316] ^= sc->count2; - rk[317] ^= sc->count3; - rk[318] ^= sc->count0; - rk[319] ^= SPH_T32(~sc->count1); - } - u += 4; - } - if (u == 448) - break; - for (s = 0; s < 8; s ++) { - rk[u + 0] = rk[u - 32] ^ rk[u - 7]; - rk[u + 1] = rk[u - 31] ^ rk[u - 6]; - rk[u + 2] = rk[u - 30] ^ rk[u - 5]; - rk[u + 3] = rk[u - 29] ^ rk[u - 4]; - u += 4; - } - } - - p0 = sc->h[0x0]; - p1 = sc->h[0x1]; - p2 = sc->h[0x2]; - p3 = sc->h[0x3]; - p4 = sc->h[0x4]; - p5 = sc->h[0x5]; - p6 = sc->h[0x6]; - p7 = sc->h[0x7]; - p8 = sc->h[0x8]; - p9 = sc->h[0x9]; - pA = sc->h[0xA]; - pB = sc->h[0xB]; - pC = sc->h[0xC]; - pD = sc->h[0xD]; - pE = sc->h[0xE]; - pF = sc->h[0xF]; - u = 0; - for (r = 0; r < 14; r ++) { -#define C512_ELT(l0, l1, l2, l3, r0, r1, r2, r3) do { \ - sph_u32 x0, x1, x2, x3; \ - x0 = r0 ^ rk[u ++]; \ - x1 = r1 ^ rk[u ++]; \ - x2 = r2 ^ rk[u ++]; \ - x3 = r3 ^ rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - l0 ^= x0; \ - l1 ^= x1; \ - l2 ^= x2; \ - l3 ^= x3; \ - } while (0) - -#define WROT(a, b, c, d) do { \ - sph_u32 t = d; \ - d = c; \ - c = b; \ - b = a; \ - a = t; \ - } while (0) - - C512_ELT(p0, p1, p2, p3, p4, p5, p6, p7); - C512_ELT(p8, p9, pA, pB, pC, pD, pE, pF); - - WROT(p0, p4, p8, pC); - WROT(p1, p5, p9, pD); - WROT(p2, p6, pA, pE); - WROT(p3, p7, pB, pF); - -#undef C512_ELT -#undef WROT - } - sc->h[0x0] ^= p0; - sc->h[0x1] ^= p1; - sc->h[0x2] ^= p2; - sc->h[0x3] ^= p3; - sc->h[0x4] ^= p4; - sc->h[0x5] ^= p5; - sc->h[0x6] ^= p6; - sc->h[0x7] ^= p7; - sc->h[0x8] ^= p8; - sc->h[0x9] ^= p9; - sc->h[0xA] ^= pA; - sc->h[0xB] ^= pB; - sc->h[0xC] ^= pC; - sc->h[0xD] ^= pD; - sc->h[0xE] ^= pE; - sc->h[0xF] ^= pF; -} - -#else - -/* - * This function assumes that "msg" is aligned for 32-bit access. - */ -static void -c512(sph_shavite_big_context *sc, const void *msg) -{ - sph_u32 p0, p1, p2, p3, p4, p5, p6, p7; - sph_u32 p8, p9, pA, pB, pC, pD, pE, pF; - sph_u32 x0, x1, x2, x3; - sph_u32 rk00, rk01, rk02, rk03, rk04, rk05, rk06, rk07; - sph_u32 rk08, rk09, rk0A, rk0B, rk0C, rk0D, rk0E, rk0F; - sph_u32 rk10, rk11, rk12, rk13, rk14, rk15, rk16, rk17; - sph_u32 rk18, rk19, rk1A, rk1B, rk1C, rk1D, rk1E, rk1F; - int r; - - p0 = sc->h[0x0]; - p1 = sc->h[0x1]; - p2 = sc->h[0x2]; - p3 = sc->h[0x3]; - p4 = sc->h[0x4]; - p5 = sc->h[0x5]; - p6 = sc->h[0x6]; - p7 = sc->h[0x7]; - p8 = sc->h[0x8]; - p9 = sc->h[0x9]; - pA = sc->h[0xA]; - pB = sc->h[0xB]; - pC = sc->h[0xC]; - pD = sc->h[0xD]; - pE = sc->h[0xE]; - pF = sc->h[0xF]; - /* round 0 */ - rk00 = sph_dec32le_aligned((const unsigned char *)msg + 0); - x0 = p4 ^ rk00; - rk01 = sph_dec32le_aligned((const unsigned char *)msg + 4); - x1 = p5 ^ rk01; - rk02 = sph_dec32le_aligned((const unsigned char *)msg + 8); - x2 = p6 ^ rk02; - rk03 = sph_dec32le_aligned((const unsigned char *)msg + 12); - x3 = p7 ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk04 = sph_dec32le_aligned((const unsigned char *)msg + 16); - x0 ^= rk04; - rk05 = sph_dec32le_aligned((const unsigned char *)msg + 20); - x1 ^= rk05; - rk06 = sph_dec32le_aligned((const unsigned char *)msg + 24); - x2 ^= rk06; - rk07 = sph_dec32le_aligned((const unsigned char *)msg + 28); - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk08 = sph_dec32le_aligned((const unsigned char *)msg + 32); - x0 ^= rk08; - rk09 = sph_dec32le_aligned((const unsigned char *)msg + 36); - x1 ^= rk09; - rk0A = sph_dec32le_aligned((const unsigned char *)msg + 40); - x2 ^= rk0A; - rk0B = sph_dec32le_aligned((const unsigned char *)msg + 44); - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk0C = sph_dec32le_aligned((const unsigned char *)msg + 48); - x0 ^= rk0C; - rk0D = sph_dec32le_aligned((const unsigned char *)msg + 52); - x1 ^= rk0D; - rk0E = sph_dec32le_aligned((const unsigned char *)msg + 56); - x2 ^= rk0E; - rk0F = sph_dec32le_aligned((const unsigned char *)msg + 60); - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p0 ^= x0; - p1 ^= x1; - p2 ^= x2; - p3 ^= x3; - rk10 = sph_dec32le_aligned((const unsigned char *)msg + 64); - x0 = pC ^ rk10; - rk11 = sph_dec32le_aligned((const unsigned char *)msg + 68); - x1 = pD ^ rk11; - rk12 = sph_dec32le_aligned((const unsigned char *)msg + 72); - x2 = pE ^ rk12; - rk13 = sph_dec32le_aligned((const unsigned char *)msg + 76); - x3 = pF ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk14 = sph_dec32le_aligned((const unsigned char *)msg + 80); - x0 ^= rk14; - rk15 = sph_dec32le_aligned((const unsigned char *)msg + 84); - x1 ^= rk15; - rk16 = sph_dec32le_aligned((const unsigned char *)msg + 88); - x2 ^= rk16; - rk17 = sph_dec32le_aligned((const unsigned char *)msg + 92); - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk18 = sph_dec32le_aligned((const unsigned char *)msg + 96); - x0 ^= rk18; - rk19 = sph_dec32le_aligned((const unsigned char *)msg + 100); - x1 ^= rk19; - rk1A = sph_dec32le_aligned((const unsigned char *)msg + 104); - x2 ^= rk1A; - rk1B = sph_dec32le_aligned((const unsigned char *)msg + 108); - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk1C = sph_dec32le_aligned((const unsigned char *)msg + 112); - x0 ^= rk1C; - rk1D = sph_dec32le_aligned((const unsigned char *)msg + 116); - x1 ^= rk1D; - rk1E = sph_dec32le_aligned((const unsigned char *)msg + 120); - x2 ^= rk1E; - rk1F = sph_dec32le_aligned((const unsigned char *)msg + 124); - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p8 ^= x0; - p9 ^= x1; - pA ^= x2; - pB ^= x3; - - for (r = 0; r < 3; r ++) { - /* round 1, 5, 9 */ - KEY_EXPAND_ELT(rk00, rk01, rk02, rk03); - rk00 ^= rk1C; - rk01 ^= rk1D; - rk02 ^= rk1E; - rk03 ^= rk1F; - if (r == 0) { - rk00 ^= sc->count0; - rk01 ^= sc->count1; - rk02 ^= sc->count2; - rk03 ^= SPH_T32(~sc->count3); - } - x0 = p0 ^ rk00; - x1 = p1 ^ rk01; - x2 = p2 ^ rk02; - x3 = p3 ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk04, rk05, rk06, rk07); - rk04 ^= rk00; - rk05 ^= rk01; - rk06 ^= rk02; - rk07 ^= rk03; - if (r == 1) { - rk04 ^= sc->count3; - rk05 ^= sc->count2; - rk06 ^= sc->count1; - rk07 ^= SPH_T32(~sc->count0); - } - x0 ^= rk04; - x1 ^= rk05; - x2 ^= rk06; - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk08, rk09, rk0A, rk0B); - rk08 ^= rk04; - rk09 ^= rk05; - rk0A ^= rk06; - rk0B ^= rk07; - x0 ^= rk08; - x1 ^= rk09; - x2 ^= rk0A; - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk0C, rk0D, rk0E, rk0F); - rk0C ^= rk08; - rk0D ^= rk09; - rk0E ^= rk0A; - rk0F ^= rk0B; - x0 ^= rk0C; - x1 ^= rk0D; - x2 ^= rk0E; - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - pC ^= x0; - pD ^= x1; - pE ^= x2; - pF ^= x3; - KEY_EXPAND_ELT(rk10, rk11, rk12, rk13); - rk10 ^= rk0C; - rk11 ^= rk0D; - rk12 ^= rk0E; - rk13 ^= rk0F; - x0 = p8 ^ rk10; - x1 = p9 ^ rk11; - x2 = pA ^ rk12; - x3 = pB ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk14, rk15, rk16, rk17); - rk14 ^= rk10; - rk15 ^= rk11; - rk16 ^= rk12; - rk17 ^= rk13; - x0 ^= rk14; - x1 ^= rk15; - x2 ^= rk16; - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk18, rk19, rk1A, rk1B); - rk18 ^= rk14; - rk19 ^= rk15; - rk1A ^= rk16; - rk1B ^= rk17; - x0 ^= rk18; - x1 ^= rk19; - x2 ^= rk1A; - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk1C, rk1D, rk1E, rk1F); - rk1C ^= rk18; - rk1D ^= rk19; - rk1E ^= rk1A; - rk1F ^= rk1B; - if (r == 2) { - rk1C ^= sc->count2; - rk1D ^= sc->count3; - rk1E ^= sc->count0; - rk1F ^= SPH_T32(~sc->count1); - } - x0 ^= rk1C; - x1 ^= rk1D; - x2 ^= rk1E; - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p4 ^= x0; - p5 ^= x1; - p6 ^= x2; - p7 ^= x3; - /* round 2, 6, 10 */ - rk00 ^= rk19; - x0 = pC ^ rk00; - rk01 ^= rk1A; - x1 = pD ^ rk01; - rk02 ^= rk1B; - x2 = pE ^ rk02; - rk03 ^= rk1C; - x3 = pF ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk04 ^= rk1D; - x0 ^= rk04; - rk05 ^= rk1E; - x1 ^= rk05; - rk06 ^= rk1F; - x2 ^= rk06; - rk07 ^= rk00; - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk08 ^= rk01; - x0 ^= rk08; - rk09 ^= rk02; - x1 ^= rk09; - rk0A ^= rk03; - x2 ^= rk0A; - rk0B ^= rk04; - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk0C ^= rk05; - x0 ^= rk0C; - rk0D ^= rk06; - x1 ^= rk0D; - rk0E ^= rk07; - x2 ^= rk0E; - rk0F ^= rk08; - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p8 ^= x0; - p9 ^= x1; - pA ^= x2; - pB ^= x3; - rk10 ^= rk09; - x0 = p4 ^ rk10; - rk11 ^= rk0A; - x1 = p5 ^ rk11; - rk12 ^= rk0B; - x2 = p6 ^ rk12; - rk13 ^= rk0C; - x3 = p7 ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk14 ^= rk0D; - x0 ^= rk14; - rk15 ^= rk0E; - x1 ^= rk15; - rk16 ^= rk0F; - x2 ^= rk16; - rk17 ^= rk10; - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk18 ^= rk11; - x0 ^= rk18; - rk19 ^= rk12; - x1 ^= rk19; - rk1A ^= rk13; - x2 ^= rk1A; - rk1B ^= rk14; - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk1C ^= rk15; - x0 ^= rk1C; - rk1D ^= rk16; - x1 ^= rk1D; - rk1E ^= rk17; - x2 ^= rk1E; - rk1F ^= rk18; - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p0 ^= x0; - p1 ^= x1; - p2 ^= x2; - p3 ^= x3; - /* round 3, 7, 11 */ - KEY_EXPAND_ELT(rk00, rk01, rk02, rk03); - rk00 ^= rk1C; - rk01 ^= rk1D; - rk02 ^= rk1E; - rk03 ^= rk1F; - x0 = p8 ^ rk00; - x1 = p9 ^ rk01; - x2 = pA ^ rk02; - x3 = pB ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk04, rk05, rk06, rk07); - rk04 ^= rk00; - rk05 ^= rk01; - rk06 ^= rk02; - rk07 ^= rk03; - x0 ^= rk04; - x1 ^= rk05; - x2 ^= rk06; - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk08, rk09, rk0A, rk0B); - rk08 ^= rk04; - rk09 ^= rk05; - rk0A ^= rk06; - rk0B ^= rk07; - x0 ^= rk08; - x1 ^= rk09; - x2 ^= rk0A; - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk0C, rk0D, rk0E, rk0F); - rk0C ^= rk08; - rk0D ^= rk09; - rk0E ^= rk0A; - rk0F ^= rk0B; - x0 ^= rk0C; - x1 ^= rk0D; - x2 ^= rk0E; - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p4 ^= x0; - p5 ^= x1; - p6 ^= x2; - p7 ^= x3; - KEY_EXPAND_ELT(rk10, rk11, rk12, rk13); - rk10 ^= rk0C; - rk11 ^= rk0D; - rk12 ^= rk0E; - rk13 ^= rk0F; - x0 = p0 ^ rk10; - x1 = p1 ^ rk11; - x2 = p2 ^ rk12; - x3 = p3 ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk14, rk15, rk16, rk17); - rk14 ^= rk10; - rk15 ^= rk11; - rk16 ^= rk12; - rk17 ^= rk13; - x0 ^= rk14; - x1 ^= rk15; - x2 ^= rk16; - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk18, rk19, rk1A, rk1B); - rk18 ^= rk14; - rk19 ^= rk15; - rk1A ^= rk16; - rk1B ^= rk17; - x0 ^= rk18; - x1 ^= rk19; - x2 ^= rk1A; - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk1C, rk1D, rk1E, rk1F); - rk1C ^= rk18; - rk1D ^= rk19; - rk1E ^= rk1A; - rk1F ^= rk1B; - x0 ^= rk1C; - x1 ^= rk1D; - x2 ^= rk1E; - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - pC ^= x0; - pD ^= x1; - pE ^= x2; - pF ^= x3; - /* round 4, 8, 12 */ - rk00 ^= rk19; - x0 = p4 ^ rk00; - rk01 ^= rk1A; - x1 = p5 ^ rk01; - rk02 ^= rk1B; - x2 = p6 ^ rk02; - rk03 ^= rk1C; - x3 = p7 ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk04 ^= rk1D; - x0 ^= rk04; - rk05 ^= rk1E; - x1 ^= rk05; - rk06 ^= rk1F; - x2 ^= rk06; - rk07 ^= rk00; - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk08 ^= rk01; - x0 ^= rk08; - rk09 ^= rk02; - x1 ^= rk09; - rk0A ^= rk03; - x2 ^= rk0A; - rk0B ^= rk04; - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk0C ^= rk05; - x0 ^= rk0C; - rk0D ^= rk06; - x1 ^= rk0D; - rk0E ^= rk07; - x2 ^= rk0E; - rk0F ^= rk08; - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p0 ^= x0; - p1 ^= x1; - p2 ^= x2; - p3 ^= x3; - rk10 ^= rk09; - x0 = pC ^ rk10; - rk11 ^= rk0A; - x1 = pD ^ rk11; - rk12 ^= rk0B; - x2 = pE ^ rk12; - rk13 ^= rk0C; - x3 = pF ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk14 ^= rk0D; - x0 ^= rk14; - rk15 ^= rk0E; - x1 ^= rk15; - rk16 ^= rk0F; - x2 ^= rk16; - rk17 ^= rk10; - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk18 ^= rk11; - x0 ^= rk18; - rk19 ^= rk12; - x1 ^= rk19; - rk1A ^= rk13; - x2 ^= rk1A; - rk1B ^= rk14; - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - rk1C ^= rk15; - x0 ^= rk1C; - rk1D ^= rk16; - x1 ^= rk1D; - rk1E ^= rk17; - x2 ^= rk1E; - rk1F ^= rk18; - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p8 ^= x0; - p9 ^= x1; - pA ^= x2; - pB ^= x3; - } - /* round 13 */ - KEY_EXPAND_ELT(rk00, rk01, rk02, rk03); - rk00 ^= rk1C; - rk01 ^= rk1D; - rk02 ^= rk1E; - rk03 ^= rk1F; - x0 = p0 ^ rk00; - x1 = p1 ^ rk01; - x2 = p2 ^ rk02; - x3 = p3 ^ rk03; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk04, rk05, rk06, rk07); - rk04 ^= rk00; - rk05 ^= rk01; - rk06 ^= rk02; - rk07 ^= rk03; - x0 ^= rk04; - x1 ^= rk05; - x2 ^= rk06; - x3 ^= rk07; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk08, rk09, rk0A, rk0B); - rk08 ^= rk04; - rk09 ^= rk05; - rk0A ^= rk06; - rk0B ^= rk07; - x0 ^= rk08; - x1 ^= rk09; - x2 ^= rk0A; - x3 ^= rk0B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk0C, rk0D, rk0E, rk0F); - rk0C ^= rk08; - rk0D ^= rk09; - rk0E ^= rk0A; - rk0F ^= rk0B; - x0 ^= rk0C; - x1 ^= rk0D; - x2 ^= rk0E; - x3 ^= rk0F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - pC ^= x0; - pD ^= x1; - pE ^= x2; - pF ^= x3; - KEY_EXPAND_ELT(rk10, rk11, rk12, rk13); - rk10 ^= rk0C; - rk11 ^= rk0D; - rk12 ^= rk0E; - rk13 ^= rk0F; - x0 = p8 ^ rk10; - x1 = p9 ^ rk11; - x2 = pA ^ rk12; - x3 = pB ^ rk13; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk14, rk15, rk16, rk17); - rk14 ^= rk10; - rk15 ^= rk11; - rk16 ^= rk12; - rk17 ^= rk13; - x0 ^= rk14; - x1 ^= rk15; - x2 ^= rk16; - x3 ^= rk17; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk18, rk19, rk1A, rk1B); - rk18 ^= rk14 ^ sc->count1; - rk19 ^= rk15 ^ sc->count0; - rk1A ^= rk16 ^ sc->count3; - rk1B ^= rk17 ^ SPH_T32(~sc->count2); - x0 ^= rk18; - x1 ^= rk19; - x2 ^= rk1A; - x3 ^= rk1B; - AES_ROUND_NOKEY(x0, x1, x2, x3); - KEY_EXPAND_ELT(rk1C, rk1D, rk1E, rk1F); - rk1C ^= rk18; - rk1D ^= rk19; - rk1E ^= rk1A; - rk1F ^= rk1B; - x0 ^= rk1C; - x1 ^= rk1D; - x2 ^= rk1E; - x3 ^= rk1F; - AES_ROUND_NOKEY(x0, x1, x2, x3); - p4 ^= x0; - p5 ^= x1; - p6 ^= x2; - p7 ^= x3; - sc->h[0x0] ^= p8; - sc->h[0x1] ^= p9; - sc->h[0x2] ^= pA; - sc->h[0x3] ^= pB; - sc->h[0x4] ^= pC; - sc->h[0x5] ^= pD; - sc->h[0x6] ^= pE; - sc->h[0x7] ^= pF; - sc->h[0x8] ^= p0; - sc->h[0x9] ^= p1; - sc->h[0xA] ^= p2; - sc->h[0xB] ^= p3; - sc->h[0xC] ^= p4; - sc->h[0xD] ^= p5; - sc->h[0xE] ^= p6; - sc->h[0xF] ^= p7; -} - -#endif - -static void -shavite_big_init(sph_shavite_big_context *sc, const sph_u32 *iv) -{ - memcpy(sc->h, iv, sizeof sc->h); - sc->ptr = 0; - sc->count0 = 0; - sc->count1 = 0; - sc->count2 = 0; - sc->count3 = 0; -} - -static void -shavite_big_core(sph_shavite_big_context *sc, const void *data, size_t len) -{ - unsigned char *buf; - size_t ptr; - - buf = sc->buf; - ptr = sc->ptr; - while (len > 0) { - size_t clen; - - clen = (sizeof sc->buf) - ptr; - if (clen > len) - clen = len; - memcpy(buf + ptr, data, clen); - data = (const unsigned char *)data + clen; - ptr += clen; - len -= clen; - if (ptr == sizeof sc->buf) { - if ((sc->count0 = SPH_T32(sc->count0 + 1024)) == 0) { - sc->count1 = SPH_T32(sc->count1 + 1); - if (sc->count1 == 0) { - sc->count2 = SPH_T32(sc->count2 + 1); - if (sc->count2 == 0) { - sc->count3 = SPH_T32( - sc->count3 + 1); - } - } - } - c512(sc, buf); - ptr = 0; - } - } - sc->ptr = ptr; -} - -static void -shavite_big_close(sph_shavite_big_context *sc, - unsigned ub, unsigned n, void *dst, size_t out_size_w32) -{ - unsigned char *buf; - size_t ptr, u; - unsigned z; - sph_u32 count0, count1, count2, count3; - - buf = sc->buf; - ptr = sc->ptr; - count0 = (sc->count0 += (ptr << 3) + n); - count1 = sc->count1; - count2 = sc->count2; - count3 = sc->count3; - z = 0x80 >> n; - z = ((ub & -z) | z) & 0xFF; - if (ptr == 0 && n == 0) { - buf[0] = 0x80; - memset(buf + 1, 0, 109); - sc->count0 = sc->count1 = sc->count2 = sc->count3 = 0; - } else if (ptr < 110) { - buf[ptr ++] = z; - memset(buf + ptr, 0, 110 - ptr); - } else { - buf[ptr ++] = z; - memset(buf + ptr, 0, 128 - ptr); - c512(sc, buf); - memset(buf, 0, 110); - sc->count0 = sc->count1 = sc->count2 = sc->count3 = 0; - } - sph_enc32le(buf + 110, count0); - sph_enc32le(buf + 114, count1); - sph_enc32le(buf + 118, count2); - sph_enc32le(buf + 122, count3); - buf[126] = out_size_w32 << 5; - buf[127] = out_size_w32 >> 3; - c512(sc, buf); - for (u = 0; u < out_size_w32; u ++) - sph_enc32le((unsigned char *)dst + (u << 2), sc->h[u]); -} - -/* see sph_shavite.h */ -void -sph_shavite512_init(void *cc) -{ - shavite_big_init(cc, IV512); -} - -/* see sph_shavite.h */ -void -sph_shavite512(void *cc, const void *data, size_t len) -{ - shavite_big_core(cc, data, len); -} - -/* see sph_shavite.h */ -void -sph_shavite512_close(void *cc, void *dst) -{ - shavite_big_close(cc, 0, 0, dst, 16); - shavite_big_init(cc, IV512); -} - -/* see sph_shavite.h */ -void -sph_shavite512_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst) -{ - shavite_big_close(cc, ub, n, dst, 16); - shavite_big_init(cc, IV512); -} - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/src/crypto/x11/shavite.cpp b/src/crypto/x11/shavite.cpp new file mode 100644 index 000000000000..5e8d1c4185fe --- /dev/null +++ b/src/crypto/x11/shavite.cpp @@ -0,0 +1,369 @@ +/* $Id: shavite.c 227 2010-06-16 17:28:38Z tp $ */ +/* + * SHAvite-3 implementation. + * + * ==========================(LICENSE BEGIN)============================ + * + * Copyright (c) 2007-2010 Projet RNRT SAPHIR + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * ===========================(LICENSE END)============================= + * + * @author Thomas Pornin + */ + +#include +#include + +#include "sph_shavite.h" + +#ifdef _MSC_VER +#pragma warning (disable: 4146) +#endif + +#define C32 SPH_C32 + +/* + * As of round 2 of the SHA-3 competition, the published reference + * implementation and test vectors are wrong, because they use + * big-endian AES tables while the internal decoding uses little-endian. + * The code below follows the specification. To turn it into a code + * which follows the reference implementation (the one called "BugFix" + * on the SHAvite-3 web site, published on Nov 23rd, 2009), comment out + * the code below (from the '#define AES_BIG_ENDIAN...' to the definition + * of the AES_ROUND_NOKEY macro) and replace it with the version which + * is commented out afterwards. + */ + +#include "aes_helper.c" + +static const sph_u32 IV512[] = { + C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), + C32(0xD1901A06), C32(0x430AE307), C32(0xB29F5CD1), C32(0xDF07FBFC), + C32(0x8E45D73D), C32(0x681AB538), C32(0xBDE86578), C32(0xDD577E47), + C32(0xE275EADE), C32(0x502D9FCD), C32(0xB9357178), C32(0x022A4B9A) +}; + +#define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ + sph_u32 t0 = (x0); \ + sph_u32 t1 = (x1); \ + sph_u32 t2 = (x2); \ + sph_u32 t3 = (x3); \ + AES_ROUND_NOKEY_LE(t0, t1, t2, t3, x0, x1, x2, x3); \ + } while (0) + +#define KEY_EXPAND_ELT(k0, k1, k2, k3) do { \ + sph_u32 kt; \ + AES_ROUND_NOKEY(k1, k2, k3, k0); \ + kt = (k0); \ + (k0) = (k1); \ + (k1) = (k2); \ + (k2) = (k3); \ + (k3) = kt; \ + } while (0) + +/* + * This function assumes that "msg" is aligned for 32-bit access. + */ +static void +c512(sph_shavite_big_context *sc, const void *msg) +{ + sph_u32 p0, p1, p2, p3, p4, p5, p6, p7; + sph_u32 p8, p9, pA, pB, pC, pD, pE, pF; + sph_u32 rk[448]; + size_t u; + int r, s; + +#if SPH_LITTLE_ENDIAN + memcpy(rk, msg, 128); +#else + for (u = 0; u < 32; u += 4) { + rk[u + 0] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 0); + rk[u + 1] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 4); + rk[u + 2] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 8); + rk[u + 3] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 12); + } +#endif + u = 32; + for (;;) { + for (s = 0; s < 4; s ++) { + sph_u32 x0, x1, x2, x3; + + x0 = rk[u - 31]; + x1 = rk[u - 30]; + x2 = rk[u - 29]; + x3 = rk[u - 32]; + AES_ROUND_NOKEY(x0, x1, x2, x3); + rk[u + 0] = x0 ^ rk[u - 4]; + rk[u + 1] = x1 ^ rk[u - 3]; + rk[u + 2] = x2 ^ rk[u - 2]; + rk[u + 3] = x3 ^ rk[u - 1]; + if (u == 32) { + rk[ 32] ^= sc->count0; + rk[ 33] ^= sc->count1; + rk[ 34] ^= sc->count2; + rk[ 35] ^= SPH_T32(~sc->count3); + } else if (u == 440) { + rk[440] ^= sc->count1; + rk[441] ^= sc->count0; + rk[442] ^= sc->count3; + rk[443] ^= SPH_T32(~sc->count2); + } + u += 4; + + x0 = rk[u - 31]; + x1 = rk[u - 30]; + x2 = rk[u - 29]; + x3 = rk[u - 32]; + AES_ROUND_NOKEY(x0, x1, x2, x3); + rk[u + 0] = x0 ^ rk[u - 4]; + rk[u + 1] = x1 ^ rk[u - 3]; + rk[u + 2] = x2 ^ rk[u - 2]; + rk[u + 3] = x3 ^ rk[u - 1]; + if (u == 164) { + rk[164] ^= sc->count3; + rk[165] ^= sc->count2; + rk[166] ^= sc->count1; + rk[167] ^= SPH_T32(~sc->count0); + } else if (u == 316) { + rk[316] ^= sc->count2; + rk[317] ^= sc->count3; + rk[318] ^= sc->count0; + rk[319] ^= SPH_T32(~sc->count1); + } + u += 4; + } + if (u == 448) + break; + for (s = 0; s < 8; s ++) { + rk[u + 0] = rk[u - 32] ^ rk[u - 7]; + rk[u + 1] = rk[u - 31] ^ rk[u - 6]; + rk[u + 2] = rk[u - 30] ^ rk[u - 5]; + rk[u + 3] = rk[u - 29] ^ rk[u - 4]; + u += 4; + } + } + + p0 = sc->h[0x0]; + p1 = sc->h[0x1]; + p2 = sc->h[0x2]; + p3 = sc->h[0x3]; + p4 = sc->h[0x4]; + p5 = sc->h[0x5]; + p6 = sc->h[0x6]; + p7 = sc->h[0x7]; + p8 = sc->h[0x8]; + p9 = sc->h[0x9]; + pA = sc->h[0xA]; + pB = sc->h[0xB]; + pC = sc->h[0xC]; + pD = sc->h[0xD]; + pE = sc->h[0xE]; + pF = sc->h[0xF]; + u = 0; + for (r = 0; r < 14; r ++) { +#define C512_ELT(l0, l1, l2, l3, r0, r1, r2, r3) do { \ + sph_u32 x0, x1, x2, x3; \ + x0 = r0 ^ rk[u ++]; \ + x1 = r1 ^ rk[u ++]; \ + x2 = r2 ^ rk[u ++]; \ + x3 = r3 ^ rk[u ++]; \ + AES_ROUND_NOKEY(x0, x1, x2, x3); \ + x0 ^= rk[u ++]; \ + x1 ^= rk[u ++]; \ + x2 ^= rk[u ++]; \ + x3 ^= rk[u ++]; \ + AES_ROUND_NOKEY(x0, x1, x2, x3); \ + x0 ^= rk[u ++]; \ + x1 ^= rk[u ++]; \ + x2 ^= rk[u ++]; \ + x3 ^= rk[u ++]; \ + AES_ROUND_NOKEY(x0, x1, x2, x3); \ + x0 ^= rk[u ++]; \ + x1 ^= rk[u ++]; \ + x2 ^= rk[u ++]; \ + x3 ^= rk[u ++]; \ + AES_ROUND_NOKEY(x0, x1, x2, x3); \ + l0 ^= x0; \ + l1 ^= x1; \ + l2 ^= x2; \ + l3 ^= x3; \ + } while (0) + +#define WROT(a, b, c, d) do { \ + sph_u32 t = d; \ + d = c; \ + c = b; \ + b = a; \ + a = t; \ + } while (0) + + C512_ELT(p0, p1, p2, p3, p4, p5, p6, p7); + C512_ELT(p8, p9, pA, pB, pC, pD, pE, pF); + + WROT(p0, p4, p8, pC); + WROT(p1, p5, p9, pD); + WROT(p2, p6, pA, pE); + WROT(p3, p7, pB, pF); + +#undef C512_ELT +#undef WROT + } + sc->h[0x0] ^= p0; + sc->h[0x1] ^= p1; + sc->h[0x2] ^= p2; + sc->h[0x3] ^= p3; + sc->h[0x4] ^= p4; + sc->h[0x5] ^= p5; + sc->h[0x6] ^= p6; + sc->h[0x7] ^= p7; + sc->h[0x8] ^= p8; + sc->h[0x9] ^= p9; + sc->h[0xA] ^= pA; + sc->h[0xB] ^= pB; + sc->h[0xC] ^= pC; + sc->h[0xD] ^= pD; + sc->h[0xE] ^= pE; + sc->h[0xF] ^= pF; +} + +static void +shavite_big_init(sph_shavite_big_context *sc, const sph_u32 *iv) +{ + memcpy(sc->h, iv, sizeof sc->h); + sc->ptr = 0; + sc->count0 = 0; + sc->count1 = 0; + sc->count2 = 0; + sc->count3 = 0; +} + +static void +shavite_big_core(sph_shavite_big_context *sc, const void *data, size_t len) +{ + unsigned char *buf; + size_t ptr; + + buf = sc->buf; + ptr = sc->ptr; + while (len > 0) { + size_t clen; + + clen = (sizeof sc->buf) - ptr; + if (clen > len) + clen = len; + memcpy(buf + ptr, data, clen); + data = (const unsigned char *)data + clen; + ptr += clen; + len -= clen; + if (ptr == sizeof sc->buf) { + if ((sc->count0 = SPH_T32(sc->count0 + 1024)) == 0) { + sc->count1 = SPH_T32(sc->count1 + 1); + if (sc->count1 == 0) { + sc->count2 = SPH_T32(sc->count2 + 1); + if (sc->count2 == 0) { + sc->count3 = SPH_T32( + sc->count3 + 1); + } + } + } + c512(sc, buf); + ptr = 0; + } + } + sc->ptr = ptr; +} + +static void +shavite_big_close(sph_shavite_big_context *sc, + unsigned ub, unsigned n, void *dst, size_t out_size_w32) +{ + unsigned char *buf; + size_t ptr, u; + unsigned z; + sph_u32 count0, count1, count2, count3; + + buf = sc->buf; + ptr = sc->ptr; + count0 = (sc->count0 += (ptr << 3) + n); + count1 = sc->count1; + count2 = sc->count2; + count3 = sc->count3; + z = 0x80 >> n; + z = ((ub & -z) | z) & 0xFF; + if (ptr == 0 && n == 0) { + buf[0] = 0x80; + memset(buf + 1, 0, 109); + sc->count0 = sc->count1 = sc->count2 = sc->count3 = 0; + } else if (ptr < 110) { + buf[ptr ++] = z; + memset(buf + ptr, 0, 110 - ptr); + } else { + buf[ptr ++] = z; + memset(buf + ptr, 0, 128 - ptr); + c512(sc, buf); + memset(buf, 0, 110); + sc->count0 = sc->count1 = sc->count2 = sc->count3 = 0; + } + sph_enc32le(buf + 110, count0); + sph_enc32le(buf + 114, count1); + sph_enc32le(buf + 118, count2); + sph_enc32le(buf + 122, count3); + buf[126] = out_size_w32 << 5; + buf[127] = out_size_w32 >> 3; + c512(sc, buf); + for (u = 0; u < out_size_w32; u ++) + sph_enc32le((unsigned char *)dst + (u << 2), sc->h[u]); +} + +/* see sph_shavite.h */ +void +sph_shavite512_init(sph_shavite512_context *cc) +{ + shavite_big_init(cc, IV512); +} + +/* see sph_shavite.h */ +void +sph_shavite512(sph_shavite512_context *cc, const void *data, size_t len) +{ + shavite_big_core(cc, data, len); +} + +/* see sph_shavite.h */ +void +sph_shavite512_close(sph_shavite512_context *cc, void *dst) +{ + shavite_big_close(cc, 0, 0, dst, 16); + shavite_big_init(cc, IV512); +} + +/* see sph_shavite.h */ +void +sph_shavite512_addbits_and_close(sph_shavite512_context *cc, unsigned ub, unsigned n, void *dst) +{ + shavite_big_close(cc, ub, n, dst, 16); + shavite_big_init(cc, IV512); +} diff --git a/src/crypto/x11/sph_shavite.h b/src/crypto/x11/sph_shavite.h index 16803411ffed..a5faf65e3e62 100644 --- a/src/crypto/x11/sph_shavite.h +++ b/src/crypto/x11/sph_shavite.h @@ -41,10 +41,6 @@ #include #include "sph_types.h" -#ifdef __cplusplus -extern "C"{ -#endif - /** * Output size (in bits) for SHAvite-512. */ @@ -79,7 +75,7 @@ typedef sph_shavite_big_context sph_shavite512_context; * @param cc the SHAvite-512 context (pointer to a * sph_shavite512_context) */ -void sph_shavite512_init(void *cc); +void sph_shavite512_init(sph_shavite512_context *cc); /** * Process some data bytes. It is acceptable that len is zero @@ -89,7 +85,7 @@ void sph_shavite512_init(void *cc); * @param data the input data * @param len the input data length (in bytes) */ -void sph_shavite512(void *cc, const void *data, size_t len); +void sph_shavite512(sph_shavite512_context *cc, const void *data, size_t len); /** * Terminate the current SHAvite-512 computation and output the result into @@ -100,7 +96,7 @@ void sph_shavite512(void *cc, const void *data, size_t len); * @param cc the SHAvite-512 context * @param dst the destination buffer */ -void sph_shavite512_close(void *cc, void *dst); +void sph_shavite512_close(sph_shavite512_context *cc, void *dst); /** * Add a few additional bits (0 to 7) to the current computation, then @@ -116,10 +112,6 @@ void sph_shavite512_close(void *cc, void *dst); * @param dst the destination buffer */ void sph_shavite512_addbits_and_close( - void *cc, unsigned ub, unsigned n, void *dst); - -#ifdef __cplusplus -} -#endif + sph_shavite512_context *cc, unsigned ub, unsigned n, void *dst); #endif From 20fc998ee039fc648753fb6689a5a7a2f4f10757 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:21:05 +0000 Subject: [PATCH 05/21] refactor: move software AES round to header --- src/Makefile.am | 2 +- .../x11/{aes_helper.c => aes_helper.hpp} | 61 +++++++++++-------- src/crypto/x11/echo.cpp | 2 +- src/crypto/x11/shavite.cpp | 18 +----- 4 files changed, 39 insertions(+), 44 deletions(-) rename src/crypto/x11/{aes_helper.c => aes_helper.hpp} (95%) diff --git a/src/Makefile.am b/src/Makefile.am index 5fe041d81bf1..da21ca85dbed 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -735,7 +735,7 @@ crypto_libbitcoin_crypto_sph_la_CPPFLAGS += \ -DSPH_SMALL_FOOTPRINT_CUBEHASH=1 \ -DSPH_SMALL_FOOTPRINT_JH=1 crypto_libbitcoin_crypto_sph_la_SOURCES = \ - crypto/x11/aes_helper.c \ + crypto/x11/aes_helper.hpp \ crypto/x11/blake.c \ crypto/x11/bmw.c \ crypto/x11/cubehash.c \ diff --git a/src/crypto/x11/aes_helper.c b/src/crypto/x11/aes_helper.hpp similarity index 95% rename from src/crypto/x11/aes_helper.c rename to src/crypto/x11/aes_helper.hpp index 0bce4d4c4ad6..f8c7f0978493 100644 --- a/src/crypto/x11/aes_helper.c +++ b/src/crypto/x11/aes_helper.hpp @@ -43,13 +43,16 @@ * @author Thomas Pornin */ +#ifndef SPH_AES_HELPER_HPP__ +#define SPH_AES_HELPER_HPP__ + +#include + #include "sph_types.h" -#ifdef __cplusplus -extern "C"{ -#endif #define AESx(x) SPH_C32(x) +namespace impl { /* * The AES*[] tables allow us to perform a fast evaluation of an AES * round; table AESi[] combines SubBytes for a byte at row i, and @@ -323,29 +326,35 @@ static const sph_u32 AES3[256] = { AESx(0x82C34141), AESx(0x29B09999), AESx(0x5A772D2D), AESx(0x1E110F0F), AESx(0x7BCBB0B0), AESx(0xA8FC5454), AESx(0x6DD6BBBB), AESx(0x2C3A1616) }; +} // namespace impl -#define AES_ROUND_LE(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3) do { \ - (Y0) = AES0[(X0) & 0xFF] \ - ^ AES1[((X1) >> 8) & 0xFF] \ - ^ AES2[((X2) >> 16) & 0xFF] \ - ^ AES3[((X3) >> 24) & 0xFF] ^ (K0); \ - (Y1) = AES0[(X1) & 0xFF] \ - ^ AES1[((X2) >> 8) & 0xFF] \ - ^ AES2[((X3) >> 16) & 0xFF] \ - ^ AES3[((X0) >> 24) & 0xFF] ^ (K1); \ - (Y2) = AES0[(X2) & 0xFF] \ - ^ AES1[((X3) >> 8) & 0xFF] \ - ^ AES2[((X0) >> 16) & 0xFF] \ - ^ AES3[((X1) >> 24) & 0xFF] ^ (K2); \ - (Y3) = AES0[(X3) & 0xFF] \ - ^ AES1[((X0) >> 8) & 0xFF] \ - ^ AES2[((X1) >> 16) & 0xFF] \ - ^ AES3[((X2) >> 24) & 0xFF] ^ (K3); \ - } while (0) - -#define AES_ROUND_NOKEY_LE(X0, X1, X2, X3, Y0, Y1, Y2, Y3) \ - AES_ROUND_LE(X0, X1, X2, X3, 0, 0, 0, 0, Y0, Y1, Y2, Y3) +void ALWAYS_INLINE AES_ROUND_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, + sph_u32 k0, sph_u32 k1, sph_u32 k2, sph_u32 k3, + sph_u32& y0, sph_u32& y1, sph_u32& y2, sph_u32& y3) +{ + using namespace impl; + (y0) = AES0[(x0) & 0xFF] + ^ AES1[((x1) >> 8) & 0xFF] + ^ AES2[((x2) >> 16) & 0xFF] + ^ AES3[((x3) >> 24) & 0xFF] ^ (k0); + (y1) = AES0[(x1) & 0xFF] + ^ AES1[((x2) >> 8) & 0xFF] + ^ AES2[((x3) >> 16) & 0xFF] + ^ AES3[((x0) >> 24) & 0xFF] ^ (k1); + (y2) = AES0[(x2) & 0xFF] + ^ AES1[((x3) >> 8) & 0xFF] + ^ AES2[((x0) >> 16) & 0xFF] + ^ AES3[((x1) >> 24) & 0xFF] ^ (k2); + (y3) = AES0[(x3) & 0xFF] + ^ AES1[((x0) >> 8) & 0xFF] + ^ AES2[((x1) >> 16) & 0xFF] + ^ AES3[((x2) >> 24) & 0xFF] ^ (k3); +} -#ifdef __cplusplus +void ALWAYS_INLINE AES_ROUND_NOKEY_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, + sph_u32& y0, sph_u32& y1, sph_u32& y2, sph_u32& y3) +{ + AES_ROUND_LE(x0, x1, x2, x3, /*k0=*/0, /*k1=*/0, /*k2=*/0, /*k3=*/0, y0, y1, y2, y3); } -#endif + +#endif // SPH_AES_HELPER_HPP__ diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index 7de8f20b4a78..fc9e31254f51 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -45,7 +45,7 @@ #define T32 SPH_T32 #define C64 SPH_C64 -#include "aes_helper.c" +#include "aes_helper.hpp" #define DECL_STATE_BIG \ sph_u64 W[16][2]; diff --git a/src/crypto/x11/shavite.cpp b/src/crypto/x11/shavite.cpp index 5e8d1c4185fe..38ea357040cb 100644 --- a/src/crypto/x11/shavite.cpp +++ b/src/crypto/x11/shavite.cpp @@ -53,7 +53,7 @@ * is commented out afterwards. */ -#include "aes_helper.c" +#include "aes_helper.hpp" static const sph_u32 IV512[] = { C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), @@ -63,21 +63,7 @@ static const sph_u32 IV512[] = { }; #define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ - sph_u32 t0 = (x0); \ - sph_u32 t1 = (x1); \ - sph_u32 t2 = (x2); \ - sph_u32 t3 = (x3); \ - AES_ROUND_NOKEY_LE(t0, t1, t2, t3, x0, x1, x2, x3); \ - } while (0) - -#define KEY_EXPAND_ELT(k0, k1, k2, k3) do { \ - sph_u32 kt; \ - AES_ROUND_NOKEY(k1, k2, k3, k0); \ - kt = (k0); \ - (k0) = (k1); \ - (k1) = (k2); \ - (k2) = (k3); \ - (k3) = kt; \ + AES_ROUND_NOKEY_LE(x0, x1, x2, x3, x0, x1, x2, x3); \ } while (0) /* From d6d3518fbecda447f3d97c3b8b81e124370e1c04 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 15 Sep 2025 23:00:27 +0000 Subject: [PATCH 06/21] crypto: replace hardcoded AES transform tables with constexpr tables --- src/crypto/x11/aes_helper.hpp | 358 +++++++--------------------------- 1 file changed, 72 insertions(+), 286 deletions(-) diff --git a/src/crypto/x11/aes_helper.hpp b/src/crypto/x11/aes_helper.hpp index f8c7f0978493..41f0d75cf040 100644 --- a/src/crypto/x11/aes_helper.hpp +++ b/src/crypto/x11/aes_helper.hpp @@ -50,282 +50,80 @@ #include "sph_types.h" -#define AESx(x) SPH_C32(x) +#include +#include namespace impl { -/* - * The AES*[] tables allow us to perform a fast evaluation of an AES - * round; table AESi[] combines SubBytes for a byte at row i, and - * MixColumns for the column where that byte goes after ShiftRows. - */ +constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0) noexcept +{ + return (static_cast(b3) << 24) | + (static_cast(b2) << 16) | + (static_cast(b1) << 8) | + (static_cast(b0)); +} -static const sph_u32 AES0[256] = { - AESx(0xA56363C6), AESx(0x847C7CF8), AESx(0x997777EE), AESx(0x8D7B7BF6), - AESx(0x0DF2F2FF), AESx(0xBD6B6BD6), AESx(0xB16F6FDE), AESx(0x54C5C591), - AESx(0x50303060), AESx(0x03010102), AESx(0xA96767CE), AESx(0x7D2B2B56), - AESx(0x19FEFEE7), AESx(0x62D7D7B5), AESx(0xE6ABAB4D), AESx(0x9A7676EC), - AESx(0x45CACA8F), AESx(0x9D82821F), AESx(0x40C9C989), AESx(0x877D7DFA), - AESx(0x15FAFAEF), AESx(0xEB5959B2), AESx(0xC947478E), AESx(0x0BF0F0FB), - AESx(0xECADAD41), AESx(0x67D4D4B3), AESx(0xFDA2A25F), AESx(0xEAAFAF45), - AESx(0xBF9C9C23), AESx(0xF7A4A453), AESx(0x967272E4), AESx(0x5BC0C09B), - AESx(0xC2B7B775), AESx(0x1CFDFDE1), AESx(0xAE93933D), AESx(0x6A26264C), - AESx(0x5A36366C), AESx(0x413F3F7E), AESx(0x02F7F7F5), AESx(0x4FCCCC83), - AESx(0x5C343468), AESx(0xF4A5A551), AESx(0x34E5E5D1), AESx(0x08F1F1F9), - AESx(0x937171E2), AESx(0x73D8D8AB), AESx(0x53313162), AESx(0x3F15152A), - AESx(0x0C040408), AESx(0x52C7C795), AESx(0x65232346), AESx(0x5EC3C39D), - AESx(0x28181830), AESx(0xA1969637), AESx(0x0F05050A), AESx(0xB59A9A2F), - AESx(0x0907070E), AESx(0x36121224), AESx(0x9B80801B), AESx(0x3DE2E2DF), - AESx(0x26EBEBCD), AESx(0x6927274E), AESx(0xCDB2B27F), AESx(0x9F7575EA), - AESx(0x1B090912), AESx(0x9E83831D), AESx(0x742C2C58), AESx(0x2E1A1A34), - AESx(0x2D1B1B36), AESx(0xB26E6EDC), AESx(0xEE5A5AB4), AESx(0xFBA0A05B), - AESx(0xF65252A4), AESx(0x4D3B3B76), AESx(0x61D6D6B7), AESx(0xCEB3B37D), - AESx(0x7B292952), AESx(0x3EE3E3DD), AESx(0x712F2F5E), AESx(0x97848413), - AESx(0xF55353A6), AESx(0x68D1D1B9), AESx(0x00000000), AESx(0x2CEDEDC1), - AESx(0x60202040), AESx(0x1FFCFCE3), AESx(0xC8B1B179), AESx(0xED5B5BB6), - AESx(0xBE6A6AD4), AESx(0x46CBCB8D), AESx(0xD9BEBE67), AESx(0x4B393972), - AESx(0xDE4A4A94), AESx(0xD44C4C98), AESx(0xE85858B0), AESx(0x4ACFCF85), - AESx(0x6BD0D0BB), AESx(0x2AEFEFC5), AESx(0xE5AAAA4F), AESx(0x16FBFBED), - AESx(0xC5434386), AESx(0xD74D4D9A), AESx(0x55333366), AESx(0x94858511), - AESx(0xCF45458A), AESx(0x10F9F9E9), AESx(0x06020204), AESx(0x817F7FFE), - AESx(0xF05050A0), AESx(0x443C3C78), AESx(0xBA9F9F25), AESx(0xE3A8A84B), - AESx(0xF35151A2), AESx(0xFEA3A35D), AESx(0xC0404080), AESx(0x8A8F8F05), - AESx(0xAD92923F), AESx(0xBC9D9D21), AESx(0x48383870), AESx(0x04F5F5F1), - AESx(0xDFBCBC63), AESx(0xC1B6B677), AESx(0x75DADAAF), AESx(0x63212142), - AESx(0x30101020), AESx(0x1AFFFFE5), AESx(0x0EF3F3FD), AESx(0x6DD2D2BF), - AESx(0x4CCDCD81), AESx(0x140C0C18), AESx(0x35131326), AESx(0x2FECECC3), - AESx(0xE15F5FBE), AESx(0xA2979735), AESx(0xCC444488), AESx(0x3917172E), - AESx(0x57C4C493), AESx(0xF2A7A755), AESx(0x827E7EFC), AESx(0x473D3D7A), - AESx(0xAC6464C8), AESx(0xE75D5DBA), AESx(0x2B191932), AESx(0x957373E6), - AESx(0xA06060C0), AESx(0x98818119), AESx(0xD14F4F9E), AESx(0x7FDCDCA3), - AESx(0x66222244), AESx(0x7E2A2A54), AESx(0xAB90903B), AESx(0x8388880B), - AESx(0xCA46468C), AESx(0x29EEEEC7), AESx(0xD3B8B86B), AESx(0x3C141428), - AESx(0x79DEDEA7), AESx(0xE25E5EBC), AESx(0x1D0B0B16), AESx(0x76DBDBAD), - AESx(0x3BE0E0DB), AESx(0x56323264), AESx(0x4E3A3A74), AESx(0x1E0A0A14), - AESx(0xDB494992), AESx(0x0A06060C), AESx(0x6C242448), AESx(0xE45C5CB8), - AESx(0x5DC2C29F), AESx(0x6ED3D3BD), AESx(0xEFACAC43), AESx(0xA66262C4), - AESx(0xA8919139), AESx(0xA4959531), AESx(0x37E4E4D3), AESx(0x8B7979F2), - AESx(0x32E7E7D5), AESx(0x43C8C88B), AESx(0x5937376E), AESx(0xB76D6DDA), - AESx(0x8C8D8D01), AESx(0x64D5D5B1), AESx(0xD24E4E9C), AESx(0xE0A9A949), - AESx(0xB46C6CD8), AESx(0xFA5656AC), AESx(0x07F4F4F3), AESx(0x25EAEACF), - AESx(0xAF6565CA), AESx(0x8E7A7AF4), AESx(0xE9AEAE47), AESx(0x18080810), - AESx(0xD5BABA6F), AESx(0x887878F0), AESx(0x6F25254A), AESx(0x722E2E5C), - AESx(0x241C1C38), AESx(0xF1A6A657), AESx(0xC7B4B473), AESx(0x51C6C697), - AESx(0x23E8E8CB), AESx(0x7CDDDDA1), AESx(0x9C7474E8), AESx(0x211F1F3E), - AESx(0xDD4B4B96), AESx(0xDCBDBD61), AESx(0x868B8B0D), AESx(0x858A8A0F), - AESx(0x907070E0), AESx(0x423E3E7C), AESx(0xC4B5B571), AESx(0xAA6666CC), - AESx(0xD8484890), AESx(0x05030306), AESx(0x01F6F6F7), AESx(0x120E0E1C), - AESx(0xA36161C2), AESx(0x5F35356A), AESx(0xF95757AE), AESx(0xD0B9B969), - AESx(0x91868617), AESx(0x58C1C199), AESx(0x271D1D3A), AESx(0xB99E9E27), - AESx(0x38E1E1D9), AESx(0x13F8F8EB), AESx(0xB398982B), AESx(0x33111122), - AESx(0xBB6969D2), AESx(0x70D9D9A9), AESx(0x898E8E07), AESx(0xA7949433), - AESx(0xB69B9B2D), AESx(0x221E1E3C), AESx(0x92878715), AESx(0x20E9E9C9), - AESx(0x49CECE87), AESx(0xFF5555AA), AESx(0x78282850), AESx(0x7ADFDFA5), - AESx(0x8F8C8C03), AESx(0xF8A1A159), AESx(0x80898909), AESx(0x170D0D1A), - AESx(0xDABFBF65), AESx(0x31E6E6D7), AESx(0xC6424284), AESx(0xB86868D0), - AESx(0xC3414182), AESx(0xB0999929), AESx(0x772D2D5A), AESx(0x110F0F1E), - AESx(0xCBB0B07B), AESx(0xFC5454A8), AESx(0xD6BBBB6D), AESx(0x3A16162C) -}; +constexpr inline uint8_t gf8_mul2(uint8_t x) noexcept { return (x << 1) ^ ((x & 0x80) ? 0x1b : 0x00); } + +constexpr auto gf8_log = []() constexpr { + std::array ret{}; + uint8_t x{1}; + for (size_t idx{0}; idx < ret.size(); ++idx) { + ret[x] = static_cast(idx); + x = (x ^ gf8_mul2(x)) & 0xff; + } + return ret; +}(); + +constexpr auto gf8_pow = []() constexpr { + std::array ret{}; + uint8_t x{1}; + for (size_t idx{0}; idx < ret.size(); ++idx) { + ret[idx] = x; + x = (x ^ gf8_mul2(x)) & 0xff; + } + return ret; +}(); + +constexpr inline uint8_t gf8_mul(uint8_t x, uint8_t y) noexcept { + if (x != 0 && y != 0) { + return gf8_pow[(gf8_log[x] + gf8_log[y]) % 255]; + } + return 0; +} -static const sph_u32 AES1[256] = { - AESx(0x6363C6A5), AESx(0x7C7CF884), AESx(0x7777EE99), AESx(0x7B7BF68D), - AESx(0xF2F2FF0D), AESx(0x6B6BD6BD), AESx(0x6F6FDEB1), AESx(0xC5C59154), - AESx(0x30306050), AESx(0x01010203), AESx(0x6767CEA9), AESx(0x2B2B567D), - AESx(0xFEFEE719), AESx(0xD7D7B562), AESx(0xABAB4DE6), AESx(0x7676EC9A), - AESx(0xCACA8F45), AESx(0x82821F9D), AESx(0xC9C98940), AESx(0x7D7DFA87), - AESx(0xFAFAEF15), AESx(0x5959B2EB), AESx(0x47478EC9), AESx(0xF0F0FB0B), - AESx(0xADAD41EC), AESx(0xD4D4B367), AESx(0xA2A25FFD), AESx(0xAFAF45EA), - AESx(0x9C9C23BF), AESx(0xA4A453F7), AESx(0x7272E496), AESx(0xC0C09B5B), - AESx(0xB7B775C2), AESx(0xFDFDE11C), AESx(0x93933DAE), AESx(0x26264C6A), - AESx(0x36366C5A), AESx(0x3F3F7E41), AESx(0xF7F7F502), AESx(0xCCCC834F), - AESx(0x3434685C), AESx(0xA5A551F4), AESx(0xE5E5D134), AESx(0xF1F1F908), - AESx(0x7171E293), AESx(0xD8D8AB73), AESx(0x31316253), AESx(0x15152A3F), - AESx(0x0404080C), AESx(0xC7C79552), AESx(0x23234665), AESx(0xC3C39D5E), - AESx(0x18183028), AESx(0x969637A1), AESx(0x05050A0F), AESx(0x9A9A2FB5), - AESx(0x07070E09), AESx(0x12122436), AESx(0x80801B9B), AESx(0xE2E2DF3D), - AESx(0xEBEBCD26), AESx(0x27274E69), AESx(0xB2B27FCD), AESx(0x7575EA9F), - AESx(0x0909121B), AESx(0x83831D9E), AESx(0x2C2C5874), AESx(0x1A1A342E), - AESx(0x1B1B362D), AESx(0x6E6EDCB2), AESx(0x5A5AB4EE), AESx(0xA0A05BFB), - AESx(0x5252A4F6), AESx(0x3B3B764D), AESx(0xD6D6B761), AESx(0xB3B37DCE), - AESx(0x2929527B), AESx(0xE3E3DD3E), AESx(0x2F2F5E71), AESx(0x84841397), - AESx(0x5353A6F5), AESx(0xD1D1B968), AESx(0x00000000), AESx(0xEDEDC12C), - AESx(0x20204060), AESx(0xFCFCE31F), AESx(0xB1B179C8), AESx(0x5B5BB6ED), - AESx(0x6A6AD4BE), AESx(0xCBCB8D46), AESx(0xBEBE67D9), AESx(0x3939724B), - AESx(0x4A4A94DE), AESx(0x4C4C98D4), AESx(0x5858B0E8), AESx(0xCFCF854A), - AESx(0xD0D0BB6B), AESx(0xEFEFC52A), AESx(0xAAAA4FE5), AESx(0xFBFBED16), - AESx(0x434386C5), AESx(0x4D4D9AD7), AESx(0x33336655), AESx(0x85851194), - AESx(0x45458ACF), AESx(0xF9F9E910), AESx(0x02020406), AESx(0x7F7FFE81), - AESx(0x5050A0F0), AESx(0x3C3C7844), AESx(0x9F9F25BA), AESx(0xA8A84BE3), - AESx(0x5151A2F3), AESx(0xA3A35DFE), AESx(0x404080C0), AESx(0x8F8F058A), - AESx(0x92923FAD), AESx(0x9D9D21BC), AESx(0x38387048), AESx(0xF5F5F104), - AESx(0xBCBC63DF), AESx(0xB6B677C1), AESx(0xDADAAF75), AESx(0x21214263), - AESx(0x10102030), AESx(0xFFFFE51A), AESx(0xF3F3FD0E), AESx(0xD2D2BF6D), - AESx(0xCDCD814C), AESx(0x0C0C1814), AESx(0x13132635), AESx(0xECECC32F), - AESx(0x5F5FBEE1), AESx(0x979735A2), AESx(0x444488CC), AESx(0x17172E39), - AESx(0xC4C49357), AESx(0xA7A755F2), AESx(0x7E7EFC82), AESx(0x3D3D7A47), - AESx(0x6464C8AC), AESx(0x5D5DBAE7), AESx(0x1919322B), AESx(0x7373E695), - AESx(0x6060C0A0), AESx(0x81811998), AESx(0x4F4F9ED1), AESx(0xDCDCA37F), - AESx(0x22224466), AESx(0x2A2A547E), AESx(0x90903BAB), AESx(0x88880B83), - AESx(0x46468CCA), AESx(0xEEEEC729), AESx(0xB8B86BD3), AESx(0x1414283C), - AESx(0xDEDEA779), AESx(0x5E5EBCE2), AESx(0x0B0B161D), AESx(0xDBDBAD76), - AESx(0xE0E0DB3B), AESx(0x32326456), AESx(0x3A3A744E), AESx(0x0A0A141E), - AESx(0x494992DB), AESx(0x06060C0A), AESx(0x2424486C), AESx(0x5C5CB8E4), - AESx(0xC2C29F5D), AESx(0xD3D3BD6E), AESx(0xACAC43EF), AESx(0x6262C4A6), - AESx(0x919139A8), AESx(0x959531A4), AESx(0xE4E4D337), AESx(0x7979F28B), - AESx(0xE7E7D532), AESx(0xC8C88B43), AESx(0x37376E59), AESx(0x6D6DDAB7), - AESx(0x8D8D018C), AESx(0xD5D5B164), AESx(0x4E4E9CD2), AESx(0xA9A949E0), - AESx(0x6C6CD8B4), AESx(0x5656ACFA), AESx(0xF4F4F307), AESx(0xEAEACF25), - AESx(0x6565CAAF), AESx(0x7A7AF48E), AESx(0xAEAE47E9), AESx(0x08081018), - AESx(0xBABA6FD5), AESx(0x7878F088), AESx(0x25254A6F), AESx(0x2E2E5C72), - AESx(0x1C1C3824), AESx(0xA6A657F1), AESx(0xB4B473C7), AESx(0xC6C69751), - AESx(0xE8E8CB23), AESx(0xDDDDA17C), AESx(0x7474E89C), AESx(0x1F1F3E21), - AESx(0x4B4B96DD), AESx(0xBDBD61DC), AESx(0x8B8B0D86), AESx(0x8A8A0F85), - AESx(0x7070E090), AESx(0x3E3E7C42), AESx(0xB5B571C4), AESx(0x6666CCAA), - AESx(0x484890D8), AESx(0x03030605), AESx(0xF6F6F701), AESx(0x0E0E1C12), - AESx(0x6161C2A3), AESx(0x35356A5F), AESx(0x5757AEF9), AESx(0xB9B969D0), - AESx(0x86861791), AESx(0xC1C19958), AESx(0x1D1D3A27), AESx(0x9E9E27B9), - AESx(0xE1E1D938), AESx(0xF8F8EB13), AESx(0x98982BB3), AESx(0x11112233), - AESx(0x6969D2BB), AESx(0xD9D9A970), AESx(0x8E8E0789), AESx(0x949433A7), - AESx(0x9B9B2DB6), AESx(0x1E1E3C22), AESx(0x87871592), AESx(0xE9E9C920), - AESx(0xCECE8749), AESx(0x5555AAFF), AESx(0x28285078), AESx(0xDFDFA57A), - AESx(0x8C8C038F), AESx(0xA1A159F8), AESx(0x89890980), AESx(0x0D0D1A17), - AESx(0xBFBF65DA), AESx(0xE6E6D731), AESx(0x424284C6), AESx(0x6868D0B8), - AESx(0x414182C3), AESx(0x999929B0), AESx(0x2D2D5A77), AESx(0x0F0F1E11), - AESx(0xB0B07BCB), AESx(0x5454A8FC), AESx(0xBBBB6DD6), AESx(0x16162C3A) -}; +constexpr auto aes_sbox_f = []() constexpr { + std::array ret{}; + uint8_t x{0}, y{0}; + ret[0x00] = 0x63; + for (size_t idx{1}; idx < ret.size(); ++idx) { + x = gf8_pow[255 - gf8_log[idx]]; + y = x; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y ^ 0x63; + ret[idx] = x; + } + return ret; +}(); -static const sph_u32 AES2[256] = { - AESx(0x63C6A563), AESx(0x7CF8847C), AESx(0x77EE9977), AESx(0x7BF68D7B), - AESx(0xF2FF0DF2), AESx(0x6BD6BD6B), AESx(0x6FDEB16F), AESx(0xC59154C5), - AESx(0x30605030), AESx(0x01020301), AESx(0x67CEA967), AESx(0x2B567D2B), - AESx(0xFEE719FE), AESx(0xD7B562D7), AESx(0xAB4DE6AB), AESx(0x76EC9A76), - AESx(0xCA8F45CA), AESx(0x821F9D82), AESx(0xC98940C9), AESx(0x7DFA877D), - AESx(0xFAEF15FA), AESx(0x59B2EB59), AESx(0x478EC947), AESx(0xF0FB0BF0), - AESx(0xAD41ECAD), AESx(0xD4B367D4), AESx(0xA25FFDA2), AESx(0xAF45EAAF), - AESx(0x9C23BF9C), AESx(0xA453F7A4), AESx(0x72E49672), AESx(0xC09B5BC0), - AESx(0xB775C2B7), AESx(0xFDE11CFD), AESx(0x933DAE93), AESx(0x264C6A26), - AESx(0x366C5A36), AESx(0x3F7E413F), AESx(0xF7F502F7), AESx(0xCC834FCC), - AESx(0x34685C34), AESx(0xA551F4A5), AESx(0xE5D134E5), AESx(0xF1F908F1), - AESx(0x71E29371), AESx(0xD8AB73D8), AESx(0x31625331), AESx(0x152A3F15), - AESx(0x04080C04), AESx(0xC79552C7), AESx(0x23466523), AESx(0xC39D5EC3), - AESx(0x18302818), AESx(0x9637A196), AESx(0x050A0F05), AESx(0x9A2FB59A), - AESx(0x070E0907), AESx(0x12243612), AESx(0x801B9B80), AESx(0xE2DF3DE2), - AESx(0xEBCD26EB), AESx(0x274E6927), AESx(0xB27FCDB2), AESx(0x75EA9F75), - AESx(0x09121B09), AESx(0x831D9E83), AESx(0x2C58742C), AESx(0x1A342E1A), - AESx(0x1B362D1B), AESx(0x6EDCB26E), AESx(0x5AB4EE5A), AESx(0xA05BFBA0), - AESx(0x52A4F652), AESx(0x3B764D3B), AESx(0xD6B761D6), AESx(0xB37DCEB3), - AESx(0x29527B29), AESx(0xE3DD3EE3), AESx(0x2F5E712F), AESx(0x84139784), - AESx(0x53A6F553), AESx(0xD1B968D1), AESx(0x00000000), AESx(0xEDC12CED), - AESx(0x20406020), AESx(0xFCE31FFC), AESx(0xB179C8B1), AESx(0x5BB6ED5B), - AESx(0x6AD4BE6A), AESx(0xCB8D46CB), AESx(0xBE67D9BE), AESx(0x39724B39), - AESx(0x4A94DE4A), AESx(0x4C98D44C), AESx(0x58B0E858), AESx(0xCF854ACF), - AESx(0xD0BB6BD0), AESx(0xEFC52AEF), AESx(0xAA4FE5AA), AESx(0xFBED16FB), - AESx(0x4386C543), AESx(0x4D9AD74D), AESx(0x33665533), AESx(0x85119485), - AESx(0x458ACF45), AESx(0xF9E910F9), AESx(0x02040602), AESx(0x7FFE817F), - AESx(0x50A0F050), AESx(0x3C78443C), AESx(0x9F25BA9F), AESx(0xA84BE3A8), - AESx(0x51A2F351), AESx(0xA35DFEA3), AESx(0x4080C040), AESx(0x8F058A8F), - AESx(0x923FAD92), AESx(0x9D21BC9D), AESx(0x38704838), AESx(0xF5F104F5), - AESx(0xBC63DFBC), AESx(0xB677C1B6), AESx(0xDAAF75DA), AESx(0x21426321), - AESx(0x10203010), AESx(0xFFE51AFF), AESx(0xF3FD0EF3), AESx(0xD2BF6DD2), - AESx(0xCD814CCD), AESx(0x0C18140C), AESx(0x13263513), AESx(0xECC32FEC), - AESx(0x5FBEE15F), AESx(0x9735A297), AESx(0x4488CC44), AESx(0x172E3917), - AESx(0xC49357C4), AESx(0xA755F2A7), AESx(0x7EFC827E), AESx(0x3D7A473D), - AESx(0x64C8AC64), AESx(0x5DBAE75D), AESx(0x19322B19), AESx(0x73E69573), - AESx(0x60C0A060), AESx(0x81199881), AESx(0x4F9ED14F), AESx(0xDCA37FDC), - AESx(0x22446622), AESx(0x2A547E2A), AESx(0x903BAB90), AESx(0x880B8388), - AESx(0x468CCA46), AESx(0xEEC729EE), AESx(0xB86BD3B8), AESx(0x14283C14), - AESx(0xDEA779DE), AESx(0x5EBCE25E), AESx(0x0B161D0B), AESx(0xDBAD76DB), - AESx(0xE0DB3BE0), AESx(0x32645632), AESx(0x3A744E3A), AESx(0x0A141E0A), - AESx(0x4992DB49), AESx(0x060C0A06), AESx(0x24486C24), AESx(0x5CB8E45C), - AESx(0xC29F5DC2), AESx(0xD3BD6ED3), AESx(0xAC43EFAC), AESx(0x62C4A662), - AESx(0x9139A891), AESx(0x9531A495), AESx(0xE4D337E4), AESx(0x79F28B79), - AESx(0xE7D532E7), AESx(0xC88B43C8), AESx(0x376E5937), AESx(0x6DDAB76D), - AESx(0x8D018C8D), AESx(0xD5B164D5), AESx(0x4E9CD24E), AESx(0xA949E0A9), - AESx(0x6CD8B46C), AESx(0x56ACFA56), AESx(0xF4F307F4), AESx(0xEACF25EA), - AESx(0x65CAAF65), AESx(0x7AF48E7A), AESx(0xAE47E9AE), AESx(0x08101808), - AESx(0xBA6FD5BA), AESx(0x78F08878), AESx(0x254A6F25), AESx(0x2E5C722E), - AESx(0x1C38241C), AESx(0xA657F1A6), AESx(0xB473C7B4), AESx(0xC69751C6), - AESx(0xE8CB23E8), AESx(0xDDA17CDD), AESx(0x74E89C74), AESx(0x1F3E211F), - AESx(0x4B96DD4B), AESx(0xBD61DCBD), AESx(0x8B0D868B), AESx(0x8A0F858A), - AESx(0x70E09070), AESx(0x3E7C423E), AESx(0xB571C4B5), AESx(0x66CCAA66), - AESx(0x4890D848), AESx(0x03060503), AESx(0xF6F701F6), AESx(0x0E1C120E), - AESx(0x61C2A361), AESx(0x356A5F35), AESx(0x57AEF957), AESx(0xB969D0B9), - AESx(0x86179186), AESx(0xC19958C1), AESx(0x1D3A271D), AESx(0x9E27B99E), - AESx(0xE1D938E1), AESx(0xF8EB13F8), AESx(0x982BB398), AESx(0x11223311), - AESx(0x69D2BB69), AESx(0xD9A970D9), AESx(0x8E07898E), AESx(0x9433A794), - AESx(0x9B2DB69B), AESx(0x1E3C221E), AESx(0x87159287), AESx(0xE9C920E9), - AESx(0xCE8749CE), AESx(0x55AAFF55), AESx(0x28507828), AESx(0xDFA57ADF), - AESx(0x8C038F8C), AESx(0xA159F8A1), AESx(0x89098089), AESx(0x0D1A170D), - AESx(0xBF65DABF), AESx(0xE6D731E6), AESx(0x4284C642), AESx(0x68D0B868), - AESx(0x4182C341), AESx(0x9929B099), AESx(0x2D5A772D), AESx(0x0F1E110F), - AESx(0xB07BCBB0), AESx(0x54A8FC54), AESx(0xBB6DD6BB), AESx(0x162C3A16) -}; +constexpr auto aes_tbox_le = []() constexpr { + std::array, 4> ret{}; + for (size_t idx{0}; idx < 256; ++idx) { + const uint8_t byte{aes_sbox_f[idx]}; + uint32_t word{pack_le(gf8_mul(byte, 3), byte, byte, gf8_mul(byte, 2))}; + for (size_t jdx{0}; jdx < 4; ++jdx) { + ret[jdx][idx] = word; + word = std::rotl(word, 8); + } + } + return ret; +}(); -static const sph_u32 AES3[256] = { - AESx(0xC6A56363), AESx(0xF8847C7C), AESx(0xEE997777), AESx(0xF68D7B7B), - AESx(0xFF0DF2F2), AESx(0xD6BD6B6B), AESx(0xDEB16F6F), AESx(0x9154C5C5), - AESx(0x60503030), AESx(0x02030101), AESx(0xCEA96767), AESx(0x567D2B2B), - AESx(0xE719FEFE), AESx(0xB562D7D7), AESx(0x4DE6ABAB), AESx(0xEC9A7676), - AESx(0x8F45CACA), AESx(0x1F9D8282), AESx(0x8940C9C9), AESx(0xFA877D7D), - AESx(0xEF15FAFA), AESx(0xB2EB5959), AESx(0x8EC94747), AESx(0xFB0BF0F0), - AESx(0x41ECADAD), AESx(0xB367D4D4), AESx(0x5FFDA2A2), AESx(0x45EAAFAF), - AESx(0x23BF9C9C), AESx(0x53F7A4A4), AESx(0xE4967272), AESx(0x9B5BC0C0), - AESx(0x75C2B7B7), AESx(0xE11CFDFD), AESx(0x3DAE9393), AESx(0x4C6A2626), - AESx(0x6C5A3636), AESx(0x7E413F3F), AESx(0xF502F7F7), AESx(0x834FCCCC), - AESx(0x685C3434), AESx(0x51F4A5A5), AESx(0xD134E5E5), AESx(0xF908F1F1), - AESx(0xE2937171), AESx(0xAB73D8D8), AESx(0x62533131), AESx(0x2A3F1515), - AESx(0x080C0404), AESx(0x9552C7C7), AESx(0x46652323), AESx(0x9D5EC3C3), - AESx(0x30281818), AESx(0x37A19696), AESx(0x0A0F0505), AESx(0x2FB59A9A), - AESx(0x0E090707), AESx(0x24361212), AESx(0x1B9B8080), AESx(0xDF3DE2E2), - AESx(0xCD26EBEB), AESx(0x4E692727), AESx(0x7FCDB2B2), AESx(0xEA9F7575), - AESx(0x121B0909), AESx(0x1D9E8383), AESx(0x58742C2C), AESx(0x342E1A1A), - AESx(0x362D1B1B), AESx(0xDCB26E6E), AESx(0xB4EE5A5A), AESx(0x5BFBA0A0), - AESx(0xA4F65252), AESx(0x764D3B3B), AESx(0xB761D6D6), AESx(0x7DCEB3B3), - AESx(0x527B2929), AESx(0xDD3EE3E3), AESx(0x5E712F2F), AESx(0x13978484), - AESx(0xA6F55353), AESx(0xB968D1D1), AESx(0x00000000), AESx(0xC12CEDED), - AESx(0x40602020), AESx(0xE31FFCFC), AESx(0x79C8B1B1), AESx(0xB6ED5B5B), - AESx(0xD4BE6A6A), AESx(0x8D46CBCB), AESx(0x67D9BEBE), AESx(0x724B3939), - AESx(0x94DE4A4A), AESx(0x98D44C4C), AESx(0xB0E85858), AESx(0x854ACFCF), - AESx(0xBB6BD0D0), AESx(0xC52AEFEF), AESx(0x4FE5AAAA), AESx(0xED16FBFB), - AESx(0x86C54343), AESx(0x9AD74D4D), AESx(0x66553333), AESx(0x11948585), - AESx(0x8ACF4545), AESx(0xE910F9F9), AESx(0x04060202), AESx(0xFE817F7F), - AESx(0xA0F05050), AESx(0x78443C3C), AESx(0x25BA9F9F), AESx(0x4BE3A8A8), - AESx(0xA2F35151), AESx(0x5DFEA3A3), AESx(0x80C04040), AESx(0x058A8F8F), - AESx(0x3FAD9292), AESx(0x21BC9D9D), AESx(0x70483838), AESx(0xF104F5F5), - AESx(0x63DFBCBC), AESx(0x77C1B6B6), AESx(0xAF75DADA), AESx(0x42632121), - AESx(0x20301010), AESx(0xE51AFFFF), AESx(0xFD0EF3F3), AESx(0xBF6DD2D2), - AESx(0x814CCDCD), AESx(0x18140C0C), AESx(0x26351313), AESx(0xC32FECEC), - AESx(0xBEE15F5F), AESx(0x35A29797), AESx(0x88CC4444), AESx(0x2E391717), - AESx(0x9357C4C4), AESx(0x55F2A7A7), AESx(0xFC827E7E), AESx(0x7A473D3D), - AESx(0xC8AC6464), AESx(0xBAE75D5D), AESx(0x322B1919), AESx(0xE6957373), - AESx(0xC0A06060), AESx(0x19988181), AESx(0x9ED14F4F), AESx(0xA37FDCDC), - AESx(0x44662222), AESx(0x547E2A2A), AESx(0x3BAB9090), AESx(0x0B838888), - AESx(0x8CCA4646), AESx(0xC729EEEE), AESx(0x6BD3B8B8), AESx(0x283C1414), - AESx(0xA779DEDE), AESx(0xBCE25E5E), AESx(0x161D0B0B), AESx(0xAD76DBDB), - AESx(0xDB3BE0E0), AESx(0x64563232), AESx(0x744E3A3A), AESx(0x141E0A0A), - AESx(0x92DB4949), AESx(0x0C0A0606), AESx(0x486C2424), AESx(0xB8E45C5C), - AESx(0x9F5DC2C2), AESx(0xBD6ED3D3), AESx(0x43EFACAC), AESx(0xC4A66262), - AESx(0x39A89191), AESx(0x31A49595), AESx(0xD337E4E4), AESx(0xF28B7979), - AESx(0xD532E7E7), AESx(0x8B43C8C8), AESx(0x6E593737), AESx(0xDAB76D6D), - AESx(0x018C8D8D), AESx(0xB164D5D5), AESx(0x9CD24E4E), AESx(0x49E0A9A9), - AESx(0xD8B46C6C), AESx(0xACFA5656), AESx(0xF307F4F4), AESx(0xCF25EAEA), - AESx(0xCAAF6565), AESx(0xF48E7A7A), AESx(0x47E9AEAE), AESx(0x10180808), - AESx(0x6FD5BABA), AESx(0xF0887878), AESx(0x4A6F2525), AESx(0x5C722E2E), - AESx(0x38241C1C), AESx(0x57F1A6A6), AESx(0x73C7B4B4), AESx(0x9751C6C6), - AESx(0xCB23E8E8), AESx(0xA17CDDDD), AESx(0xE89C7474), AESx(0x3E211F1F), - AESx(0x96DD4B4B), AESx(0x61DCBDBD), AESx(0x0D868B8B), AESx(0x0F858A8A), - AESx(0xE0907070), AESx(0x7C423E3E), AESx(0x71C4B5B5), AESx(0xCCAA6666), - AESx(0x90D84848), AESx(0x06050303), AESx(0xF701F6F6), AESx(0x1C120E0E), - AESx(0xC2A36161), AESx(0x6A5F3535), AESx(0xAEF95757), AESx(0x69D0B9B9), - AESx(0x17918686), AESx(0x9958C1C1), AESx(0x3A271D1D), AESx(0x27B99E9E), - AESx(0xD938E1E1), AESx(0xEB13F8F8), AESx(0x2BB39898), AESx(0x22331111), - AESx(0xD2BB6969), AESx(0xA970D9D9), AESx(0x07898E8E), AESx(0x33A79494), - AESx(0x2DB69B9B), AESx(0x3C221E1E), AESx(0x15928787), AESx(0xC920E9E9), - AESx(0x8749CECE), AESx(0xAAFF5555), AESx(0x50782828), AESx(0xA57ADFDF), - AESx(0x038F8C8C), AESx(0x59F8A1A1), AESx(0x09808989), AESx(0x1A170D0D), - AESx(0x65DABFBF), AESx(0xD731E6E6), AESx(0x84C64242), AESx(0xD0B86868), - AESx(0x82C34141), AESx(0x29B09999), AESx(0x5A772D2D), AESx(0x1E110F0F), - AESx(0x7BCBB0B0), AESx(0xA8FC5454), AESx(0x6DD6BBBB), AESx(0x2C3A1616) -}; +static_assert(aes_tbox_le[0][ 0] == 0xa56363c6 && + aes_tbox_le[0][255] == 0x3a16162c && + aes_tbox_le[3][ 0] == 0xc6a56363 && + aes_tbox_le[3][255] == 0x2c3a1616, "Bad little-endian AES transform table"); } // namespace impl void ALWAYS_INLINE AES_ROUND_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, @@ -333,22 +131,10 @@ void ALWAYS_INLINE AES_ROUND_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, sph_u32& y0, sph_u32& y1, sph_u32& y2, sph_u32& y3) { using namespace impl; - (y0) = AES0[(x0) & 0xFF] - ^ AES1[((x1) >> 8) & 0xFF] - ^ AES2[((x2) >> 16) & 0xFF] - ^ AES3[((x3) >> 24) & 0xFF] ^ (k0); - (y1) = AES0[(x1) & 0xFF] - ^ AES1[((x2) >> 8) & 0xFF] - ^ AES2[((x3) >> 16) & 0xFF] - ^ AES3[((x0) >> 24) & 0xFF] ^ (k1); - (y2) = AES0[(x2) & 0xFF] - ^ AES1[((x3) >> 8) & 0xFF] - ^ AES2[((x0) >> 16) & 0xFF] - ^ AES3[((x1) >> 24) & 0xFF] ^ (k2); - (y3) = AES0[(x3) & 0xFF] - ^ AES1[((x0) >> 8) & 0xFF] - ^ AES2[((x1) >> 16) & 0xFF] - ^ AES3[((x2) >> 24) & 0xFF] ^ (k3); + y0 = aes_tbox_le[0][(x0) & 0xff] ^ aes_tbox_le[1][((x1) >> 8) & 0xff] ^ aes_tbox_le[2][((x2) >> 16) & 0xff] ^ aes_tbox_le[3][((x3) >> 24) & 0xff] ^ (k0); + y1 = aes_tbox_le[0][(x1) & 0xff] ^ aes_tbox_le[1][((x2) >> 8) & 0xff] ^ aes_tbox_le[2][((x3) >> 16) & 0xff] ^ aes_tbox_le[3][((x0) >> 24) & 0xff] ^ (k1); + y2 = aes_tbox_le[0][(x2) & 0xff] ^ aes_tbox_le[1][((x3) >> 8) & 0xff] ^ aes_tbox_le[2][((x0) >> 16) & 0xff] ^ aes_tbox_le[3][((x1) >> 24) & 0xff] ^ (k2); + y3 = aes_tbox_le[0][(x3) & 0xff] ^ aes_tbox_le[1][((x0) >> 8) & 0xff] ^ aes_tbox_le[2][((x1) >> 16) & 0xff] ^ aes_tbox_le[3][((x2) >> 24) & 0xff] ^ (k3); } void ALWAYS_INLINE AES_ROUND_NOKEY_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, From cba39c5b3732dce95bf0db9d50d3f62f7dcba6e8 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 15:19:08 +0000 Subject: [PATCH 07/21] const: use function pointer to allow for switching implementation --- src/Makefile.am | 8 +- src/bench/bench_bitcoin.cpp | 2 + src/crypto/x11/aes.cpp | 32 +++++++ src/crypto/x11/aes_helper.hpp | 146 ----------------------------- src/crypto/x11/dispatch.cpp | 26 +++++ src/crypto/x11/dispatch.h | 22 +++++ src/crypto/x11/echo.cpp | 15 +-- src/crypto/x11/shavite.cpp | 12 ++- src/crypto/x11/util/consts_aes.hpp | 85 +++++++++++++++++ src/crypto/x11/util/util.hpp | 22 +++++ src/init/common.cpp | 2 + src/test/util/setup_common.cpp | 2 + 12 files changed, 215 insertions(+), 159 deletions(-) create mode 100644 src/crypto/x11/aes.cpp delete mode 100644 src/crypto/x11/aes_helper.hpp create mode 100644 src/crypto/x11/dispatch.cpp create mode 100644 src/crypto/x11/dispatch.h create mode 100644 src/crypto/x11/util/consts_aes.hpp create mode 100644 src/crypto/x11/util/util.hpp diff --git a/src/Makefile.am b/src/Makefile.am index da21ca85dbed..777dbf1c7e7d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -735,10 +735,12 @@ crypto_libbitcoin_crypto_sph_la_CPPFLAGS += \ -DSPH_SMALL_FOOTPRINT_CUBEHASH=1 \ -DSPH_SMALL_FOOTPRINT_JH=1 crypto_libbitcoin_crypto_sph_la_SOURCES = \ - crypto/x11/aes_helper.hpp \ + crypto/x11/aes.cpp \ crypto/x11/blake.c \ crypto/x11/bmw.c \ crypto/x11/cubehash.c \ + crypto/x11/dispatch.cpp \ + crypto/x11/dispatch.h \ crypto/x11/echo.cpp \ crypto/x11/groestl.c \ crypto/x11/jh.c \ @@ -758,7 +760,9 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/sph_shavite.h \ crypto/x11/sph_simd.h \ crypto/x11/sph_skein.h \ - crypto/x11/sph_types.h + crypto/x11/sph_types.h \ + crypto/x11/util/consts_aes.hpp \ + crypto/x11/util/util.hpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index a52021cf0a27..c5fe3fbdfddb 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,7 @@ int main(int argc, char** argv) { ArgsManager argsman; SetupBenchArgs(argsman); + SapphireAutoDetect(); SHA256AutoDetect(); std::string error; if (!argsman.ParseParameters(argc, argv, error)) { diff --git a/src/crypto/x11/aes.cpp b/src/crypto/x11/aes.cpp new file mode 100644 index 000000000000..c874cbab05e8 --- /dev/null +++ b/src/crypto/x11/aes.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include + +namespace sapphire { +namespace soft_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + using namespace consts; + y0 = aes_tbox_le[0][(x0) & 0xff] ^ aes_tbox_le[1][((x1) >> 8) & 0xff] ^ aes_tbox_le[2][((x2) >> 16) & 0xff] ^ aes_tbox_le[3][((x3) >> 24) & 0xff] ^ (k0); + y1 = aes_tbox_le[0][(x1) & 0xff] ^ aes_tbox_le[1][((x2) >> 8) & 0xff] ^ aes_tbox_le[2][((x3) >> 16) & 0xff] ^ aes_tbox_le[3][((x0) >> 24) & 0xff] ^ (k1); + y2 = aes_tbox_le[0][(x2) & 0xff] ^ aes_tbox_le[1][((x3) >> 8) & 0xff] ^ aes_tbox_le[2][((x0) >> 16) & 0xff] ^ aes_tbox_le[3][((x1) >> 24) & 0xff] ^ (k2); + y3 = aes_tbox_le[0][(x3) & 0xff] ^ aes_tbox_le[1][((x0) >> 8) & 0xff] ^ aes_tbox_le[2][((x1) >> 16) & 0xff] ^ aes_tbox_le[3][((x2) >> 24) & 0xff] ^ (k3); +} + +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + Round(x0, x1, x2, x3, /*k0=*/0, /*k1=*/0, /*k2=*/0, /*k3=*/0, y0, y1, y2, y3); +} +} // namespace soft_aes +} // namespace sapphire + +sapphire::dispatch::AESRoundFn aes_round = sapphire::soft_aes::Round; +sapphire::dispatch::AESRoundFnNk aes_round_nk = sapphire::soft_aes::RoundKeyless; diff --git a/src/crypto/x11/aes_helper.hpp b/src/crypto/x11/aes_helper.hpp deleted file mode 100644 index 41f0d75cf040..000000000000 --- a/src/crypto/x11/aes_helper.hpp +++ /dev/null @@ -1,146 +0,0 @@ -/* $Id: aes_helper.c 220 2010-06-09 09:21:50Z tp $ */ -/* - * AES tables. This file is not meant to be compiled by itself; it - * is included by some hash function implementations. It contains - * the precomputed tables and helper macros for evaluating an AES - * round, optionally with a final XOR with a subkey. - * - * By default, this file defines the tables and macros for little-endian - * processing (i.e. it is assumed that the input bytes have been read - * from memory and assembled with the little-endian convention). If - * the 'AES_BIG_ENDIAN' macro is defined (to a non-zero integer value) - * when this file is included, then the tables and macros for big-endian - * processing are defined instead. The big-endian tables and macros have - * names distinct from the little-endian tables and macros, hence it is - * possible to have both simultaneously, by including this file twice - * (with and without the AES_BIG_ENDIAN macro). - * - * ==========================(LICENSE BEGIN)============================ - * - * Copyright (c) 2007-2010 Projet RNRT SAPHIR - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ===========================(LICENSE END)============================= - * - * @author Thomas Pornin - */ - -#ifndef SPH_AES_HELPER_HPP__ -#define SPH_AES_HELPER_HPP__ - -#include - -#include "sph_types.h" - -#include -#include - -namespace impl { -constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0) noexcept -{ - return (static_cast(b3) << 24) | - (static_cast(b2) << 16) | - (static_cast(b1) << 8) | - (static_cast(b0)); -} - -constexpr inline uint8_t gf8_mul2(uint8_t x) noexcept { return (x << 1) ^ ((x & 0x80) ? 0x1b : 0x00); } - -constexpr auto gf8_log = []() constexpr { - std::array ret{}; - uint8_t x{1}; - for (size_t idx{0}; idx < ret.size(); ++idx) { - ret[x] = static_cast(idx); - x = (x ^ gf8_mul2(x)) & 0xff; - } - return ret; -}(); - -constexpr auto gf8_pow = []() constexpr { - std::array ret{}; - uint8_t x{1}; - for (size_t idx{0}; idx < ret.size(); ++idx) { - ret[idx] = x; - x = (x ^ gf8_mul2(x)) & 0xff; - } - return ret; -}(); - -constexpr inline uint8_t gf8_mul(uint8_t x, uint8_t y) noexcept { - if (x != 0 && y != 0) { - return gf8_pow[(gf8_log[x] + gf8_log[y]) % 255]; - } - return 0; -} - -constexpr auto aes_sbox_f = []() constexpr { - std::array ret{}; - uint8_t x{0}, y{0}; - ret[0x00] = 0x63; - for (size_t idx{1}; idx < ret.size(); ++idx) { - x = gf8_pow[255 - gf8_log[idx]]; - y = x; y = std::rotl(y, 1); - x ^= y; y = std::rotl(y, 1); - x ^= y; y = std::rotl(y, 1); - x ^= y; y = std::rotl(y, 1); - x ^= y ^ 0x63; - ret[idx] = x; - } - return ret; -}(); - -constexpr auto aes_tbox_le = []() constexpr { - std::array, 4> ret{}; - for (size_t idx{0}; idx < 256; ++idx) { - const uint8_t byte{aes_sbox_f[idx]}; - uint32_t word{pack_le(gf8_mul(byte, 3), byte, byte, gf8_mul(byte, 2))}; - for (size_t jdx{0}; jdx < 4; ++jdx) { - ret[jdx][idx] = word; - word = std::rotl(word, 8); - } - } - return ret; -}(); - -static_assert(aes_tbox_le[0][ 0] == 0xa56363c6 && - aes_tbox_le[0][255] == 0x3a16162c && - aes_tbox_le[3][ 0] == 0xc6a56363 && - aes_tbox_le[3][255] == 0x2c3a1616, "Bad little-endian AES transform table"); -} // namespace impl - -void ALWAYS_INLINE AES_ROUND_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, - sph_u32 k0, sph_u32 k1, sph_u32 k2, sph_u32 k3, - sph_u32& y0, sph_u32& y1, sph_u32& y2, sph_u32& y3) -{ - using namespace impl; - y0 = aes_tbox_le[0][(x0) & 0xff] ^ aes_tbox_le[1][((x1) >> 8) & 0xff] ^ aes_tbox_le[2][((x2) >> 16) & 0xff] ^ aes_tbox_le[3][((x3) >> 24) & 0xff] ^ (k0); - y1 = aes_tbox_le[0][(x1) & 0xff] ^ aes_tbox_le[1][((x2) >> 8) & 0xff] ^ aes_tbox_le[2][((x3) >> 16) & 0xff] ^ aes_tbox_le[3][((x0) >> 24) & 0xff] ^ (k1); - y2 = aes_tbox_le[0][(x2) & 0xff] ^ aes_tbox_le[1][((x3) >> 8) & 0xff] ^ aes_tbox_le[2][((x0) >> 16) & 0xff] ^ aes_tbox_le[3][((x1) >> 24) & 0xff] ^ (k2); - y3 = aes_tbox_le[0][(x3) & 0xff] ^ aes_tbox_le[1][((x0) >> 8) & 0xff] ^ aes_tbox_le[2][((x1) >> 16) & 0xff] ^ aes_tbox_le[3][((x2) >> 24) & 0xff] ^ (k3); -} - -void ALWAYS_INLINE AES_ROUND_NOKEY_LE(sph_u32 x0, sph_u32 x1, sph_u32 x2, sph_u32 x3, - sph_u32& y0, sph_u32& y1, sph_u32& y2, sph_u32& y3) -{ - AES_ROUND_LE(x0, x1, x2, x3, /*k0=*/0, /*k1=*/0, /*k2=*/0, /*k3=*/0, y0, y1, y2, y3); -} - -#endif // SPH_AES_HELPER_HPP__ diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp new file mode 100644 index 000000000000..6e4b263c8836 --- /dev/null +++ b/src/crypto/x11/dispatch.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include + +namespace sapphire { +namespace soft_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +} // namespace soft_aes +} // namespace sapphire + +extern sapphire::dispatch::AESRoundFn aes_round; +extern sapphire::dispatch::AESRoundFnNk aes_round_nk; + +void SapphireAutoDetect() +{ + aes_round = sapphire::soft_aes::Round; + aes_round_nk = sapphire::soft_aes::RoundKeyless; +} diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h new file mode 100644 index 000000000000..f9c96428b11e --- /dev/null +++ b/src/crypto/x11/dispatch.h @@ -0,0 +1,22 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_X11_DISPATCH_H +#define BITCOIN_CRYPTO_X11_DISPATCH_H + +#include + +namespace sapphire { +namespace dispatch { +typedef void (*AESRoundFn)(uint32_t, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, + uint32_t&, uint32_t&, uint32_t&, uint32_t&); +typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, + uint32_t&, uint32_t&, uint32_t&, uint32_t&); +} // namespace dispatch +} // namespace sapphire + +void SapphireAutoDetect(); + +#endif // BITCOIN_CRYPTO_X11_DISPATCH_H diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index fc9e31254f51..f9a9c46c8469 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -30,11 +30,16 @@ * @author Thomas Pornin */ -#include -#include +#include + +#include +#include #include "sph_echo.h" +extern sapphire::dispatch::AESRoundFn aes_round; +extern sapphire::dispatch::AESRoundFnNk aes_round_nk; + /* * We can use a 64-bit implementation only if a 64-bit type is available. */ @@ -45,8 +50,6 @@ #define T32 SPH_T32 #define C64 SPH_C64 -#include "aes_helper.hpp" - #define DECL_STATE_BIG \ sph_u64 W[16][2]; @@ -79,8 +82,8 @@ aes_2rounds_all(sph_u64 W[16][2], sph_u32 X2 = (sph_u32)Wh; sph_u32 X3 = (sph_u32)(Wh >> 32); sph_u32 Y0, Y1, Y2, Y3; \ - AES_ROUND_LE(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); - AES_ROUND_NOKEY_LE(Y0, Y1, Y2, Y3, X0, X1, X2, X3); + aes_round(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); + aes_round_nk(Y0, Y1, Y2, Y3, X0, X1, X2, X3); W[n][0] = (sph_u64)X0 | ((sph_u64)X1 << 32); W[n][1] = (sph_u64)X2 | ((sph_u64)X3 << 32); if ((K0 = T32(K0 + 1)) == 0) { diff --git a/src/crypto/x11/shavite.cpp b/src/crypto/x11/shavite.cpp index 38ea357040cb..abd0e74c4536 100644 --- a/src/crypto/x11/shavite.cpp +++ b/src/crypto/x11/shavite.cpp @@ -30,11 +30,15 @@ * @author Thomas Pornin */ -#include -#include +#include + +#include +#include #include "sph_shavite.h" +extern sapphire::dispatch::AESRoundFnNk aes_round_nk; + #ifdef _MSC_VER #pragma warning (disable: 4146) #endif @@ -53,8 +57,6 @@ * is commented out afterwards. */ -#include "aes_helper.hpp" - static const sph_u32 IV512[] = { C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), C32(0xD1901A06), C32(0x430AE307), C32(0xB29F5CD1), C32(0xDF07FBFC), @@ -63,7 +65,7 @@ static const sph_u32 IV512[] = { }; #define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ - AES_ROUND_NOKEY_LE(x0, x1, x2, x3, x0, x1, x2, x3); \ + aes_round_nk(x0, x1, x2, x3, x0, x1, x2, x3); \ } while (0) /* diff --git a/src/crypto/x11/util/consts_aes.hpp b/src/crypto/x11/util/consts_aes.hpp new file mode 100644 index 000000000000..a06b3011787a --- /dev/null +++ b/src/crypto/x11/util/consts_aes.hpp @@ -0,0 +1,85 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_X11_UTIL_CONSTS_AES_HPP +#define BITCOIN_CRYPTO_X11_UTIL_CONSTS_AES_HPP + +#include + +#include +#include +#include + +namespace sapphire { +namespace util { +constexpr inline uint8_t gf8_mul2(uint8_t x) noexcept { + return (x << 1) ^ ((x & 0x80) ? 0x1b : 0x00); +} + +constexpr auto gf8_log = []() constexpr { + std::array ret{}; + uint8_t x{1}; + for (size_t idx{0}; idx < ret.size(); ++idx) { + ret[x] = static_cast(idx); + x = (x ^ gf8_mul2(x)) & 0xff; + } + return ret; +}(); + +constexpr auto gf8_pow = []() constexpr { + std::array ret{}; + uint8_t x{1}; + for (size_t idx{0}; idx < ret.size(); ++idx) { + ret[idx] = x; + x = (x ^ gf8_mul2(x)) & 0xff; + } + return ret; +}(); + +constexpr inline uint8_t gf8_mul(uint8_t x, uint8_t y) noexcept { + if (x != 0 && y != 0) { + return gf8_pow[(gf8_log[x] + gf8_log[y]) % 255]; + } + return 0; +} +} // namespace util + +namespace consts { +constexpr auto aes_sbox_f = []() constexpr { + std::array ret{}; + uint8_t x{0}, y{0}; + ret[0x00] = 0x63; + for (size_t idx{1}; idx < ret.size(); ++idx) { + x = util::gf8_pow[255 - util::gf8_log[idx]]; + y = x; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y; y = std::rotl(y, 1); + x ^= y ^ 0x63; + ret[idx] = x; + } + return ret; +}(); + +constexpr auto aes_tbox_le = []() constexpr { + std::array, 4> ret{}; + for (size_t idx{0}; idx < 256; ++idx) { + const uint8_t byte{aes_sbox_f[idx]}; + uint32_t word{util::pack_le(util::gf8_mul(byte, 3), byte, byte, util::gf8_mul(byte, 2))}; + for (size_t jdx{0}; jdx < 4; ++jdx) { + ret[jdx][idx] = word; + word = std::rotl(word, 8); + } + } + return ret; +}(); + +static_assert(aes_tbox_le[0][ 0] == 0xa56363c6 && + aes_tbox_le[0][255] == 0x3a16162c && + aes_tbox_le[3][ 0] == 0xc6a56363 && + aes_tbox_le[3][255] == 0x2c3a1616, "Bad little-endian AES transform table"); +} // namespace consts +} // namespace sapphire + +#endif // BITCOIN_CRYPTO_X11_UTIL_CONSTS_AES_HPP diff --git a/src/crypto/x11/util/util.hpp b/src/crypto/x11/util/util.hpp new file mode 100644 index 000000000000..52e06b1773e6 --- /dev/null +++ b/src/crypto/x11/util/util.hpp @@ -0,0 +1,22 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_X11_UTIL_UTIL_HPP +#define BITCOIN_CRYPTO_X11_UTIL_UTIL_HPP + +#include + +namespace sapphire { +namespace util { +constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0) noexcept +{ + return (static_cast(b3) << 24) | + (static_cast(b2) << 16) | + (static_cast(b1) << 8) | + (static_cast(b0)); +} +} // namespace util +} // namespace sapphire + +#endif // BITCOIN_CRYPTO_X11_UTIL_UTIL_HPP diff --git a/src/init/common.cpp b/src/init/common.cpp index a48417411692..8ca786eaa045 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,7 @@ namespace init { void SetGlobals() { + SapphireAutoDetect(); std::string sha256_algo = SHA256AutoDetect(); LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 5ff670fab0c9..a6d5fb0a969e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -178,6 +179,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve InitLogging(*m_node.args); AppInitParameterInteraction(*m_node.args); LogInstance().StartLogging(); + SapphireAutoDetect(); SHA256AutoDetect(); ECC_Start(); BLSInit(); From 1f2eb21d7608e89de1ea9d0c5209115f61c5463b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 09:37:52 +0000 Subject: [PATCH 08/21] crypto: implement naive AES-NI backend for simple rounds --- configure.ac | 21 ++++++++++++++++ src/Makefile.am | 13 ++++++++++ src/crypto/x11/dispatch.cpp | 42 ++++++++++++++++++++++++++++++++ src/crypto/x11/util/util.hpp | 27 ++++++++++++++++++++ src/crypto/x11/x86_aesni/aes.cpp | 32 ++++++++++++++++++++++++ 5 files changed, 135 insertions(+) create mode 100644 src/crypto/x11/x86_aesni/aes.cpp diff --git a/configure.ac b/configure.ac index b85fcc46d003..94cac2d3631f 100644 --- a/configure.ac +++ b/configure.ac @@ -540,6 +540,7 @@ enable_arm_shani=no enable_sse42=no enable_sse41=no enable_avx2=no +enable_x86_aesni=no enable_x86_shani=no dnl Check for optional instruction set support. Enabling these does _not_ imply that all code will @@ -550,6 +551,7 @@ dnl x86 AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4.1 -maes], [X86_AESNI_CXXFLAGS="-msse4.1 -maes"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-msse4 -msha], [X86_SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) enable_clmul= @@ -640,6 +642,23 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ) CXXFLAGS="$TEMP_CXXFLAGS" +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$X86_AESNI_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for x86 AES-NI intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + #include + ]],[[ + __m128i x = _mm_setzero_si128(); + x = _mm_aesenc_si128(x, _mm_setzero_si128()); + return _mm_extract_epi32(x, 0); + ]])], + [ AC_MSG_RESULT([yes]); enable_x86_aesni=yes; AC_DEFINE([ENABLE_X86_AESNI], [1], [Define this symbol to build code that uses x86 AES-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" + # ARM AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) @@ -1823,6 +1842,7 @@ AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) +AM_CONDITIONAL([ENABLE_X86_AESNI], [test "$enable_x86_aesni" = "yes"]) AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) @@ -1882,6 +1902,7 @@ AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) +AC_SUBST(X86_AESNI_CXXFLAGS) AC_SUBST(X86_SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) AC_SUBST(ARM_SHANI_CXXFLAGS) diff --git a/src/Makefile.am b/src/Makefile.am index 777dbf1c7e7d..7d719a13d8c5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -78,6 +78,10 @@ LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SPH) if ENABLE_SSE41 LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) +if ENABLE_X86_AESNI +LIBBITCOIN_CRYPTO_X86_AESNI = crypto/libbitcoin_crypto_x86_aesni.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_AESNI) +endif if ENABLE_X86_SHANI LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) @@ -764,6 +768,15 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/util/consts_aes.hpp \ crypto/x11/util/util.hpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_x86_aesni_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS += $(X86_AESNI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS += -DENABLE_SSE41 -DENABLE_X86_AESNI +crypto_libbitcoin_crypto_x86_aesni_la_SOURCES = crypto/x11/x86_aesni/aes.cpp + # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above crypto_libbitcoin_crypto_x86_shani_la_LDFLAGS = $(AM_LDFLAGS) -static diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 6e4b263c8836..f82fd3d1a5b2 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -2,11 +2,31 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#if defined(HAVE_CONFIG_H) +#include +#endif + #include +#if !defined(DISABLE_OPTIMIZED_SHA256) +#include +#endif // !DISABLE_OPTIMIZED_SHA256 + #include namespace sapphire { +#if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +namespace x86_aesni_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +} // namespace x86_aesni_aes +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // !DISABLE_OPTIMIZED_SHA256 + namespace soft_aes { void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, @@ -16,6 +36,19 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, } // namespace soft_aes } // namespace sapphire +namespace { +[[maybe_unused]] bool use_aes_ni = []() { +#if !defined(DISABLE_OPTIMIZED_SHA256) && defined(HAVE_GETCPUID) + uint32_t eax, ebx, ecx, edx; + GetCPUID(1, 0, eax, ebx, ecx, edx); + return (/*has_sse4_1=*/((ecx >> 19) & 1) && + /*has_aes_ni=*/((ecx >> 25) & 1)); +#else + return false; +#endif // !DISABLE_OPTIMIZED_SHA256 && HAVE_GETCPUID +}(); +} // anonymous namespace + extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; @@ -23,4 +56,13 @@ void SapphireAutoDetect() { aes_round = sapphire::soft_aes::Round; aes_round_nk = sapphire::soft_aes::RoundKeyless; + +#if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) + if (use_aes_ni) { + aes_round = sapphire::x86_aesni_aes::Round; + aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; + } +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // !DISABLE_OPTIMIZED_SHA256 } diff --git a/src/crypto/x11/util/util.hpp b/src/crypto/x11/util/util.hpp index 52e06b1773e6..de924fd1e0b9 100644 --- a/src/crypto/x11/util/util.hpp +++ b/src/crypto/x11/util/util.hpp @@ -7,6 +7,14 @@ #include +#if !defined(DISABLE_OPTIMIZED_SHA256) +#include + +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#include +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // !DISABLE_OPTIMIZED_SHA256 + namespace sapphire { namespace util { constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0) noexcept @@ -16,6 +24,25 @@ constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0 (static_cast(b1) << 8) | (static_cast(b0)); } + +#if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +__m128i ALWAYS_INLINE aes_round(const __m128i& input, const __m128i& key) { return _mm_aesenc_si128(input, key); } + +__m128i ALWAYS_INLINE pack_le(const uint32_t& w0, const uint32_t& w1, const uint32_t& w2, const uint32_t& w3) +{ + return _mm_set_epi32(w3, w2, w1, w0); +} + +void ALWAYS_INLINE unpack_le(const __m128i& i, uint32_t& w0, uint32_t& w1, uint32_t& w2, uint32_t& w3) +{ + w0 = _mm_extract_epi32(i, 0); + w1 = _mm_extract_epi32(i, 1); + w2 = _mm_extract_epi32(i, 2); + w3 = _mm_extract_epi32(i, 3); +} +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // !DISABLE_OPTIMIZED_SHA256 } // namespace util } // namespace sapphire diff --git a/src/crypto/x11/x86_aesni/aes.cpp b/src/crypto/x11/x86_aesni/aes.cpp new file mode 100644 index 000000000000..aee045faa173 --- /dev/null +++ b/src/crypto/x11/x86_aesni/aes.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#include + +#include + +#include +#include + +namespace sapphire { +namespace x86_aesni_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + __m128i block = util::aes_round(util::pack_le(x0, x1, x2, x3), util::pack_le(k0, k1, k2, k3)); + util::unpack_le(block, y0, y1, y2, y3); +} + +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + __m128i block = util::aes_round(util::pack_le(x0, x1, x2, x3), _mm_setzero_si128()); + util::unpack_le(block, y0, y1, y2, y3); +} +} // namespace x86_aesni_aes +} // namespace sapphire + +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI From 31a673241b8b862937b46ed238b6c88b7d78d192 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 14:24:26 +0000 Subject: [PATCH 09/21] crypto: implement AES-NI backend for Echo512's `FullStateRound()` --- src/Makefile.am | 5 ++- src/crypto/x11/aes.h | 20 +++++++++ src/crypto/x11/dispatch.cpp | 9 ++++ src/crypto/x11/dispatch.h | 2 + src/crypto/x11/echo.cpp | 68 +++++++++++++------------------ src/crypto/x11/x86_aesni/echo.cpp | 39 ++++++++++++++++++ 6 files changed, 103 insertions(+), 40 deletions(-) create mode 100644 src/crypto/x11/aes.h create mode 100644 src/crypto/x11/x86_aesni/echo.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 7d719a13d8c5..b933834780c1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -740,6 +740,7 @@ crypto_libbitcoin_crypto_sph_la_CPPFLAGS += \ -DSPH_SMALL_FOOTPRINT_JH=1 crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/aes.cpp \ + crypto/x11/aes.h \ crypto/x11/blake.c \ crypto/x11/bmw.c \ crypto/x11/cubehash.c \ @@ -775,7 +776,9 @@ crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -st crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS += $(X86_AESNI_CXXFLAGS) crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS += -DENABLE_SSE41 -DENABLE_X86_AESNI -crypto_libbitcoin_crypto_x86_aesni_la_SOURCES = crypto/x11/x86_aesni/aes.cpp +crypto_libbitcoin_crypto_x86_aesni_la_SOURCES = \ + crypto/x11/x86_aesni/aes.cpp \ + crypto/x11/x86_aesni/echo.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above diff --git a/src/crypto/x11/aes.h b/src/crypto/x11/aes.h new file mode 100644 index 000000000000..0a7420d1f8ba --- /dev/null +++ b/src/crypto/x11/aes.h @@ -0,0 +1,20 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_X11_AES_H +#define BITCOIN_CRYPTO_X11_AES_H + +#include + +namespace sapphire { +namespace soft_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +} // namespace soft_aes +} // namespace sapphire + +#endif // BITCOIN_CRYPTO_X11_AES_H diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index f82fd3d1a5b2..8cd651b3f17f 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -24,6 +24,9 @@ void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); } // namespace x86_aesni_aes +namespace x86_aesni_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); +} // namespace x86_aesni_echo #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #endif // !DISABLE_OPTIMIZED_SHA256 @@ -34,6 +37,9 @@ void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); } // namespace soft_aes +namespace soft_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); +} // namespace soft_echo } // namespace sapphire namespace { @@ -51,17 +57,20 @@ namespace { extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; +extern sapphire::dispatch::EchoRoundFn echo_round; void SapphireAutoDetect() { aes_round = sapphire::soft_aes::Round; aes_round_nk = sapphire::soft_aes::RoundKeyless; + echo_round = sapphire::soft_echo::FullStateRound; #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) if (use_aes_ni) { aes_round = sapphire::x86_aesni_aes::Round; aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; + echo_round = sapphire::x86_aesni_echo::FullStateRound; } #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #endif // !DISABLE_OPTIMIZED_SHA256 diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index f9c96428b11e..efcf8a534ebe 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -14,6 +14,8 @@ typedef void (*AESRoundFn)(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t&, uint32_t&, uint32_t&, uint32_t&); typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t&, uint32_t&, uint32_t&, uint32_t&); + +typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index f9a9c46c8469..3675a1313fc8 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -30,6 +30,7 @@ * @author Thomas Pornin */ +#include #include #include @@ -37,9 +38,6 @@ #include "sph_echo.h" -extern sapphire::dispatch::AESRoundFn aes_round; -extern sapphire::dispatch::AESRoundFnNk aes_round_nk; - /* * We can use a 64-bit implementation only if a 64-bit type is available. */ @@ -50,56 +48,48 @@ extern sapphire::dispatch::AESRoundFnNk aes_round_nk; #define T32 SPH_T32 #define C64 SPH_C64 -#define DECL_STATE_BIG \ - sph_u64 W[16][2]; - -#define INPUT_BLOCK_BIG(sc) do { \ - unsigned u; \ - memcpy(W, sc->u.Vb, 16 * sizeof(sph_u64)); \ - for (u = 0; u < 8; u ++) { \ - W[u + 8][0] = sph_dec64le_aligned( \ - sc->buf + 16 * u); \ - W[u + 8][1] = sph_dec64le_aligned( \ - sc->buf + 16 * u + 8); \ - } \ - } while (0) - -static void -aes_2rounds_all(sph_u64 W[16][2], - sph_u32 *pK0, sph_u32 *pK1, sph_u32 *pK2, sph_u32 *pK3) +namespace sapphire { +namespace soft_echo { +void FullStateRound(sph_u64 W[16][2], sph_u32& K0, sph_u32& K1, sph_u32& K2, sph_u32& K3) { - int n; - sph_u32 K0 = *pK0; - sph_u32 K1 = *pK1; - sph_u32 K2 = *pK2; - sph_u32 K3 = *pK3; - - for (n = 0; n < 16; n ++) { + for (int n = 0; n < 16; n ++) { sph_u64 Wl = W[n][0]; sph_u64 Wh = W[n][1]; sph_u32 X0 = (sph_u32)Wl; sph_u32 X1 = (sph_u32)(Wl >> 32); sph_u32 X2 = (sph_u32)Wh; sph_u32 X3 = (sph_u32)(Wh >> 32); - sph_u32 Y0, Y1, Y2, Y3; \ - aes_round(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); - aes_round_nk(Y0, Y1, Y2, Y3, X0, X1, X2, X3); + sph_u32 Y0, Y1, Y2, Y3; + soft_aes::Round(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); + soft_aes::RoundKeyless(Y0, Y1, Y2, Y3, X0, X1, X2, X3); W[n][0] = (sph_u64)X0 | ((sph_u64)X1 << 32); W[n][1] = (sph_u64)X2 | ((sph_u64)X3 << 32); if ((K0 = T32(K0 + 1)) == 0) { - if ((K1 = T32(K1 + 1)) == 0) - if ((K2 = T32(K2 + 1)) == 0) + if ((K1 = T32(K1 + 1)) == 0) { + if ((K2 = T32(K2 + 1)) == 0) { K3 = T32(K3 + 1); + } + } } } - *pK0 = K0; - *pK1 = K1; - *pK2 = K2; - *pK3 = K3; } +} // namespace soft_echo +} // namespace sapphire -#define BIG_SUB_WORDS do { \ - aes_2rounds_all(W, &K0, &K1, &K2, &K3); \ +sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound; + +#define DECL_STATE_BIG \ + alignas(16) sph_u64 W[16][2]; + +#define INPUT_BLOCK_BIG(sc) do { \ + unsigned u; \ + memcpy(W, sc->u.Vb, 16 * sizeof(sph_u64)); \ + for (u = 0; u < 8; u ++) { \ + W[u + 8][0] = sph_dec64le_aligned( \ + sc->buf + 16 * u); \ + W[u + 8][1] = sph_dec64le_aligned( \ + sc->buf + 16 * u + 8); \ + } \ } while (0) #define SHIFT_ROW1(a, b, c, d) do { \ @@ -176,7 +166,7 @@ mix_column(sph_u64 W[16][2], int ia, int ib, int ic, int id) } while (0) #define BIG_ROUND do { \ - BIG_SUB_WORDS; \ + echo_round(W, K0, K1, K2, K3); \ BIG_SHIFT_ROWS; \ BIG_MIX_COLUMNS; \ } while (0) diff --git a/src/crypto/x11/x86_aesni/echo.cpp b/src/crypto/x11/x86_aesni/echo.cpp new file mode 100644 index 000000000000..bed9c74e800b --- /dev/null +++ b/src/crypto/x11/x86_aesni/echo.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#include + +#include + +#include +#include + +namespace sapphire { +namespace x86_aesni_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +{ + __m128i key = util::pack_le(k0, k1, k2, k3); + for (int n = 0; n < 16; n++) { + __m128i block = _mm_load_si128((const __m128i*)&W[n][0]); + block = util::aes_round(block, key); + block = util::aes_round(block, _mm_setzero_si128()); + _mm_store_si128((__m128i*)&W[n][0], block); + + util::unpack_le(key, k0, k1, k2, k3); + if ((k0 = (k0 + 1)) == 0) { + if ((k1 = (k1 + 1)) == 0) { + if ((k2 = (k2 + 1)) == 0) { + k3 = (k3 + 1); + } + } + } + key = util::pack_le(k0, k1, k2, k3); + } + util::unpack_le(key, k0, k1, k2, k3); +} +} // namespace x86_aesni_echo +} // namespace sapphire + +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI From da3887118a59e93d324b875ef14014da0a56daf3 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 09:39:42 +0000 Subject: [PATCH 10/21] crypto: implement SSSE3 backend for Echo512's `MixColumns()` --- configure.ac | 18 +++++++++ src/Makefile.am | 14 +++++++ src/crypto/x11/dispatch.cpp | 36 ++++++++++------- src/crypto/x11/dispatch.h | 1 + src/crypto/x11/echo.cpp | 73 ++++++++++++++++++----------------- src/crypto/x11/ssse3/echo.cpp | 65 +++++++++++++++++++++++++++++++ src/crypto/x11/util/util.hpp | 8 +++- 7 files changed, 163 insertions(+), 52 deletions(-) create mode 100644 src/crypto/x11/ssse3/echo.cpp diff --git a/configure.ac b/configure.ac index 94cac2d3631f..699777e8f47d 100644 --- a/configure.ac +++ b/configure.ac @@ -537,6 +537,7 @@ AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fsta enable_arm_crc=no enable_arm_shani=no +enable_ssse3=no enable_sse42=no enable_sse41=no enable_avx2=no @@ -548,6 +549,7 @@ dnl be compiled with them, rather that specific objects/libs may use them after dnl compatibility. dnl x86 +AX_CHECK_COMPILE_FLAG([-mssse3], [SSSE3_CXXFLAGS="-mssse3"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) @@ -572,6 +574,20 @@ if test "$enable_clmul" = "yes"; then AC_DEFINE([HAVE_CLMUL], [1], [Define this symbol if clmul instructions can be used]) fi +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$SSSE3_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for SSSE3 intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + ]],[[ + __m64 x = _mm_abs_pi32(_m_from_int(0)); + return 0; + ]])], + [ AC_MSG_RESULT([yes]); enable_ssse3=yes; AC_DEFINE([ENABLE_SSSE3], [1], [Define this symbol to build code that uses SSSE3 intrinsics]) ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" + TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$SSE42_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for SSE4.2 intrinsics]) @@ -1839,6 +1855,7 @@ AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) +AM_CONDITIONAL([ENABLE_SSSE3], [test "$enable_ssse3" = "yes"]) AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) @@ -1898,6 +1915,7 @@ AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SPHLIB_FLAGS) +AC_SUBST(SSSE3_CXXFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(CLMUL_CXXFLAGS) diff --git a/src/Makefile.am b/src/Makefile.am index b933834780c1..790bc2baab1c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -75,6 +75,10 @@ endif LIBBITCOIN_CRYPTO = $(LIBBITCOIN_CRYPTO_BASE) LIBBITCOIN_CRYPTO_SPH = crypto/libbitcoin_crypto_sph.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SPH) +if ENABLE_SSSE3 +LIBBITCOIN_CRYPTO_SSSE3 = crypto/libbitcoin_crypto_ssse3.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSSE3) +endif if ENABLE_SSE41 LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) @@ -769,6 +773,16 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/util/consts_aes.hpp \ crypto/x11/util/util.hpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_ssse3_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_ssse3_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_ssse3_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_ssse3_la_CXXFLAGS += $(SSSE3_CXXFLAGS) +crypto_libbitcoin_crypto_ssse3_la_CPPFLAGS += -DENABLE_SSSE3 +crypto_libbitcoin_crypto_ssse3_la_SOURCES = \ + crypto/x11/ssse3/echo.cpp + # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above crypto_libbitcoin_crypto_x86_aesni_la_LDFLAGS = $(AM_LDFLAGS) -static diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 8cd651b3f17f..8b59846b2385 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -16,6 +16,12 @@ namespace sapphire { #if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_SSSE3) +namespace ssse3_echo { +void MixColumns(uint64_t W[16][2]); +} // namespace ssse3_echo +#endif // ENABLE_SSSE3 + #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) namespace x86_aesni_aes { void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, @@ -39,24 +45,13 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, } // namespace soft_aes namespace soft_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); +void MixColumns(uint64_t W[16][2]); } // namespace soft_echo } // namespace sapphire -namespace { -[[maybe_unused]] bool use_aes_ni = []() { -#if !defined(DISABLE_OPTIMIZED_SHA256) && defined(HAVE_GETCPUID) - uint32_t eax, ebx, ecx, edx; - GetCPUID(1, 0, eax, ebx, ecx, edx); - return (/*has_sse4_1=*/((ecx >> 19) & 1) && - /*has_aes_ni=*/((ecx >> 25) & 1)); -#else - return false; -#endif // !DISABLE_OPTIMIZED_SHA256 && HAVE_GETCPUID -}(); -} // anonymous namespace - extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; +extern sapphire::dispatch::EchoMixCols echo_mix_columns; extern sapphire::dispatch::EchoRoundFn echo_round; void SapphireAutoDetect() @@ -64,14 +59,27 @@ void SapphireAutoDetect() aes_round = sapphire::soft_aes::Round; aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; + echo_mix_columns = sapphire::soft_echo::MixColumns; #if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(HAVE_GETCPUID) + uint32_t eax, ebx, ecx, edx; + GetCPUID(1, 0, eax, ebx, ecx, edx); #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) - if (use_aes_ni) { + const bool use_sse_4_1 = ((ecx >> 19) & 1); + const bool use_aes_ni = ((ecx >> 25) & 1); + if (use_sse_4_1 && use_aes_ni) { aes_round = sapphire::x86_aesni_aes::Round; aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; echo_round = sapphire::x86_aesni_echo::FullStateRound; } #endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#if defined(ENABLE_SSSE3) + const bool use_ssse3 = ((ecx >> 9) & 1); + if (use_ssse3) { + echo_mix_columns = sapphire::ssse3_echo::MixColumns; + } +#endif // ENABLE_SSSE3 +#endif // HAVE_GETCPUID #endif // !DISABLE_OPTIMIZED_SHA256 } diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index efcf8a534ebe..b3e309afa59a 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -15,6 +15,7 @@ typedef void (*AESRoundFn)(uint32_t, uint32_t, uint32_t, uint32_t, typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t&, uint32_t&, uint32_t&, uint32_t&); +typedef void (*EchoMixCols)(uint64_t[16][2]); typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index 3675a1313fc8..b948a8906de0 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -33,6 +33,8 @@ #include #include +#include + #include #include @@ -50,6 +52,31 @@ namespace sapphire { namespace soft_echo { +namespace { +void ALWAYS_INLINE MixColumn(sph_u64 W[16][2], int ia, int ib, int ic, int id) +{ + for (int n = 0; n < 2; n ++) { + sph_u64 a = W[ia][n]; + sph_u64 b = W[ib][n]; + sph_u64 c = W[ic][n]; + sph_u64 d = W[id][n]; + sph_u64 ab = a ^ b; + sph_u64 bc = b ^ c; + sph_u64 cd = c ^ d; + sph_u64 abx = ((ab & C64(0x8080808080808080)) >> 7) * 27U + ^ ((ab & C64(0x7F7F7F7F7F7F7F7F)) << 1); + sph_u64 bcx = ((bc & C64(0x8080808080808080)) >> 7) * 27U + ^ ((bc & C64(0x7F7F7F7F7F7F7F7F)) << 1); + sph_u64 cdx = ((cd & C64(0x8080808080808080)) >> 7) * 27U + ^ ((cd & C64(0x7F7F7F7F7F7F7F7F)) << 1); + W[ia][n] = abx ^ bc ^ d; + W[ib][n] = bcx ^ a ^ cd; + W[ic][n] = cdx ^ ab ^ d; + W[id][n] = abx ^ bcx ^ cdx ^ ab ^ c; + } +} +} // anonymous namespace + void FullStateRound(sph_u64 W[16][2], sph_u32& K0, sph_u32& K1, sph_u32& K2, sph_u32& K3) { for (int n = 0; n < 16; n ++) { @@ -73,9 +100,18 @@ void FullStateRound(sph_u64 W[16][2], sph_u32& K0, sph_u32& K1, sph_u32& K2, sph } } } + +void MixColumns(uint64_t W[16][2]) +{ + MixColumn(W, 0, 1, 2, 3); + MixColumn(W, 4, 5, 6, 7); + MixColumn(W, 8, 9, 10, 11); + MixColumn(W, 12, 13, 14, 15); +} } // namespace soft_echo } // namespace sapphire +sapphire::dispatch::EchoMixCols echo_mix_columns = sapphire::soft_echo::MixColumns; sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound; #define DECL_STATE_BIG \ @@ -130,45 +166,10 @@ sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound SHIFT_ROW3(3, 7, 11, 15); \ } while (0) -static void -mix_column(sph_u64 W[16][2], int ia, int ib, int ic, int id) -{ - int n; - - for (n = 0; n < 2; n ++) { - sph_u64 a = W[ia][n]; - sph_u64 b = W[ib][n]; - sph_u64 c = W[ic][n]; - sph_u64 d = W[id][n]; - sph_u64 ab = a ^ b; - sph_u64 bc = b ^ c; - sph_u64 cd = c ^ d; - sph_u64 abx = ((ab & C64(0x8080808080808080)) >> 7) * 27U - ^ ((ab & C64(0x7F7F7F7F7F7F7F7F)) << 1); - sph_u64 bcx = ((bc & C64(0x8080808080808080)) >> 7) * 27U - ^ ((bc & C64(0x7F7F7F7F7F7F7F7F)) << 1); - sph_u64 cdx = ((cd & C64(0x8080808080808080)) >> 7) * 27U - ^ ((cd & C64(0x7F7F7F7F7F7F7F7F)) << 1); - W[ia][n] = abx ^ bc ^ d; - W[ib][n] = bcx ^ a ^ cd; - W[ic][n] = cdx ^ ab ^ d; - W[id][n] = abx ^ bcx ^ cdx ^ ab ^ c; - } -} - -#define MIX_COLUMN(a, b, c, d) mix_column(W, a, b, c, d) - -#define BIG_MIX_COLUMNS do { \ - MIX_COLUMN(0, 1, 2, 3); \ - MIX_COLUMN(4, 5, 6, 7); \ - MIX_COLUMN(8, 9, 10, 11); \ - MIX_COLUMN(12, 13, 14, 15); \ - } while (0) - #define BIG_ROUND do { \ echo_round(W, K0, K1, K2, K3); \ BIG_SHIFT_ROWS; \ - BIG_MIX_COLUMNS; \ + echo_mix_columns(W); \ } while (0) #define FINAL_BIG do { \ diff --git a/src/crypto/x11/ssse3/echo.cpp b/src/crypto/x11/ssse3/echo.cpp new file mode 100644 index 000000000000..4415d0445b30 --- /dev/null +++ b/src/crypto/x11/ssse3/echo.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_SSSE3) +#include +#include + +#include + +#include + +namespace sapphire { +namespace { +__m128i ALWAYS_INLINE gf8_mul2(const __m128i& x) +{ + // (x << 1) + const __m128i lhs = _mm_add_epi8(x, x); + // (x & 0x80) ? 0xff : 0x00 + const __m128i msb_set = _mm_and_si128(x, _mm_set1_epi8(0x80)); + const __m128i mask = _mm_cmpeq_epi8(msb_set, _mm_set1_epi8(0x80)); + // Replace 0xff with 0x1b + const __m128i rhs = _mm_and_si128(mask, _mm_set1_epi8(0x1b)); + // (x << 1) ^ ((x & 0x80) ? 0x1b : 0x00) + return util::Xor(lhs, rhs); +} + +void ALWAYS_INLINE MixColumn(uint64_t W[16][2], int ia, int ib, int ic, int id) +{ + const __m128i a = _mm_load_si128((const __m128i*)&W[ia][0]); + const __m128i b = _mm_load_si128((const __m128i*)&W[ib][0]); + const __m128i c = _mm_load_si128((const __m128i*)&W[ic][0]); + const __m128i d = _mm_load_si128((const __m128i*)&W[id][0]); + + const __m128i ab = util::Xor(a, b); + const __m128i bc = util::Xor(b, c); + const __m128i cd = util::Xor(c, d); + + const __m128i abx = gf8_mul2(ab); + const __m128i bcx = gf8_mul2(bc); + const __m128i cdx = gf8_mul2(cd); + + // W[ia] = abx ^ bc ^ d + _mm_store_si128((__m128i*)&W[ia][0], util::Xor(util::Xor(abx, bc), d)); + // W[ib] = bcx ^ a ^ cd + _mm_store_si128((__m128i*)&W[ib][0], util::Xor(util::Xor(bcx, a), cd)); + // W[ic] = cdx ^ ab ^ d + _mm_store_si128((__m128i*)&W[ic][0], util::Xor(util::Xor(cdx, ab), d)); + // W[id] = abx ^ bcx ^ cdx ^ ab ^ c + _mm_store_si128((__m128i*)&W[id][0], util::Xor(util::Xor(util::Xor(util::Xor(abx, bcx), cdx), ab), c)); +} +} // anonymous namespace + +namespace ssse3_echo { +void MixColumns(uint64_t W[16][2]) +{ + MixColumn(W, 0, 1, 2, 3); + MixColumn(W, 4, 5, 6, 7); + MixColumn(W, 8, 9, 10, 11); + MixColumn(W, 12, 13, 14, 15); +} +} // namespace ssse3_echo +} // namespace sapphire + +#endif // ENABLE_SSSE3 diff --git a/src/crypto/x11/util/util.hpp b/src/crypto/x11/util/util.hpp index de924fd1e0b9..a3971cd19c5c 100644 --- a/src/crypto/x11/util/util.hpp +++ b/src/crypto/x11/util/util.hpp @@ -10,9 +10,9 @@ #if !defined(DISABLE_OPTIMIZED_SHA256) #include -#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) #include -#endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // ENABLE_SSSE3 || (ENABLE_SSE41 && ENABLE_X86_AESNI) #endif // !DISABLE_OPTIMIZED_SHA256 namespace sapphire { @@ -26,6 +26,9 @@ constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0 } #if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) +__m128i ALWAYS_INLINE Xor(const __m128i& x, const __m128i& y) { return _mm_xor_si128(x, y); } + #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) __m128i ALWAYS_INLINE aes_round(const __m128i& input, const __m128i& key) { return _mm_aesenc_si128(input, key); } @@ -42,6 +45,7 @@ void ALWAYS_INLINE unpack_le(const __m128i& i, uint32_t& w0, uint32_t& w1, uint3 w3 = _mm_extract_epi32(i, 3); } #endif // ENABLE_SSE41 && ENABLE_X86_AESNI +#endif // ENABLE_SSSE3 || (ENABLE_SSE41 && ENABLE_X86_AESNI) #endif // !DISABLE_OPTIMIZED_SHA256 } // namespace util } // namespace sapphire From 71d6ef98cf75a9dd54e8e1e917ca9c4089e305a6 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 19 Sep 2025 22:54:09 +0000 Subject: [PATCH 11/21] crypto: implement SSSE3 backend for Echo512's `ShiftRows()` --- src/crypto/x11/dispatch.cpp | 5 +++ src/crypto/x11/dispatch.h | 1 + src/crypto/x11/echo.cpp | 54 +++++++++++++++------------ src/crypto/x11/ssse3/echo.cpp | 70 +++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 24 deletions(-) diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 8b59846b2385..dd6108906694 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -19,6 +19,7 @@ namespace sapphire { #if defined(ENABLE_SSSE3) namespace ssse3_echo { void MixColumns(uint64_t W[16][2]); +void ShiftRows(uint64_t W[16][2]); } // namespace ssse3_echo #endif // ENABLE_SSSE3 @@ -46,6 +47,7 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, namespace soft_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); void MixColumns(uint64_t W[16][2]); +void ShiftRows(uint64_t W[16][2]); } // namespace soft_echo } // namespace sapphire @@ -53,6 +55,7 @@ extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; extern sapphire::dispatch::EchoMixCols echo_mix_columns; extern sapphire::dispatch::EchoRoundFn echo_round; +extern sapphire::dispatch::EchoShiftRows echo_shift_rows; void SapphireAutoDetect() { @@ -60,6 +63,7 @@ void SapphireAutoDetect() aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; echo_mix_columns = sapphire::soft_echo::MixColumns; + echo_shift_rows = sapphire::soft_echo::ShiftRows; #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(HAVE_GETCPUID) @@ -78,6 +82,7 @@ void SapphireAutoDetect() const bool use_ssse3 = ((ecx >> 9) & 1); if (use_ssse3) { echo_mix_columns = sapphire::ssse3_echo::MixColumns; + echo_shift_rows = sapphire::ssse3_echo::ShiftRows; } #endif // ENABLE_SSSE3 #endif // HAVE_GETCPUID diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index b3e309afa59a..0b9197dba395 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -17,6 +17,7 @@ typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, typedef void (*EchoMixCols)(uint64_t[16][2]); typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); +typedef void (*EchoShiftRows)(uint64_t[16][2]); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index b948a8906de0..513c75bc6276 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -108,26 +108,9 @@ void MixColumns(uint64_t W[16][2]) MixColumn(W, 8, 9, 10, 11); MixColumn(W, 12, 13, 14, 15); } -} // namespace soft_echo -} // namespace sapphire - -sapphire::dispatch::EchoMixCols echo_mix_columns = sapphire::soft_echo::MixColumns; -sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound; - -#define DECL_STATE_BIG \ - alignas(16) sph_u64 W[16][2]; - -#define INPUT_BLOCK_BIG(sc) do { \ - unsigned u; \ - memcpy(W, sc->u.Vb, 16 * sizeof(sph_u64)); \ - for (u = 0; u < 8; u ++) { \ - W[u + 8][0] = sph_dec64le_aligned( \ - sc->buf + 16 * u); \ - W[u + 8][1] = sph_dec64le_aligned( \ - sc->buf + 16 * u + 8); \ - } \ - } while (0) +void ShiftRows(uint64_t W[16][2]) +{ #define SHIFT_ROW1(a, b, c, d) do { \ sph_u64 tmp; \ tmp = W[a][0]; \ @@ -160,15 +143,38 @@ sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound #define SHIFT_ROW3(a, b, c, d) SHIFT_ROW1(d, c, b, a) -#define BIG_SHIFT_ROWS do { \ - SHIFT_ROW1(1, 5, 9, 13); \ - SHIFT_ROW2(2, 6, 10, 14); \ - SHIFT_ROW3(3, 7, 11, 15); \ + SHIFT_ROW1(1, 5, 9, 13); + SHIFT_ROW2(2, 6, 10, 14); + SHIFT_ROW3(3, 7, 11, 15); + +#undef SHIFT_ROW1 +#undef SHIFT_ROW2 +#undef SHIFT_ROW3 +} +} // namespace soft_echo +} // namespace sapphire + +sapphire::dispatch::EchoMixCols echo_mix_columns = sapphire::soft_echo::MixColumns; +sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound; +sapphire::dispatch::EchoShiftRows echo_shift_rows = sapphire::soft_echo::ShiftRows; + +#define DECL_STATE_BIG \ + alignas(16) sph_u64 W[16][2]; + +#define INPUT_BLOCK_BIG(sc) do { \ + unsigned u; \ + memcpy(W, sc->u.Vb, 16 * sizeof(sph_u64)); \ + for (u = 0; u < 8; u ++) { \ + W[u + 8][0] = sph_dec64le_aligned( \ + sc->buf + 16 * u); \ + W[u + 8][1] = sph_dec64le_aligned( \ + sc->buf + 16 * u + 8); \ + } \ } while (0) #define BIG_ROUND do { \ echo_round(W, K0, K1, K2, K3); \ - BIG_SHIFT_ROWS; \ + echo_shift_rows(W); \ echo_mix_columns(W); \ } while (0) diff --git a/src/crypto/x11/ssse3/echo.cpp b/src/crypto/x11/ssse3/echo.cpp index 4415d0445b30..20cf942cff1c 100644 --- a/src/crypto/x11/ssse3/echo.cpp +++ b/src/crypto/x11/ssse3/echo.cpp @@ -49,6 +49,34 @@ void ALWAYS_INLINE MixColumn(uint64_t W[16][2], int ia, int ib, int ic, int id) // W[id] = abx ^ bcx ^ cdx ^ ab ^ c _mm_store_si128((__m128i*)&W[id][0], util::Xor(util::Xor(util::Xor(util::Xor(abx, bcx), cdx), ab), c)); } + +void ALWAYS_INLINE ShiftRow1(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) +{ + __m128i tmp = Wa; + Wa = Wb; + Wb = Wc; + Wc = Wd; + Wd = tmp; +} + +void ALWAYS_INLINE ShiftRow2(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) +{ + __m128i tmp1 = Wa; + __m128i tmp2 = Wb; + Wa = Wc; + Wb = Wd; + Wc = tmp1; + Wd = tmp2; +} + +void ALWAYS_INLINE ShiftRow3(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) +{ + __m128i tmp = Wd; + Wd = Wc; + Wc = Wb; + Wb = Wa; + Wa = tmp; +} } // anonymous namespace namespace ssse3_echo { @@ -59,6 +87,48 @@ void MixColumns(uint64_t W[16][2]) MixColumn(W, 8, 9, 10, 11); MixColumn(W, 12, 13, 14, 15); } + +void ShiftRows(uint64_t W[16][2]) +{ + alignas(16) __m128i w[16]; + w[0] = _mm_load_si128((const __m128i*)&W[0][0]); + w[1] = _mm_load_si128((const __m128i*)&W[1][0]); + w[2] = _mm_load_si128((const __m128i*)&W[2][0]); + w[3] = _mm_load_si128((const __m128i*)&W[3][0]); + w[4] = _mm_load_si128((const __m128i*)&W[4][0]); + w[5] = _mm_load_si128((const __m128i*)&W[5][0]); + w[6] = _mm_load_si128((const __m128i*)&W[6][0]); + w[7] = _mm_load_si128((const __m128i*)&W[7][0]); + w[8] = _mm_load_si128((const __m128i*)&W[8][0]); + w[9] = _mm_load_si128((const __m128i*)&W[9][0]); + w[10] = _mm_load_si128((const __m128i*)&W[10][0]); + w[11] = _mm_load_si128((const __m128i*)&W[11][0]); + w[12] = _mm_load_si128((const __m128i*)&W[12][0]); + w[13] = _mm_load_si128((const __m128i*)&W[13][0]); + w[14] = _mm_load_si128((const __m128i*)&W[14][0]); + w[15] = _mm_load_si128((const __m128i*)&W[15][0]); + + ShiftRow1(w[1], w[5], w[9], w[13]); + ShiftRow2(w[2], w[6], w[10], w[14]); + ShiftRow3(w[3], w[7], w[11], w[15]); + + _mm_store_si128((__m128i*)&W[0][0], w[0]); + _mm_store_si128((__m128i*)&W[1][0], w[1]); + _mm_store_si128((__m128i*)&W[2][0], w[2]); + _mm_store_si128((__m128i*)&W[3][0], w[3]); + _mm_store_si128((__m128i*)&W[4][0], w[4]); + _mm_store_si128((__m128i*)&W[5][0], w[5]); + _mm_store_si128((__m128i*)&W[6][0], w[6]); + _mm_store_si128((__m128i*)&W[7][0], w[7]); + _mm_store_si128((__m128i*)&W[8][0], w[8]); + _mm_store_si128((__m128i*)&W[9][0], w[9]); + _mm_store_si128((__m128i*)&W[10][0], w[10]); + _mm_store_si128((__m128i*)&W[11][0], w[11]); + _mm_store_si128((__m128i*)&W[12][0], w[12]); + _mm_store_si128((__m128i*)&W[13][0], w[13]); + _mm_store_si128((__m128i*)&W[14][0], w[14]); + _mm_store_si128((__m128i*)&W[15][0], w[15]); +} } // namespace ssse3_echo } // namespace sapphire From 250dcce314d834a385f9b369f1e57dc4cec0e304 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 19 Sep 2025 21:12:57 +0000 Subject: [PATCH 12/21] crypto: combine Echo512's Shift and Mix operations --- src/crypto/x11/dispatch.cpp | 15 +++---- src/crypto/x11/dispatch.h | 3 +- src/crypto/x11/echo.cpp | 75 +++++++++++++++++------------------ src/crypto/x11/ssse3/echo.cpp | 15 +++---- 4 files changed, 49 insertions(+), 59 deletions(-) diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index dd6108906694..55a43a6b8c64 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -18,8 +18,7 @@ namespace sapphire { #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(ENABLE_SSSE3) namespace ssse3_echo { -void MixColumns(uint64_t W[16][2]); -void ShiftRows(uint64_t W[16][2]); +void ShiftAndMix(uint64_t W[16][2]); } // namespace ssse3_echo #endif // ENABLE_SSSE3 @@ -46,24 +45,21 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, } // namespace soft_aes namespace soft_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); -void MixColumns(uint64_t W[16][2]); -void ShiftRows(uint64_t W[16][2]); +void ShiftAndMix(uint64_t W[16][2]); } // namespace soft_echo } // namespace sapphire extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; -extern sapphire::dispatch::EchoMixCols echo_mix_columns; +extern sapphire::dispatch::EchoShiftMix echo_shift_mix; extern sapphire::dispatch::EchoRoundFn echo_round; -extern sapphire::dispatch::EchoShiftRows echo_shift_rows; void SapphireAutoDetect() { aes_round = sapphire::soft_aes::Round; aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; - echo_mix_columns = sapphire::soft_echo::MixColumns; - echo_shift_rows = sapphire::soft_echo::ShiftRows; + echo_shift_mix = sapphire::soft_echo::ShiftAndMix; #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(HAVE_GETCPUID) @@ -81,8 +77,7 @@ void SapphireAutoDetect() #if defined(ENABLE_SSSE3) const bool use_ssse3 = ((ecx >> 9) & 1); if (use_ssse3) { - echo_mix_columns = sapphire::ssse3_echo::MixColumns; - echo_shift_rows = sapphire::ssse3_echo::ShiftRows; + echo_shift_mix = sapphire::ssse3_echo::ShiftAndMix; } #endif // ENABLE_SSSE3 #endif // HAVE_GETCPUID diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index 0b9197dba395..ddeca43d2735 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -15,9 +15,8 @@ typedef void (*AESRoundFn)(uint32_t, uint32_t, uint32_t, uint32_t, typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t&, uint32_t&, uint32_t&, uint32_t&); -typedef void (*EchoMixCols)(uint64_t[16][2]); typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); -typedef void (*EchoShiftRows)(uint64_t[16][2]); +typedef void (*EchoShiftMix)(uint64_t[16][2]); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/echo.cpp b/src/crypto/x11/echo.cpp index 513c75bc6276..00fb84fb2915 100644 --- a/src/crypto/x11/echo.cpp +++ b/src/crypto/x11/echo.cpp @@ -75,41 +75,8 @@ void ALWAYS_INLINE MixColumn(sph_u64 W[16][2], int ia, int ib, int ic, int id) W[id][n] = abx ^ bcx ^ cdx ^ ab ^ c; } } -} // anonymous namespace - -void FullStateRound(sph_u64 W[16][2], sph_u32& K0, sph_u32& K1, sph_u32& K2, sph_u32& K3) -{ - for (int n = 0; n < 16; n ++) { - sph_u64 Wl = W[n][0]; - sph_u64 Wh = W[n][1]; - sph_u32 X0 = (sph_u32)Wl; - sph_u32 X1 = (sph_u32)(Wl >> 32); - sph_u32 X2 = (sph_u32)Wh; - sph_u32 X3 = (sph_u32)(Wh >> 32); - sph_u32 Y0, Y1, Y2, Y3; - soft_aes::Round(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); - soft_aes::RoundKeyless(Y0, Y1, Y2, Y3, X0, X1, X2, X3); - W[n][0] = (sph_u64)X0 | ((sph_u64)X1 << 32); - W[n][1] = (sph_u64)X2 | ((sph_u64)X3 << 32); - if ((K0 = T32(K0 + 1)) == 0) { - if ((K1 = T32(K1 + 1)) == 0) { - if ((K2 = T32(K2 + 1)) == 0) { - K3 = T32(K3 + 1); - } - } - } - } -} - -void MixColumns(uint64_t W[16][2]) -{ - MixColumn(W, 0, 1, 2, 3); - MixColumn(W, 4, 5, 6, 7); - MixColumn(W, 8, 9, 10, 11); - MixColumn(W, 12, 13, 14, 15); -} -void ShiftRows(uint64_t W[16][2]) +void ALWAYS_INLINE ShiftRows(uint64_t W[16][2]) { #define SHIFT_ROW1(a, b, c, d) do { \ sph_u64 tmp; \ @@ -151,12 +118,45 @@ void ShiftRows(uint64_t W[16][2]) #undef SHIFT_ROW2 #undef SHIFT_ROW3 } +} // anonymous namespace + +void FullStateRound(sph_u64 W[16][2], sph_u32& K0, sph_u32& K1, sph_u32& K2, sph_u32& K3) +{ + for (int n = 0; n < 16; n ++) { + sph_u64 Wl = W[n][0]; + sph_u64 Wh = W[n][1]; + sph_u32 X0 = (sph_u32)Wl; + sph_u32 X1 = (sph_u32)(Wl >> 32); + sph_u32 X2 = (sph_u32)Wh; + sph_u32 X3 = (sph_u32)(Wh >> 32); + sph_u32 Y0, Y1, Y2, Y3; + soft_aes::Round(X0, X1, X2, X3, K0, K1, K2, K3, Y0, Y1, Y2, Y3); + soft_aes::RoundKeyless(Y0, Y1, Y2, Y3, X0, X1, X2, X3); + W[n][0] = (sph_u64)X0 | ((sph_u64)X1 << 32); + W[n][1] = (sph_u64)X2 | ((sph_u64)X3 << 32); + if ((K0 = T32(K0 + 1)) == 0) { + if ((K1 = T32(K1 + 1)) == 0) { + if ((K2 = T32(K2 + 1)) == 0) { + K3 = T32(K3 + 1); + } + } + } + } +} + +void ShiftAndMix(uint64_t W[16][2]) +{ + ShiftRows(W); + MixColumn(W, 0, 1, 2, 3); + MixColumn(W, 4, 5, 6, 7); + MixColumn(W, 8, 9, 10, 11); + MixColumn(W, 12, 13, 14, 15); +} } // namespace soft_echo } // namespace sapphire -sapphire::dispatch::EchoMixCols echo_mix_columns = sapphire::soft_echo::MixColumns; sapphire::dispatch::EchoRoundFn echo_round = sapphire::soft_echo::FullStateRound; -sapphire::dispatch::EchoShiftRows echo_shift_rows = sapphire::soft_echo::ShiftRows; +sapphire::dispatch::EchoShiftMix echo_shift_mix = sapphire::soft_echo::ShiftAndMix; #define DECL_STATE_BIG \ alignas(16) sph_u64 W[16][2]; @@ -174,8 +174,7 @@ sapphire::dispatch::EchoShiftRows echo_shift_rows = sapphire::soft_echo::ShiftRo #define BIG_ROUND do { \ echo_round(W, K0, K1, K2, K3); \ - echo_shift_rows(W); \ - echo_mix_columns(W); \ + echo_shift_mix(W); \ } while (0) #define FINAL_BIG do { \ diff --git a/src/crypto/x11/ssse3/echo.cpp b/src/crypto/x11/ssse3/echo.cpp index 20cf942cff1c..7ebee1054f7f 100644 --- a/src/crypto/x11/ssse3/echo.cpp +++ b/src/crypto/x11/ssse3/echo.cpp @@ -80,15 +80,7 @@ void ALWAYS_INLINE ShiftRow3(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) } // anonymous namespace namespace ssse3_echo { -void MixColumns(uint64_t W[16][2]) -{ - MixColumn(W, 0, 1, 2, 3); - MixColumn(W, 4, 5, 6, 7); - MixColumn(W, 8, 9, 10, 11); - MixColumn(W, 12, 13, 14, 15); -} - -void ShiftRows(uint64_t W[16][2]) +void ShiftAndMix(uint64_t W[16][2]) { alignas(16) __m128i w[16]; w[0] = _mm_load_si128((const __m128i*)&W[0][0]); @@ -128,6 +120,11 @@ void ShiftRows(uint64_t W[16][2]) _mm_store_si128((__m128i*)&W[13][0], w[13]); _mm_store_si128((__m128i*)&W[14][0], w[14]); _mm_store_si128((__m128i*)&W[15][0], w[15]); + + MixColumn(W, 0, 1, 2, 3); + MixColumn(W, 4, 5, 6, 7); + MixColumn(W, 8, 9, 10, 11); + MixColumn(W, 12, 13, 14, 15); } } // namespace ssse3_echo } // namespace sapphire From e1bbec4c39152339267a01bef3277aadd0e1826b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 19 Sep 2025 21:57:25 +0000 Subject: [PATCH 13/21] crypto: avoid extra load/store in Echo512's `ShiftAndMix()` --- src/crypto/x11/ssse3/echo.cpp | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/crypto/x11/ssse3/echo.cpp b/src/crypto/x11/ssse3/echo.cpp index 7ebee1054f7f..5968bb81a5ba 100644 --- a/src/crypto/x11/ssse3/echo.cpp +++ b/src/crypto/x11/ssse3/echo.cpp @@ -25,12 +25,12 @@ __m128i ALWAYS_INLINE gf8_mul2(const __m128i& x) return util::Xor(lhs, rhs); } -void ALWAYS_INLINE MixColumn(uint64_t W[16][2], int ia, int ib, int ic, int id) +void ALWAYS_INLINE MixColumn(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) { - const __m128i a = _mm_load_si128((const __m128i*)&W[ia][0]); - const __m128i b = _mm_load_si128((const __m128i*)&W[ib][0]); - const __m128i c = _mm_load_si128((const __m128i*)&W[ic][0]); - const __m128i d = _mm_load_si128((const __m128i*)&W[id][0]); + const __m128i a = Wa; + const __m128i b = Wb; + const __m128i c = Wc; + const __m128i d = Wd; const __m128i ab = util::Xor(a, b); const __m128i bc = util::Xor(b, c); @@ -40,14 +40,14 @@ void ALWAYS_INLINE MixColumn(uint64_t W[16][2], int ia, int ib, int ic, int id) const __m128i bcx = gf8_mul2(bc); const __m128i cdx = gf8_mul2(cd); - // W[ia] = abx ^ bc ^ d - _mm_store_si128((__m128i*)&W[ia][0], util::Xor(util::Xor(abx, bc), d)); - // W[ib] = bcx ^ a ^ cd - _mm_store_si128((__m128i*)&W[ib][0], util::Xor(util::Xor(bcx, a), cd)); - // W[ic] = cdx ^ ab ^ d - _mm_store_si128((__m128i*)&W[ic][0], util::Xor(util::Xor(cdx, ab), d)); - // W[id] = abx ^ bcx ^ cdx ^ ab ^ c - _mm_store_si128((__m128i*)&W[id][0], util::Xor(util::Xor(util::Xor(util::Xor(abx, bcx), cdx), ab), c)); + // Wa = abx ^ bc ^ d + Wa = util::Xor(util::Xor(abx, bc), d); + // Wb = bcx ^ a ^ cd + Wb = util::Xor(util::Xor(bcx, a), cd); + // Wc = cdx ^ ab ^ d + Wc = util::Xor(util::Xor(cdx, ab), d); + // Wd = abx ^ bcx ^ cdx ^ ab ^ c + Wd = util::Xor(util::Xor(util::Xor(util::Xor(abx, bcx), cdx), ab), c); } void ALWAYS_INLINE ShiftRow1(__m128i& Wa, __m128i& Wb, __m128i& Wc, __m128i& Wd) @@ -104,6 +104,11 @@ void ShiftAndMix(uint64_t W[16][2]) ShiftRow2(w[2], w[6], w[10], w[14]); ShiftRow3(w[3], w[7], w[11], w[15]); + MixColumn(w[0], w[1], w[2], w[3]); + MixColumn(w[4], w[5], w[6], w[7]); + MixColumn(w[8], w[9], w[10], w[11]); + MixColumn(w[12], w[13], w[14], w[15]); + _mm_store_si128((__m128i*)&W[0][0], w[0]); _mm_store_si128((__m128i*)&W[1][0], w[1]); _mm_store_si128((__m128i*)&W[2][0], w[2]); @@ -120,11 +125,6 @@ void ShiftAndMix(uint64_t W[16][2]) _mm_store_si128((__m128i*)&W[13][0], w[13]); _mm_store_si128((__m128i*)&W[14][0], w[14]); _mm_store_si128((__m128i*)&W[15][0], w[15]); - - MixColumn(W, 0, 1, 2, 3); - MixColumn(W, 4, 5, 6, 7); - MixColumn(W, 8, 9, 10, 11); - MixColumn(W, 12, 13, 14, 15); } } // namespace ssse3_echo } // namespace sapphire From f2ececc757fea28b49735225036780d139d0c1a2 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 20 Sep 2025 00:47:26 +0000 Subject: [PATCH 14/21] crypto: implement AES-NI backend for Shavite512's `CompressElement()` --- src/Makefile.am | 3 +- src/crypto/x11/dispatch.cpp | 11 +++++ src/crypto/x11/dispatch.h | 3 ++ src/crypto/x11/shavite.cpp | 65 +++++++++++++++++----------- src/crypto/x11/x86_aesni/shavite.cpp | 41 ++++++++++++++++++ 5 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 src/crypto/x11/x86_aesni/shavite.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 790bc2baab1c..69efbcdf7f31 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -792,7 +792,8 @@ crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS += $(X86_AESNI_CXXFLAGS) crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS += -DENABLE_SSE41 -DENABLE_X86_AESNI crypto_libbitcoin_crypto_x86_aesni_la_SOURCES = \ crypto/x11/x86_aesni/aes.cpp \ - crypto/x11/x86_aesni/echo.cpp + crypto/x11/x86_aesni/echo.cpp \ + crypto/x11/x86_aesni/shavite.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 55a43a6b8c64..151f7859c197 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -33,6 +33,10 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, namespace x86_aesni_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace x86_aesni_echo +namespace x86_aesni_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +} // namespace x86_aesni_shavite #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #endif // !DISABLE_OPTIMIZED_SHA256 @@ -47,12 +51,17 @@ namespace soft_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); void ShiftAndMix(uint64_t W[16][2]); } // namespace soft_echo +namespace soft_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +} // namespace soft_shavite } // namespace sapphire extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; extern sapphire::dispatch::EchoShiftMix echo_shift_mix; extern sapphire::dispatch::EchoRoundFn echo_round; +extern sapphire::dispatch::ShaviteCompressFn shavite_c512e; void SapphireAutoDetect() { @@ -60,6 +69,7 @@ void SapphireAutoDetect() aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; echo_shift_mix = sapphire::soft_echo::ShiftAndMix; + shavite_c512e = sapphire::soft_shavite::CompressElement; #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(HAVE_GETCPUID) @@ -72,6 +82,7 @@ void SapphireAutoDetect() aes_round = sapphire::x86_aesni_aes::Round; aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; echo_round = sapphire::x86_aesni_echo::FullStateRound; + shavite_c512e = sapphire::x86_aesni_shavite::CompressElement; } #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #if defined(ENABLE_SSSE3) diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index ddeca43d2735..a648b055d5bb 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -17,6 +17,9 @@ typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); typedef void (*EchoShiftMix)(uint64_t[16][2]); + +typedef void (*ShaviteCompressFn)(uint32_t&, uint32_t&, uint32_t&, uint32_t&, + uint32_t, uint32_t, uint32_t, uint32_t, const uint32_t*); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/shavite.cpp b/src/crypto/x11/shavite.cpp index abd0e74c4536..9cd773d085ba 100644 --- a/src/crypto/x11/shavite.cpp +++ b/src/crypto/x11/shavite.cpp @@ -30,6 +30,7 @@ * @author Thomas Pornin */ +#include #include #include @@ -45,6 +46,41 @@ extern sapphire::dispatch::AESRoundFnNk aes_round_nk; #define C32 SPH_C32 +namespace sapphire { +namespace soft_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) +{ + uint32_t x0{r0 ^ rk[0]}; + uint32_t x1{r1 ^ rk[1]}; + uint32_t x2{r2 ^ rk[2]}; + uint32_t x3{r3 ^ rk[3]}; + soft_aes::RoundKeyless(x0, x1, x2, x3, x0, x1, x2, x3); + x0 ^= rk[4]; + x1 ^= rk[5]; + x2 ^= rk[6]; + x3 ^= rk[7]; + soft_aes::RoundKeyless(x0, x1, x2, x3, x0, x1, x2, x3); + x0 ^= rk[8]; + x1 ^= rk[9]; + x2 ^= rk[10]; + x3 ^= rk[11]; + soft_aes::RoundKeyless(x0, x1, x2, x3, x0, x1, x2, x3); + x0 ^= rk[12]; + x1 ^= rk[13]; + x2 ^= rk[14]; + x3 ^= rk[15]; + soft_aes::RoundKeyless(x0, x1, x2, x3, x0, x1, x2, x3); + l0 ^= x0; + l1 ^= x1; + l2 ^= x2; + l3 ^= x3; +} +} // namespace shavite_soft +} // namespace sapphire + +sapphire::dispatch::ShaviteCompressFn shavite_c512e = sapphire::soft_shavite::CompressElement; + /* * As of round 2 of the SHA-3 competition, the published reference * implementation and test vectors are wrong, because they use @@ -76,7 +112,7 @@ c512(sph_shavite_big_context *sc, const void *msg) { sph_u32 p0, p1, p2, p3, p4, p5, p6, p7; sph_u32 p8, p9, pA, pB, pC, pD, pE, pF; - sph_u32 rk[448]; + alignas(16) sph_u32 rk[448]; size_t u; int r, s; @@ -173,31 +209,8 @@ c512(sph_shavite_big_context *sc, const void *msg) u = 0; for (r = 0; r < 14; r ++) { #define C512_ELT(l0, l1, l2, l3, r0, r1, r2, r3) do { \ - sph_u32 x0, x1, x2, x3; \ - x0 = r0 ^ rk[u ++]; \ - x1 = r1 ^ rk[u ++]; \ - x2 = r2 ^ rk[u ++]; \ - x3 = r3 ^ rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - x0 ^= rk[u ++]; \ - x1 ^= rk[u ++]; \ - x2 ^= rk[u ++]; \ - x3 ^= rk[u ++]; \ - AES_ROUND_NOKEY(x0, x1, x2, x3); \ - l0 ^= x0; \ - l1 ^= x1; \ - l2 ^= x2; \ - l3 ^= x3; \ + shavite_c512e(l0, l1, l2, l3, r0, r1, r2, r3, &rk[u]); \ + u += 16; \ } while (0) #define WROT(a, b, c, d) do { \ diff --git a/src/crypto/x11/x86_aesni/shavite.cpp b/src/crypto/x11/x86_aesni/shavite.cpp new file mode 100644 index 000000000000..f1810a6d6863 --- /dev/null +++ b/src/crypto/x11/x86_aesni/shavite.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#include + +#include + +#include +#include + +namespace sapphire { +namespace x86_aesni_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) +{ + // Load block + XOR with round key 1 + __m128i block = util::pack_le(r0, r1, r2, r3); + block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[0])); + // AES round + XOR with round key 2 + block = util::aes_round(block, _mm_setzero_si128()); + block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[4])); + // AES round + XOR with round key 3 + block = util::aes_round(block, _mm_setzero_si128()); + block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[8])); + // AES Round + XOR with round key 4 + block = util::aes_round(block, _mm_setzero_si128()); + block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[12])); + // AES round + block = util::aes_round(block, _mm_setzero_si128()); + // Unpack + XOR with l values + l0 ^= _mm_extract_epi32(block, 0); + l1 ^= _mm_extract_epi32(block, 1); + l2 ^= _mm_extract_epi32(block, 2); + l3 ^= _mm_extract_epi32(block, 3); +} +} // namespace x86_aesni_shavite +} // namespace sapphire + +#endif // ENABLE_SSE41 && ENABLE_X86_AESNI From 76bd23671476ded7e83f8271f937783bc7762c9a Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 11:49:31 +0000 Subject: [PATCH 15/21] crypto: implement naive ARM AES backend for simple rounds --- configure.ac | 20 ++++++++++- src/Makefile.am | 14 ++++++++ src/crypto/x11/arm_crypto/aes.cpp | 31 +++++++++++++++++ src/crypto/x11/dispatch.cpp | 58 ++++++++++++++++++++++++++++++- src/crypto/x11/util/util.hpp | 35 +++++++++++++++++++ 5 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/crypto/x11/arm_crypto/aes.cpp diff --git a/configure.ac b/configure.ac index 699777e8f47d..d44f42c84049 100644 --- a/configure.ac +++ b/configure.ac @@ -535,6 +535,7 @@ dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, se dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fstack-reuse=none"]) +enable_arm_aes=no enable_arm_crc=no enable_arm_shani=no enable_ssse3=no @@ -677,7 +678,22 @@ CXXFLAGS="$TEMP_CXXFLAGS" # ARM AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_AES_CXXFLAGS="-march=armv8-a+crypto"; ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) + +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$ARM_AES_CXXFLAGS $CXXFLAGS" +AC_MSG_CHECKING([for ARMv8 AES intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + ]],[[ + uint8x16_t a, b; + vaesmcq_u8(vaeseq_u8(a, b)); + return 0; + ]])], + [ AC_MSG_RESULT([yes]); enable_arm_aes=yes; AC_DEFINE([ENABLE_ARM_AES], [1], [Define this symbol to build code that uses ARMv8 AES intrinsics]) ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$ARM_CRC_CXXFLAGS $CXXFLAGS" @@ -1861,6 +1877,7 @@ AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) AM_CONDITIONAL([ENABLE_X86_AESNI], [test "$enable_x86_aesni" = "yes"]) AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_AES], [test "$enable_arm_aes" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) @@ -1922,6 +1939,7 @@ AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) AC_SUBST(X86_AESNI_CXXFLAGS) AC_SUBST(X86_SHANI_CXXFLAGS) +AC_SUBST(ARM_AES_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) AC_SUBST(ARM_SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) diff --git a/src/Makefile.am b/src/Makefile.am index 69efbcdf7f31..7bf2c85e0e42 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -95,6 +95,10 @@ if ENABLE_AVX2 LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif +if ENABLE_ARM_AES +LIBBITCOIN_CRYPTO_ARM_AES = crypto/libbitcoin_crypto_arm_aes.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_AES) +endif if ENABLE_ARM_SHANI LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) @@ -773,6 +777,16 @@ crypto_libbitcoin_crypto_sph_la_SOURCES = \ crypto/x11/util/consts_aes.hpp \ crypto/x11/util/util.hpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_aes_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_aes_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_aes_la_CXXFLAGS += $(ARM_AES_CXXFLAGS) +crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS += -DENABLE_ARM_AES +crypto_libbitcoin_crypto_arm_aes_la_SOURCES = \ + crypto/x11/arm_crypto/aes.cpp + # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above crypto_libbitcoin_crypto_ssse3_la_LDFLAGS = $(AM_LDFLAGS) -static diff --git a/src/crypto/x11/arm_crypto/aes.cpp b/src/crypto/x11/arm_crypto/aes.cpp new file mode 100644 index 000000000000..fd5ee31e7aa2 --- /dev/null +++ b/src/crypto/x11/arm_crypto/aes.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_ARM_AES) +#include + +#include + +#include + +namespace sapphire { +namespace arm_crypto_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + uint8x16_t block = util::aes_round(util::pack_le(x0, x1, x2, x3), util::pack_le(k0, k1, k2, k3)); + util::unpack_le(block, y0, y1, y2, y3); +} + +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) +{ + uint8x16_t block = util::aes_round_nk(util::pack_le(x0, x1, x2, x3)); + util::unpack_le(block, y0, y1, y2, y3); +} +} // namespace arm_crypto_aes +} // namespace sapphire + +#endif // ENABLE_ARM_AES diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 151f7859c197..6b717a8cc761 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -10,12 +10,39 @@ #if !defined(DISABLE_OPTIMIZED_SHA256) #include + +#if defined(ENABLE_ARM_AES) +#if defined(__APPLE__) +#include +#include +#endif // __APPLE__ + +#if defined(__linux__) +#include +#include +#endif // __linux__ + +#if defined(_WIN32) +#include +#include +#endif // _WIN32 +#endif // ENABLE_ARM_AES #endif // !DISABLE_OPTIMIZED_SHA256 -#include +#include namespace sapphire { #if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_ARM_AES) +namespace arm_crypto_aes { +void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, + uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); +} // namespace arm_crypto_aes +#endif // ENABLE_ARM_AES + #if defined(ENABLE_SSSE3) namespace ssse3_echo { void ShiftAndMix(uint64_t W[16][2]); @@ -92,5 +119,34 @@ void SapphireAutoDetect() } #endif // ENABLE_SSSE3 #endif // HAVE_GETCPUID + +#if defined(ENABLE_ARM_AES) + bool have_arm_aes = false; +#if defined(__APPLE__) + int val = 0; + size_t len = sizeof(val); + if (::sysctlbyname("hw.optional.arm.FEAT_AES", &val, &len, nullptr, 0) == 0) { + have_arm_aes = val != 0; + } +#endif // __APPLE__ + +#if defined(__linux__) +#if defined(__arm__) + have_arm_aes = (::getauxval(AT_HWCAP2) & HWCAP2_AES); +#endif // __arm__ +#if defined(__aarch64__) + have_arm_aes = (::getauxval(AT_HWCAP) & HWCAP_AES); +#endif // __aarch64__ +#endif // __linux__ + +#if defined(_WIN32) + have_arm_aes = ::IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#endif // _WIN32 + + if (have_arm_aes) { + aes_round = sapphire::arm_crypto_aes::Round; + aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; + } +#endif // ENABLE_ARM_AES #endif // !DISABLE_OPTIMIZED_SHA256 } diff --git a/src/crypto/x11/util/util.hpp b/src/crypto/x11/util/util.hpp index a3971cd19c5c..b49507f840ab 100644 --- a/src/crypto/x11/util/util.hpp +++ b/src/crypto/x11/util/util.hpp @@ -10,6 +10,10 @@ #if !defined(DISABLE_OPTIMIZED_SHA256) #include +#if defined(ENABLE_ARM_AES) +#include +#endif // ENABLE_ARM_AES + #if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) #include #endif // ENABLE_SSSE3 || (ENABLE_SSE41 && ENABLE_X86_AESNI) @@ -26,6 +30,37 @@ constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0 } #if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_ARM_AES) +uint8x16_t ALWAYS_INLINE Xor(const uint8x16_t& x, const uint8x16_t& y) { return veorq_u8(x, y); } + +uint8x16_t ALWAYS_INLINE pack_le(const uint32_t& w0, const uint32_t& w1, const uint32_t& w2, const uint32_t& w3) +{ + return vreinterpretq_u8_u32(uint32x4_t{w0, w1, w2, w3}); +} + +void ALWAYS_INLINE unpack_le(const uint8x16_t& i, uint32_t& w0, uint32_t& w1, uint32_t& w2, uint32_t& w3) +{ + const uint32x4_t r = vreinterpretq_u32_u8(i); + w0 = vgetq_lane_u32(r, 0); + w1 = vgetq_lane_u32(r, 1); + w2 = vgetq_lane_u32(r, 2); + w3 = vgetq_lane_u32(r, 3); +} + +uint8x16_t ALWAYS_INLINE aes_round(const uint8x16_t& input, const uint8x16_t& key) +{ + // See "Emulating x86 AES Intrinsics on ARMv8-A" by Michael Brase for _mm_aesenc_si128 + // https://blog.michaelbrase.com/2018/05/08/emulating-x86-aes-intrinsics-on-armv8-a/ + return Xor(vaesmcq_u8(vaeseq_u8(input, vmovq_n_u8(0))), key); +} + +uint8x16_t ALWAYS_INLINE aes_round_nk(const uint8x16_t& input) +{ + // We can skip the XOR when we don't have a key + return vaesmcq_u8(vaeseq_u8(input, vmovq_n_u8(0))); +} +#endif // ENABLE_ARM_AES + #if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) __m128i ALWAYS_INLINE Xor(const __m128i& x, const __m128i& y) { return _mm_xor_si128(x, y); } From 959c9ee6c02762696d1294cb1ea3c7364cdf37af Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 07:47:34 +0000 Subject: [PATCH 16/21] crypto: implement ARM AES backend for Echo512's `FullStateRound()` --- src/Makefile.am | 3 ++- src/crypto/x11/arm_crypto/echo.cpp | 38 ++++++++++++++++++++++++++++++ src/crypto/x11/dispatch.cpp | 19 +++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/crypto/x11/arm_crypto/echo.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 7bf2c85e0e42..f5825efcc57b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -785,7 +785,8 @@ crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_arm_aes_la_CXXFLAGS += $(ARM_AES_CXXFLAGS) crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS += -DENABLE_ARM_AES crypto_libbitcoin_crypto_arm_aes_la_SOURCES = \ - crypto/x11/arm_crypto/aes.cpp + crypto/x11/arm_crypto/aes.cpp \ + crypto/x11/arm_crypto/echo.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above diff --git a/src/crypto/x11/arm_crypto/echo.cpp b/src/crypto/x11/arm_crypto/echo.cpp new file mode 100644 index 000000000000..c18f8d77dbc4 --- /dev/null +++ b/src/crypto/x11/arm_crypto/echo.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_ARM_AES) +#include + +#include + +#include + +namespace sapphire { +namespace arm_crypto_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +{ + uint8x16_t key = util::pack_le(k0, k1, k2, k3); + for (int n = 0; n < 16; n++) { + uint8x16_t block = vreinterpretq_u8_u64(vld1q_u64(&W[n][0])); + block = util::aes_round(block, key); + block = util::aes_round_nk(block); + vst1q_u64(&W[n][0], vreinterpretq_u64_u8(block)); + + util::unpack_le(key, k0, k1, k2, k3); + if ((k0 = (k0 + 1)) == 0) { + if ((k1 = (k1 + 1)) == 0) { + if ((k2 = (k2 + 1)) == 0) { + k3 = (k3 + 1); + } + } + } + key = util::pack_le(k0, k1, k2, k3); + } + util::unpack_le(key, k0, k1, k2, k3); +} +} // namespace arm_crypto_echo +} // namespace sapphire + +#endif // ENABLE_ARM_AES diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 6b717a8cc761..4a99e3330829 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -22,6 +22,11 @@ #include #endif // __linux__ +#if defined(__FreeBSD__) +#include +#include +#endif // __FreeBSD__ + #if defined(_WIN32) #include #include @@ -41,6 +46,9 @@ void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); } // namespace arm_crypto_aes +namespace arm_crypto_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); +} // namespace arm_crypto_echo #endif // ENABLE_ARM_AES #if defined(ENABLE_SSSE3) @@ -139,6 +147,16 @@ void SapphireAutoDetect() #endif // __aarch64__ #endif // __linux__ +#if defined(__FreeBSD__) + [[maybe_unused]] unsigned long hwcap{0}; +#if defined(__arm__) + have_arm_aes = ((::elf_aux_info(AT_HWCAP2, &hwcap, sizeof(hwcap)) == 0) && ((hwcap & HWCAP2_AES) != 0)); +#endif // __arm__ +#if defined(__aarch64__) + have_arm_aes = ((::elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) && ((hwcap & HWCAP_AES) != 0)); +#endif // __aarch64__ +#endif // __FreeBSD__ + #if defined(_WIN32) have_arm_aes = ::IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); #endif // _WIN32 @@ -146,6 +164,7 @@ void SapphireAutoDetect() if (have_arm_aes) { aes_round = sapphire::arm_crypto_aes::Round; aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; + echo_round = sapphire::arm_crypto_echo::FullStateRound; } #endif // ENABLE_ARM_AES #endif // !DISABLE_OPTIMIZED_SHA256 From 963215bf9172ac71f7ad1f3f588b6aaae7798a82 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 10:46:18 +0000 Subject: [PATCH 17/21] crypto: implement ARM AES backend for Shavite512's `CompressElement()` --- src/Makefile.am | 3 ++- src/crypto/x11/arm_crypto/shavite.cpp | 38 +++++++++++++++++++++++++++ src/crypto/x11/dispatch.cpp | 5 ++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/crypto/x11/arm_crypto/shavite.cpp diff --git a/src/Makefile.am b/src/Makefile.am index f5825efcc57b..2c846aa1595a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -786,7 +786,8 @@ crypto_libbitcoin_crypto_arm_aes_la_CXXFLAGS += $(ARM_AES_CXXFLAGS) crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS += -DENABLE_ARM_AES crypto_libbitcoin_crypto_arm_aes_la_SOURCES = \ crypto/x11/arm_crypto/aes.cpp \ - crypto/x11/arm_crypto/echo.cpp + crypto/x11/arm_crypto/echo.cpp \ + crypto/x11/arm_crypto/shavite.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above diff --git a/src/crypto/x11/arm_crypto/shavite.cpp b/src/crypto/x11/arm_crypto/shavite.cpp new file mode 100644 index 000000000000..41f2551b342f --- /dev/null +++ b/src/crypto/x11/arm_crypto/shavite.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_ARM_AES) +#include + +#include + +#include + +namespace sapphire { +namespace arm_crypto_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) +{ + // Pack block + XOR with round key 1 + uint8x16_t block = util::pack_le(r0, r1, r2, r3); + block = util::Xor(block, vreinterpretq_u8_u32(vld1q_u32(&rk[0]))); + // AES round + XOR with round key 2 + block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[4]))); + // AES round + XOR with round key 3 + block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[8]))); + // AES Round + XOR with round key 4 + block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[12]))); + // AES round + block = util::aes_round_nk(block); + // Unpack + XOR with l values + uint32x4_t result = vreinterpretq_u32_u8(block); + l0 ^= vgetq_lane_u32(result, 0); + l1 ^= vgetq_lane_u32(result, 1); + l2 ^= vgetq_lane_u32(result, 2); + l3 ^= vgetq_lane_u32(result, 3); +} +} // namespace arm_crypto_shavite +} // namespace sapphire + +#endif // ENABLE_ARM_AES diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 4a99e3330829..6a349d8ed05a 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -49,6 +49,10 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, namespace arm_crypto_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace arm_crypto_echo +namespace arm_crypto_shavite { +void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +} // namespace arm_crypto_shavite #endif // ENABLE_ARM_AES #if defined(ENABLE_SSSE3) @@ -165,6 +169,7 @@ void SapphireAutoDetect() aes_round = sapphire::arm_crypto_aes::Round; aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; echo_round = sapphire::arm_crypto_echo::FullStateRound; + shavite_c512e = sapphire::arm_crypto_shavite::CompressElement; } #endif // ENABLE_ARM_AES #endif // !DISABLE_OPTIMIZED_SHA256 From fa68c704de7ae3de9ff6e40f70f578efe66baa4a Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 07:59:52 +0000 Subject: [PATCH 18/21] crypto: implement ARM NEON backend for Echo512's `ShiftAndMix()` --- configure.ac | 24 ++++++ src/Makefile.am | 14 ++++ src/crypto/x11/arm_neon/echo.cpp | 132 +++++++++++++++++++++++++++++++ src/crypto/x11/dispatch.cpp | 60 +++++++++++--- src/crypto/x11/util/util.hpp | 8 +- 5 files changed, 223 insertions(+), 15 deletions(-) create mode 100644 src/crypto/x11/arm_neon/echo.cpp diff --git a/configure.ac b/configure.ac index d44f42c84049..1faf45f182e8 100644 --- a/configure.ac +++ b/configure.ac @@ -537,6 +537,7 @@ AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fsta enable_arm_aes=no enable_arm_crc=no +enable_arm_neon=no enable_arm_shani=no enable_ssse3=no enable_sse42=no @@ -714,6 +715,27 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ) CXXFLAGS="$TEMP_CXXFLAGS" +ARM_NEON_CXXFLAGS="" +TEMP_CXXFLAGS="$CXXFLAGS" +for flag in "-march=armv8-a" "-march=armv7-a -mfpu=neon"; do + AX_CHECK_COMPILE_FLAG([$flag], [ + CXXFLAGS="$CXXFLAGS $flag" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + ]], [[ + float32x4_t f = vdupq_n_f32(0.0); + return 0; + ]])], [ + ARM_NEON_CXXFLAGS="$flag" + enable_arm_neon=yes + AC_DEFINE([ENABLE_ARM_NEON], [1], [Define this symbol to build code that uses ARM NEON intrinsics]) + break + ]) + CXXFLAGS="$TEMP_CXXFLAGS" + ]) +done +CXXFLAGS="$TEMP_CXXFLAGS" + TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$ARM_SHANI_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics]) @@ -1879,6 +1901,7 @@ AM_CONDITIONAL([ENABLE_X86_AESNI], [test "$enable_x86_aesni" = "yes"]) AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_AES], [test "$enable_arm_aes" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_NEON], [test "$enable_arm_neon" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) AM_CONDITIONAL([USE_NATPMP], [test "$use_natpmp" = "yes"]) @@ -1941,6 +1964,7 @@ AC_SUBST(X86_AESNI_CXXFLAGS) AC_SUBST(X86_SHANI_CXXFLAGS) AC_SUBST(ARM_AES_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) +AC_SUBST(ARM_NEON_CXXFLAGS) AC_SUBST(ARM_SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) diff --git a/src/Makefile.am b/src/Makefile.am index 2c846aa1595a..562f4a3afe46 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -99,6 +99,10 @@ if ENABLE_ARM_AES LIBBITCOIN_CRYPTO_ARM_AES = crypto/libbitcoin_crypto_arm_aes.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_AES) endif +if ENABLE_ARM_NEON +LIBBITCOIN_CRYPTO_ARM_NEON = crypto/libbitcoin_crypto_arm_neon.la +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_NEON) +endif if ENABLE_ARM_SHANI LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) @@ -789,6 +793,16 @@ crypto_libbitcoin_crypto_arm_aes_la_SOURCES = \ crypto/x11/arm_crypto/echo.cpp \ crypto/x11/arm_crypto/shavite.cpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_neon_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_neon_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_neon_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_neon_la_CXXFLAGS += $(ARM_NEON_CXXFLAGS) +crypto_libbitcoin_crypto_arm_neon_la_CPPFLAGS += -DENABLE_ARM_NEON +crypto_libbitcoin_crypto_arm_neon_la_SOURCES = \ + crypto/x11/arm_neon/echo.cpp + # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above crypto_libbitcoin_crypto_ssse3_la_LDFLAGS = $(AM_LDFLAGS) -static diff --git a/src/crypto/x11/arm_neon/echo.cpp b/src/crypto/x11/arm_neon/echo.cpp new file mode 100644 index 000000000000..ed85910341b7 --- /dev/null +++ b/src/crypto/x11/arm_neon/echo.cpp @@ -0,0 +1,132 @@ +// Copyright (c) 2025 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(ENABLE_ARM_NEON) +#include +#include + +#include + +#include + +namespace sapphire { +namespace { +uint8x16_t ALWAYS_INLINE gf8_mul2(const uint8x16_t& x) +{ + // (x << 1) + const uint8x16_t lhs = vshlq_n_u8(x, 1); + // (x & 0x80) ? 0xff : 0x00 + const uint8x16_t msb_set = vandq_u8(x, vmovq_n_u8(0x80)); + const uint8x16_t mask = vceqq_u8(msb_set, vmovq_n_u8(0x80)); + // Replace 0xff with 0x1b + const uint8x16_t rhs = vandq_u8(mask, vmovq_n_u8(0x1b)); + // (x << 1) ^ ((x & 0x80) ? 0x1b : 0x00)) + return util::Xor(lhs, rhs); +} + +void ALWAYS_INLINE MixColumn(uint8x16_t& Wa, uint8x16_t& Wb, uint8x16_t& Wc, uint8x16_t& Wd) +{ + const uint8x16_t a = Wa; + const uint8x16_t b = Wb; + const uint8x16_t c = Wc; + const uint8x16_t d = Wd; + + const uint8x16_t ab = util::Xor(a, b); + const uint8x16_t bc = util::Xor(b, c); + const uint8x16_t cd = util::Xor(c, d); + + const uint8x16_t abx = gf8_mul2(ab); + const uint8x16_t bcx = gf8_mul2(bc); + const uint8x16_t cdx = gf8_mul2(cd); + + // Wa = abx ^ bc ^ d + Wa = util::Xor(util::Xor(abx, bc), d); + // Wb = bcx ^ a ^ cd + Wb = util::Xor(util::Xor(bcx, a), cd); + // Wc = cdx ^ ab ^ d + Wc = util::Xor(util::Xor(cdx, ab), d); + // Wd = abx ^ bcx ^ cdx ^ ab ^ c + Wd = util::Xor(util::Xor(util::Xor(util::Xor(abx, bcx), cdx), ab), c); +} + +void ALWAYS_INLINE ShiftRow1(uint8x16_t& Wa, uint8x16_t& Wb, uint8x16_t& Wc, uint8x16_t& Wd) +{ + uint8x16_t tmp = Wa; + Wa = Wb; + Wb = Wc; + Wc = Wd; + Wd = tmp; +} + +void ALWAYS_INLINE ShiftRow2(uint8x16_t& Wa, uint8x16_t& Wb, uint8x16_t& Wc, uint8x16_t& Wd) +{ + uint8x16_t tmp1 = Wa; + uint8x16_t tmp2 = Wb; + Wa = Wc; + Wb = Wd; + Wc = tmp1; + Wd = tmp2; +} + +void ALWAYS_INLINE ShiftRow3(uint8x16_t& Wa, uint8x16_t& Wb, uint8x16_t& Wc, uint8x16_t& Wd) +{ + uint8x16_t tmp = Wd; + Wd = Wc; + Wc = Wb; + Wb = Wa; + Wa = tmp; +} +} // anonymous namespace + +namespace arm_neon_echo { +void ShiftAndMix(uint64_t W[16][2]) +{ + alignas(16) uint8x16_t w[16]; + w[0] = vreinterpretq_u8_u64(vld1q_u64(&W[0][0])); + w[1] = vreinterpretq_u8_u64(vld1q_u64(&W[1][0])); + w[2] = vreinterpretq_u8_u64(vld1q_u64(&W[2][0])); + w[3] = vreinterpretq_u8_u64(vld1q_u64(&W[3][0])); + w[4] = vreinterpretq_u8_u64(vld1q_u64(&W[4][0])); + w[5] = vreinterpretq_u8_u64(vld1q_u64(&W[5][0])); + w[6] = vreinterpretq_u8_u64(vld1q_u64(&W[6][0])); + w[7] = vreinterpretq_u8_u64(vld1q_u64(&W[7][0])); + w[8] = vreinterpretq_u8_u64(vld1q_u64(&W[8][0])); + w[9] = vreinterpretq_u8_u64(vld1q_u64(&W[9][0])); + w[10] = vreinterpretq_u8_u64(vld1q_u64(&W[10][0])); + w[11] = vreinterpretq_u8_u64(vld1q_u64(&W[11][0])); + w[12] = vreinterpretq_u8_u64(vld1q_u64(&W[12][0])); + w[13] = vreinterpretq_u8_u64(vld1q_u64(&W[13][0])); + w[14] = vreinterpretq_u8_u64(vld1q_u64(&W[14][0])); + w[15] = vreinterpretq_u8_u64(vld1q_u64(&W[15][0])); + + ShiftRow1(w[1], w[5], w[9], w[13]); + ShiftRow2(w[2], w[6], w[10], w[14]); + ShiftRow3(w[3], w[7], w[11], w[15]); + + MixColumn(w[0], w[1], w[2], w[3]); + MixColumn(w[4], w[5], w[6], w[7]); + MixColumn(w[8], w[9], w[10], w[11]); + MixColumn(w[12], w[13], w[14], w[15]); + + vst1q_u64(&W[0][0], vreinterpretq_u64_u8(w[0])); + vst1q_u64(&W[1][0], vreinterpretq_u64_u8(w[1])); + vst1q_u64(&W[2][0], vreinterpretq_u64_u8(w[2])); + vst1q_u64(&W[3][0], vreinterpretq_u64_u8(w[3])); + vst1q_u64(&W[4][0], vreinterpretq_u64_u8(w[4])); + vst1q_u64(&W[5][0], vreinterpretq_u64_u8(w[5])); + vst1q_u64(&W[6][0], vreinterpretq_u64_u8(w[6])); + vst1q_u64(&W[7][0], vreinterpretq_u64_u8(w[7])); + vst1q_u64(&W[8][0], vreinterpretq_u64_u8(w[8])); + vst1q_u64(&W[9][0], vreinterpretq_u64_u8(w[9])); + vst1q_u64(&W[10][0], vreinterpretq_u64_u8(w[10])); + vst1q_u64(&W[11][0], vreinterpretq_u64_u8(w[11])); + vst1q_u64(&W[12][0], vreinterpretq_u64_u8(w[12])); + vst1q_u64(&W[13][0], vreinterpretq_u64_u8(w[13])); + vst1q_u64(&W[14][0], vreinterpretq_u64_u8(w[14])); + vst1q_u64(&W[15][0], vreinterpretq_u64_u8(w[15])); +} +} // namespace arm_neon_echo +} // namespace sapphire + +#endif // ENABLE_ARM_NEON diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 6a349d8ed05a..65b795daad72 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -11,7 +11,7 @@ #if !defined(DISABLE_OPTIMIZED_SHA256) #include -#if defined(ENABLE_ARM_AES) +#if defined(ENABLE_ARM_AES) || defined(ENABLE_ARM_NEON) #if defined(__APPLE__) #include #include @@ -31,7 +31,7 @@ #include #include #endif // _WIN32 -#endif // ENABLE_ARM_AES +#endif // ENABLE_ARM_AES || ENABLE_ARM_NEON #endif // !DISABLE_OPTIMIZED_SHA256 #include @@ -55,6 +55,12 @@ void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, } // namespace arm_crypto_shavite #endif // ENABLE_ARM_AES +#if defined(ENABLE_ARM_NEON) +namespace arm_neon_echo { +void ShiftAndMix(uint64_t W[16][2]); +} // namespace arm_neon_echo +#endif // ENABLE_ARM_NEON + #if defined(ENABLE_SSSE3) namespace ssse3_echo { void ShiftAndMix(uint64_t W[16][2]); @@ -96,6 +102,21 @@ void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, } // namespace soft_shavite } // namespace sapphire +namespace { +#if !defined(DISABLE_OPTIMIZED_SHA256) +#if defined(ENABLE_ARM_AES) || defined(ENABLE_ARM_NEON) +#if defined(__APPLE__) +bool IsSysCtlNonZero(const char* name) +{ + int val = 0; + size_t len = sizeof(val); + return ::sysctlbyname(name, &val, &len, nullptr, 0) == 0 && val != 0; +} +#endif // __APPLE__ +#endif // ENABLE_ARM_AES || ENABLE_ARM_NEON +#endif // !DISABLE_OPTIMIZED_SHA256 +} // anonymous namespace + extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; extern sapphire::dispatch::EchoShiftMix echo_shift_mix; @@ -132,39 +153,47 @@ void SapphireAutoDetect() #endif // ENABLE_SSSE3 #endif // HAVE_GETCPUID -#if defined(ENABLE_ARM_AES) - bool have_arm_aes = false; +#if defined(ENABLE_ARM_AES) || defined(ENABLE_ARM_NEON) + [[maybe_unused]] bool have_arm_aes = false; + [[maybe_unused]] bool have_arm_neon = false; + #if defined(__APPLE__) - int val = 0; - size_t len = sizeof(val); - if (::sysctlbyname("hw.optional.arm.FEAT_AES", &val, &len, nullptr, 0) == 0) { - have_arm_aes = val != 0; - } + have_arm_aes = IsSysCtlNonZero("hw.optional.arm.FEAT_AES"); + have_arm_neon = IsSysCtlNonZero("hw.optional.neon") || IsSysCtlNonZero("hw.optional.AdvSIMD") || + IsSysCtlNonZero("hw.optional.arm.AdvSIMD"); // See https://github.com/google/cpu_features/issues/390 #endif // __APPLE__ #if defined(__linux__) #if defined(__arm__) have_arm_aes = (::getauxval(AT_HWCAP2) & HWCAP2_AES); + have_arm_neon = (::getauxval(AT_HWCAP) & HWCAP_NEON); #endif // __arm__ #if defined(__aarch64__) have_arm_aes = (::getauxval(AT_HWCAP) & HWCAP_AES); + have_arm_neon = (::getauxval(AT_HWCAP) & HWCAP_ASIMD); #endif // __aarch64__ #endif // __linux__ #if defined(__FreeBSD__) - [[maybe_unused]] unsigned long hwcap{0}; + [[maybe_unused]] unsigned long hwcap{0}, hwcap2{0}; #if defined(__arm__) - have_arm_aes = ((::elf_aux_info(AT_HWCAP2, &hwcap, sizeof(hwcap)) == 0) && ((hwcap & HWCAP2_AES) != 0)); + have_arm_aes = ((::elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2)) == 0) && ((hwcap2 & HWCAP2_AES) != 0)); + have_arm_neon = ((::elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) && ((hwcap & HWCAP_NEON) != 0)); #endif // __arm__ #if defined(__aarch64__) - have_arm_aes = ((::elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) && ((hwcap & HWCAP_AES) != 0)); + if (::elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) { + have_arm_aes = ((hwcap & HWCAP_AES) != 0); + have_arm_neon = ((hwcap & HWCAP_ASIMD) != 0); + } #endif // __aarch64__ #endif // __FreeBSD__ #if defined(_WIN32) have_arm_aes = ::IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); + have_arm_neon = ::IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE); #endif // _WIN32 +#if defined(ENABLE_ARM_AES) if (have_arm_aes) { aes_round = sapphire::arm_crypto_aes::Round; aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; @@ -172,5 +201,12 @@ void SapphireAutoDetect() shavite_c512e = sapphire::arm_crypto_shavite::CompressElement; } #endif // ENABLE_ARM_AES + +#if defined (ENABLE_ARM_NEON) + if (have_arm_neon) { + echo_shift_mix = sapphire::arm_neon_echo::ShiftAndMix; + } +#endif // ENABLE_ARM_NEON +#endif // ENABLE_ARM_AES || ENABLE_ARM_NEON #endif // !DISABLE_OPTIMIZED_SHA256 } diff --git a/src/crypto/x11/util/util.hpp b/src/crypto/x11/util/util.hpp index b49507f840ab..ca0defedfeeb 100644 --- a/src/crypto/x11/util/util.hpp +++ b/src/crypto/x11/util/util.hpp @@ -10,9 +10,9 @@ #if !defined(DISABLE_OPTIMIZED_SHA256) #include -#if defined(ENABLE_ARM_AES) +#if defined(ENABLE_ARM_AES) || defined(ENABLE_ARM_NEON) #include -#endif // ENABLE_ARM_AES +#endif // ENABLE_ARM_AES || ENABLE_ARM_NEON #if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) #include @@ -30,7 +30,7 @@ constexpr inline uint32_t pack_le(uint8_t b3, uint8_t b2, uint8_t b1, uint8_t b0 } #if !defined(DISABLE_OPTIMIZED_SHA256) -#if defined(ENABLE_ARM_AES) +#if defined(ENABLE_ARM_AES) || defined(ENABLE_ARM_NEON) uint8x16_t ALWAYS_INLINE Xor(const uint8x16_t& x, const uint8x16_t& y) { return veorq_u8(x, y); } uint8x16_t ALWAYS_INLINE pack_le(const uint32_t& w0, const uint32_t& w1, const uint32_t& w2, const uint32_t& w3) @@ -47,6 +47,7 @@ void ALWAYS_INLINE unpack_le(const uint8x16_t& i, uint32_t& w0, uint32_t& w1, ui w3 = vgetq_lane_u32(r, 3); } +#if defined(ENABLE_ARM_AES) uint8x16_t ALWAYS_INLINE aes_round(const uint8x16_t& input, const uint8x16_t& key) { // See "Emulating x86 AES Intrinsics on ARMv8-A" by Michael Brase for _mm_aesenc_si128 @@ -60,6 +61,7 @@ uint8x16_t ALWAYS_INLINE aes_round_nk(const uint8x16_t& input) return vaesmcq_u8(vaeseq_u8(input, vmovq_n_u8(0))); } #endif // ENABLE_ARM_AES +#endif // ENABLE_ARM_AES || ENABLE_ARM_NEON #if defined(ENABLE_SSSE3) || (defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI)) __m128i ALWAYS_INLINE Xor(const __m128i& x, const __m128i& y) { return _mm_xor_si128(x, y); } From 31f93ad93e09d5c8a8be42e4512c49e7705568f3 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 11:50:09 +0000 Subject: [PATCH 19/21] crypto: unroll Echo512's `FullStateRound()` Also lets us get rid of an extra unnecessary pack/unpack --- src/crypto/x11/arm_crypto/echo.cpp | 67 ++++++++++++++++++++++-------- src/crypto/x11/x86_aesni/echo.cpp | 67 ++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 34 deletions(-) diff --git a/src/crypto/x11/arm_crypto/echo.cpp b/src/crypto/x11/arm_crypto/echo.cpp index c18f8d77dbc4..2ffb8bd67543 100644 --- a/src/crypto/x11/arm_crypto/echo.cpp +++ b/src/crypto/x11/arm_crypto/echo.cpp @@ -5,32 +5,65 @@ #if defined(ENABLE_ARM_AES) #include -#include +#include #include namespace sapphire { -namespace arm_crypto_echo { -void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +namespace { +void ALWAYS_INLINE StateRound(uint64_t W[16][2], size_t idx, uint8x16_t& key, uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) { - uint8x16_t key = util::pack_le(k0, k1, k2, k3); - for (int n = 0; n < 16; n++) { - uint8x16_t block = vreinterpretq_u8_u64(vld1q_u64(&W[n][0])); - block = util::aes_round(block, key); - block = util::aes_round_nk(block); - vst1q_u64(&W[n][0], vreinterpretq_u64_u8(block)); + uint8x16_t block = vreinterpretq_u8_u64(vld1q_u64(&W[idx][0])); + block = util::aes_round(block, key); + block = util::aes_round_nk(block); + vst1q_u64(&W[idx][0], vreinterpretq_u64_u8(block)); - util::unpack_le(key, k0, k1, k2, k3); - if ((k0 = (k0 + 1)) == 0) { - if ((k1 = (k1 + 1)) == 0) { - if ((k2 = (k2 + 1)) == 0) { - k3 = (k3 + 1); - } + util::unpack_le(key, k0, k1, k2, k3); + if ((k0 = (k0 + 1)) == 0) { + if ((k1 = (k1 + 1)) == 0) { + if ((k2 = (k2 + 1)) == 0) { + k3 = (k3 + 1); } } - key = util::pack_le(k0, k1, k2, k3); } - util::unpack_le(key, k0, k1, k2, k3); +} +} // anonymous namespace + +namespace arm_crypto_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +{ + uint8x16_t key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 0, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 1, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 2, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 3, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 4, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 5, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 6, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 7, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 8, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 9, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 10, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 11, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 12, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 13, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 14, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 15, key, k0, k1, k2, k3); } } // namespace arm_crypto_echo } // namespace sapphire diff --git a/src/crypto/x11/x86_aesni/echo.cpp b/src/crypto/x11/x86_aesni/echo.cpp index bed9c74e800b..b8b1be82a87e 100644 --- a/src/crypto/x11/x86_aesni/echo.cpp +++ b/src/crypto/x11/x86_aesni/echo.cpp @@ -5,33 +5,66 @@ #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) #include -#include +#include #include #include namespace sapphire { -namespace x86_aesni_echo { -void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +namespace { +void ALWAYS_INLINE StateRound(uint64_t W[16][2], size_t idx, __m128i& key, uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) { - __m128i key = util::pack_le(k0, k1, k2, k3); - for (int n = 0; n < 16; n++) { - __m128i block = _mm_load_si128((const __m128i*)&W[n][0]); - block = util::aes_round(block, key); - block = util::aes_round(block, _mm_setzero_si128()); - _mm_store_si128((__m128i*)&W[n][0], block); + __m128i block = _mm_load_si128((const __m128i*)&W[idx][0]); + block = util::aes_round(block, key); + block = util::aes_round(block, _mm_setzero_si128()); + _mm_store_si128((__m128i*)&W[idx][0], block); - util::unpack_le(key, k0, k1, k2, k3); - if ((k0 = (k0 + 1)) == 0) { - if ((k1 = (k1 + 1)) == 0) { - if ((k2 = (k2 + 1)) == 0) { - k3 = (k3 + 1); - } + util::unpack_le(key, k0, k1, k2, k3); + if ((k0 = (k0 + 1)) == 0) { + if ((k1 = (k1 + 1)) == 0) { + if ((k2 = (k2 + 1)) == 0) { + k3 = (k3 + 1); } } - key = util::pack_le(k0, k1, k2, k3); } - util::unpack_le(key, k0, k1, k2, k3); +} +} // anonymous namespace + +namespace x86_aesni_echo { +void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3) +{ + __m128i key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 0, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 1, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 2, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 3, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 4, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 5, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 6, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 7, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 8, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 9, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 10, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 11, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 12, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 13, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 14, key, k0, k1, k2, k3); + key = util::pack_le(k0, k1, k2, k3); + StateRound(W, 15, key, k0, k1, k2, k3); } } // namespace x86_aesni_echo } // namespace sapphire From d131fccbdf09d43e96e9baa0eda19e4c2deeab2c Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 22 Sep 2025 06:12:34 +0000 Subject: [PATCH 20/21] crypto: implement Shavite512's full `Compress()` routine --- src/crypto/x11/arm_crypto/shavite.cpp | 171 +++++++++++++++++++++++-- src/crypto/x11/dispatch.cpp | 17 ++- src/crypto/x11/dispatch.h | 5 +- src/crypto/x11/shavite.cpp | 71 ++++++----- src/crypto/x11/x86_aesni/shavite.cpp | 174 +++++++++++++++++++++++--- 5 files changed, 363 insertions(+), 75 deletions(-) diff --git a/src/crypto/x11/arm_crypto/shavite.cpp b/src/crypto/x11/arm_crypto/shavite.cpp index 41f2551b342f..a19cc033c58d 100644 --- a/src/crypto/x11/arm_crypto/shavite.cpp +++ b/src/crypto/x11/arm_crypto/shavite.cpp @@ -3,35 +3,182 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(ENABLE_ARM_AES) +#include #include #include +#include #include namespace sapphire { -namespace arm_crypto_shavite { +namespace { void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, - uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint8x16_t* rk_words) { - // Pack block + XOR with round key 1 uint8x16_t block = util::pack_le(r0, r1, r2, r3); - block = util::Xor(block, vreinterpretq_u8_u32(vld1q_u32(&rk[0]))); - // AES round + XOR with round key 2 - block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[4]))); - // AES round + XOR with round key 3 - block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[8]))); - // AES Round + XOR with round key 4 - block = util::Xor(util::aes_round_nk(block), vreinterpretq_u8_u32(vld1q_u32(&rk[12]))); - // AES round + block = util::Xor(block, rk_words[0]); + block = util::Xor(util::aes_round_nk(block), rk_words[1]); + block = util::Xor(util::aes_round_nk(block), rk_words[2]); + block = util::Xor(util::aes_round_nk(block), rk_words[3]); block = util::aes_round_nk(block); - // Unpack + XOR with l values uint32x4_t result = vreinterpretq_u32_u8(block); l0 ^= vgetq_lane_u32(result, 0); l1 ^= vgetq_lane_u32(result, 1); l2 ^= vgetq_lane_u32(result, 2); l3 ^= vgetq_lane_u32(result, 3); } +} // anonymous namespace + +namespace arm_crypto_shavite { +void Compress(sph_shavite_big_context *sc, const void *msg) +{ + uint32_t p0, p1, p2, p3, p4, p5, p6, p7; + uint32_t p8, p9, pA, pB, pC, pD, pE, pF; + + alignas(16) uint8x16_t rk_words[448/4]; + alignas(16) uint32_t rk[448]; + +#if SPH_LITTLE_ENDIAN + memcpy(rk, msg, 128); +#else + for (size_t u{0}; u < 32; u += 4) { + rk[u + 0] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 0); + rk[u + 1] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 4); + rk[u + 2] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 8); + rk[u + 3] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 12); + } +#endif + + size_t u{32}; + for (;;) { + for (int s{0}; s < 4; s++) { + uint32_t x0 = rk[u - 31]; + uint32_t x1 = rk[u - 30]; + uint32_t x2 = rk[u - 29]; + uint32_t x3 = rk[u - 32]; + + uint32x4_t block = vreinterpretq_u32_u8(util::aes_round_nk(util::pack_le(x0, x1, x2, x3))); + rk[u + 0] = vgetq_lane_u32(block, 0) ^ rk[u - 4]; + rk[u + 1] = vgetq_lane_u32(block, 1) ^ rk[u - 3]; + rk[u + 2] = vgetq_lane_u32(block, 2) ^ rk[u - 2]; + rk[u + 3] = vgetq_lane_u32(block, 3) ^ rk[u - 1]; + + if (u == 32) { + rk[32] ^= sc->count0; + rk[33] ^= sc->count1; + rk[34] ^= sc->count2; + rk[35] ^= SPH_T32(~sc->count3); + } else if (u == 440) { + rk[440] ^= sc->count1; + rk[441] ^= sc->count0; + rk[442] ^= sc->count3; + rk[443] ^= SPH_T32(~sc->count2); + } + u += 4; + + x0 = rk[u - 31]; + x1 = rk[u - 30]; + x2 = rk[u - 29]; + x3 = rk[u - 32]; + + block = vreinterpretq_u32_u8(util::aes_round_nk(util::pack_le(x0, x1, x2, x3))); + rk[u + 0] = vgetq_lane_u32(block, 0) ^ rk[u - 4]; + rk[u + 1] = vgetq_lane_u32(block, 1) ^ rk[u - 3]; + rk[u + 2] = vgetq_lane_u32(block, 2) ^ rk[u - 2]; + rk[u + 3] = vgetq_lane_u32(block, 3) ^ rk[u - 1]; + + if (u == 164) { + rk[164] ^= sc->count3; + rk[165] ^= sc->count2; + rk[166] ^= sc->count1; + rk[167] ^= SPH_T32(~sc->count0); + } else if (u == 316) { + rk[316] ^= sc->count2; + rk[317] ^= sc->count3; + rk[318] ^= sc->count0; + rk[319] ^= SPH_T32(~sc->count1); + } + u += 4; + } + if (u == 448) + break; + for (int s = 0; s < 8; s++) { + rk[u + 0] = rk[u - 32] ^ rk[u - 7]; + rk[u + 1] = rk[u - 31] ^ rk[u - 6]; + rk[u + 2] = rk[u - 30] ^ rk[u - 5]; + rk[u + 3] = rk[u - 29] ^ rk[u - 4]; + u += 4; + if (u == 448) + break; + } + } + + for (int i{0}; i < (448/4); i++) { + rk_words[i] = vreinterpretq_u8_u32(vld1q_u32(&rk[i*4])); + } + + p0 = sc->h[0x0]; + p1 = sc->h[0x1]; + p2 = sc->h[0x2]; + p3 = sc->h[0x3]; + p4 = sc->h[0x4]; + p5 = sc->h[0x5]; + p6 = sc->h[0x6]; + p7 = sc->h[0x7]; + p8 = sc->h[0x8]; + p9 = sc->h[0x9]; + pA = sc->h[0xA]; + pB = sc->h[0xB]; + pC = sc->h[0xC]; + pD = sc->h[0xD]; + pE = sc->h[0xE]; + pF = sc->h[0xF]; + + size_t u_words{0}; + for (size_t r{0}; r < 14; r++) { + CompressElement(p0, p1, p2, p3, p4, p5, p6, p7, &rk_words[u_words]); + u_words += 4; + CompressElement(p8, p9, pA, pB, pC, pD, pE, pF, &rk_words[u_words]); + u_words += 4; + +#define WROT(a, b, c, d) do { \ + uint32_t t = d; \ + d = c; \ + c = b; \ + b = a; \ + a = t; \ + } while (0) + + WROT(p0, p4, p8, pC); + WROT(p1, p5, p9, pD); + WROT(p2, p6, pA, pE); + WROT(p3, p7, pB, pF); + +#undef WROT + } + + sc->h[0x0] ^= p0; + sc->h[0x1] ^= p1; + sc->h[0x2] ^= p2; + sc->h[0x3] ^= p3; + sc->h[0x4] ^= p4; + sc->h[0x5] ^= p5; + sc->h[0x6] ^= p6; + sc->h[0x7] ^= p7; + sc->h[0x8] ^= p8; + sc->h[0x9] ^= p9; + sc->h[0xA] ^= pA; + sc->h[0xB] ^= pB; + sc->h[0xC] ^= pC; + sc->h[0xD] ^= pD; + sc->h[0xE] ^= pE; + sc->h[0xF] ^= pF; +} } // namespace arm_crypto_shavite } // namespace sapphire diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index 65b795daad72..cbc5d8133b4e 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -50,8 +50,7 @@ namespace arm_crypto_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace arm_crypto_echo namespace arm_crypto_shavite { -void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, - uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +void Compress(sph_shavite_big_context *sc, const void *msg); } // namespace arm_crypto_shavite #endif // ENABLE_ARM_AES @@ -79,8 +78,7 @@ namespace x86_aesni_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace x86_aesni_echo namespace x86_aesni_shavite { -void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, - uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +void Compress(sph_shavite_big_context *sc, const void *msg); } // namespace x86_aesni_shavite #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #endif // !DISABLE_OPTIMIZED_SHA256 @@ -97,8 +95,7 @@ void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, void ShiftAndMix(uint64_t W[16][2]); } // namespace soft_echo namespace soft_shavite { -void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, - uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk); +void Compress(sph_shavite_big_context *sc, const void *msg); } // namespace soft_shavite } // namespace sapphire @@ -121,7 +118,7 @@ extern sapphire::dispatch::AESRoundFn aes_round; extern sapphire::dispatch::AESRoundFnNk aes_round_nk; extern sapphire::dispatch::EchoShiftMix echo_shift_mix; extern sapphire::dispatch::EchoRoundFn echo_round; -extern sapphire::dispatch::ShaviteCompressFn shavite_c512e; +extern sapphire::dispatch::ShaviteCompressFn shavite_c512; void SapphireAutoDetect() { @@ -129,7 +126,7 @@ void SapphireAutoDetect() aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; echo_shift_mix = sapphire::soft_echo::ShiftAndMix; - shavite_c512e = sapphire::soft_shavite::CompressElement; + shavite_c512 = sapphire::soft_shavite::Compress; #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(HAVE_GETCPUID) @@ -142,7 +139,7 @@ void SapphireAutoDetect() aes_round = sapphire::x86_aesni_aes::Round; aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; echo_round = sapphire::x86_aesni_echo::FullStateRound; - shavite_c512e = sapphire::x86_aesni_shavite::CompressElement; + shavite_c512 = sapphire::x86_aesni_shavite::Compress; } #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #if defined(ENABLE_SSSE3) @@ -198,7 +195,7 @@ void SapphireAutoDetect() aes_round = sapphire::arm_crypto_aes::Round; aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; echo_round = sapphire::arm_crypto_echo::FullStateRound; - shavite_c512e = sapphire::arm_crypto_shavite::CompressElement; + shavite_c512 = sapphire::arm_crypto_shavite::Compress; } #endif // ENABLE_ARM_AES diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index a648b055d5bb..4cc5edac0808 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CRYPTO_X11_DISPATCH_H #define BITCOIN_CRYPTO_X11_DISPATCH_H +#include + #include namespace sapphire { @@ -18,8 +20,7 @@ typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); typedef void (*EchoShiftMix)(uint64_t[16][2]); -typedef void (*ShaviteCompressFn)(uint32_t&, uint32_t&, uint32_t&, uint32_t&, - uint32_t, uint32_t, uint32_t, uint32_t, const uint32_t*); +typedef void (*ShaviteCompressFn)(sph_shavite_big_context*, const void *); } // namespace dispatch } // namespace sapphire diff --git a/src/crypto/x11/shavite.cpp b/src/crypto/x11/shavite.cpp index 9cd773d085ba..e928668d52b6 100644 --- a/src/crypto/x11/shavite.cpp +++ b/src/crypto/x11/shavite.cpp @@ -38,16 +38,37 @@ #include "sph_shavite.h" -extern sapphire::dispatch::AESRoundFnNk aes_round_nk; - #ifdef _MSC_VER #pragma warning (disable: 4146) #endif #define C32 SPH_C32 +/* + * As of round 2 of the SHA-3 competition, the published reference + * implementation and test vectors are wrong, because they use + * big-endian AES tables while the internal decoding uses little-endian. + * The code below follows the specification. To turn it into a code + * which follows the reference implementation (the one called "BugFix" + * on the SHAvite-3 web site, published on Nov 23rd, 2009), comment out + * the code below (from the '#define AES_BIG_ENDIAN...' to the definition + * of the AES_ROUND_NOKEY macro) and replace it with the version which + * is commented out afterwards. + */ + +static const sph_u32 IV512[] = { + C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), + C32(0xD1901A06), C32(0x430AE307), C32(0xB29F5CD1), C32(0xDF07FBFC), + C32(0x8E45D73D), C32(0x681AB538), C32(0xBDE86578), C32(0xDD577E47), + C32(0xE275EADE), C32(0x502D9FCD), C32(0xB9357178), C32(0x022A4B9A) +}; + +#define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ + sapphire::soft_aes::RoundKeyless(x0, x1, x2, x3, x0, x1, x2, x3); \ + } while (0) + namespace sapphire { -namespace soft_shavite { +namespace { void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) { @@ -76,39 +97,13 @@ void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, l2 ^= x2; l3 ^= x3; } -} // namespace shavite_soft -} // namespace sapphire - -sapphire::dispatch::ShaviteCompressFn shavite_c512e = sapphire::soft_shavite::CompressElement; - -/* - * As of round 2 of the SHA-3 competition, the published reference - * implementation and test vectors are wrong, because they use - * big-endian AES tables while the internal decoding uses little-endian. - * The code below follows the specification. To turn it into a code - * which follows the reference implementation (the one called "BugFix" - * on the SHAvite-3 web site, published on Nov 23rd, 2009), comment out - * the code below (from the '#define AES_BIG_ENDIAN...' to the definition - * of the AES_ROUND_NOKEY macro) and replace it with the version which - * is commented out afterwards. - */ - -static const sph_u32 IV512[] = { - C32(0x72FCCDD8), C32(0x79CA4727), C32(0x128A077B), C32(0x40D55AEC), - C32(0xD1901A06), C32(0x430AE307), C32(0xB29F5CD1), C32(0xDF07FBFC), - C32(0x8E45D73D), C32(0x681AB538), C32(0xBDE86578), C32(0xDD577E47), - C32(0xE275EADE), C32(0x502D9FCD), C32(0xB9357178), C32(0x022A4B9A) -}; - -#define AES_ROUND_NOKEY(x0, x1, x2, x3) do { \ - aes_round_nk(x0, x1, x2, x3, x0, x1, x2, x3); \ - } while (0) +} // anonymous namespace +namespace soft_shavite { /* * This function assumes that "msg" is aligned for 32-bit access. */ -static void -c512(sph_shavite_big_context *sc, const void *msg) +void Compress(sph_shavite_big_context *sc, const void *msg) { sph_u32 p0, p1, p2, p3, p4, p5, p6, p7; sph_u32 p8, p9, pA, pB, pC, pD, pE, pF; @@ -209,7 +204,7 @@ c512(sph_shavite_big_context *sc, const void *msg) u = 0; for (r = 0; r < 14; r ++) { #define C512_ELT(l0, l1, l2, l3, r0, r1, r2, r3) do { \ - shavite_c512e(l0, l1, l2, l3, r0, r1, r2, r3, &rk[u]); \ + CompressElement(l0, l1, l2, l3, r0, r1, r2, r3, &rk[u]); \ u += 16; \ } while (0) @@ -249,6 +244,10 @@ c512(sph_shavite_big_context *sc, const void *msg) sc->h[0xE] ^= pE; sc->h[0xF] ^= pF; } +} // namespace shavite_soft +} // namespace sapphire + +sapphire::dispatch::ShaviteCompressFn shavite_c512 = sapphire::soft_shavite::Compress; static void shavite_big_init(sph_shavite_big_context *sc, const sph_u32 *iv) @@ -290,7 +289,7 @@ shavite_big_core(sph_shavite_big_context *sc, const void *data, size_t len) } } } - c512(sc, buf); + shavite_c512(sc, buf); ptr = 0; } } @@ -324,7 +323,7 @@ shavite_big_close(sph_shavite_big_context *sc, } else { buf[ptr ++] = z; memset(buf + ptr, 0, 128 - ptr); - c512(sc, buf); + shavite_c512(sc, buf); memset(buf, 0, 110); sc->count0 = sc->count1 = sc->count2 = sc->count3 = 0; } @@ -334,7 +333,7 @@ shavite_big_close(sph_shavite_big_context *sc, sph_enc32le(buf + 122, count3); buf[126] = out_size_w32 << 5; buf[127] = out_size_w32 >> 3; - c512(sc, buf); + shavite_c512(sc, buf); for (u = 0; u < out_size_w32; u ++) sph_enc32le((unsigned char *)dst + (u << 2), sc->h[u]); } diff --git a/src/crypto/x11/x86_aesni/shavite.cpp b/src/crypto/x11/x86_aesni/shavite.cpp index f1810a6d6863..3bb8d9d16ba5 100644 --- a/src/crypto/x11/x86_aesni/shavite.cpp +++ b/src/crypto/x11/x86_aesni/shavite.cpp @@ -3,38 +3,182 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) +#include #include #include +#include #include #include namespace sapphire { -namespace x86_aesni_shavite { +namespace { void CompressElement(uint32_t& l0, uint32_t& l1, uint32_t& l2, uint32_t& l3, - uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const uint32_t* rk) + uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3, const __m128i* rk_words) { - // Load block + XOR with round key 1 __m128i block = util::pack_le(r0, r1, r2, r3); - block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[0])); - // AES round + XOR with round key 2 - block = util::aes_round(block, _mm_setzero_si128()); - block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[4])); - // AES round + XOR with round key 3 - block = util::aes_round(block, _mm_setzero_si128()); - block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[8])); - // AES Round + XOR with round key 4 + block = util::Xor(block, rk_words[0]); + block = util::Xor(util::aes_round(block, _mm_setzero_si128()), rk_words[1]); + block = util::Xor(util::aes_round(block, _mm_setzero_si128()), rk_words[2]); + block = util::Xor(util::aes_round(block, _mm_setzero_si128()), rk_words[3]); block = util::aes_round(block, _mm_setzero_si128()); - block = util::Xor(block, _mm_load_si128((const __m128i*)&rk[12])); - // AES round - block = util::aes_round(block, _mm_setzero_si128()); - // Unpack + XOR with l values l0 ^= _mm_extract_epi32(block, 0); l1 ^= _mm_extract_epi32(block, 1); l2 ^= _mm_extract_epi32(block, 2); l3 ^= _mm_extract_epi32(block, 3); } +} // anonymous namespace + +namespace x86_aesni_shavite { +void Compress(sph_shavite_big_context *sc, const void *msg) +{ + uint32_t p0, p1, p2, p3, p4, p5, p6, p7; + uint32_t p8, p9, pA, pB, pC, pD, pE, pF; + + alignas(16) __m128i rk_words[448/4]; + alignas(16) uint32_t rk[448]; + +#if SPH_LITTLE_ENDIAN + memcpy(rk, msg, 128); +#else + for (size_t u{0}; u < 32; u += 4) { + rk[u + 0] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 0); + rk[u + 1] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 4); + rk[u + 2] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 8); + rk[u + 3] = sph_dec32le_aligned( + (const unsigned char *)msg + (u << 2) + 12); + } +#endif + + size_t u{32}; + for (;;) { + for (int s{0}; s < 4; s++) { + uint32_t x0 = rk[u - 31]; + uint32_t x1 = rk[u - 30]; + uint32_t x2 = rk[u - 29]; + uint32_t x3 = rk[u - 32]; + + __m128i block = util::aes_round(util::pack_le(x0, x1, x2, x3), _mm_setzero_si128()); + rk[u + 0] = _mm_extract_epi32(block, 0) ^ rk[u - 4]; + rk[u + 1] = _mm_extract_epi32(block, 1) ^ rk[u - 3]; + rk[u + 2] = _mm_extract_epi32(block, 2) ^ rk[u - 2]; + rk[u + 3] = _mm_extract_epi32(block, 3) ^ rk[u - 1]; + + if (u == 32) { + rk[32] ^= sc->count0; + rk[33] ^= sc->count1; + rk[34] ^= sc->count2; + rk[35] ^= SPH_T32(~sc->count3); + } else if (u == 440) { + rk[440] ^= sc->count1; + rk[441] ^= sc->count0; + rk[442] ^= sc->count3; + rk[443] ^= SPH_T32(~sc->count2); + } + u += 4; + + x0 = rk[u - 31]; + x1 = rk[u - 30]; + x2 = rk[u - 29]; + x3 = rk[u - 32]; + + block = util::aes_round(util::pack_le(x0, x1, x2, x3), _mm_setzero_si128()); + rk[u + 0] = _mm_extract_epi32(block, 0) ^ rk[u - 4]; + rk[u + 1] = _mm_extract_epi32(block, 1) ^ rk[u - 3]; + rk[u + 2] = _mm_extract_epi32(block, 2) ^ rk[u - 2]; + rk[u + 3] = _mm_extract_epi32(block, 3) ^ rk[u - 1]; + + if (u == 164) { + rk[164] ^= sc->count3; + rk[165] ^= sc->count2; + rk[166] ^= sc->count1; + rk[167] ^= SPH_T32(~sc->count0); + } else if (u == 316) { + rk[316] ^= sc->count2; + rk[317] ^= sc->count3; + rk[318] ^= sc->count0; + rk[319] ^= SPH_T32(~sc->count1); + } + u += 4; + } + if (u == 448) + break; + for (int s = 0; s < 8; s++) { + rk[u + 0] = rk[u - 32] ^ rk[u - 7]; + rk[u + 1] = rk[u - 31] ^ rk[u - 6]; + rk[u + 2] = rk[u - 30] ^ rk[u - 5]; + rk[u + 3] = rk[u - 29] ^ rk[u - 4]; + u += 4; + if (u == 448) + break; + } + } + + for (int i{0}; i < (448/4); i++) { + rk_words[i] = _mm_load_si128((const __m128i*)&rk[i*4]); + } + + p0 = sc->h[0x0]; + p1 = sc->h[0x1]; + p2 = sc->h[0x2]; + p3 = sc->h[0x3]; + p4 = sc->h[0x4]; + p5 = sc->h[0x5]; + p6 = sc->h[0x6]; + p7 = sc->h[0x7]; + p8 = sc->h[0x8]; + p9 = sc->h[0x9]; + pA = sc->h[0xA]; + pB = sc->h[0xB]; + pC = sc->h[0xC]; + pD = sc->h[0xD]; + pE = sc->h[0xE]; + pF = sc->h[0xF]; + + size_t u_words{0}; + for (size_t r{0}; r < 14; r++) { + CompressElement(p0, p1, p2, p3, p4, p5, p6, p7, &rk_words[u_words]); + u_words += 4; + CompressElement(p8, p9, pA, pB, pC, pD, pE, pF, &rk_words[u_words]); + u_words += 4; + +#define WROT(a, b, c, d) do { \ + uint32_t t = d; \ + d = c; \ + c = b; \ + b = a; \ + a = t; \ + } while (0) + + WROT(p0, p4, p8, pC); + WROT(p1, p5, p9, pD); + WROT(p2, p6, pA, pE); + WROT(p3, p7, pB, pF); + +#undef WROT + } + + sc->h[0x0] ^= p0; + sc->h[0x1] ^= p1; + sc->h[0x2] ^= p2; + sc->h[0x3] ^= p3; + sc->h[0x4] ^= p4; + sc->h[0x5] ^= p5; + sc->h[0x6] ^= p6; + sc->h[0x7] ^= p7; + sc->h[0x8] ^= p8; + sc->h[0x9] ^= p9; + sc->h[0xA] ^= pA; + sc->h[0xB] ^= pB; + sc->h[0xC] ^= pC; + sc->h[0xD] ^= pD; + sc->h[0xE] ^= pE; + sc->h[0xF] ^= pF; +} } // namespace x86_aesni_shavite } // namespace sapphire From 005967b4d823ce28bd88b3d9009782b1b3ff1848 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 21 Sep 2025 14:30:50 +0000 Subject: [PATCH 21/21] crypto: drop naive AES backends --- src/Makefile.am | 2 -- src/crypto/x11/aes.cpp | 3 --- src/crypto/x11/arm_crypto/aes.cpp | 31 ------------------------------ src/crypto/x11/dispatch.cpp | 29 ---------------------------- src/crypto/x11/dispatch.h | 6 ------ src/crypto/x11/x86_aesni/aes.cpp | 32 ------------------------------- 6 files changed, 103 deletions(-) delete mode 100644 src/crypto/x11/arm_crypto/aes.cpp delete mode 100644 src/crypto/x11/x86_aesni/aes.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 562f4a3afe46..137f6974aa8a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -789,7 +789,6 @@ crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_arm_aes_la_CXXFLAGS += $(ARM_AES_CXXFLAGS) crypto_libbitcoin_crypto_arm_aes_la_CPPFLAGS += -DENABLE_ARM_AES crypto_libbitcoin_crypto_arm_aes_la_SOURCES = \ - crypto/x11/arm_crypto/aes.cpp \ crypto/x11/arm_crypto/echo.cpp \ crypto/x11/arm_crypto/shavite.cpp @@ -821,7 +820,6 @@ crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_x86_aesni_la_CXXFLAGS += $(X86_AESNI_CXXFLAGS) crypto_libbitcoin_crypto_x86_aesni_la_CPPFLAGS += -DENABLE_SSE41 -DENABLE_X86_AESNI crypto_libbitcoin_crypto_x86_aesni_la_SOURCES = \ - crypto/x11/x86_aesni/aes.cpp \ crypto/x11/x86_aesni/echo.cpp \ crypto/x11/x86_aesni/shavite.cpp diff --git a/src/crypto/x11/aes.cpp b/src/crypto/x11/aes.cpp index c874cbab05e8..dc3188971ce0 100644 --- a/src/crypto/x11/aes.cpp +++ b/src/crypto/x11/aes.cpp @@ -27,6 +27,3 @@ void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, } } // namespace soft_aes } // namespace sapphire - -sapphire::dispatch::AESRoundFn aes_round = sapphire::soft_aes::Round; -sapphire::dispatch::AESRoundFnNk aes_round_nk = sapphire::soft_aes::RoundKeyless; diff --git a/src/crypto/x11/arm_crypto/aes.cpp b/src/crypto/x11/arm_crypto/aes.cpp deleted file mode 100644 index fd5ee31e7aa2..000000000000 --- a/src/crypto/x11/arm_crypto/aes.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2025 The Dash Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(ENABLE_ARM_AES) -#include - -#include - -#include - -namespace sapphire { -namespace arm_crypto_aes { -void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) -{ - uint8x16_t block = util::aes_round(util::pack_le(x0, x1, x2, x3), util::pack_le(k0, k1, k2, k3)); - util::unpack_le(block, y0, y1, y2, y3); -} - -void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) -{ - uint8x16_t block = util::aes_round_nk(util::pack_le(x0, x1, x2, x3)); - util::unpack_le(block, y0, y1, y2, y3); -} -} // namespace arm_crypto_aes -} // namespace sapphire - -#endif // ENABLE_ARM_AES diff --git a/src/crypto/x11/dispatch.cpp b/src/crypto/x11/dispatch.cpp index cbc5d8133b4e..5bc413c32dd6 100644 --- a/src/crypto/x11/dispatch.cpp +++ b/src/crypto/x11/dispatch.cpp @@ -39,13 +39,6 @@ namespace sapphire { #if !defined(DISABLE_OPTIMIZED_SHA256) #if defined(ENABLE_ARM_AES) -namespace arm_crypto_aes { -void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -} // namespace arm_crypto_aes namespace arm_crypto_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace arm_crypto_echo @@ -67,13 +60,6 @@ void ShiftAndMix(uint64_t W[16][2]); #endif // ENABLE_SSSE3 #if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) -namespace x86_aesni_aes { -void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -} // namespace x86_aesni_aes namespace x86_aesni_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); } // namespace x86_aesni_echo @@ -83,13 +69,6 @@ void Compress(sph_shavite_big_context *sc, const void *msg); #endif // ENABLE_SSE41 && ENABLE_X86_AESNI #endif // !DISABLE_OPTIMIZED_SHA256 -namespace soft_aes { -void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3); -} // namespace soft_aes namespace soft_echo { void FullStateRound(uint64_t W[16][2], uint32_t& k0, uint32_t& k1, uint32_t& k2, uint32_t& k3); void ShiftAndMix(uint64_t W[16][2]); @@ -114,16 +93,12 @@ bool IsSysCtlNonZero(const char* name) #endif // !DISABLE_OPTIMIZED_SHA256 } // anonymous namespace -extern sapphire::dispatch::AESRoundFn aes_round; -extern sapphire::dispatch::AESRoundFnNk aes_round_nk; extern sapphire::dispatch::EchoShiftMix echo_shift_mix; extern sapphire::dispatch::EchoRoundFn echo_round; extern sapphire::dispatch::ShaviteCompressFn shavite_c512; void SapphireAutoDetect() { - aes_round = sapphire::soft_aes::Round; - aes_round_nk = sapphire::soft_aes::RoundKeyless; echo_round = sapphire::soft_echo::FullStateRound; echo_shift_mix = sapphire::soft_echo::ShiftAndMix; shavite_c512 = sapphire::soft_shavite::Compress; @@ -136,8 +111,6 @@ void SapphireAutoDetect() const bool use_sse_4_1 = ((ecx >> 19) & 1); const bool use_aes_ni = ((ecx >> 25) & 1); if (use_sse_4_1 && use_aes_ni) { - aes_round = sapphire::x86_aesni_aes::Round; - aes_round_nk = sapphire::x86_aesni_aes::RoundKeyless; echo_round = sapphire::x86_aesni_echo::FullStateRound; shavite_c512 = sapphire::x86_aesni_shavite::Compress; } @@ -192,8 +165,6 @@ void SapphireAutoDetect() #if defined(ENABLE_ARM_AES) if (have_arm_aes) { - aes_round = sapphire::arm_crypto_aes::Round; - aes_round_nk = sapphire::arm_crypto_aes::RoundKeyless; echo_round = sapphire::arm_crypto_echo::FullStateRound; shavite_c512 = sapphire::arm_crypto_shavite::Compress; } diff --git a/src/crypto/x11/dispatch.h b/src/crypto/x11/dispatch.h index 4cc5edac0808..c46562b38b90 100644 --- a/src/crypto/x11/dispatch.h +++ b/src/crypto/x11/dispatch.h @@ -11,12 +11,6 @@ namespace sapphire { namespace dispatch { -typedef void (*AESRoundFn)(uint32_t, uint32_t, uint32_t, uint32_t, - uint32_t, uint32_t, uint32_t, uint32_t, - uint32_t&, uint32_t&, uint32_t&, uint32_t&); -typedef void (*AESRoundFnNk)(uint32_t, uint32_t, uint32_t, uint32_t, - uint32_t&, uint32_t&, uint32_t&, uint32_t&); - typedef void (*EchoRoundFn)(uint64_t[16][2], uint32_t&, uint32_t&, uint32_t&, uint32_t&); typedef void (*EchoShiftMix)(uint64_t[16][2]); diff --git a/src/crypto/x11/x86_aesni/aes.cpp b/src/crypto/x11/x86_aesni/aes.cpp deleted file mode 100644 index aee045faa173..000000000000 --- a/src/crypto/x11/x86_aesni/aes.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2025 The Dash Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(ENABLE_SSE41) && defined(ENABLE_X86_AESNI) -#include - -#include - -#include -#include - -namespace sapphire { -namespace x86_aesni_aes { -void Round(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t k0, uint32_t k1, uint32_t k2, uint32_t k3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) -{ - __m128i block = util::aes_round(util::pack_le(x0, x1, x2, x3), util::pack_le(k0, k1, k2, k3)); - util::unpack_le(block, y0, y1, y2, y3); -} - -void RoundKeyless(uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3, - uint32_t& y0, uint32_t& y1, uint32_t& y2, uint32_t& y3) -{ - __m128i block = util::aes_round(util::pack_le(x0, x1, x2, x3), _mm_setzero_si128()); - util::unpack_le(block, y0, y1, y2, y3); -} -} // namespace x86_aesni_aes -} // namespace sapphire - -#endif // ENABLE_SSE41 && ENABLE_X86_AESNI