From 7e67392a3d1cf0a113388b2cff40d0c395b70561 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Tue, 26 Oct 2021 01:57:20 +0300 Subject: [PATCH 01/13] Add ARM64 detection via asm test test --- CMakeLists.txt | 8 +- include/cpu_features_macros.h | 2 +- include/cpuinfo_aarch64.h | 1 - include/cpuinfo_core_aarch64.h | 179 +++++++++++ include/internal/cputype_aarch64.h | 123 ++++++++ src/cpuinfo_core_aarch64.c | 471 +++++++++++++++++++++++++++++ 6 files changed, 780 insertions(+), 4 deletions(-) create mode 100644 include/cpuinfo_core_aarch64.h create mode 100644 include/internal/cputype_aarch64.h create mode 100644 src/cpuinfo_core_aarch64.c diff --git a/CMakeLists.txt b/CMakeLists.txt index f5d5fc7d..57dad59a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,10 +52,10 @@ set(PROCESSOR_IS_POWER FALSE) if(CMAKE_SYSTEM_PROCESSOR MATCHES "^mips") set(PROCESSOR_IS_MIPS TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(^aarch64)|(^arm64)") + set(PROCESSOR_IS_AARCH64 TRUE) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") set(PROCESSOR_IS_ARM TRUE) -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64") - set(PROCESSOR_IS_AARCH64 TRUE) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") set(PROCESSOR_IS_X86 TRUE) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)") @@ -73,6 +73,10 @@ macro(add_cpu_features_headers_and_sources HDRS_LIST_NAME SRCS_LIST_NAME) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_arm.h) elseif(PROCESSOR_IS_AARCH64) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_aarch64.h) + list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cputype_aarch64.h) + list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_core_aarch64.h) + list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/src/cpuinfo_aarch64.c) + list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/src/cpuinfo_core_aarch64.c) elseif(PROCESSOR_IS_X86) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_x86.h) list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_x86.h) diff --git a/include/cpu_features_macros.h b/include/cpu_features_macros.h index 8f5c38ca..384f31c5 100644 --- a/include/cpu_features_macros.h +++ b/include/cpu_features_macros.h @@ -39,7 +39,7 @@ #define CPU_FEATURES_ARCH_ARM #endif -#if defined(__aarch64__) +#if (defined(__aarch64__) || defined(__arm64__)) #define CPU_FEATURES_ARCH_AARCH64 #endif diff --git a/include/cpuinfo_aarch64.h b/include/cpuinfo_aarch64.h index 1b57d218..75efd1cf 100644 --- a/include/cpuinfo_aarch64.h +++ b/include/cpuinfo_aarch64.h @@ -72,7 +72,6 @@ typedef struct { int rng : 1; // True random number generator support. int bti : 1; // Branch target identification. int mte : 1; // Memory tagging extension. - // Make sure to update Aarch64FeaturesEnum below if you add a field here. } Aarch64Features; diff --git a/include/cpuinfo_core_aarch64.h b/include/cpuinfo_core_aarch64.h new file mode 100644 index 00000000..de60c729 --- /dev/null +++ b/include/cpuinfo_core_aarch64.h @@ -0,0 +1,179 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CPU_FEATURES_CPUINFO_CORE_AARCH64_H +#define CPU_FEATURES_CPUINFO_CORE_AARCH64_H + +#include "cpu_features_macros.h" +#include "internal/cputype_aarch64.h" + +CPU_FEATURES_START_CPP_NAMESPACE + +/////////////////////////////////////////////////////////////////////////////// +// Aarch64 info via mrs instruction +/////////////////////////////////////////////////////////////////////////////// + +typedef enum { + UNKNOWN_IMPL_AARCH64, + RESERVED, + ARM, + CAVIUM, + BRCM, + HISI, + DEQ, + FUJITSU, + INFINEON, + MOTOROLA, + NVIDIA, + APM, + QCOM, + MARVELL, + APPLE, + INTEL, + AMPERE +} Aarch64Implementer; + +typedef enum { + UNKNOWN_PART_NUM_AARCH64, + AEM_V8, + FOUNDATION, + CORTEX_A35, + CORTEX_A53, + CORTEX_A55, + CORTEX_A57, + CORTEX_A72, + CORTEX_A73, + CORTEX_A75, + CORTEX_A76, + CORTEX_A77, + NEOVERSE_N1, + POTENZA, + THUNDERX, + THUNDERX_81XX, + THUNDERX_83XX, + THUNDERX2, + BRAHMA_B53, + VULCAN, + FALKOR_V1, + FALKOR, + KRYO, + KRYO_2XX_GOLD, + KRYO_2XX_SILVER, + KRYO_3XX_SILVER, + KRYO_4XX_GOLD, + KRYO_4XX_SILVER, + DENVER, + CARMEL, + A64FX, + TSV110, + M1_ICESTORM, + M1_FIRESTORM, + M1_ICESTORM_TONGA, + M1_FIRESTORM_TONGA +} Aarch64PartNum; + +typedef enum { + UNKNOWN_ARCH_AARCH64, + ARMv4, + ARMv4T, + ARMv5, + ARMv5T, + ARMv5TE, + ARMv5TEJ, + ARMv6, + ARMv7, + ARMv7F, + ARMv7S, + ARMv7K, + ARMv8, + ARMv8E +} Aarch64Architecture; + +typedef struct { + int fp; // Floating-point. + int fphp; // Half-precision floating-point arithmetic. + int i8mm; // Int8 matrix multiplication instructions. + int bf16; // BFloat16 instructions. + int gpi; // Indicates support for an IMPLEMENTATION DEFINED. + int gpa; // Indicates whether the QARMA5 algorithm is implemented. + int lrcpc; // Support for weaker release consistency. + int fcma; // Floating point complex numbers. + int jscvt; // Support for JavaScript conversion. + int api; // Indicates whether an IMPLEMENTATION DEFINED algorithm. + int apa; // Indicates whether the QARMA5 algorithm is implemented. + int dpb; // Indicates support for the DC CVAP and DC CVADP. + int ls64; // Indicates support for LD64B and ST64B* instructions. + int xs; // Indicates support for the XS attribute. + int specres; // Indicates support for prediction invalid instructions. + int sb; // Speculation barrier. + int frintts; // Indicates support for the FRINT32Z,FRINT32X,FRINT64Z. + int dp; // Indicates support for Dot Product instructions. + int dgh; // Data Gathering Hint instruction. + int fhm; // Indicates support for FMLAL and FMLSL instructions. + int ts; // Indicates support for flag manipulation instructions. + int tlb; // Indicates support for Outer shareable and TLB range. + int rndr; // Indicates support for Random Number instructions. + int amuv1; // Indicates support for Activity Monitors Extension. + int amuv1p1; // Support for virtualization of the activity monitor + int mpam; // Indicates support for MPAM Extension. + int sve; // Scalable Vector Extension. + int dit; // Data independent timing. + int sel2; // Secure EL2. + int ras; // RAS Extension version. + int rasv1p1; // FEAT_RASv1p1. + int double_fault; // FEAT_DoubleFault. + int gicv4; // System register interface to versions 3.0 and 4.0 of the + // GIC CPU interface is supported. + int gicv41; // System register interface to version 4.1 of the GIC CPU + // interface is supported. + int asimd; // Advanced SIMD. + int asimdfhm; // Additional half-precision instructions. + int el3_64; // EL3 can be executed in AArch64 state only. + int el3_32_64; // EL3 can be executed in either AArch64 or AArch32 state. + int el2_64; // EL2 can be executed in AArch64 state only. + int el2_32_64; // EL2 can be executed in either AArch64 or AArch32 state. + int el1; // EL1 Exception level handling. + int el0; // EL0 Exception level handling. + int aes; // Hardware-accelerated Advanced Encryption Standard. + int sha1; // Hardware-accelerated SHA1. + int sha2; // Hardware-accelerated SHA2. + int crc32; // Hardware-accelerated CRC-32. + int atomics; // Armv8.1 atomic instructions. + int rdm; // Indicates support for SQRDMLAH and SQRDMLSH + int sha3; // Hardware-accelerated SHA3. + int sm3; // Hardware-accelerated SM3. + int sm4; // Hardware-accelerated SM4. +} Aarch64FeaturesCpuid; + +typedef struct { + Aarch64FeaturesCpuid features; + Aarch64Implementer implementer; + Aarch64PartNum part; + Aarch64Architecture architecture; + int variant; + int revision; +} Aarch64InfoCpuid; + +Aarch64InfoCpuid GetAarch64InfoCpuid(void); +const char* GetAarch64ImplementerString(const Aarch64InfoCpuid* info); +const char* GetAarch64PartNumString(const Aarch64InfoCpuid* info); +const char* GetAarch64ArchitectureString(const Aarch64InfoCpuid* info); + +CPU_FEATURES_END_CPP_NAMESPACE + +#if !defined(CPU_FEATURES_ARCH_AARCH64) +#error "Including cpuinfo_core_aarch64.h from a non-aarch64 target." +#endif + +#endif // CPU_FEATURES_CPUINFO_CORE_AARCH64_H diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h new file mode 100644 index 00000000..b1bc657f --- /dev/null +++ b/include/internal/cputype_aarch64.h @@ -0,0 +1,123 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H +#define CPU_FEATURES_CPUTYPE_AARCH64_H + +#include "cpu_features_macros.h" +#include "stdint.h" + +CPU_FEATURES_START_CPP_NAMESPACE + +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +#define CPU_ARCH_ARMv4 0x1 +#define CPU_ARCH_ARMv4T 0x2 +#define CPU_ARCH_ARMv5 0x3 +#define CPU_ARCH_ARMv5T 0x4 +#define CPU_ARCH_ARMv5TE 0x5 +#define CPU_ARCH_ARMv5TEJ 0x6 +#define CPU_ARCH_ARMv6 0x7 +#define CPU_ARCH_ARMv7 0x8 +#define CPU_ARCH_ARMv7F 0x9 +#define CPU_ARCH_ARMv7S 0xA +#define CPU_ARCH_ARMv7K 0xB +#define CPU_ARCH_ARMv8 0xC +#define CPU_ARCH_ARMv8E 0xD + +// Special code indicating we need to look somewhere else for +// the architecture version. +#define CPU_ARCH_ARM_EXTENDED 0xF + +#define ARM_CPU_IMP_RESERVED 0x00 +#define ARM_CPU_IMP_ARM 0x41 +#define ARM_CPU_IMP_BRCM 0x42 +#define ARM_CPU_IMP_CAVIUM 0x43 +#define ARM_CPU_IMP_DEQ 0x44 +#define ARM_CPU_IMP_FUJITSU 0x46 +#define ARM_CPU_IMP_HISI 0x48 +#define ARM_CPU_IMP_INFINEON 0x49 +#define ARM_CPU_IMP_MOTOROLA 0x4D +#define ARM_CPU_IMP_NVIDIA 0x4E +#define ARM_CPU_IMP_APM 0x50 +#define ARM_CPU_IMP_QCOM 0x51 +#define ARM_CPU_IMP_MARVELL 0x56 +#define ARM_CPU_IMP_APPLE 0x61 +#define ARM_CPU_IMP_INTEL 0x69 +#define ARM_CPU_IMP_AMPERE 0xC0 + +#define ARM_CPU_PART_AEM_V8 0xD0F +#define ARM_CPU_PART_FOUNDATION 0xD00 +#define ARM_CPU_PART_CORTEX_A57 0xD07 +#define ARM_CPU_PART_CORTEX_A72 0xD08 +#define ARM_CPU_PART_CORTEX_A53 0xD03 +#define ARM_CPU_PART_CORTEX_A73 0xD09 +#define ARM_CPU_PART_CORTEX_A75 0xD0A +#define ARM_CPU_PART_CORTEX_A35 0xD04 +#define ARM_CPU_PART_CORTEX_A55 0xD05 +#define ARM_CPU_PART_CORTEX_A76 0xD0B +#define ARM_CPU_PART_NEOVERSE_N1 0xD0C +#define ARM_CPU_PART_CORTEX_A77 0xD0D + +#define APM_CPU_PART_POTENZA 0x000 + +#define CAVIUM_CPU_PART_THUNDERX 0x0A1 +#define CAVIUM_CPU_PART_THUNDERX_81XX 0x0A2 +#define CAVIUM_CPU_PART_THUNDERX_83XX 0x0A3 +#define CAVIUM_CPU_PART_THUNDERX2 0x0AF + +#define BRCM_CPU_PART_BRAHMA_B53 0x100 +#define BRCM_CPU_PART_VULCAN 0x516 + +#define QCOM_CPU_PART_FALKOR_V1 0x800 +#define QCOM_CPU_PART_FALKOR 0xC00 +#define QCOM_CPU_PART_KRYO 0x200 +#define QCOM_CPU_PART_KRYO_2XX_GOLD 0x800 +#define QCOM_CPU_PART_KRYO_2XX_SILVER 0x801 +#define QCOM_CPU_PART_KRYO_3XX_SILVER 0x803 +#define QCOM_CPU_PART_KRYO_4XX_GOLD 0x804 +#define QCOM_CPU_PART_KRYO_4XX_SILVER 0x805 + +#define NVIDIA_CPU_PART_DENVER 0x003 +#define NVIDIA_CPU_PART_CARMEL 0x004 + +#define FUJITSU_CPU_PART_A64FX 0x001 + +#define HISI_CPU_PART_TSV110 0xD01 + +#define APPLE_CPU_PART_M1_ICESTORM 0x020 +#define APPLE_CPU_PART_M1_FIRESTORM 0x021 +#define APPLE_CPU_PART_M1_ICESTORM_TONGA 0x022 +#define APPLE_CPU_PART_M1_FIRESTORM_TONGA 0x023 + +#define READ_MRS(reg) \ + ({ \ + uint64_t __val; \ + asm volatile("mrs %0, "__stringify(reg) : "=r"(__val)); \ + __val; \ + }) + +uint64_t GetCpuid_MIDR_EL1(); +uint64_t GetCpuid_AA64ISAR0_EL1(); +uint64_t GetCpuid_AA64ISAR1_EL1(); +uint64_t GetCpuid_AA64PFR0_EL1(); + +CPU_FEATURES_END_CPP_NAMESPACE + +#if !defined(CPU_FEATURES_ARCH_AARCH64) +#error "Including internal/cputype_aarch64.h from a non-aarch64 target." +#endif + +#endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/src/cpuinfo_core_aarch64.c b/src/cpuinfo_core_aarch64.c new file mode 100644 index 00000000..998bef11 --- /dev/null +++ b/src/cpuinfo_core_aarch64.c @@ -0,0 +1,471 @@ +#include "cpuinfo_core_aarch64.h" + +#include "bit_utils.h" +#include "cputype_aarch64.h" + +/////////////////////////////////////////////////////////////////////////////// +// Aarch64 info via mrs instruction +/////////////////////////////////////////////////////////////////////////////// + +uint64_t GetCpuid_MIDR_EL1() { return READ_MRS(MIDR_EL1); } +uint64_t GetCpuid_AA64ISAR0_EL1() { return READ_MRS(ID_AA64ISAR0_EL1); } +uint64_t GetCpuid_AA64ISAR1_EL1() { return READ_MRS(ID_AA64ISAR1_EL1); } +uint64_t GetCpuid_AA64PFR0_EL1() { return READ_MRS(ID_AA64PFR0_EL1); } + +// Floating-point. Defined values are: 0b0000, 0b0001, 0b1111. +// All other values are reserved. +// Floating-point is not implemented when fp_value = 0b1111 or reserved. +// This field must have the same value as the AdvSimd field. +static void SetFpInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { + const uint64_t fp_value = ExtractBitRange(pfr0, 19, 16); + + // fp_value = 0b0000: + // Floating-point is implemented, and includes support for: + // * Single-precision and double-precision floating-point types. + // * Conversions between single-precision and half-precision data types, + // and double-precision and half-precision data types. + if (fp_value == 0) { + info->features.fp = 1; + return; + } + + // fp_value = 0b0001: + // As for 0b0000, and also includes support for half-precision + // floating-point arithmetic. + if (fp_value == 1) { + info->features.fp = 1; + info->features.fphp = 1; + } +} + +// Advanced SIMD. Defined values are: 0b0000, 0b0001, 0b1111. +// All other values are reserved. +// This field must have the same value as the FP field. +static void SetAdvSimdInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { + const uint64_t asimd_value = ExtractBitRange(pfr0, 23, 20); + + // Advanced SIMD is implemented, including support for the following SISD and + // SIMD operations: + // * Integer byte, halfword, word and doubleword element operations. + // * Single-precision and double-precision floating-point arithmetic. + // * Conversions between single-precision and half-precision data types, + // and double-precision and half-precision data types. + if (asimd_value == 0) { + info->features.asimd = 1; + return; + } + + // asimd_value = 0b0001: + // As for 0b0000, and also includes support for half-precision + // floating-point arithmetic. + if (asimd_value == 1) { + info->features.asimd = 1; + info->features.asimdfhm = 1; + } +} + +// AMU indicates support for Activity Monitors Extension. Defined values are: +// 0b0000, 0b0001, 0b1111 +static void SetAmuInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { + const uint64_t amu = ExtractBitRange(pfr0, 47, 44); + + // FEAT_AMUv1 is implemented. + if (amu == 1) { + info->features.amuv1 = 1; + return; + } + + // FEAT_AMUv1p1 is implemented. As 0b0001 and adds support for virtualization + // of the activity monitor event counters. + if (amu == 2) { + info->features.amuv1 = 1; + info->features.amuv1p1 = 1; + } +} + +static void SetInstructionSetAttribute0Aarch64(Aarch64InfoCpuid* info) { + const uint64_t isa0 = GetCpuid_AA64ISAR0_EL1(); + info->features.aes = ExtractBitRange(isa0, 7, 4); + info->features.sha1 = ExtractBitRange(isa0, 11, 8); + info->features.sha2 = ExtractBitRange(isa0, 15, 12); + info->features.crc32 = ExtractBitRange(isa0, 19, 16); + info->features.atomics = ExtractBitRange(isa0, 23, 20); + info->features.rdm = ExtractBitRange(isa0, 31, 28); + info->features.atomics = ExtractBitRange(isa0, 31, 28); + info->features.sha3 = ExtractBitRange(isa0, 35, 32); + info->features.sm3 = ExtractBitRange(isa0, 39, 36); + info->features.sm4 = ExtractBitRange(isa0, 43, 40); + info->features.dp = ExtractBitRange(isa0, 47, 44); + info->features.fhm = ExtractBitRange(isa0, 51, 48); + info->features.ts = ExtractBitRange(isa0, 55, 52); + info->features.tlb = ExtractBitRange(isa0, 59, 56); + info->features.rndr = ExtractBitRange(isa0, 63, 60); +} + +static void SetInstructionSetAttribute1Aarch64(Aarch64InfoCpuid* info) { + const uint64_t isa1 = GetCpuid_AA64ISAR1_EL1(); + info->features.ls64 = ExtractBitRange(isa1, 63, 60); + info->features.xs = ExtractBitRange(isa1, 59, 56); + info->features.i8mm = ExtractBitRange(isa1, 55, 52); + info->features.dgh = ExtractBitRange(isa1, 51, 48); + info->features.bf16 = ExtractBitRange(isa1, 47, 44); + info->features.specres = ExtractBitRange(isa1, 43, 40); + info->features.sb = ExtractBitRange(isa1, 39, 36); + info->features.frintts = ExtractBitRange(isa1, 35, 32); + info->features.gpi = ExtractBitRange(isa1, 31, 28); + info->features.gpa = ExtractBitRange(isa1, 27, 24); + info->features.lrcpc = ExtractBitRange(isa1, 23, 20); + info->features.fcma = ExtractBitRange(isa1, 19, 16); + info->features.jscvt = ExtractBitRange(isa1, 15, 12); + info->features.api = ExtractBitRange(isa1, 11, 8); + info->features.apa = ExtractBitRange(isa1, 7, 4); + info->features.dpb = ExtractBitRange(isa1, 3, 0); +} + +static void SetProcessorFeatureRegister0Aarch64(Aarch64InfoCpuid* info) { + const uint64_t pfr0 = GetCpuid_AA64PFR0_EL1(); + info->features.el0 = ExtractBitRange(pfr0, 3, 0); + info->features.el1 = ExtractBitRange(pfr0, 7, 4); + + const uint64_t el2 = ExtractBitRange(pfr0, 11, 8); + if (el2 == 1) info->features.el2_64 = 1; + if (el2 == 2) info->features.el2_32_64 = 1; + + const uint64_t el3 = ExtractBitRange(pfr0, 15, 12); + if (el3 == 1) info->features.el3_64 = 1; + if (el3 == 2) info->features.el3_32_64 = 1; + + const uint64_t gic = ExtractBitRange(pfr0, 27, 24); + if (gic == 1) info->features.gicv4 = 1; + if (gic == 2) info->features.gicv41 = 1; + + const uint64_t ras = ExtractBitRange(pfr0, 31, 28); + if (ras == 1) info->features.ras = 1; + if (ras == 2) { + info->features.ras = 1; + info->features.rasv1p1 = 1; + } + if (ras >= 2 && info->features.el3_64 == 1) info->features.double_fault = 1; + + info->features.sve = ExtractBitRange(pfr0, 35, 32); + info->features.sel2 = ExtractBitRange(pfr0, 39, 36); + info->features.mpam = ExtractBitRange(pfr0, 43, 40); + info->features.dit = ExtractBitRange(pfr0, 51, 48); + + SetFpInfoAarch64(info, pfr0); + SetAdvSimdInfoAarch64(info, pfr0); + SetAmuInfoAarch64(info, pfr0); +} + +static Aarch64PartNum GetPartNumAarch64(const uint64_t midr) { + const uint64_t midr_part_num = ExtractBitRange(midr, 15, 4); + switch (midr_part_num) { + case ARM_CPU_PART_AEM_V8: + return AEM_V8; + case ARM_CPU_PART_FOUNDATION: + return FOUNDATION; + case ARM_CPU_PART_NEOVERSE_N1: + return NEOVERSE_N1; + case ARM_CPU_PART_CORTEX_A35: + return CORTEX_A35; + case ARM_CPU_PART_CORTEX_A53: + return CORTEX_A53; + case ARM_CPU_PART_CORTEX_A55: + return CORTEX_A55; + case ARM_CPU_PART_CORTEX_A57: + return CORTEX_A57; + case ARM_CPU_PART_CORTEX_A72: + return CORTEX_A72; + case ARM_CPU_PART_CORTEX_A73: + return CORTEX_A73; + case ARM_CPU_PART_CORTEX_A75: + return CORTEX_A75; + case ARM_CPU_PART_CORTEX_A76: + return CORTEX_A76; + case ARM_CPU_PART_CORTEX_A77: + return CORTEX_A77; + case APM_CPU_PART_POTENZA: + return POTENZA; + case CAVIUM_CPU_PART_THUNDERX: + return THUNDERX; + case CAVIUM_CPU_PART_THUNDERX_81XX: + return THUNDERX_81XX; + case CAVIUM_CPU_PART_THUNDERX_83XX: + return THUNDERX_83XX; + case CAVIUM_CPU_PART_THUNDERX2: + return THUNDERX2; + case BRCM_CPU_PART_BRAHMA_B53: + return BRAHMA_B53; + case BRCM_CPU_PART_VULCAN: + return VULCAN; + case QCOM_CPU_PART_FALKOR_V1: + return FALKOR_V1; + case QCOM_CPU_PART_FALKOR: + return FALKOR; + case QCOM_CPU_PART_KRYO: + return KRYO; + case QCOM_CPU_PART_KRYO_2XX_SILVER: + return KRYO_2XX_SILVER; + case QCOM_CPU_PART_KRYO_3XX_SILVER: + return KRYO_3XX_SILVER; + case QCOM_CPU_PART_KRYO_4XX_GOLD: + return KRYO_4XX_GOLD; + case QCOM_CPU_PART_KRYO_4XX_SILVER: + return KRYO_4XX_SILVER; + case NVIDIA_CPU_PART_DENVER: + return DENVER; + case NVIDIA_CPU_PART_CARMEL: + return CARMEL; + case FUJITSU_CPU_PART_A64FX: + return A64FX; + case HISI_CPU_PART_TSV110: + return TSV110; + case APPLE_CPU_PART_M1_FIRESTORM: + return M1_FIRESTORM; + case APPLE_CPU_PART_M1_ICESTORM: + return M1_ICESTORM; + case APPLE_CPU_PART_M1_FIRESTORM_TONGA: + return M1_FIRESTORM_TONGA; + case APPLE_CPU_PART_M1_ICESTORM_TONGA: + return M1_ICESTORM_TONGA; + default: + return UNKNOWN_PART_NUM_AARCH64; + } +} + +static Aarch64Implementer GetImplementerAarch64(const uint64_t midr) { + const uint64_t midr_impl = ExtractBitRange(midr, 31, 24); + switch (midr_impl) { + case ARM_CPU_IMP_RESERVED: + return RESERVED; + case ARM_CPU_IMP_ARM: + return ARM; + case ARM_CPU_IMP_APM: + return APM; + case ARM_CPU_IMP_APPLE: + return APPLE; + case ARM_CPU_IMP_BRCM: + return BRCM; + case ARM_CPU_IMP_CAVIUM: + return CAVIUM; + case ARM_CPU_IMP_FUJITSU: + return FUJITSU; + case ARM_CPU_IMP_HISI: + return HISI; + case ARM_CPU_IMP_NVIDIA: + return NVIDIA; + case ARM_CPU_IMP_QCOM: + return QCOM; + case ARM_CPU_IMP_DEQ: + return DEQ; + case ARM_CPU_IMP_INFINEON: + return INFINEON; + case ARM_CPU_IMP_MOTOROLA: + return MOTOROLA; + case ARM_CPU_IMP_MARVELL: + return MARVELL; + case ARM_CPU_IMP_INTEL: + return INTEL; + case ARM_CPU_IMP_AMPERE: + return AMPERE; + default: + return UNKNOWN_IMPL_AARCH64; + } +} + +static Aarch64Architecture GetArchitectureAarch64(const uint64_t midr) { + const uint64_t architecture = ExtractBitRange(midr, 19, 16); + switch (architecture) { + case CPU_ARCH_ARMv4: + return ARMv4; + case CPU_ARCH_ARMv4T: + return ARMv4T; + case CPU_ARCH_ARMv5: + return ARMv5; + case CPU_ARCH_ARMv5T: + return ARMv5T; + case CPU_ARCH_ARMv5TE: + return ARMv5TE; + case CPU_ARCH_ARMv5TEJ: + return ARMv5TEJ; + case CPU_ARCH_ARMv6: + return ARMv6; + case CPU_ARCH_ARMv7: + return ARMv7; + case CPU_ARCH_ARMv7F: + return ARMv7F; + case CPU_ARCH_ARMv7S: + return ARMv7S; + case CPU_ARCH_ARMv7K: + return ARMv7K; + case CPU_ARCH_ARMv8: + return ARMv8; + case CPU_ARCH_ARMv8E: + return ARMv8E; + default: + return UNKNOWN_ARCH_AARCH64; + } +} + +static const Aarch64InfoCpuid kEmptyAarch64InfoMrs; + +Aarch64InfoCpuid GetAarch64InfoCpuid(void) { + Aarch64InfoCpuid info = kEmptyAarch64InfoMrs; + const uint64_t midr = GetCpuid_MIDR_EL1(); + info.implementer = GetImplementerAarch64(midr); + info.variant = ExtractBitRange(midr, 23, 20); + info.architecture = GetArchitectureAarch64(midr); + info.part = GetPartNumAarch64(midr); + info.revision = ExtractBitRange(midr, 3, 0); + SetProcessorFeatureRegister0Aarch64(&info); + SetInstructionSetAttribute0Aarch64(&info); + SetInstructionSetAttribute1Aarch64(&info); + return info; +} + +const char* GetAarch64ImplementerString(const Aarch64InfoCpuid* info) { + switch (info->implementer) { + case RESERVED: + return "RESERVED"; + case ARM: + return "ARM"; + case CAVIUM: + return "CAVIUM"; + case BRCM: + return "BRCM"; + case HISI: + return "HISI"; + case DEQ: + return "DEQ"; + case FUJITSU: + return "FUJITSU"; + case INFINEON: + return "INFINEON"; + case MOTOROLA: + return "MOTOROLA"; + case NVIDIA: + return "NVIDIA"; + case APM: + return "APM"; + case QCOM: + return "QCOM"; + case MARVELL: + return "MARVELL"; + case APPLE: + return "APPLE"; + case INTEL: + return "INTEL"; + case AMPERE: + return "AMPERE"; + case UNKNOWN_IMPL_AARCH64: + return "UNKNOWN_IMPL_AARCH64"; + } +} +const char* GetAarch64PartNumString(const Aarch64InfoCpuid* info) { + switch (info->part) { + case UNKNOWN_PART_NUM_AARCH64: + return "UNKNOWN_PART_NUM_AARCH64"; + case AEM_V8: + return "AEM_V8"; + case FOUNDATION: + return "FOUNDATION"; + case CORTEX_A35: + return "CORTEX_A35"; + case CORTEX_A53: + return "CORTEX_A53"; + case CORTEX_A55: + return "CORTEX_A55"; + case CORTEX_A57: + return "CORTEX_A57"; + case CORTEX_A72: + return "CORTEX_A72"; + case CORTEX_A73: + return "CORTEX_A73"; + case CORTEX_A75: + return "CORTEX_A75"; + case CORTEX_A76: + return "CORTEX_A76"; + case CORTEX_A77: + return "CORTEX_A77"; + case NEOVERSE_N1: + return "NEOVERSE_N1"; + case POTENZA: + return "POTENZA"; + case THUNDERX: + return "THUNDERX"; + case THUNDERX_81XX: + return "THUNDERX_81XX"; + case THUNDERX_83XX: + return "THUNDERX_83XX"; + case THUNDERX2: + return "THUNDERX2"; + case BRAHMA_B53: + return "BRAHMA_B53"; + case VULCAN: + return "VULCAN"; + case FALKOR_V1: + return "FALKOR_V1"; + case FALKOR: + return "FALKOR"; + case KRYO: + return "KRYO"; + case KRYO_2XX_GOLD: + return "KRYO_2XX_GOLD"; + case KRYO_2XX_SILVER: + return "KRYO_2XX_SILVER"; + case KRYO_3XX_SILVER: + return "KRYO_3XX_SILVER"; + case KRYO_4XX_GOLD: + return "KRYO_4XX_GOLD"; + case KRYO_4XX_SILVER: + return "KRYO_4XX_SILVER"; + case DENVER: + return "DENVER"; + case CARMEL: + return "CARMEL"; + case A64FX: + return "A64FX"; + case TSV110: + return "TSV110"; + case M1_ICESTORM: + return "M1_ICESTORM"; + case M1_FIRESTORM: + return "M1_FIRESTORM"; + case M1_ICESTORM_TONGA: + return "M1_ICESTORM_TONGA"; + case M1_FIRESTORM_TONGA: + return "M1_FIRESTORM_TONGA"; + } +} +const char* GetAarch64ArchitectureString(const Aarch64InfoCpuid* info) { + switch (info->architecture) { + case ARMv4: + return "ARMv4"; + case ARMv4T: + return "ARMv4T"; + case ARMv5: + return "ARMv5"; + case ARMv5T: + return "ARMv5T"; + case ARMv5TE: + return "ARMv5TE"; + case ARMv5TEJ: + return "ARMv5TEJ"; + case ARMv6: + return "ARMv6"; + case ARMv7: + return "ARMv7"; + case ARMv7F: + return "ARMv7F"; + case ARMv7S: + return "ARMv7S"; + case ARMv7K: + return "ARMv7K"; + case ARMv8: + return "ARMv8"; + case ARMv8E: + return "ARMv8E"; + default: + return "UNKNOWN_ARCH_AARCH64"; + } +} \ No newline at end of file From 83158099c6d13fcf9b0f1ad2818357665ba999a8 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Wed, 27 Oct 2021 02:50:50 +0300 Subject: [PATCH 02/13] add _M_ARM64 macro for detection arm64 --- include/cpu_features_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/cpu_features_macros.h b/include/cpu_features_macros.h index 384f31c5..d8545a54 100644 --- a/include/cpu_features_macros.h +++ b/include/cpu_features_macros.h @@ -39,7 +39,7 @@ #define CPU_FEATURES_ARCH_ARM #endif -#if (defined(__aarch64__) || defined(__arm64__)) +#if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)) #define CPU_FEATURES_ARCH_AARCH64 #endif From f637e8d9a4ba5783b9f32e7589dd549a3163cdf0 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Fri, 29 Oct 2021 05:41:14 +0300 Subject: [PATCH 03/13] add aarch64_base inline file --- CMakeLists.txt | 2 -- ...c => impl_aarch64__base_implementation.inl} | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) rename src/{cpuinfo_core_aarch64.c => impl_aarch64__base_implementation.inl} (95%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57dad59a..322fb4b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,8 +75,6 @@ macro(add_cpu_features_headers_and_sources HDRS_LIST_NAME SRCS_LIST_NAME) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_aarch64.h) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cputype_aarch64.h) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_core_aarch64.h) - list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/src/cpuinfo_aarch64.c) - list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/src/cpuinfo_core_aarch64.c) elseif(PROCESSOR_IS_X86) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_x86.h) list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_x86.h) diff --git a/src/cpuinfo_core_aarch64.c b/src/impl_aarch64__base_implementation.inl similarity index 95% rename from src/cpuinfo_core_aarch64.c rename to src/impl_aarch64__base_implementation.inl index 998bef11..0a395ef4 100644 --- a/src/cpuinfo_core_aarch64.c +++ b/src/impl_aarch64__base_implementation.inl @@ -1,8 +1,26 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "cpuinfo_core_aarch64.h" #include "bit_utils.h" #include "cputype_aarch64.h" +#if !defined(CPU_FEATURES_) +#error "Cannot compile cpuinfo_aarch64 on a non x86 platform." +#endif + /////////////////////////////////////////////////////////////////////////////// // Aarch64 info via mrs instruction /////////////////////////////////////////////////////////////////////////////// From cd988f5a27556d8045b87dafb560ec3055d3ac96 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Sun, 31 Oct 2021 03:20:23 +0300 Subject: [PATCH 04/13] Add read sys reg --- CMakeLists.txt | 3 +- include/cpuinfo_core_aarch64.h | 179 --------- include/internal/cpuid_aarch64.h | 83 ++++ include/internal/cputype_aarch64.h | 123 ------ src/impl_aarch64__base_implementation.inl | 436 ++-------------------- src/impl_aarch64_freebsd.c | 27 ++ src/impl_aarch64_linux_or_android.c | 27 +- src/impl_aarch64_windows.c | 27 ++ 8 files changed, 184 insertions(+), 721 deletions(-) delete mode 100644 include/cpuinfo_core_aarch64.h create mode 100644 include/internal/cpuid_aarch64.h delete mode 100644 include/internal/cputype_aarch64.h create mode 100644 src/impl_aarch64_freebsd.c create mode 100644 src/impl_aarch64_windows.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 322fb4b3..28023aa9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,8 +73,7 @@ macro(add_cpu_features_headers_and_sources HDRS_LIST_NAME SRCS_LIST_NAME) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_arm.h) elseif(PROCESSOR_IS_AARCH64) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_aarch64.h) - list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cputype_aarch64.h) - list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_core_aarch64.h) + list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_aarch64.h) elseif(PROCESSOR_IS_X86) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_x86.h) list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_x86.h) diff --git a/include/cpuinfo_core_aarch64.h b/include/cpuinfo_core_aarch64.h deleted file mode 100644 index de60c729..00000000 --- a/include/cpuinfo_core_aarch64.h +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CPU_FEATURES_CPUINFO_CORE_AARCH64_H -#define CPU_FEATURES_CPUINFO_CORE_AARCH64_H - -#include "cpu_features_macros.h" -#include "internal/cputype_aarch64.h" - -CPU_FEATURES_START_CPP_NAMESPACE - -/////////////////////////////////////////////////////////////////////////////// -// Aarch64 info via mrs instruction -/////////////////////////////////////////////////////////////////////////////// - -typedef enum { - UNKNOWN_IMPL_AARCH64, - RESERVED, - ARM, - CAVIUM, - BRCM, - HISI, - DEQ, - FUJITSU, - INFINEON, - MOTOROLA, - NVIDIA, - APM, - QCOM, - MARVELL, - APPLE, - INTEL, - AMPERE -} Aarch64Implementer; - -typedef enum { - UNKNOWN_PART_NUM_AARCH64, - AEM_V8, - FOUNDATION, - CORTEX_A35, - CORTEX_A53, - CORTEX_A55, - CORTEX_A57, - CORTEX_A72, - CORTEX_A73, - CORTEX_A75, - CORTEX_A76, - CORTEX_A77, - NEOVERSE_N1, - POTENZA, - THUNDERX, - THUNDERX_81XX, - THUNDERX_83XX, - THUNDERX2, - BRAHMA_B53, - VULCAN, - FALKOR_V1, - FALKOR, - KRYO, - KRYO_2XX_GOLD, - KRYO_2XX_SILVER, - KRYO_3XX_SILVER, - KRYO_4XX_GOLD, - KRYO_4XX_SILVER, - DENVER, - CARMEL, - A64FX, - TSV110, - M1_ICESTORM, - M1_FIRESTORM, - M1_ICESTORM_TONGA, - M1_FIRESTORM_TONGA -} Aarch64PartNum; - -typedef enum { - UNKNOWN_ARCH_AARCH64, - ARMv4, - ARMv4T, - ARMv5, - ARMv5T, - ARMv5TE, - ARMv5TEJ, - ARMv6, - ARMv7, - ARMv7F, - ARMv7S, - ARMv7K, - ARMv8, - ARMv8E -} Aarch64Architecture; - -typedef struct { - int fp; // Floating-point. - int fphp; // Half-precision floating-point arithmetic. - int i8mm; // Int8 matrix multiplication instructions. - int bf16; // BFloat16 instructions. - int gpi; // Indicates support for an IMPLEMENTATION DEFINED. - int gpa; // Indicates whether the QARMA5 algorithm is implemented. - int lrcpc; // Support for weaker release consistency. - int fcma; // Floating point complex numbers. - int jscvt; // Support for JavaScript conversion. - int api; // Indicates whether an IMPLEMENTATION DEFINED algorithm. - int apa; // Indicates whether the QARMA5 algorithm is implemented. - int dpb; // Indicates support for the DC CVAP and DC CVADP. - int ls64; // Indicates support for LD64B and ST64B* instructions. - int xs; // Indicates support for the XS attribute. - int specres; // Indicates support for prediction invalid instructions. - int sb; // Speculation barrier. - int frintts; // Indicates support for the FRINT32Z,FRINT32X,FRINT64Z. - int dp; // Indicates support for Dot Product instructions. - int dgh; // Data Gathering Hint instruction. - int fhm; // Indicates support for FMLAL and FMLSL instructions. - int ts; // Indicates support for flag manipulation instructions. - int tlb; // Indicates support for Outer shareable and TLB range. - int rndr; // Indicates support for Random Number instructions. - int amuv1; // Indicates support for Activity Monitors Extension. - int amuv1p1; // Support for virtualization of the activity monitor - int mpam; // Indicates support for MPAM Extension. - int sve; // Scalable Vector Extension. - int dit; // Data independent timing. - int sel2; // Secure EL2. - int ras; // RAS Extension version. - int rasv1p1; // FEAT_RASv1p1. - int double_fault; // FEAT_DoubleFault. - int gicv4; // System register interface to versions 3.0 and 4.0 of the - // GIC CPU interface is supported. - int gicv41; // System register interface to version 4.1 of the GIC CPU - // interface is supported. - int asimd; // Advanced SIMD. - int asimdfhm; // Additional half-precision instructions. - int el3_64; // EL3 can be executed in AArch64 state only. - int el3_32_64; // EL3 can be executed in either AArch64 or AArch32 state. - int el2_64; // EL2 can be executed in AArch64 state only. - int el2_32_64; // EL2 can be executed in either AArch64 or AArch32 state. - int el1; // EL1 Exception level handling. - int el0; // EL0 Exception level handling. - int aes; // Hardware-accelerated Advanced Encryption Standard. - int sha1; // Hardware-accelerated SHA1. - int sha2; // Hardware-accelerated SHA2. - int crc32; // Hardware-accelerated CRC-32. - int atomics; // Armv8.1 atomic instructions. - int rdm; // Indicates support for SQRDMLAH and SQRDMLSH - int sha3; // Hardware-accelerated SHA3. - int sm3; // Hardware-accelerated SM3. - int sm4; // Hardware-accelerated SM4. -} Aarch64FeaturesCpuid; - -typedef struct { - Aarch64FeaturesCpuid features; - Aarch64Implementer implementer; - Aarch64PartNum part; - Aarch64Architecture architecture; - int variant; - int revision; -} Aarch64InfoCpuid; - -Aarch64InfoCpuid GetAarch64InfoCpuid(void); -const char* GetAarch64ImplementerString(const Aarch64InfoCpuid* info); -const char* GetAarch64PartNumString(const Aarch64InfoCpuid* info); -const char* GetAarch64ArchitectureString(const Aarch64InfoCpuid* info); - -CPU_FEATURES_END_CPP_NAMESPACE - -#if !defined(CPU_FEATURES_ARCH_AARCH64) -#error "Including cpuinfo_core_aarch64.h from a non-aarch64 target." -#endif - -#endif // CPU_FEATURES_CPUINFO_CORE_AARCH64_H diff --git a/include/internal/cpuid_aarch64.h b/include/internal/cpuid_aarch64.h new file mode 100644 index 00000000..844751b2 --- /dev/null +++ b/include/internal/cpuid_aarch64.h @@ -0,0 +1,83 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H +#define CPU_FEATURES_CPUTYPE_AARCH64_H + +#include "cpu_features_macros.h" +#include "stdint.h" + +CPU_FEATURES_START_CPP_NAMESPACE + +uint64_t GetCpuid_MIDR_EL1(); +uint64_t GetCpuid_ID_AA64ISAR0_EL1(); +uint64_t GetCpuid_ID_AA64ISAR1_EL1(); +uint64_t GetCpuid_ID_AA64PFR0_EL1(); +uint64_t GetCpuid_ID_AA64ZFR0_EL1(); + +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +#define __emit_inst(x) ".inst " __stringify((x)) "\n\t" + +#define OP0_SHIFT 19 +#define OP1_SHIFT 16 +#define CRN_SHIFT 12 +#define CRM_SHIFT 8 +#define OP2_SHIFT 5 + +#define SYS_REG(op0, op1, crn, crm, op2) \ + (((op0) << OP0_SHIFT) | ((op1) << OP1_SHIFT) | ((crn) << CRN_SHIFT) | \ + ((crm) << CRM_SHIFT) | ((op2) << OP2_SHIFT)) + +#define SYS_MIDR_EL1 SYS_REG(3, 0, 0, 0, 0) +#define SYS_ID_AA64ZFR0_EL1 SYS_REG(3, 0, 0, 4, 4) + +#define __DEFINE_MRS_MSR_S_REGNUM \ + " .irp " \ + "num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25, " \ + "26,27,28,29,30\n" \ + " .equ .L__reg_num_x\\num, \\num\n" \ + " .endr\n" \ + " .equ .L__reg_num_xzr, 31\n" + +#define DEFINE_MRS_S \ + __DEFINE_MRS_MSR_S_REGNUM \ + " .macro mrs_s, rt, sreg\n" __emit_inst( \ + 0xD5200000 | (\\sreg) | (.L__reg_num_\\rt)) " .endm\n" + +#define UNDEFINE_MRS_S " .purgem mrs_s\n" + +#define __mrs_s(v, r) \ + DEFINE_MRS_S \ + " mrs_s " v ", " __stringify(r) "\n" UNDEFINE_MRS_S + +// For registers without architectural names. +#define READ_SYS_REG_S(r) \ + ({ \ + uint64_t __val; \ + asm volatile(__mrs_s("%0", r) : "=r"(__val)); \ + __val; \ + }) + +#define READ_SYS_REG(reg) \ + ({ \ + uint64_t __val; \ + asm volatile("mrs %0, "__stringify(reg) : "=r"(__val)); \ + __val; \ + }) + +CPU_FEATURES_END_CPP_NAMESPACE + +#endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h deleted file mode 100644 index b1bc657f..00000000 --- a/include/internal/cputype_aarch64.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H -#define CPU_FEATURES_CPUTYPE_AARCH64_H - -#include "cpu_features_macros.h" -#include "stdint.h" - -CPU_FEATURES_START_CPP_NAMESPACE - -#define __stringify_1(x...) #x -#define __stringify(x...) __stringify_1(x) - -#define CPU_ARCH_ARMv4 0x1 -#define CPU_ARCH_ARMv4T 0x2 -#define CPU_ARCH_ARMv5 0x3 -#define CPU_ARCH_ARMv5T 0x4 -#define CPU_ARCH_ARMv5TE 0x5 -#define CPU_ARCH_ARMv5TEJ 0x6 -#define CPU_ARCH_ARMv6 0x7 -#define CPU_ARCH_ARMv7 0x8 -#define CPU_ARCH_ARMv7F 0x9 -#define CPU_ARCH_ARMv7S 0xA -#define CPU_ARCH_ARMv7K 0xB -#define CPU_ARCH_ARMv8 0xC -#define CPU_ARCH_ARMv8E 0xD - -// Special code indicating we need to look somewhere else for -// the architecture version. -#define CPU_ARCH_ARM_EXTENDED 0xF - -#define ARM_CPU_IMP_RESERVED 0x00 -#define ARM_CPU_IMP_ARM 0x41 -#define ARM_CPU_IMP_BRCM 0x42 -#define ARM_CPU_IMP_CAVIUM 0x43 -#define ARM_CPU_IMP_DEQ 0x44 -#define ARM_CPU_IMP_FUJITSU 0x46 -#define ARM_CPU_IMP_HISI 0x48 -#define ARM_CPU_IMP_INFINEON 0x49 -#define ARM_CPU_IMP_MOTOROLA 0x4D -#define ARM_CPU_IMP_NVIDIA 0x4E -#define ARM_CPU_IMP_APM 0x50 -#define ARM_CPU_IMP_QCOM 0x51 -#define ARM_CPU_IMP_MARVELL 0x56 -#define ARM_CPU_IMP_APPLE 0x61 -#define ARM_CPU_IMP_INTEL 0x69 -#define ARM_CPU_IMP_AMPERE 0xC0 - -#define ARM_CPU_PART_AEM_V8 0xD0F -#define ARM_CPU_PART_FOUNDATION 0xD00 -#define ARM_CPU_PART_CORTEX_A57 0xD07 -#define ARM_CPU_PART_CORTEX_A72 0xD08 -#define ARM_CPU_PART_CORTEX_A53 0xD03 -#define ARM_CPU_PART_CORTEX_A73 0xD09 -#define ARM_CPU_PART_CORTEX_A75 0xD0A -#define ARM_CPU_PART_CORTEX_A35 0xD04 -#define ARM_CPU_PART_CORTEX_A55 0xD05 -#define ARM_CPU_PART_CORTEX_A76 0xD0B -#define ARM_CPU_PART_NEOVERSE_N1 0xD0C -#define ARM_CPU_PART_CORTEX_A77 0xD0D - -#define APM_CPU_PART_POTENZA 0x000 - -#define CAVIUM_CPU_PART_THUNDERX 0x0A1 -#define CAVIUM_CPU_PART_THUNDERX_81XX 0x0A2 -#define CAVIUM_CPU_PART_THUNDERX_83XX 0x0A3 -#define CAVIUM_CPU_PART_THUNDERX2 0x0AF - -#define BRCM_CPU_PART_BRAHMA_B53 0x100 -#define BRCM_CPU_PART_VULCAN 0x516 - -#define QCOM_CPU_PART_FALKOR_V1 0x800 -#define QCOM_CPU_PART_FALKOR 0xC00 -#define QCOM_CPU_PART_KRYO 0x200 -#define QCOM_CPU_PART_KRYO_2XX_GOLD 0x800 -#define QCOM_CPU_PART_KRYO_2XX_SILVER 0x801 -#define QCOM_CPU_PART_KRYO_3XX_SILVER 0x803 -#define QCOM_CPU_PART_KRYO_4XX_GOLD 0x804 -#define QCOM_CPU_PART_KRYO_4XX_SILVER 0x805 - -#define NVIDIA_CPU_PART_DENVER 0x003 -#define NVIDIA_CPU_PART_CARMEL 0x004 - -#define FUJITSU_CPU_PART_A64FX 0x001 - -#define HISI_CPU_PART_TSV110 0xD01 - -#define APPLE_CPU_PART_M1_ICESTORM 0x020 -#define APPLE_CPU_PART_M1_FIRESTORM 0x021 -#define APPLE_CPU_PART_M1_ICESTORM_TONGA 0x022 -#define APPLE_CPU_PART_M1_FIRESTORM_TONGA 0x023 - -#define READ_MRS(reg) \ - ({ \ - uint64_t __val; \ - asm volatile("mrs %0, "__stringify(reg) : "=r"(__val)); \ - __val; \ - }) - -uint64_t GetCpuid_MIDR_EL1(); -uint64_t GetCpuid_AA64ISAR0_EL1(); -uint64_t GetCpuid_AA64ISAR1_EL1(); -uint64_t GetCpuid_AA64PFR0_EL1(); - -CPU_FEATURES_END_CPP_NAMESPACE - -#if !defined(CPU_FEATURES_ARCH_AARCH64) -#error "Including internal/cputype_aarch64.h from a non-aarch64 target." -#endif - -#endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 0a395ef4..0f6bc3b4 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -12,29 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cpuinfo_core_aarch64.h" +#include "internal/bit_utils.h" +#include "internal/cpuid_aarch64.h" -#include "bit_utils.h" -#include "cputype_aarch64.h" +#include "cpuinfo_aarch64.h" -#if !defined(CPU_FEATURES_) -#error "Cannot compile cpuinfo_aarch64 on a non x86 platform." +#if !defined(CPU_FEATURES_ARCH_AARCH64) +#error "Cannot compile cpuinfo_aarch64 on a non aarch64 platform." #endif /////////////////////////////////////////////////////////////////////////////// // Aarch64 info via mrs instruction /////////////////////////////////////////////////////////////////////////////// -uint64_t GetCpuid_MIDR_EL1() { return READ_MRS(MIDR_EL1); } -uint64_t GetCpuid_AA64ISAR0_EL1() { return READ_MRS(ID_AA64ISAR0_EL1); } -uint64_t GetCpuid_AA64ISAR1_EL1() { return READ_MRS(ID_AA64ISAR1_EL1); } -uint64_t GetCpuid_AA64PFR0_EL1() { return READ_MRS(ID_AA64PFR0_EL1); } +uint64_t GetCpuid_MIDR_EL1() {return READ_SYS_REG_S(SYS_MIDR_EL1 ); } +uint64_t GetCpuid_ID_AA64ISAR0_EL1() { return READ_SYS_REG(ID_AA64ISAR0_EL1); } +uint64_t GetCpuid_ID_AA64ISAR1_EL1() { return READ_SYS_REG(ID_AA64ISAR1_EL1); } +uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG(ID_AA64PFR0_EL1); } +uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); } + +static void DetectFeaturesBase(Aarch64Info* info) { + const uint64_t pfr0 = GetCpuid_ID_AA64PFR0_EL1(); + + info->features.sve = ExtractBitRange(pfr0, 35, 32); + info->features.dit = ExtractBitRange(pfr0, 51, 48); -// Floating-point. Defined values are: 0b0000, 0b0001, 0b1111. -// All other values are reserved. -// Floating-point is not implemented when fp_value = 0b1111 or reserved. -// This field must have the same value as the AdvSimd field. -static void SetFpInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { const uint64_t fp_value = ExtractBitRange(pfr0, 19, 16); // fp_value = 0b0000: @@ -54,12 +56,7 @@ static void SetFpInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { info->features.fp = 1; info->features.fphp = 1; } -} -// Advanced SIMD. Defined values are: 0b0000, 0b0001, 0b1111. -// All other values are reserved. -// This field must have the same value as the FP field. -static void SetAdvSimdInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { const uint64_t asimd_value = ExtractBitRange(pfr0, 23, 20); // Advanced SIMD is implemented, including support for the following SISD and @@ -80,410 +77,55 @@ static void SetAdvSimdInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { info->features.asimd = 1; info->features.asimdfhm = 1; } -} - -// AMU indicates support for Activity Monitors Extension. Defined values are: -// 0b0000, 0b0001, 0b1111 -static void SetAmuInfoAarch64(Aarch64InfoCpuid* info, uint64_t pfr0) { - const uint64_t amu = ExtractBitRange(pfr0, 47, 44); - - // FEAT_AMUv1 is implemented. - if (amu == 1) { - info->features.amuv1 = 1; - return; - } - // FEAT_AMUv1p1 is implemented. As 0b0001 and adds support for virtualization - // of the activity monitor event counters. - if (amu == 2) { - info->features.amuv1 = 1; - info->features.amuv1p1 = 1; + const uint64_t isa0 = GetCpuid_ID_AA64ISAR0_EL1(); + const uint64_t aes = ExtractBitRange(isa0, 7, 4); + if (aes == 1) info->features.aes = 1; + if (aes == 2) { + info->features.aes = 1; + info->features.pmull = 1; } -} - -static void SetInstructionSetAttribute0Aarch64(Aarch64InfoCpuid* info) { - const uint64_t isa0 = GetCpuid_AA64ISAR0_EL1(); - info->features.aes = ExtractBitRange(isa0, 7, 4); info->features.sha1 = ExtractBitRange(isa0, 11, 8); info->features.sha2 = ExtractBitRange(isa0, 15, 12); info->features.crc32 = ExtractBitRange(isa0, 19, 16); info->features.atomics = ExtractBitRange(isa0, 23, 20); - info->features.rdm = ExtractBitRange(isa0, 31, 28); info->features.atomics = ExtractBitRange(isa0, 31, 28); info->features.sha3 = ExtractBitRange(isa0, 35, 32); info->features.sm3 = ExtractBitRange(isa0, 39, 36); info->features.sm4 = ExtractBitRange(isa0, 43, 40); - info->features.dp = ExtractBitRange(isa0, 47, 44); - info->features.fhm = ExtractBitRange(isa0, 51, 48); - info->features.ts = ExtractBitRange(isa0, 55, 52); - info->features.tlb = ExtractBitRange(isa0, 59, 56); - info->features.rndr = ExtractBitRange(isa0, 63, 60); -} + info->features.rng = ExtractBitRange(isa0, 63, 60); -static void SetInstructionSetAttribute1Aarch64(Aarch64InfoCpuid* info) { - const uint64_t isa1 = GetCpuid_AA64ISAR1_EL1(); - info->features.ls64 = ExtractBitRange(isa1, 63, 60); - info->features.xs = ExtractBitRange(isa1, 59, 56); + const uint64_t isa1 = GetCpuid_ID_AA64ISAR1_EL1(); info->features.i8mm = ExtractBitRange(isa1, 55, 52); info->features.dgh = ExtractBitRange(isa1, 51, 48); info->features.bf16 = ExtractBitRange(isa1, 47, 44); - info->features.specres = ExtractBitRange(isa1, 43, 40); info->features.sb = ExtractBitRange(isa1, 39, 36); - info->features.frintts = ExtractBitRange(isa1, 35, 32); - info->features.gpi = ExtractBitRange(isa1, 31, 28); - info->features.gpa = ExtractBitRange(isa1, 27, 24); info->features.lrcpc = ExtractBitRange(isa1, 23, 20); info->features.fcma = ExtractBitRange(isa1, 19, 16); info->features.jscvt = ExtractBitRange(isa1, 15, 12); - info->features.api = ExtractBitRange(isa1, 11, 8); - info->features.apa = ExtractBitRange(isa1, 7, 4); - info->features.dpb = ExtractBitRange(isa1, 3, 0); -} - -static void SetProcessorFeatureRegister0Aarch64(Aarch64InfoCpuid* info) { - const uint64_t pfr0 = GetCpuid_AA64PFR0_EL1(); - info->features.el0 = ExtractBitRange(pfr0, 3, 0); - info->features.el1 = ExtractBitRange(pfr0, 7, 4); - const uint64_t el2 = ExtractBitRange(pfr0, 11, 8); - if (el2 == 1) info->features.el2_64 = 1; - if (el2 == 2) info->features.el2_32_64 = 1; - - const uint64_t el3 = ExtractBitRange(pfr0, 15, 12); - if (el3 == 1) info->features.el3_64 = 1; - if (el3 == 2) info->features.el3_32_64 = 1; - - const uint64_t gic = ExtractBitRange(pfr0, 27, 24); - if (gic == 1) info->features.gicv4 = 1; - if (gic == 2) info->features.gicv41 = 1; - - const uint64_t ras = ExtractBitRange(pfr0, 31, 28); - if (ras == 1) info->features.ras = 1; - if (ras == 2) { - info->features.ras = 1; - info->features.rasv1p1 = 1; - } - if (ras >= 2 && info->features.el3_64 == 1) info->features.double_fault = 1; - - info->features.sve = ExtractBitRange(pfr0, 35, 32); - info->features.sel2 = ExtractBitRange(pfr0, 39, 36); - info->features.mpam = ExtractBitRange(pfr0, 43, 40); - info->features.dit = ExtractBitRange(pfr0, 51, 48); - - SetFpInfoAarch64(info, pfr0); - SetAdvSimdInfoAarch64(info, pfr0); - SetAmuInfoAarch64(info, pfr0); -} - -static Aarch64PartNum GetPartNumAarch64(const uint64_t midr) { - const uint64_t midr_part_num = ExtractBitRange(midr, 15, 4); - switch (midr_part_num) { - case ARM_CPU_PART_AEM_V8: - return AEM_V8; - case ARM_CPU_PART_FOUNDATION: - return FOUNDATION; - case ARM_CPU_PART_NEOVERSE_N1: - return NEOVERSE_N1; - case ARM_CPU_PART_CORTEX_A35: - return CORTEX_A35; - case ARM_CPU_PART_CORTEX_A53: - return CORTEX_A53; - case ARM_CPU_PART_CORTEX_A55: - return CORTEX_A55; - case ARM_CPU_PART_CORTEX_A57: - return CORTEX_A57; - case ARM_CPU_PART_CORTEX_A72: - return CORTEX_A72; - case ARM_CPU_PART_CORTEX_A73: - return CORTEX_A73; - case ARM_CPU_PART_CORTEX_A75: - return CORTEX_A75; - case ARM_CPU_PART_CORTEX_A76: - return CORTEX_A76; - case ARM_CPU_PART_CORTEX_A77: - return CORTEX_A77; - case APM_CPU_PART_POTENZA: - return POTENZA; - case CAVIUM_CPU_PART_THUNDERX: - return THUNDERX; - case CAVIUM_CPU_PART_THUNDERX_81XX: - return THUNDERX_81XX; - case CAVIUM_CPU_PART_THUNDERX_83XX: - return THUNDERX_83XX; - case CAVIUM_CPU_PART_THUNDERX2: - return THUNDERX2; - case BRCM_CPU_PART_BRAHMA_B53: - return BRAHMA_B53; - case BRCM_CPU_PART_VULCAN: - return VULCAN; - case QCOM_CPU_PART_FALKOR_V1: - return FALKOR_V1; - case QCOM_CPU_PART_FALKOR: - return FALKOR; - case QCOM_CPU_PART_KRYO: - return KRYO; - case QCOM_CPU_PART_KRYO_2XX_SILVER: - return KRYO_2XX_SILVER; - case QCOM_CPU_PART_KRYO_3XX_SILVER: - return KRYO_3XX_SILVER; - case QCOM_CPU_PART_KRYO_4XX_GOLD: - return KRYO_4XX_GOLD; - case QCOM_CPU_PART_KRYO_4XX_SILVER: - return KRYO_4XX_SILVER; - case NVIDIA_CPU_PART_DENVER: - return DENVER; - case NVIDIA_CPU_PART_CARMEL: - return CARMEL; - case FUJITSU_CPU_PART_A64FX: - return A64FX; - case HISI_CPU_PART_TSV110: - return TSV110; - case APPLE_CPU_PART_M1_FIRESTORM: - return M1_FIRESTORM; - case APPLE_CPU_PART_M1_ICESTORM: - return M1_ICESTORM; - case APPLE_CPU_PART_M1_FIRESTORM_TONGA: - return M1_FIRESTORM_TONGA; - case APPLE_CPU_PART_M1_ICESTORM_TONGA: - return M1_ICESTORM_TONGA; - default: - return UNKNOWN_PART_NUM_AARCH64; - } -} - -static Aarch64Implementer GetImplementerAarch64(const uint64_t midr) { - const uint64_t midr_impl = ExtractBitRange(midr, 31, 24); - switch (midr_impl) { - case ARM_CPU_IMP_RESERVED: - return RESERVED; - case ARM_CPU_IMP_ARM: - return ARM; - case ARM_CPU_IMP_APM: - return APM; - case ARM_CPU_IMP_APPLE: - return APPLE; - case ARM_CPU_IMP_BRCM: - return BRCM; - case ARM_CPU_IMP_CAVIUM: - return CAVIUM; - case ARM_CPU_IMP_FUJITSU: - return FUJITSU; - case ARM_CPU_IMP_HISI: - return HISI; - case ARM_CPU_IMP_NVIDIA: - return NVIDIA; - case ARM_CPU_IMP_QCOM: - return QCOM; - case ARM_CPU_IMP_DEQ: - return DEQ; - case ARM_CPU_IMP_INFINEON: - return INFINEON; - case ARM_CPU_IMP_MOTOROLA: - return MOTOROLA; - case ARM_CPU_IMP_MARVELL: - return MARVELL; - case ARM_CPU_IMP_INTEL: - return INTEL; - case ARM_CPU_IMP_AMPERE: - return AMPERE; - default: - return UNKNOWN_IMPL_AARCH64; + if (info->features.sve) { + const uint64_t zfr0 = GetCpuid_ID_AA64ZFR0_EL1(); + const uint64_t svef64mm = ExtractBitRange(zfr0, 59, 56); + const uint64_t svef32mm = ExtractBitRange(zfr0, 55, 52); + const uint64_t svei8mm = ExtractBitRange(zfr0, 47, 44); + if (svef64mm == 1) info->features.svef64mm = 1; + if (svef32mm == 1) info->features.svef32mm = 1; + if (svei8mm == 1) info->features.svei8mm = 1; } } -static Aarch64Architecture GetArchitectureAarch64(const uint64_t midr) { - const uint64_t architecture = ExtractBitRange(midr, 19, 16); - switch (architecture) { - case CPU_ARCH_ARMv4: - return ARMv4; - case CPU_ARCH_ARMv4T: - return ARMv4T; - case CPU_ARCH_ARMv5: - return ARMv5; - case CPU_ARCH_ARMv5T: - return ARMv5T; - case CPU_ARCH_ARMv5TE: - return ARMv5TE; - case CPU_ARCH_ARMv5TEJ: - return ARMv5TEJ; - case CPU_ARCH_ARMv6: - return ARMv6; - case CPU_ARCH_ARMv7: - return ARMv7; - case CPU_ARCH_ARMv7F: - return ARMv7F; - case CPU_ARCH_ARMv7S: - return ARMv7S; - case CPU_ARCH_ARMv7K: - return ARMv7K; - case CPU_ARCH_ARMv8: - return ARMv8; - case CPU_ARCH_ARMv8E: - return ARMv8E; - default: - return UNKNOWN_ARCH_AARCH64; - } -} +static void DetectFeatures(Aarch64Info* info); -static const Aarch64InfoCpuid kEmptyAarch64InfoMrs; +static const Aarch64Info kEmptyAarch64Info; -Aarch64InfoCpuid GetAarch64InfoCpuid(void) { - Aarch64InfoCpuid info = kEmptyAarch64InfoMrs; +Aarch64Info GetAarch64Info(void) { + Aarch64Info info = kEmptyAarch64Info; const uint64_t midr = GetCpuid_MIDR_EL1(); - info.implementer = GetImplementerAarch64(midr); + info.implementer = ExtractBitRange(midr, 31, 24); info.variant = ExtractBitRange(midr, 23, 20); - info.architecture = GetArchitectureAarch64(midr); - info.part = GetPartNumAarch64(midr); + info.part = ExtractBitRange(midr, 15, 4); info.revision = ExtractBitRange(midr, 3, 0); - SetProcessorFeatureRegister0Aarch64(&info); - SetInstructionSetAttribute0Aarch64(&info); - SetInstructionSetAttribute1Aarch64(&info); + DetectFeatures(&info); return info; -} - -const char* GetAarch64ImplementerString(const Aarch64InfoCpuid* info) { - switch (info->implementer) { - case RESERVED: - return "RESERVED"; - case ARM: - return "ARM"; - case CAVIUM: - return "CAVIUM"; - case BRCM: - return "BRCM"; - case HISI: - return "HISI"; - case DEQ: - return "DEQ"; - case FUJITSU: - return "FUJITSU"; - case INFINEON: - return "INFINEON"; - case MOTOROLA: - return "MOTOROLA"; - case NVIDIA: - return "NVIDIA"; - case APM: - return "APM"; - case QCOM: - return "QCOM"; - case MARVELL: - return "MARVELL"; - case APPLE: - return "APPLE"; - case INTEL: - return "INTEL"; - case AMPERE: - return "AMPERE"; - case UNKNOWN_IMPL_AARCH64: - return "UNKNOWN_IMPL_AARCH64"; - } -} -const char* GetAarch64PartNumString(const Aarch64InfoCpuid* info) { - switch (info->part) { - case UNKNOWN_PART_NUM_AARCH64: - return "UNKNOWN_PART_NUM_AARCH64"; - case AEM_V8: - return "AEM_V8"; - case FOUNDATION: - return "FOUNDATION"; - case CORTEX_A35: - return "CORTEX_A35"; - case CORTEX_A53: - return "CORTEX_A53"; - case CORTEX_A55: - return "CORTEX_A55"; - case CORTEX_A57: - return "CORTEX_A57"; - case CORTEX_A72: - return "CORTEX_A72"; - case CORTEX_A73: - return "CORTEX_A73"; - case CORTEX_A75: - return "CORTEX_A75"; - case CORTEX_A76: - return "CORTEX_A76"; - case CORTEX_A77: - return "CORTEX_A77"; - case NEOVERSE_N1: - return "NEOVERSE_N1"; - case POTENZA: - return "POTENZA"; - case THUNDERX: - return "THUNDERX"; - case THUNDERX_81XX: - return "THUNDERX_81XX"; - case THUNDERX_83XX: - return "THUNDERX_83XX"; - case THUNDERX2: - return "THUNDERX2"; - case BRAHMA_B53: - return "BRAHMA_B53"; - case VULCAN: - return "VULCAN"; - case FALKOR_V1: - return "FALKOR_V1"; - case FALKOR: - return "FALKOR"; - case KRYO: - return "KRYO"; - case KRYO_2XX_GOLD: - return "KRYO_2XX_GOLD"; - case KRYO_2XX_SILVER: - return "KRYO_2XX_SILVER"; - case KRYO_3XX_SILVER: - return "KRYO_3XX_SILVER"; - case KRYO_4XX_GOLD: - return "KRYO_4XX_GOLD"; - case KRYO_4XX_SILVER: - return "KRYO_4XX_SILVER"; - case DENVER: - return "DENVER"; - case CARMEL: - return "CARMEL"; - case A64FX: - return "A64FX"; - case TSV110: - return "TSV110"; - case M1_ICESTORM: - return "M1_ICESTORM"; - case M1_FIRESTORM: - return "M1_FIRESTORM"; - case M1_ICESTORM_TONGA: - return "M1_ICESTORM_TONGA"; - case M1_FIRESTORM_TONGA: - return "M1_FIRESTORM_TONGA"; - } -} -const char* GetAarch64ArchitectureString(const Aarch64InfoCpuid* info) { - switch (info->architecture) { - case ARMv4: - return "ARMv4"; - case ARMv4T: - return "ARMv4T"; - case ARMv5: - return "ARMv5"; - case ARMv5T: - return "ARMv5T"; - case ARMv5TE: - return "ARMv5TE"; - case ARMv5TEJ: - return "ARMv5TEJ"; - case ARMv6: - return "ARMv6"; - case ARMv7: - return "ARMv7"; - case ARMv7F: - return "ARMv7F"; - case ARMv7S: - return "ARMv7S"; - case ARMv7K: - return "ARMv7K"; - case ARMv8: - return "ARMv8"; - case ARMv8E: - return "ARMv8E"; - default: - return "UNKNOWN_ARCH_AARCH64"; - } } \ No newline at end of file diff --git a/src/impl_aarch64_freebsd.c b/src/impl_aarch64_freebsd.c new file mode 100644 index 00000000..fcc33ac6 --- /dev/null +++ b/src/impl_aarch64_freebsd.c @@ -0,0 +1,27 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cpu_features_macros.h" + +#ifdef CPU_FEATURES_ARCH_AARCH64 +#ifdef CPU_FEATURES_OS_FREEBSD +#include "impl_aarch64__base_implementation.inl" + +static void DetectFeatures(Aarch64Info* info) { + DetectFeaturesBase(info); +} + +#endif + +#endif diff --git a/src/impl_aarch64_linux_or_android.c b/src/impl_aarch64_linux_or_android.c index 745beb9c..5ae39d60 100644 --- a/src/impl_aarch64_linux_or_android.c +++ b/src/impl_aarch64_linux_or_android.c @@ -78,6 +78,7 @@ #define INTROSPECTION_PREFIX Aarch64 #define INTROSPECTION_ENUM_PREFIX AARCH64 #include "define_introspection_and_hwcaps.inl" +#include "impl_aarch64__base_implementation.inl" //////////////////////////////////////////////////////////////////////////////// // Implementation. @@ -100,14 +101,6 @@ static bool HandleAarch64Line(const LineResult result, kSetters[i](&info->features, CpuFeatures_StringView_HasWord( value, kCpuInfoFlags[i], ' ')); } - } else if (CpuFeatures_StringView_IsEquals(key, str("CPU implementer"))) { - info->implementer = CpuFeatures_StringView_ParsePositiveNumber(value); - } else if (CpuFeatures_StringView_IsEquals(key, str("CPU variant"))) { - info->variant = CpuFeatures_StringView_ParsePositiveNumber(value); - } else if (CpuFeatures_StringView_IsEquals(key, str("CPU part"))) { - info->part = CpuFeatures_StringView_ParsePositiveNumber(value); - } else if (CpuFeatures_StringView_IsEquals(key, str("CPU revision"))) { - info->revision = CpuFeatures_StringView_ParsePositiveNumber(value); } } return !result.eof; @@ -127,23 +120,17 @@ static void FillProcCpuInfoData(Aarch64Info* const info) { } } -static const Aarch64Info kEmptyAarch64Info; - -Aarch64Info GetAarch64Info(void) { - // capabilities are fetched from both getauxval and /proc/cpuinfo so we can - // have some information if the executable is sandboxed (aka no access to - // /proc/cpuinfo). - Aarch64Info info = kEmptyAarch64Info; - - FillProcCpuInfoData(&info); +// capabilities are fetched from both getauxval and /proc/cpuinfo so we can +// have some information if the executable is sandboxed (aka no access to +// /proc/cpuinfo). +static void DetectFeatures(Aarch64Info* info) { + FillProcCpuInfoData(info); const HardwareCapabilities hwcaps = CpuFeatures_GetHardwareCapabilities(); for (size_t i = 0; i < AARCH64_LAST_; ++i) { if (CpuFeatures_IsHwCapsSet(kHardwareCapabilities[i], hwcaps)) { - kSetters[i](&info.features, true); + kSetters[i](&info->features, true); } } - - return info; } #endif // defined(CPU_FEATURES_OS_LINUX) || defined(CPU_FEATURES_OS_ANDROID) diff --git a/src/impl_aarch64_windows.c b/src/impl_aarch64_windows.c new file mode 100644 index 00000000..480a8345 --- /dev/null +++ b/src/impl_aarch64_windows.c @@ -0,0 +1,27 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cpu_features_macros.h" + +#ifdef CPU_FEATURES_ARCH_AARCH64 +#ifdef CPU_FEATURES_OS_WINDOWS +#include "impl_aarch64__base_implementation.inl" + +static void DetectFeatures(Aarch64Info* info) { + DetectFeaturesBase(info); +} + +#endif + +#endif From f63e96d806b1ecbfc4c873dd64e408d7e4c2a4fa Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Sun, 31 Oct 2021 18:52:51 +0200 Subject: [PATCH 05/13] Add sve hwcap --- include/cpuinfo_aarch64.h | 102 +++++++++++----------- include/internal/cpuid_aarch64.h | 54 +----------- src/define_cpuid_aarch64.inl | 64 ++++++++++++++ src/define_introspection.inl | 4 +- src/impl_aarch64__base_implementation.inl | 37 +++++--- src/impl_aarch64_linux_or_android.c | 2 - src/stringize.h | 16 ++++ 7 files changed, 158 insertions(+), 121 deletions(-) create mode 100644 src/define_cpuid_aarch64.inl create mode 100644 src/stringize.h diff --git a/include/cpuinfo_aarch64.h b/include/cpuinfo_aarch64.h index 75efd1cf..d322462f 100644 --- a/include/cpuinfo_aarch64.h +++ b/include/cpuinfo_aarch64.h @@ -21,57 +21,57 @@ CPU_FEATURES_START_CPP_NAMESPACE typedef struct { - int fp : 1; // Floating-point. - int asimd : 1; // Advanced SIMD. - int evtstrm : 1; // Generic timer generated events. - int aes : 1; // Hardware-accelerated Advanced Encryption Standard. - int pmull : 1; // Polynomial multiply long. - int sha1 : 1; // Hardware-accelerated SHA1. - int sha2 : 1; // Hardware-accelerated SHA2-256. - int crc32 : 1; // Hardware-accelerated CRC-32. - int atomics : 1; // Armv8.1 atomic instructions. - int fphp : 1; // Half-precision floating point support. - int asimdhp : 1; // Advanced SIMD half-precision support. - int cpuid : 1; // Access to certain ID registers. - int asimdrdm : 1; // Rounding Double Multiply Accumulate/Subtract. - int jscvt : 1; // Support for JavaScript conversion. - int fcma : 1; // Floating point complex numbers. - int lrcpc : 1; // Support for weaker release consistency. - int dcpop : 1; // Data persistence writeback. - int sha3 : 1; // Hardware-accelerated SHA3. - int sm3 : 1; // Hardware-accelerated SM3. - int sm4 : 1; // Hardware-accelerated SM4. - int asimddp : 1; // Dot product instruction. - int sha512 : 1; // Hardware-accelerated SHA512. - int sve : 1; // Scalable Vector Extension. - int asimdfhm : 1; // Additional half-precision instructions. - int dit : 1; // Data independent timing. - int uscat : 1; // Unaligned atomics support. - int ilrcpc : 1; // Additional support for weaker release consistency. - int flagm : 1; // Flag manipulation instructions. - int ssbs : 1; // Speculative Store Bypass Safe PSTATE bit. - int sb : 1; // Speculation barrier. - int paca : 1; // Address authentication. - int pacg : 1; // Generic authentication. - int dcpodp : 1; // Data cache clean to point of persistence. - int sve2 : 1; // Scalable Vector Extension (version 2). - int sveaes : 1; // SVE AES instructions. - int svepmull : 1; // SVE polynomial multiply long instructions. - int svebitperm : 1; // SVE bit permute instructions. - int svesha3 : 1; // SVE SHA3 instructions. - int svesm4 : 1; // SVE SM4 instructions. - int flagm2 : 1; // Additional flag manipulation instructions. - int frint : 1; // Floating point to integer rounding. - int svei8mm : 1; // SVE Int8 matrix multiplication instructions. - int svef32mm : 1; // SVE FP32 matrix multiplication instruction. - int svef64mm : 1; // SVE FP64 matrix multiplication instructions. - int svebf16 : 1; // SVE BFloat16 instructions. - int i8mm : 1; // Int8 matrix multiplication instructions. - int bf16 : 1; // BFloat16 instructions. - int dgh : 1; // Data Gathering Hint instruction. - int rng : 1; // True random number generator support. - int bti : 1; // Branch target identification. - int mte : 1; // Memory tagging extension. + int fp; // Floating-point. + int asimd; // Advanced SIMD. + int evtstrm; // Generic timer generated events. + int aes; // Hardware-accelerated Advanced Encryption Standard. + int pmull; // Polynomial multiply long. + int sha1; // Hardware-accelerated SHA1. + int sha2; // Hardware-accelerated SHA2-256. + int crc32; // Hardware-accelerated CRC-32. + int atomics; // Armv8.1 atomic instructions. + int fphp; // Half-precision floating point support. + int asimdhp; // Advanced SIMD half-precision support. + int cpuid; // Access to certain ID registers. + int asimdrdm; // Rounding Double Multiply Accumulate/Subtract. + int jscvt; // Support for JavaScript conversion. + int fcma; // Floating point complex numbers. + int lrcpc; // Support for weaker release consistency. + int dcpop; // Data persistence writeback. + int sha3; // Hardware-accelerated SHA3. + int sm3; // Hardware-accelerated SM3. + int sm4; // Hardware-accelerated SM4. + int asimddp; // Dot product instruction. + int sha512; // Hardware-accelerated SHA512. + int sve; // Scalable Vector Extension. + int asimdfhm; // Additional half-precision instructions. + int dit; // Data independent timing. + int uscat; // Unaligned atomics support. + int ilrcpc; // Additional support for weaker release consistency. + int flagm; // Flag manipulation instructions. + int ssbs; // Speculative Store Bypass Safe PSTATE bit. + int sb; // Speculation barrier. + int paca; // Address authentication. + int pacg; // Generic authentication. + int dcpodp; // Data cache clean to point of persistence. + int sve2; // Scalable Vector Extension (version 2). + int sveaes; // SVE AES instructions. + int svepmull; // SVE polynomial multiply long instructions. + int svebitperm; // SVE bit permute instructions. + int svesha3; // SVE SHA3 instructions. + int svesm4; // SVE SM4 instructions. + int flagm2; // Additional flag manipulation instructions. + int frint; // Floating point to integer rounding. + int svei8mm; // SVE Int8 matrix multiplication instructions. + int svef32mm; // SVE FP32 matrix multiplication instruction. + int svef64mm; // SVE FP64 matrix multiplication instructions. + int svebf16; // SVE BFloat16 instructions. + int i8mm; // Int8 matrix multiplication instructions. + int bf16; // BFloat16 instructions. + int dgh; // Data Gathering Hint instruction. + int rng; // True random number generator support. + int bti; // Branch target identification. + int mte; // Memory tagging extension. // Make sure to update Aarch64FeaturesEnum below if you add a field here. } Aarch64Features; diff --git a/include/internal/cpuid_aarch64.h b/include/internal/cpuid_aarch64.h index 844751b2..c3e34ccc 100644 --- a/include/internal/cpuid_aarch64.h +++ b/include/internal/cpuid_aarch64.h @@ -16,7 +16,7 @@ #define CPU_FEATURES_CPUTYPE_AARCH64_H #include "cpu_features_macros.h" -#include "stdint.h" +#include CPU_FEATURES_START_CPP_NAMESPACE @@ -26,58 +26,6 @@ uint64_t GetCpuid_ID_AA64ISAR1_EL1(); uint64_t GetCpuid_ID_AA64PFR0_EL1(); uint64_t GetCpuid_ID_AA64ZFR0_EL1(); -#define __stringify_1(x...) #x -#define __stringify(x...) __stringify_1(x) - -#define __emit_inst(x) ".inst " __stringify((x)) "\n\t" - -#define OP0_SHIFT 19 -#define OP1_SHIFT 16 -#define CRN_SHIFT 12 -#define CRM_SHIFT 8 -#define OP2_SHIFT 5 - -#define SYS_REG(op0, op1, crn, crm, op2) \ - (((op0) << OP0_SHIFT) | ((op1) << OP1_SHIFT) | ((crn) << CRN_SHIFT) | \ - ((crm) << CRM_SHIFT) | ((op2) << OP2_SHIFT)) - -#define SYS_MIDR_EL1 SYS_REG(3, 0, 0, 0, 0) -#define SYS_ID_AA64ZFR0_EL1 SYS_REG(3, 0, 0, 4, 4) - -#define __DEFINE_MRS_MSR_S_REGNUM \ - " .irp " \ - "num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25, " \ - "26,27,28,29,30\n" \ - " .equ .L__reg_num_x\\num, \\num\n" \ - " .endr\n" \ - " .equ .L__reg_num_xzr, 31\n" - -#define DEFINE_MRS_S \ - __DEFINE_MRS_MSR_S_REGNUM \ - " .macro mrs_s, rt, sreg\n" __emit_inst( \ - 0xD5200000 | (\\sreg) | (.L__reg_num_\\rt)) " .endm\n" - -#define UNDEFINE_MRS_S " .purgem mrs_s\n" - -#define __mrs_s(v, r) \ - DEFINE_MRS_S \ - " mrs_s " v ", " __stringify(r) "\n" UNDEFINE_MRS_S - -// For registers without architectural names. -#define READ_SYS_REG_S(r) \ - ({ \ - uint64_t __val; \ - asm volatile(__mrs_s("%0", r) : "=r"(__val)); \ - __val; \ - }) - -#define READ_SYS_REG(reg) \ - ({ \ - uint64_t __val; \ - asm volatile("mrs %0, "__stringify(reg) : "=r"(__val)); \ - __val; \ - }) - CPU_FEATURES_END_CPP_NAMESPACE #endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl new file mode 100644 index 00000000..061357ec --- /dev/null +++ b/src/define_cpuid_aarch64.inl @@ -0,0 +1,64 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stringize.h" + +#define EMIT_INST(x) ".inst " STRINGIZE((x)) "\n\t" + +#define OP0_SHIFT 19 +#define OP1_SHIFT 16 +#define CRN_SHIFT 12 +#define CRM_SHIFT 8 +#define OP2_SHIFT 5 + +#define SYS_REG(op0, op1, crn, crm, op2) \ + (((op0) << OP0_SHIFT) | ((op1) << OP1_SHIFT) | ((crn) << CRN_SHIFT) | \ + ((crm) << CRM_SHIFT) | ((op2) << OP2_SHIFT)) + +#define SYS_MIDR_EL1 SYS_REG(3, 0, 0, 0, 0) +#define SYS_ID_AA64ZFR0_EL1 SYS_REG(3, 0, 0, 4, 4) + +#define DEFINE_MRS_MSR_S_REGNUM \ + " .irp " \ + "num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25, " \ + "26,27,28,29,30\n" \ + " .equ .L__reg_num_x\\num, \\num\n" \ + " .endr\n" \ + " .equ .L__reg_num_xzr, 31\n" + +#define DEFINE_MRS_S \ + DEFINE_MRS_MSR_S_REGNUM \ + " .macro mrs_s, rt, sreg\n" EMIT_INST( \ + 0xD5200000 | (\\sreg) | (.L__reg_num_\\rt)) " .endm\n" + +#define UNDEFINE_MRS_S " .purgem mrs_s\n" + +#define MRS_S(v, r) \ + DEFINE_MRS_S \ + " mrs_s " v ", " STRINGIZE(r) "\n" UNDEFINE_MRS_S + +// For registers without architectural names. +#define READ_SYS_REG_S(r) \ + ({ \ + uint64_t __val; \ + asm volatile(MRS_S("%0", r) : "=r"(__val)); \ + __val; \ + }) + +#define READ_SYS_REG(reg) \ + ({ \ + uint64_t __val; \ + asm volatile("mrs %0, " STRINGIZE(reg) : "=r"(__val)); \ + __val; \ + }) diff --git a/src/define_introspection.inl b/src/define_introspection.inl index c0eb916d..435284b8 100644 --- a/src/define_introspection.inl +++ b/src/define_introspection.inl @@ -23,9 +23,7 @@ #endif #include - -#define STRINGIZE_(s) #s -#define STRINGIZE(s) STRINGIZE_(s) +#include "stringize.h" #define FEAT_TYPE_NAME__(X) X##Features #define FEAT_TYPE_NAME_(X) FEAT_TYPE_NAME__(X) diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 0f6bc3b4..1064c549 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "internal/bit_utils.h" -#include "internal/cpuid_aarch64.h" +#include #include "cpuinfo_aarch64.h" +#include "define_cpuid_aarch64.inl" +#include "internal/bit_utils.h" +#include "internal/cpuid_aarch64.h" #if !defined(CPU_FEATURES_ARCH_AARCH64) #error "Cannot compile cpuinfo_aarch64 on a non aarch64 platform." @@ -25,11 +27,13 @@ // Aarch64 info via mrs instruction /////////////////////////////////////////////////////////////////////////////// -uint64_t GetCpuid_MIDR_EL1() {return READ_SYS_REG_S(SYS_MIDR_EL1 ); } +uint64_t GetCpuid_MIDR_EL1() { return READ_SYS_REG_S(SYS_MIDR_EL1); } uint64_t GetCpuid_ID_AA64ISAR0_EL1() { return READ_SYS_REG(ID_AA64ISAR0_EL1); } uint64_t GetCpuid_ID_AA64ISAR1_EL1() { return READ_SYS_REG(ID_AA64ISAR1_EL1); } uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG(ID_AA64PFR0_EL1); } -uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); } +uint64_t GetCpuid_ID_AA64ZFR0_EL1() { + return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); +} static void DetectFeaturesBase(Aarch64Info* info) { const uint64_t pfr0 = GetCpuid_ID_AA64PFR0_EL1(); @@ -46,7 +50,6 @@ static void DetectFeaturesBase(Aarch64Info* info) { // and double-precision and half-precision data types. if (fp_value == 0) { info->features.fp = 1; - return; } // fp_value = 0b0001: @@ -67,7 +70,6 @@ static void DetectFeaturesBase(Aarch64Info* info) { // and double-precision and half-precision data types. if (asimd_value == 0) { info->features.asimd = 1; - return; } // asimd_value = 0b0001: @@ -106,12 +108,23 @@ static void DetectFeaturesBase(Aarch64Info* info) { if (info->features.sve) { const uint64_t zfr0 = GetCpuid_ID_AA64ZFR0_EL1(); - const uint64_t svef64mm = ExtractBitRange(zfr0, 59, 56); - const uint64_t svef32mm = ExtractBitRange(zfr0, 55, 52); - const uint64_t svei8mm = ExtractBitRange(zfr0, 47, 44); - if (svef64mm == 1) info->features.svef64mm = 1; - if (svef32mm == 1) info->features.svef32mm = 1; - if (svei8mm == 1) info->features.svei8mm = 1; + info->features.svef64mm = ExtractBitRange(zfr0, 59, 56); + info->features.svef32mm = ExtractBitRange(zfr0, 55, 52); + info->features.svei8mm = ExtractBitRange(zfr0, 47, 44); + info->features.svesm4 = ExtractBitRange(zfr0, 43, 40); + info->features.svesha3 = ExtractBitRange(zfr0, 35, 32); + info->features.svebf16 = ExtractBitRange(zfr0, 23, 20); + info->features.svebitperm = ExtractBitRange(zfr0, 19, 16); + info->features.sve2 = ExtractBitRange(zfr0, 0, 3); + + const uint64_t sveaes = ExtractBitRange(zfr0, 7, 4); + if (sveaes == 1) { + info->features.sveaes = 1; + } + if (sveaes == 2) { + info->features.sveaes = 1; + info->features.svepmull = 1; + } } } diff --git a/src/impl_aarch64_linux_or_android.c b/src/impl_aarch64_linux_or_android.c index 5ae39d60..d19a4b08 100644 --- a/src/impl_aarch64_linux_or_android.c +++ b/src/impl_aarch64_linux_or_android.c @@ -86,10 +86,8 @@ #include -#include "internal/bit_utils.h" #include "internal/filesystem.h" #include "internal/stack_line_reader.h" -#include "internal/string_view.h" static bool HandleAarch64Line(const LineResult result, Aarch64Info* const info) { diff --git a/src/stringize.h b/src/stringize.h new file mode 100644 index 00000000..c13eea0b --- /dev/null +++ b/src/stringize.h @@ -0,0 +1,16 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define STRINGIZE_(s) #s +#define STRINGIZE(s) STRINGIZE_(s) From a8cd6417066e3e37360e3b5f78534568f15361b9 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Mon, 1 Nov 2021 04:00:26 +0200 Subject: [PATCH 06/13] Update aarch64 tests add define linux or android --- CMakeLists.txt | 1 + include/cpuinfo_aarch64.h | 105 +++++++-------- include/internal/cpuid_aarch64.h | 5 +- include/internal/cputype_aarch64.h | 153 ++++++++++++++++++++++ src/impl_aarch64__base_implementation.inl | 8 +- src/impl_aarch64_freebsd.c | 10 +- src/impl_aarch64_macos.c | 25 ++++ src/impl_aarch64_windows.c | 10 +- test/CMakeLists.txt | 1 + test/cpuinfo_aarch64_test.cc | 85 +++++++++++- 10 files changed, 333 insertions(+), 70 deletions(-) create mode 100644 include/internal/cputype_aarch64.h create mode 100644 src/impl_aarch64_macos.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 28023aa9..e21abbd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ macro(add_cpu_features_headers_and_sources HDRS_LIST_NAME SRCS_LIST_NAME) elseif(PROCESSOR_IS_AARCH64) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_aarch64.h) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_aarch64.h) + list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cputype_aarch64.h) elseif(PROCESSOR_IS_X86) list(APPEND ${HDRS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/cpuinfo_x86.h) list(APPEND ${SRCS_LIST_NAME} ${PROJECT_SOURCE_DIR}/include/internal/cpuid_x86.h) diff --git a/include/cpuinfo_aarch64.h b/include/cpuinfo_aarch64.h index d322462f..a587e113 100644 --- a/include/cpuinfo_aarch64.h +++ b/include/cpuinfo_aarch64.h @@ -15,63 +15,66 @@ #ifndef CPU_FEATURES_INCLUDE_CPUINFO_AARCH64_H_ #define CPU_FEATURES_INCLUDE_CPUINFO_AARCH64_H_ +#include + #include "cpu_features_cache_info.h" #include "cpu_features_macros.h" CPU_FEATURES_START_CPP_NAMESPACE typedef struct { - int fp; // Floating-point. - int asimd; // Advanced SIMD. - int evtstrm; // Generic timer generated events. - int aes; // Hardware-accelerated Advanced Encryption Standard. - int pmull; // Polynomial multiply long. - int sha1; // Hardware-accelerated SHA1. - int sha2; // Hardware-accelerated SHA2-256. - int crc32; // Hardware-accelerated CRC-32. - int atomics; // Armv8.1 atomic instructions. - int fphp; // Half-precision floating point support. - int asimdhp; // Advanced SIMD half-precision support. - int cpuid; // Access to certain ID registers. - int asimdrdm; // Rounding Double Multiply Accumulate/Subtract. - int jscvt; // Support for JavaScript conversion. - int fcma; // Floating point complex numbers. - int lrcpc; // Support for weaker release consistency. - int dcpop; // Data persistence writeback. - int sha3; // Hardware-accelerated SHA3. - int sm3; // Hardware-accelerated SM3. - int sm4; // Hardware-accelerated SM4. - int asimddp; // Dot product instruction. - int sha512; // Hardware-accelerated SHA512. - int sve; // Scalable Vector Extension. - int asimdfhm; // Additional half-precision instructions. - int dit; // Data independent timing. - int uscat; // Unaligned atomics support. - int ilrcpc; // Additional support for weaker release consistency. - int flagm; // Flag manipulation instructions. - int ssbs; // Speculative Store Bypass Safe PSTATE bit. - int sb; // Speculation barrier. - int paca; // Address authentication. - int pacg; // Generic authentication. - int dcpodp; // Data cache clean to point of persistence. - int sve2; // Scalable Vector Extension (version 2). - int sveaes; // SVE AES instructions. - int svepmull; // SVE polynomial multiply long instructions. - int svebitperm; // SVE bit permute instructions. - int svesha3; // SVE SHA3 instructions. - int svesm4; // SVE SM4 instructions. - int flagm2; // Additional flag manipulation instructions. - int frint; // Floating point to integer rounding. - int svei8mm; // SVE Int8 matrix multiplication instructions. - int svef32mm; // SVE FP32 matrix multiplication instruction. - int svef64mm; // SVE FP64 matrix multiplication instructions. - int svebf16; // SVE BFloat16 instructions. - int i8mm; // Int8 matrix multiplication instructions. - int bf16; // BFloat16 instructions. - int dgh; // Data Gathering Hint instruction. - int rng; // True random number generator support. - int bti; // Branch target identification. - int mte; // Memory tagging extension. + uint8_t fp; // Floating-point. + uint8_t asimd; // Advanced SIMD. + uint8_t evtstrm; // Generic timer generated events. + uint8_t aes; // Hardware-accelerated Advanced Encryption Standard. + uint8_t pmull; // Polynomial multiply long. + uint8_t sha1; // Hardware-accelerated SHA1. + uint8_t sha2; // Hardware-accelerated SHA2-256. + uint8_t crc32; // Hardware-accelerated CRC-32. + uint8_t atomics; // Armv8.1 atomic instructions. + uint8_t fphp; // Half-precision floating point support. + uint8_t asimdhp; // Advanced SIMD half-precision support. + uint8_t cpuid; // Access to certain ID registers. + uint8_t asimdrdm; // Rounding Double Multiply Accumulate/Subtract. + uint8_t jscvt; // Support for JavaScript conversion. + uint8_t fcma; // Floating point complex numbers. + uint8_t lrcpc; // Support for weaker release consistency. + uint8_t dcpop; // Data persistence writeback. + uint8_t sha3; // Hardware-accelerated SHA3. + uint8_t sm3; // Hardware-accelerated SM3. + uint8_t sm4; // Hardware-accelerated SM4. + uint8_t asimddp; // Dot product instruction. + uint8_t sha512; // Hardware-accelerated SHA512. + uint8_t sve; // Scalable Vector Extension. + uint8_t asimdfhm; // Additional half-precision instructions. + uint8_t dit; // Data independent timing. + uint8_t uscat; // Unaligned atomics support. + uint8_t ilrcpc; // Additional support for weaker release consistency. + uint8_t flagm; // Flag manipulation instructions. + uint8_t ssbs; // Speculative Store Bypass Safe PSTATE bit. + uint8_t sb; // Speculation barrier. + uint8_t paca; // Address authentication. + uint8_t pacg; // Generic authentication. + uint8_t dcpodp; // Data cache clean to point of persistence. + uint8_t sve2; // Scalable Vector Extension (version 2). + uint8_t sveaes; // SVE AES instructions. + uint8_t svepmull; // SVE polynomial multiply long instructions. + uint8_t svebitperm; // SVE bit permute instructions. + uint8_t svesha3; // SVE SHA3 instructions. + uint8_t svesm4; // SVE SM4 instructions. + uint8_t flagm2; // Additional flag manipulation instructions. + uint8_t frint; // Floating point to integer rounding. + uint8_t svei8mm; // SVE Int8 matrix multiplication instructions. + uint8_t svef32mm; // SVE FP32 matrix multiplication instruction. + uint8_t svef64mm; // SVE FP64 matrix multiplication instructions. + uint8_t svebf16; // SVE BFloat16 instructions. + uint8_t i8mm; // Int8 matrix multiplication instructions. + uint8_t bf16; // BFloat16 instructions. + uint8_t dgh; // Data Gathering Hint instruction. + uint8_t rng; // True random number generator support. + uint8_t bti; // Branch target identification. + uint8_t mte; // Memory tagging extension. + // Make sure to update Aarch64FeaturesEnum below if you add a field here. } Aarch64Features; diff --git a/include/internal/cpuid_aarch64.h b/include/internal/cpuid_aarch64.h index c3e34ccc..c8fcc751 100644 --- a/include/internal/cpuid_aarch64.h +++ b/include/internal/cpuid_aarch64.h @@ -15,9 +15,10 @@ #ifndef CPU_FEATURES_CPUTYPE_AARCH64_H #define CPU_FEATURES_CPUTYPE_AARCH64_H -#include "cpu_features_macros.h" #include +#include "cpu_features_macros.h" + CPU_FEATURES_START_CPP_NAMESPACE uint64_t GetCpuid_MIDR_EL1(); @@ -26,6 +27,8 @@ uint64_t GetCpuid_ID_AA64ISAR1_EL1(); uint64_t GetCpuid_ID_AA64PFR0_EL1(); uint64_t GetCpuid_ID_AA64ZFR0_EL1(); +static void DetectFeatures(Aarch64Info* info); + CPU_FEATURES_END_CPP_NAMESPACE #endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h new file mode 100644 index 00000000..4894e65f --- /dev/null +++ b/include/internal/cputype_aarch64.h @@ -0,0 +1,153 @@ +#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H +#define CPU_FEATURES_CPUTYPE_AARCH64_H + +// Arm Architecture Reference Manual Armv8, for A-profile architecture. D13.2.98 +// https://elixir.bootlin.com/linux/latest/source/arch/arm64/include/asm/cputype.h + +#define MIDR_REVISION_SHIFT 0 +#define MIDR_PARTNUM_SHIFT 4 +#define MIDR_ARCHITECTURE_SHIFT 16 +#define MIDR_VARIANT_SHIFT 20 +#define MIDR_IMPLEMENTOR_SHIFT 24 + +// MIDR: implementor, partnum +#define MIDR_CPU_MODEL_IP(imp, partnum) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | (0xF << MIDR_ARCHITECTURE_SHIFT) | \ + ((partnum) << MIDR_PARTNUM_SHIFT)) + +// MIDR: implementor, partnum, revision +#define MIDR_CPU_MODEL_IPR(imp, partnum, revision) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | (0xF << MIDR_ARCHITECTURE_SHIFT) | \ + ((partnum) << MIDR_PARTNUM_SHIFT) | ((revision) << MIDR_REVISION_SHIFT)) + +// MIDR: implementor, variant, partnum, revision +#define MIDR_CPU_MODEL_IPRV(imp, var, partnum, revision) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | ((var) << MIDR_VARIANT_SHIFT) | \ + (0xF << MIDR_ARCHITECTURE_SHIFT) | ((partnum) << MIDR_PARTNUM_SHIFT) | \ + ((revision) << MIDR_REVISION_SHIFT)) + +// MIDR: implementor, variant, architecture, partnum, revision +#define MIDR_CPU_MODEL_IVARV(imp, var, arch, partnum, revision) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | ((var) << MIDR_VARIANT_SHIFT) | \ + ((arch) << MIDR_ARCHITECTURE_SHIFT) | ((partnum) << MIDR_PARTNUM_SHIFT) | \ + ((revision) << MIDR_REVISION_SHIFT)) + +#define ARM_CPU_IMP_ARM 0x41 // ARM Limited. +#define ARM_CPU_IMP_BRCM 0x42 // Broadcom Corporation. +#define ARM_CPU_IMP_CAVIUM 0x43 // Cavium Inc. +#define CPU_VID_DEC 0x44 // Digital Equipment Corporation. +#define ARM_CPU_IMP_FUJITSU 0x46 // Fujitsu Ltd. +#define ARM_CPU_IMP_HISI 0x48 // HISI +#define ARM_CPU_IMP_INFINEON 0x49 // Infineon Technologies AG. +#define CPU_VID_MOTOROLA 0x4D // Motorola - Freescale Semiconductor Inc. +#define ARM_CPU_IMP_NVIDIA 0x4E // NVIDIA Corporation. +#define ARM_CPU_IMP_APM 0x50 // Applied Micro Circuits Corporation. +#define ARM_CPU_IMP_QCOM 0x51 // Qualcomm Inc. +#define ARM_CPU_IMP_MARVELL 0x56 // Marvell International Ltd. +#define ARM_CPU_IMP_APPLE 0x61 // Apple Inc. +#define ARM_CPU_IMP_INTEL 0x69 // Intel Corporation. +#define ARM_CPU_IMP_AMPERE 0xC0 // Ampere Computing. + +#define ARM_CPU_PART_AEM_V8 0xD0F +#define ARM_CPU_PART_FOUNDATION 0xD00 +#define ARM_CPU_PART_CORTEX_A57 0xD07 +#define ARM_CPU_PART_CORTEX_A72 0xD08 +#define ARM_CPU_PART_CORTEX_A53 0xD03 +#define ARM_CPU_PART_CORTEX_A73 0xD09 +#define ARM_CPU_PART_CORTEX_A75 0xD0A +#define ARM_CPU_PART_CORTEX_A35 0xD04 +#define ARM_CPU_PART_CORTEX_A55 0xD05 +#define ARM_CPU_PART_CORTEX_A76 0xD0B +#define ARM_CPU_PART_NEOVERSE_N1 0xD0C +#define ARM_CPU_PART_CORTEX_A77 0xD0D + +#define APM_CPU_PART_POTENZA 0x000 + +#define CAVIUM_CPU_PART_THUNDERX 0x0A1 +#define CAVIUM_CPU_PART_THUNDERX_81XX 0x0A2 +#define CAVIUM_CPU_PART_THUNDERX_83XX 0x0A3 +#define CAVIUM_CPU_PART_THUNDERX2 0x0AF + +#define BRCM_CPU_PART_BRAHMA_B53 0x100 +#define BRCM_CPU_PART_VULCAN 0x516 + +#define QCOM_CPU_PART_FALKOR_V1 0x800 +#define QCOM_CPU_PART_FALKOR 0xC00 +#define QCOM_CPU_PART_KRYO 0x200 +#define QCOM_CPU_PART_KRYO_2XX_GOLD 0x800 +#define QCOM_CPU_PART_KRYO_2XX_SILVER 0x801 +#define QCOM_CPU_PART_KRYO_3XX_SILVER 0x803 +#define QCOM_CPU_PART_KRYO_4XX_GOLD 0x804 +#define QCOM_CPU_PART_KRYO_4XX_SILVER 0x805 + +#define NVIDIA_CPU_PART_DENVER 0x003 +#define NVIDIA_CPU_PART_CARMEL 0x004 + +#define FUJITSU_CPU_PART_A64FX 0x001 + +#define HISI_CPU_PART_TSV110 0xD01 + +#define APPLE_CPU_PART_ICESTORM 0x020 +#define APPLE_CPU_PART_FIRESTORM 0x021 +#define APPLE_CPU_PART_M1_ICESTORM_TONGA 0x022 +#define APPLE_CPU_PART_M1_FIRESTORM_TONGA 0x023 + +#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) +#define MIDR_CORTEX_A53_R3 \ + MIDR_CPU_MODEL_IPR(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53, 3) + +#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) +#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) +#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73) +#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) +#define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) +#define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) +#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) +#define MIDR_NEOVERSE_N1 \ + MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) +#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) + +#define MIDR_THUNDERX \ + MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) +#define MIDR_THUNDERX_81XX \ + MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) +#define MIDR_THUNDERX_83XX \ + MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) +#define MIDR_CAVIUM_THUNDERX2 \ + MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2) + +#define MIDR_BRAHMA_B53 \ + MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_BRAHMA_B53) +#define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN) + +#define MIDR_QCOM_FALKOR_V1 \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1) +#define MIDR_QCOM_FALKOR MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR) +#define MIDR_QCOM_KRYO MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO) +#define MIDR_QCOM_KRYO_2XX_GOLD \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_GOLD) +#define MIDR_QCOM_KRYO_2XX_SILVER \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_SILVER) +#define MIDR_QCOM_KRYO_3XX_SILVER \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_3XX_SILVER) +#define MIDR_QCOM_KRYO_4XX_GOLD \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_GOLD) +#define MIDR_QCOM_KRYO_4XX_SILVER \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_SILVER) + +#define MIDR_NVIDIA_DENVER \ + MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_DENVER) +#define MIDR_NVIDIA_CARMEL \ + MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) + +#define MIDR_FUJITSU_A64FX \ + MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) + +#define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) + +#define MIDR_APPLE_M1_ICESTORM \ + MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) +#define MIDR_APPLE_M1_FIRESTORM \ + MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) + +#endif // CPU_FEATURES_CPUTYPE_AARCH64_H diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 1064c549..320201b7 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -19,7 +19,7 @@ #include "internal/bit_utils.h" #include "internal/cpuid_aarch64.h" -#if !defined(CPU_FEATURES_ARCH_AARCH64) +#ifndef CPU_FEATURES_ARCH_AARCH64 #error "Cannot compile cpuinfo_aarch64 on a non aarch64 platform." #endif @@ -27,6 +27,9 @@ // Aarch64 info via mrs instruction /////////////////////////////////////////////////////////////////////////////// +#ifdef CPU_FEATURES_MOCK_CPUID_AARCH64 +// Implementation will be provided by test/cpuinfo_aarch64_test.cc. +#else uint64_t GetCpuid_MIDR_EL1() { return READ_SYS_REG_S(SYS_MIDR_EL1); } uint64_t GetCpuid_ID_AA64ISAR0_EL1() { return READ_SYS_REG(ID_AA64ISAR0_EL1); } uint64_t GetCpuid_ID_AA64ISAR1_EL1() { return READ_SYS_REG(ID_AA64ISAR1_EL1); } @@ -34,6 +37,7 @@ uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG(ID_AA64PFR0_EL1); } uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); } +#endif static void DetectFeaturesBase(Aarch64Info* info) { const uint64_t pfr0 = GetCpuid_ID_AA64PFR0_EL1(); @@ -128,8 +132,6 @@ static void DetectFeaturesBase(Aarch64Info* info) { } } -static void DetectFeatures(Aarch64Info* info); - static const Aarch64Info kEmptyAarch64Info; Aarch64Info GetAarch64Info(void) { diff --git a/src/impl_aarch64_freebsd.c b/src/impl_aarch64_freebsd.c index fcc33ac6..bfc5d4a5 100644 --- a/src/impl_aarch64_freebsd.c +++ b/src/impl_aarch64_freebsd.c @@ -16,12 +16,10 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #ifdef CPU_FEATURES_OS_FREEBSD -#include "impl_aarch64__base_implementation.inl" -static void DetectFeatures(Aarch64Info* info) { - DetectFeaturesBase(info); -} +#include "impl_aarch64__base_implementation.inl" -#endif +static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } -#endif +#endif // CPU_FEATURES_OS_FREEBSD +#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/src/impl_aarch64_macos.c b/src/impl_aarch64_macos.c new file mode 100644 index 00000000..15e7802d --- /dev/null +++ b/src/impl_aarch64_macos.c @@ -0,0 +1,25 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cpu_features_macros.h" + +#ifdef CPU_FEATURES_ARCH_AARCH64 +#ifdef CPU_FEATURES_OS_MACOS + +#include "impl_aarch64__base_implementation.inl" + +static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } + +#endif // CPU_FEATURES_OS_MACOS +#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/src/impl_aarch64_windows.c b/src/impl_aarch64_windows.c index 480a8345..b6e2a892 100644 --- a/src/impl_aarch64_windows.c +++ b/src/impl_aarch64_windows.c @@ -16,12 +16,10 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #ifdef CPU_FEATURES_OS_WINDOWS -#include "impl_aarch64__base_implementation.inl" -static void DetectFeatures(Aarch64Info* info) { - DetectFeaturesBase(info); -} +#include "impl_aarch64__base_implementation.inl" -#endif +static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } -#endif +#endif // CPU_FEATURES_OS_WINDOWS +#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8e8f72af..00c20413 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -72,6 +72,7 @@ endif() ## cpuinfo_aarch64_test if(PROCESSOR_IS_AARCH64) add_executable(cpuinfo_aarch64_test cpuinfo_aarch64_test.cc ../src/impl_aarch64_linux_or_android.c) + target_compile_definitions(cpuinfo_aarch64_test PUBLIC CPU_FEATURES_MOCK_CPUID_AARCH64) target_link_libraries(cpuinfo_aarch64_test all_libraries) add_test(NAME cpuinfo_aarch64_test COMMAND cpuinfo_aarch64_test) endif() diff --git a/test/cpuinfo_aarch64_test.cc b/test/cpuinfo_aarch64_test.cc index 04b61438..30af7e74 100644 --- a/test/cpuinfo_aarch64_test.cc +++ b/test/cpuinfo_aarch64_test.cc @@ -14,16 +14,93 @@ #include "cpuinfo_aarch64.h" +#include + #include "filesystem_for_testing.h" #include "gtest/gtest.h" #include "hwcaps_for_testing.h" +#include "internal/cpuid_aarch64.h" namespace cpu_features { namespace { +class FakeCpuAarch64 { + public: + uint64_t GetCpuid_MIDR_EL1() const { return _midr_el1; } + + uint64_t GetCpuid_ID_AA64ISAR0_EL1() const { return _id_aa64isar0_el1; } + uint64_t GetCpuid_ID_AA64ISAR1_EL1() const { return _id_aa64isar1_el1; } + uint64_t GetCpuid_ID_AA64PFR0_EL1() const { return _id_aa64pfr0_el1; } + uint64_t GetCpuid_ID_AA64ZFR0_EL1() const { return _id_aa64zfr0_el1; } + + void SetCpuid_MIDR_EL1(uint64_t midr_el1) { _midr_el1 = midr_el1; } + + void SetCpuid_ID_AA64ISAR0_EL1(uint64_t id_aa64isar0_el1) { + _id_aa64isar0_el1 = id_aa64isar0_el1; + } + + void SetCpuid_ID_AA64ISAR1_EL1(uint64_t id_aa64isar1_el1) { + _id_aa64isar1_el1 = id_aa64isar1_el1; + } + + void SetCpuid_ID_AA64PFR0_EL1(uint64_t id_aa64pfr0_el1) { + _id_aa64pfr0_el1 = id_aa64pfr0_el1; + } + + void SetCpuid_ID_AA64ZFR0_EL1(uint64_t id_aa64zfr0_el1) { + _id_aa64pfr0_el1 = id_aa64zfr0_el1; + } + + private: + uint64_t _midr_el1; + uint64_t _id_aa64isar0_el1; + uint64_t _id_aa64isar1_el1; + uint64_t _id_aa64pfr0_el1; + uint64_t _id_aa64zfr0_el1; +}; + +FakeCpuAarch64* g_fake_cpu_aarch64; + +FakeCpuAarch64& cpu() { + assert(g_fake_cpu_aarch64 != nullptr); + return *g_fake_cpu_aarch64; +} + +extern "C" uint64_t GetCpuid_MIDR_EL1() { return cpu().GetCpuid_MIDR_EL1(); } + +extern "C" uint64_t GetCpuid_ID_AA64ISAR0_EL1() { + return cpu().GetCpuid_ID_AA64ISAR0_EL1(); +} + +extern "C" uint64_t GetCpuid_ID_AA64ISAR1_EL1() { + return cpu().GetCpuid_ID_AA64ISAR1_EL1(); +} + +extern "C" uint64_t GetCpuid_ID_AA64PFR0_EL1() { + return cpu().GetCpuid_ID_AA64ISAR1_EL1(); +} + +extern "C" uint64_t GetCpuid_ID_AA64ZFR0_EL1() { + return cpu().GetCpuid_ID_AA64ISAR1_EL1(); +} + void DisableHardwareCapabilities() { SetHardwareCapabilities(0, 0); } -TEST(CpuinfoAarch64Test, FromHardwareCap) { +class CpuInfoAarch64Test : public ::testing::Test { + protected: + void SetUp() override { + assert(g_fake_cpu_aarch64 == nullptr); + g_fake_cpu_aarch64 = new FakeCpuAarch64(); + } + + void TearDown() override { + delete g_fake_cpu_aarch64; + g_fake_cpu_aarch64 = nullptr; + } +}; + +#if defined(CPU_FEATURES_OS_LINUX) || defined(CPU_FEATURES_OS_ANDROID) +TEST_F(CpuInfoAarch64Test, FromHardwareCap_LINUX) { ResetHwcaps(); SetHardwareCapabilities(AARCH64_HWCAP_FP | AARCH64_HWCAP_AES, 0); GetEmptyFilesystem(); // disabling /proc/cpuinfo @@ -62,7 +139,7 @@ TEST(CpuinfoAarch64Test, FromHardwareCap) { EXPECT_FALSE(info.features.pacg); } -TEST(CpuinfoAarch64Test, FromHardwareCap2) { +TEST_F(CpuInfoAarch64Test, FromHardwareCap2_LINUX) { ResetHwcaps(); SetHardwareCapabilities(AARCH64_HWCAP_FP, AARCH64_HWCAP2_SVE2 | AARCH64_HWCAP2_BTI); @@ -91,7 +168,7 @@ TEST(CpuinfoAarch64Test, FromHardwareCap2) { EXPECT_FALSE(info.features.rng); } -TEST(CpuinfoAarch64Test, ARMCortexA53) { +TEST_F(CpuInfoAarch64Test, ARMCortexA53_LINUX) { ResetHwcaps(); auto& fs = GetEmptyFilesystem(); fs.CreateFile("/proc/cpuinfo", @@ -110,6 +187,7 @@ CPU architecture: AArch64 CPU variant : 0x0 CPU part : 0xd03 CPU revision : 3)"); + cpu().SetCpuid_MIDR_EL1(MIDR_CORTEX_A53_R3); const auto info = GetAarch64Info(); EXPECT_EQ(info.implementer, 0x41); EXPECT_EQ(info.variant, 0x0); @@ -169,6 +247,7 @@ CPU revision : 3)"); EXPECT_FALSE(info.features.bti); EXPECT_FALSE(info.features.mte); } +#endif } // namespace } // namespace cpu_features From 70055d52010ded3d2d956977117b227dbfb9f6a5 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Mon, 1 Nov 2021 23:45:26 +0200 Subject: [PATCH 07/13] Add detection midr --- src/define_cpuid_aarch64.inl | 7 ------ src/impl_aarch64__base_implementation.inl | 26 ++++++++++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index 061357ec..275eb80f 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -55,10 +55,3 @@ asm volatile(MRS_S("%0", r) : "=r"(__val)); \ __val; \ }) - -#define READ_SYS_REG(reg) \ - ({ \ - uint64_t __val; \ - asm volatile("mrs %0, " STRINGIZE(reg) : "=r"(__val)); \ - __val; \ - }) diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 320201b7..774871ce 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -31,9 +31,16 @@ // Implementation will be provided by test/cpuinfo_aarch64_test.cc. #else uint64_t GetCpuid_MIDR_EL1() { return READ_SYS_REG_S(SYS_MIDR_EL1); } -uint64_t GetCpuid_ID_AA64ISAR0_EL1() { return READ_SYS_REG(ID_AA64ISAR0_EL1); } -uint64_t GetCpuid_ID_AA64ISAR1_EL1() { return READ_SYS_REG(ID_AA64ISAR1_EL1); } -uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG(ID_AA64PFR0_EL1); } +uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG_S(ID_AA64PFR0_EL1); } + +uint64_t GetCpuid_ID_AA64ISAR0_EL1() { + return READ_SYS_REG_S(ID_AA64ISAR0_EL1); +} + +uint64_t GetCpuid_ID_AA64ISAR1_EL1() { + return READ_SYS_REG_S(ID_AA64ISAR1_EL1); +} + uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); } @@ -137,10 +144,13 @@ static const Aarch64Info kEmptyAarch64Info; Aarch64Info GetAarch64Info(void) { Aarch64Info info = kEmptyAarch64Info; const uint64_t midr = GetCpuid_MIDR_EL1(); - info.implementer = ExtractBitRange(midr, 31, 24); - info.variant = ExtractBitRange(midr, 23, 20); - info.part = ExtractBitRange(midr, 15, 4); - info.revision = ExtractBitRange(midr, 3, 0); - DetectFeatures(&info); + if (midr) { + info.features.cpuid = 1; + info.implementer = ExtractBitRange(midr, 31, 24); + info.variant = ExtractBitRange(midr, 23, 20); + info.part = ExtractBitRange(midr, 15, 4); + info.revision = ExtractBitRange(midr, 3, 0); + DetectFeatures(&info); + } return info; } \ No newline at end of file From 7ad64b1d0c4ca4916f5972ea3fcc9aa073e0bb97 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Sun, 7 Nov 2021 02:15:20 +0200 Subject: [PATCH 08/13] Add initial armasm --- include/internal/cpuid_aarch64.h | 10 +-- include/internal/cputype_aarch64.h | 20 ++++- src/define_cpuid_aarch64.inl | 33 +++++++- src/define_introspection.inl | 1 + src/impl_aarch64__base_implementation.inl | 98 +++++++++++++---------- src/impl_aarch64_freebsd.c | 2 +- src/impl_aarch64_iphone.c | 25 ++++++ src/impl_aarch64_macos.c | 2 +- src/impl_aarch64_windows.c | 2 +- test/cpuinfo_aarch64_test.cc | 6 +- 10 files changed, 139 insertions(+), 60 deletions(-) create mode 100644 src/impl_aarch64_iphone.c diff --git a/include/internal/cpuid_aarch64.h b/include/internal/cpuid_aarch64.h index c8fcc751..6d3f667c 100644 --- a/include/internal/cpuid_aarch64.h +++ b/include/internal/cpuid_aarch64.h @@ -1,4 +1,4 @@ -// Copyright 2017 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H -#define CPU_FEATURES_CPUTYPE_AARCH64_H +#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H_ +#define CPU_FEATURES_CPUTYPE_AARCH64_H_ #include @@ -27,8 +27,6 @@ uint64_t GetCpuid_ID_AA64ISAR1_EL1(); uint64_t GetCpuid_ID_AA64PFR0_EL1(); uint64_t GetCpuid_ID_AA64ZFR0_EL1(); -static void DetectFeatures(Aarch64Info* info); - CPU_FEATURES_END_CPP_NAMESPACE -#endif // CPU_FEATURES_CPUTYPE_AARCH64_H +#endif // CPU_FEATURES_CPUTYPE_AARCH64_H_ diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h index 4894e65f..112713c8 100644 --- a/include/internal/cputype_aarch64.h +++ b/include/internal/cputype_aarch64.h @@ -1,5 +1,19 @@ -#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H -#define CPU_FEATURES_CPUTYPE_AARCH64_H +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CPU_FEATURES_CPUTYPE_AARCH64_H_ +#define CPU_FEATURES_CPUTYPE_AARCH64_H_ // Arm Architecture Reference Manual Armv8, for A-profile architecture. D13.2.98 // https://elixir.bootlin.com/linux/latest/source/arch/arm64/include/asm/cputype.h @@ -150,4 +164,4 @@ #define MIDR_APPLE_M1_FIRESTORM \ MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) -#endif // CPU_FEATURES_CPUTYPE_AARCH64_H +#endif // CPU_FEATURES_CPUTYPE_AARCH64_H_ diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index 275eb80f..9857dd62 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -14,8 +14,6 @@ #include "stringize.h" -#define EMIT_INST(x) ".inst " STRINGIZE((x)) "\n\t" - #define OP0_SHIFT 19 #define OP1_SHIFT 16 #define CRN_SHIFT 12 @@ -27,8 +25,17 @@ ((crm) << CRM_SHIFT) | ((op2) << OP2_SHIFT)) #define SYS_MIDR_EL1 SYS_REG(3, 0, 0, 0, 0) + +#define SYS_ID_AA64PFR0_EL1 SYS_REG(3, 0, 0, 4, 0) #define SYS_ID_AA64ZFR0_EL1 SYS_REG(3, 0, 0, 4, 4) +#define SYS_ID_AA64ISAR0_EL1 SYS_REG(3, 0, 0, 6, 0) +#define SYS_ID_AA64ISAR1_EL1 SYS_REG(3, 0, 0, 6, 1) + +#if (defined(CPU_FEATURES_COMPILER_GCC) || defined(CPU_FEATURES_COMPILER_CLANG)) + +#define EMIT_INST(x) ".inst " STRINGIZE((x)) "\n\t" + #define DEFINE_MRS_MSR_S_REGNUM \ " .irp " \ "num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25, " \ @@ -55,3 +62,25 @@ asm volatile(MRS_S("%0", r) : "=r"(__val)); \ __val; \ }) +#elif defined(CPU_FEATURES_COMPILER_MSC) +#define READ_SYS_REG_S(r) \ + ({ \ + uint64_t sreg; \ + __asm { \ + GBLA count \ + count SETA 0 \ + WHILE count < 31 \ + count SETA count + 1 \ + GBLA __reg_num_x$count \ + __reg_num_x$count SETA count \ + WEND \ + GBLA __reg_num_xzr \ + __reg_num_xzr SETA 31 \ + \ + sreg DCI 0xD5200000 :OR: sreg :OR: __reg_num_$r \ + } \ + sreg; \ + }) +#else +#error "Cannot compile cpuinfo_aarch64 on a non supported compiler." +#endif \ No newline at end of file diff --git a/src/define_introspection.inl b/src/define_introspection.inl index 435284b8..ac80d6db 100644 --- a/src/define_introspection.inl +++ b/src/define_introspection.inl @@ -23,6 +23,7 @@ #endif #include + #include "stringize.h" #define FEAT_TYPE_NAME__(X) X##Features diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 774871ce..9eb9feb2 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -1,4 +1,4 @@ -// Copyright 2017 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include "cpuinfo_aarch64.h" #include "define_cpuid_aarch64.inl" #include "internal/bit_utils.h" @@ -31,14 +29,17 @@ // Implementation will be provided by test/cpuinfo_aarch64_test.cc. #else uint64_t GetCpuid_MIDR_EL1() { return READ_SYS_REG_S(SYS_MIDR_EL1); } -uint64_t GetCpuid_ID_AA64PFR0_EL1() { return READ_SYS_REG_S(ID_AA64PFR0_EL1); } + +uint64_t GetCpuid_ID_AA64PFR0_EL1() { + return READ_SYS_REG_S(SYS_ID_AA64PFR0_EL1); +} uint64_t GetCpuid_ID_AA64ISAR0_EL1() { - return READ_SYS_REG_S(ID_AA64ISAR0_EL1); + return READ_SYS_REG_S(SYS_ID_AA64ISAR0_EL1); } uint64_t GetCpuid_ID_AA64ISAR1_EL1() { - return READ_SYS_REG_S(ID_AA64ISAR1_EL1); + return READ_SYS_REG_S(SYS_ID_AA64ISAR1_EL1); } uint64_t GetCpuid_ID_AA64ZFR0_EL1() { @@ -46,32 +47,36 @@ uint64_t GetCpuid_ID_AA64ZFR0_EL1() { } #endif +// This function have to be implemented by the OS or +// can use base implementation DetectFeaturesBase. +static void DetectFeatures(Aarch64Info* info); + static void DetectFeaturesBase(Aarch64Info* info) { const uint64_t pfr0 = GetCpuid_ID_AA64PFR0_EL1(); info->features.sve = ExtractBitRange(pfr0, 35, 32); info->features.dit = ExtractBitRange(pfr0, 51, 48); - const uint64_t fp_value = ExtractBitRange(pfr0, 19, 16); + const uint64_t fp = ExtractBitRange(pfr0, 19, 16); // fp_value = 0b0000: // Floating-point is implemented, and includes support for: // * Single-precision and double-precision floating-point types. // * Conversions between single-precision and half-precision data types, // and double-precision and half-precision data types. - if (fp_value == 0) { + if (fp == 0) { info->features.fp = 1; } // fp_value = 0b0001: // As for 0b0000, and also includes support for half-precision // floating-point arithmetic. - if (fp_value == 1) { + if (fp == 1) { info->features.fp = 1; info->features.fphp = 1; } - const uint64_t asimd_value = ExtractBitRange(pfr0, 23, 20); + const uint64_t asimd = ExtractBitRange(pfr0, 23, 20); // Advanced SIMD is implemented, including support for the following SISD and // SIMD operations: @@ -79,63 +84,70 @@ static void DetectFeaturesBase(Aarch64Info* info) { // * Single-precision and double-precision floating-point arithmetic. // * Conversions between single-precision and half-precision data types, // and double-precision and half-precision data types. - if (asimd_value == 0) { + if (asimd == 0) { info->features.asimd = 1; } - // asimd_value = 0b0001: + // asimd = 0b0001: // As for 0b0000, and also includes support for half-precision // floating-point arithmetic. - if (asimd_value == 1) { + if (asimd == 1) { info->features.asimd = 1; - info->features.asimdfhm = 1; + info->features.asimdhp = 1; } const uint64_t isa0 = GetCpuid_ID_AA64ISAR0_EL1(); + const uint64_t aes = ExtractBitRange(isa0, 7, 4); - if (aes == 1) info->features.aes = 1; - if (aes == 2) { - info->features.aes = 1; - info->features.pmull = 1; - } + if (aes >= 1) info->features.aes = 1; + if (aes >= 2) info->features.pmull = 1; + info->features.sha1 = ExtractBitRange(isa0, 11, 8); - info->features.sha2 = ExtractBitRange(isa0, 15, 12); + const uint64_t sha2 = ExtractBitRange(isa0, 15, 12); + if (sha2 >= 1) info->features.sha2 = 1; + if (sha2 >= 2) info->features.sha512 = 1; + info->features.crc32 = ExtractBitRange(isa0, 19, 16); info->features.atomics = ExtractBitRange(isa0, 23, 20); - info->features.atomics = ExtractBitRange(isa0, 31, 28); + info->features.asimdrdm = ExtractBitRange(isa0, 31, 28); info->features.sha3 = ExtractBitRange(isa0, 35, 32); info->features.sm3 = ExtractBitRange(isa0, 39, 36); info->features.sm4 = ExtractBitRange(isa0, 43, 40); + info->features.asimddp = ExtractBitRange(isa0, 47, 44); + info->features.asimdfhm = ExtractBitRange(isa0, 51, 48); + const uint64_t ts = ExtractBitRange(isa0, 55, 52); + if (ts == 1) { + info->features.flagm = 1; + } + if (ts == 2) { + info->features.flagm = 1; + info->features.flagm2 = 1; + } info->features.rng = ExtractBitRange(isa0, 63, 60); const uint64_t isa1 = GetCpuid_ID_AA64ISAR1_EL1(); - info->features.i8mm = ExtractBitRange(isa1, 55, 52); - info->features.dgh = ExtractBitRange(isa1, 51, 48); - info->features.bf16 = ExtractBitRange(isa1, 47, 44); - info->features.sb = ExtractBitRange(isa1, 39, 36); - info->features.lrcpc = ExtractBitRange(isa1, 23, 20); - info->features.fcma = ExtractBitRange(isa1, 19, 16); info->features.jscvt = ExtractBitRange(isa1, 15, 12); + info->features.fcma = ExtractBitRange(isa1, 19, 16); + info->features.lrcpc = ExtractBitRange(isa1, 23, 20); + info->features.sb = ExtractBitRange(isa1, 39, 36); + info->features.bf16 = ExtractBitRange(isa1, 47, 44); + info->features.dgh = ExtractBitRange(isa1, 51, 48); + info->features.i8mm = ExtractBitRange(isa1, 55, 52); if (info->features.sve) { const uint64_t zfr0 = GetCpuid_ID_AA64ZFR0_EL1(); - info->features.svef64mm = ExtractBitRange(zfr0, 59, 56); - info->features.svef32mm = ExtractBitRange(zfr0, 55, 52); - info->features.svei8mm = ExtractBitRange(zfr0, 47, 44); - info->features.svesm4 = ExtractBitRange(zfr0, 43, 40); - info->features.svesha3 = ExtractBitRange(zfr0, 35, 32); - info->features.svebf16 = ExtractBitRange(zfr0, 23, 20); - info->features.svebitperm = ExtractBitRange(zfr0, 19, 16); info->features.sve2 = ExtractBitRange(zfr0, 0, 3); + info->features.svebitperm = ExtractBitRange(zfr0, 19, 16); + info->features.svebf16 = ExtractBitRange(zfr0, 23, 20); + info->features.svesha3 = ExtractBitRange(zfr0, 35, 32); + info->features.svesm4 = ExtractBitRange(zfr0, 43, 40); + info->features.svei8mm = ExtractBitRange(zfr0, 47, 44); + info->features.svef32mm = ExtractBitRange(zfr0, 55, 52); + info->features.svef64mm = ExtractBitRange(zfr0, 59, 56); const uint64_t sveaes = ExtractBitRange(zfr0, 7, 4); - if (sveaes == 1) { - info->features.sveaes = 1; - } - if (sveaes == 2) { - info->features.sveaes = 1; - info->features.svepmull = 1; - } + if (sveaes >= 1) info->features.sveaes = 1; + if (sveaes >= 2) info->features.svepmull = 1; } } @@ -150,7 +162,7 @@ Aarch64Info GetAarch64Info(void) { info.variant = ExtractBitRange(midr, 23, 20); info.part = ExtractBitRange(midr, 15, 4); info.revision = ExtractBitRange(midr, 3, 0); - DetectFeatures(&info); + DetectFeaturesBase(&info); } return info; -} \ No newline at end of file +} diff --git a/src/impl_aarch64_freebsd.c b/src/impl_aarch64_freebsd.c index bfc5d4a5..c68ee32c 100644 --- a/src/impl_aarch64_freebsd.c +++ b/src/impl_aarch64_freebsd.c @@ -1,4 +1,4 @@ -// Copyright 2017 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/impl_aarch64_iphone.c b/src/impl_aarch64_iphone.c new file mode 100644 index 00000000..d1bef200 --- /dev/null +++ b/src/impl_aarch64_iphone.c @@ -0,0 +1,25 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cpu_features_macros.h" + +#ifdef CPU_FEATURES_ARCH_AARCH64 +#ifdef CPU_FEATURES_OS_IPHONE + +#include "impl_aarch64__base_implementation.inl" + +static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } + +#endif // CPU_FEATURES_OS_IPHONE +#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/src/impl_aarch64_macos.c b/src/impl_aarch64_macos.c index 15e7802d..411a96f1 100644 --- a/src/impl_aarch64_macos.c +++ b/src/impl_aarch64_macos.c @@ -1,4 +1,4 @@ -// Copyright 2017 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/impl_aarch64_windows.c b/src/impl_aarch64_windows.c index b6e2a892..c1b603f5 100644 --- a/src/impl_aarch64_windows.c +++ b/src/impl_aarch64_windows.c @@ -1,4 +1,4 @@ -// Copyright 2017 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/cpuinfo_aarch64_test.cc b/test/cpuinfo_aarch64_test.cc index 30af7e74..b68a556e 100644 --- a/test/cpuinfo_aarch64_test.cc +++ b/test/cpuinfo_aarch64_test.cc @@ -100,7 +100,7 @@ class CpuInfoAarch64Test : public ::testing::Test { }; #if defined(CPU_FEATURES_OS_LINUX) || defined(CPU_FEATURES_OS_ANDROID) -TEST_F(CpuInfoAarch64Test, FromHardwareCap_LINUX) { +TEST_F(CpuInfoAarch64Test, FromHardwareCap_HWCAP) { ResetHwcaps(); SetHardwareCapabilities(AARCH64_HWCAP_FP | AARCH64_HWCAP_AES, 0); GetEmptyFilesystem(); // disabling /proc/cpuinfo @@ -139,7 +139,7 @@ TEST_F(CpuInfoAarch64Test, FromHardwareCap_LINUX) { EXPECT_FALSE(info.features.pacg); } -TEST_F(CpuInfoAarch64Test, FromHardwareCap2_LINUX) { +TEST_F(CpuInfoAarch64Test, FromHardwareCap2_HWCAP2) { ResetHwcaps(); SetHardwareCapabilities(AARCH64_HWCAP_FP, AARCH64_HWCAP2_SVE2 | AARCH64_HWCAP2_BTI); @@ -168,7 +168,7 @@ TEST_F(CpuInfoAarch64Test, FromHardwareCap2_LINUX) { EXPECT_FALSE(info.features.rng); } -TEST_F(CpuInfoAarch64Test, ARMCortexA53_LINUX) { +TEST_F(CpuInfoAarch64Test, ARMCortexA53_PROC_CPUINFO) { ResetHwcaps(); auto& fs = GetEmptyFilesystem(); fs.CreateFile("/proc/cpuinfo", From 3997b98560ff6a35f719abf70116244bdf6c4828 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Sun, 7 Nov 2021 15:51:47 +0200 Subject: [PATCH 09/13] Add asm file for msvc --- .../aarch64/cpuid_msvc_armasm64_windows.asm | 18 +++++++++++++++ src/define_cpuid_aarch64.inl | 22 +++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 src/asm/aarch64/cpuid_msvc_armasm64_windows.asm diff --git a/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm b/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm new file mode 100644 index 00000000..f62efa1e --- /dev/null +++ b/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm @@ -0,0 +1,18 @@ +AREA READ_MRS, CODE, READONLY +EXPORT READ_MRS_S + +count SETA 0 +WHILE count < 31 + reg_num_x$count SETA count + count SETA count + 1 +WEND +reg_num_xzr SETA 31 + +READ_MRS_S PROC + r SETA X0 + sysreg SETA X1 + cpuid SETA DCI 0xD5200000 :OR: sysreg :OR: reg_num_$r + MOV X0, cpuid +ENDP + +END \ No newline at end of file diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index 9857dd62..73f44b38 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -63,24 +63,14 @@ __val; \ }) #elif defined(CPU_FEATURES_COMPILER_MSC) +extern void READ_MRS_S(); #define READ_SYS_REG_S(r) \ ({ \ - uint64_t sreg; \ - __asm { \ - GBLA count \ - count SETA 0 \ - WHILE count < 31 \ - count SETA count + 1 \ - GBLA __reg_num_x$count \ - __reg_num_x$count SETA count \ - WEND \ - GBLA __reg_num_xzr \ - __reg_num_xzr SETA 31 \ - \ - sreg DCI 0xD5200000 :OR: sreg :OR: __reg_num_$r \ - } \ - sreg; \ + uint64_t __val; \ + READ_MRS_S(__val, r); \ + __val; \ }) #else #error "Cannot compile cpuinfo_aarch64 on a non supported compiler." -#endif \ No newline at end of file +#endif + From 1f935e91292bef6cbcbab4cd47759834c63f1166 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Tue, 9 Nov 2021 05:13:39 +0200 Subject: [PATCH 10/13] Add tests for inline asm implementation --- include/internal/cputype_aarch64.h | 84 ++++++++++--------- .../aarch64/cpuid_msvc_armasm64_windows.asm | 18 ---- src/define_cpuid_aarch64.inl | 10 --- src/impl_aarch64__base_implementation.inl | 2 +- test/cpuinfo_aarch64_test.cc | 36 +++++++- test/cputype_aarch64_for_testing.h | 54 ++++++++++++ 6 files changed, 133 insertions(+), 71 deletions(-) delete mode 100644 src/asm/aarch64/cpuid_msvc_armasm64_windows.asm create mode 100644 test/cputype_aarch64_for_testing.h diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h index 112713c8..292641f5 100644 --- a/include/internal/cputype_aarch64.h +++ b/include/internal/cputype_aarch64.h @@ -21,7 +21,6 @@ #define MIDR_REVISION_SHIFT 0 #define MIDR_PARTNUM_SHIFT 4 #define MIDR_ARCHITECTURE_SHIFT 16 -#define MIDR_VARIANT_SHIFT 20 #define MIDR_IMPLEMENTOR_SHIFT 24 // MIDR: implementor, partnum @@ -34,18 +33,6 @@ (((imp) << MIDR_IMPLEMENTOR_SHIFT) | (0xF << MIDR_ARCHITECTURE_SHIFT) | \ ((partnum) << MIDR_PARTNUM_SHIFT) | ((revision) << MIDR_REVISION_SHIFT)) -// MIDR: implementor, variant, partnum, revision -#define MIDR_CPU_MODEL_IPRV(imp, var, partnum, revision) \ - (((imp) << MIDR_IMPLEMENTOR_SHIFT) | ((var) << MIDR_VARIANT_SHIFT) | \ - (0xF << MIDR_ARCHITECTURE_SHIFT) | ((partnum) << MIDR_PARTNUM_SHIFT) | \ - ((revision) << MIDR_REVISION_SHIFT)) - -// MIDR: implementor, variant, architecture, partnum, revision -#define MIDR_CPU_MODEL_IVARV(imp, var, arch, partnum, revision) \ - (((imp) << MIDR_IMPLEMENTOR_SHIFT) | ((var) << MIDR_VARIANT_SHIFT) | \ - ((arch) << MIDR_ARCHITECTURE_SHIFT) | ((partnum) << MIDR_PARTNUM_SHIFT) | \ - ((revision) << MIDR_REVISION_SHIFT)) - #define ARM_CPU_IMP_ARM 0x41 // ARM Limited. #define ARM_CPU_IMP_BRCM 0x42 // Broadcom Corporation. #define ARM_CPU_IMP_CAVIUM 0x43 // Cavium Inc. @@ -110,58 +97,73 @@ #define MIDR_CORTEX_A53_R3 \ MIDR_CPU_MODEL_IPR(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53, 3) -#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) -#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) -#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73) -#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) -#define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) -#define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) -#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) +#define MIDR_CORTEX_A57 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) +#define MIDR_CORTEX_A72 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) +#define MIDR_CORTEX_A73 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73) +#define MIDR_CORTEX_A75 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) +#define MIDR_CORTEX_A35 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) +#define MIDR_CORTEX_A55 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) +#define MIDR_CORTEX_A76 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) #define MIDR_NEOVERSE_N1 \ MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) -#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) +#define MIDR_CORTEX_A77 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) #define MIDR_THUNDERX \ - MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX \ - MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX \ - MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) #define MIDR_CAVIUM_THUNDERX2 \ - MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2) #define MIDR_BRAHMA_B53 \ - MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_BRAHMA_B53) -#define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_BRAHMA_B53) +#define MIDR_BRCM_VULCAN \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN) #define MIDR_QCOM_FALKOR_V1 \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1) -#define MIDR_QCOM_FALKOR MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR) -#define MIDR_QCOM_KRYO MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1) +#define MIDR_QCOM_FALKOR \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR) +#define MIDR_QCOM_KRYO MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO) #define MIDR_QCOM_KRYO_2XX_GOLD \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_GOLD) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_GOLD) #define MIDR_QCOM_KRYO_2XX_SILVER \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_SILVER) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_2XX_SILVER) #define MIDR_QCOM_KRYO_3XX_SILVER \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_3XX_SILVER) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_3XX_SILVER) #define MIDR_QCOM_KRYO_4XX_GOLD \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_GOLD) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_GOLD) #define MIDR_QCOM_KRYO_4XX_SILVER \ - MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_SILVER) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO_4XX_SILVER) #define MIDR_NVIDIA_DENVER \ - MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_DENVER) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_DENVER) #define MIDR_NVIDIA_CARMEL \ - MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) #define MIDR_FUJITSU_A64FX \ - MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) -#define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) +#define MIDR_HISI_TSV110 \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) #define MIDR_APPLE_M1_ICESTORM \ - MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) #define MIDR_APPLE_M1_FIRESTORM \ - MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) +#define MIDR_APPLE_M1_ICESTORM_TONGA \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_TONGA) +#define MIDR_APPLE_M1_FIRESTORM_TONGA \ + MIDR_CPU_MODEL_IP(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_TONGA) #endif // CPU_FEATURES_CPUTYPE_AARCH64_H_ diff --git a/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm b/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm deleted file mode 100644 index f62efa1e..00000000 --- a/src/asm/aarch64/cpuid_msvc_armasm64_windows.asm +++ /dev/null @@ -1,18 +0,0 @@ -AREA READ_MRS, CODE, READONLY -EXPORT READ_MRS_S - -count SETA 0 -WHILE count < 31 - reg_num_x$count SETA count - count SETA count + 1 -WEND -reg_num_xzr SETA 31 - -READ_MRS_S PROC - r SETA X0 - sysreg SETA X1 - cpuid SETA DCI 0xD5200000 :OR: sysreg :OR: reg_num_$r - MOV X0, cpuid -ENDP - -END \ No newline at end of file diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index 73f44b38..e8962f8c 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -33,7 +33,6 @@ #define SYS_ID_AA64ISAR1_EL1 SYS_REG(3, 0, 0, 6, 1) #if (defined(CPU_FEATURES_COMPILER_GCC) || defined(CPU_FEATURES_COMPILER_CLANG)) - #define EMIT_INST(x) ".inst " STRINGIZE((x)) "\n\t" #define DEFINE_MRS_MSR_S_REGNUM \ @@ -62,15 +61,6 @@ asm volatile(MRS_S("%0", r) : "=r"(__val)); \ __val; \ }) -#elif defined(CPU_FEATURES_COMPILER_MSC) -extern void READ_MRS_S(); -#define READ_SYS_REG_S(r) \ - ({ \ - uint64_t __val; \ - READ_MRS_S(__val, r); \ - __val; \ - }) #else #error "Cannot compile cpuinfo_aarch64 on a non supported compiler." #endif - diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 9eb9feb2..c9026c98 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -162,7 +162,7 @@ Aarch64Info GetAarch64Info(void) { info.variant = ExtractBitRange(midr, 23, 20); info.part = ExtractBitRange(midr, 15, 4); info.revision = ExtractBitRange(midr, 3, 0); - DetectFeaturesBase(&info); + DetectFeatures(&info); } return info; } diff --git a/test/cpuinfo_aarch64_test.cc b/test/cpuinfo_aarch64_test.cc index b68a556e..b64c6dcd 100644 --- a/test/cpuinfo_aarch64_test.cc +++ b/test/cpuinfo_aarch64_test.cc @@ -20,6 +20,7 @@ #include "gtest/gtest.h" #include "hwcaps_for_testing.h" #include "internal/cpuid_aarch64.h" +#include "cputype_aarch64_for_testing.h" namespace cpu_features { namespace { @@ -104,6 +105,7 @@ TEST_F(CpuInfoAarch64Test, FromHardwareCap_HWCAP) { ResetHwcaps(); SetHardwareCapabilities(AARCH64_HWCAP_FP | AARCH64_HWCAP_AES, 0); GetEmptyFilesystem(); // disabling /proc/cpuinfo + cpu().SetCpuid_MIDR_EL1(1); const auto info = GetAarch64Info(); EXPECT_TRUE(info.features.fp); EXPECT_FALSE(info.features.asimd); @@ -116,7 +118,7 @@ TEST_F(CpuInfoAarch64Test, FromHardwareCap_HWCAP) { EXPECT_FALSE(info.features.atomics); EXPECT_FALSE(info.features.fphp); EXPECT_FALSE(info.features.asimdhp); - EXPECT_FALSE(info.features.cpuid); + EXPECT_TRUE(info.features.cpuid); EXPECT_FALSE(info.features.asimdrdm); EXPECT_FALSE(info.features.jscvt); EXPECT_FALSE(info.features.fcma); @@ -144,6 +146,7 @@ TEST_F(CpuInfoAarch64Test, FromHardwareCap2_HWCAP2) { SetHardwareCapabilities(AARCH64_HWCAP_FP, AARCH64_HWCAP2_SVE2 | AARCH64_HWCAP2_BTI); GetEmptyFilesystem(); // disabling /proc/cpuinfo + cpu().SetCpuid_MIDR_EL1(1); const auto info = GetAarch64Info(); EXPECT_TRUE(info.features.fp); @@ -247,6 +250,37 @@ CPU revision : 3)"); EXPECT_FALSE(info.features.bti); EXPECT_FALSE(info.features.mte); } +#else +TEST_F(CpuInfoAarch64Test, ARM_CORTEX_A72_MRS_MIDR_EL1) { + cpu().SetCpuid_MIDR_EL1(MIDR_CORTEX_A72); + const auto info = GetAarch64Info(); + EXPECT_EQ(info.implementer, 0x41); + EXPECT_EQ(info.variant, 0x0); + EXPECT_EQ(info.part, 0xD08); + EXPECT_EQ(info.revision, 0); +} + +TEST_F(CpuInfoAarch64Test, ARM_CORTEX_A53_R3_MRS_ID_AA64ISAR0_EL1) { + cpu().SetCpuid_MIDR_EL1(MIDR_CORTEX_A53_R3); + cpu().SetCpuid_ID_AA64ISAR0_EL1(ID_AA64ISAR0_EL1_ARM_CORTEX_A53_R3); + const auto info = GetAarch64Info(); + EXPECT_EQ(info.implementer, 0x41); + EXPECT_EQ(info.variant, 0x0); + EXPECT_EQ(info.part, 0xD03); + EXPECT_EQ(info.revision, 3); + + EXPECT_TRUE(info.features.cpuid); + EXPECT_TRUE(info.features.aes); + EXPECT_TRUE(info.features.pmull); + EXPECT_TRUE(info.features.sha1); + EXPECT_TRUE(info.features.sha2); + EXPECT_TRUE(info.features.crc32); + EXPECT_FALSE(info.features.atomics); + EXPECT_TRUE(info.features.cpuid); + EXPECT_FALSE(info.features.sha3); + EXPECT_FALSE(info.features.sm3); + EXPECT_FALSE(info.features.sm4); +} #endif } // namespace diff --git a/test/cputype_aarch64_for_testing.h b/test/cputype_aarch64_for_testing.h new file mode 100644 index 00000000..b6d2f56d --- /dev/null +++ b/test/cputype_aarch64_for_testing.h @@ -0,0 +1,54 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ +#define CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ + +#define ID_AA64ISAR0_EL1_AES_SHIFT 4 +#define ID_AA64ISAR0_EL1_SHA1_SHIFT 8 +#define ID_AA64ISAR0_EL1_SHA2_SHIFT 12 +#define ID_AA64ISAR0_EL1_CRC32_SHIFT 16 +#define ID_AA64ISAR0_EL1_ATOMIC_SHIFT 20 +#define ID_AA64ISAR0_EL1_RDM_SHIFT 28 +#define ID_AA64ISAR0_EL1_SHA3_SHIFT 32 +#define ID_AA64ISAR0_EL1_SM3_SHIFT 36 +#define ID_AA64ISAR0_EL1_SM4_SHIFT 40 +#define ID_AA64ISAR0_EL1_DP_SHIFT 44 +#define ID_AA64ISAR0_EL1_FHM_SHIFT 48 +#define ID_AA64ISAR0_EL1_TS_SHIFT 52 +#define ID_AA64ISAR0_EL1_TLB_SHIFT 56 +#define ID_AA64ISAR0_EL1_RNDR_SHIFT 60 + +#define ID_AA64ISAR0_EL1_MODEL(rndr, tlb, ts, fhm, dp, sm4, sm3, sha3, rdm, \ + atomic, crc32, sha2, sha1, aes) \ + (((rndr) << ID_AA64ISAR0_EL1_RNDR_SHIFT) | \ + ((tlb) << ID_AA64ISAR0_EL1_TLB_SHIFT) | \ + ((ts) << ID_AA64ISAR0_EL1_TS_SHIFT) | \ + ((fhm) << ID_AA64ISAR0_EL1_FHM_SHIFT) | \ + ((dp) << ID_AA64ISAR0_EL1_DP_SHIFT) | \ + ((sm4) << ID_AA64ISAR0_EL1_SM4_SHIFT) | \ + ((sm3) << ID_AA64ISAR0_EL1_SM3_SHIFT) | \ + ((sha3) << ID_AA64ISAR0_EL1_SHA3_SHIFT) | \ + ((rdm) << ID_AA64ISAR0_EL1_RDM_SHIFT) | \ + ((atomic) << ID_AA64ISAR0_EL1_ATOMIC_SHIFT) | \ + ((crc32) << ID_AA64ISAR0_EL1_CRC32_SHIFT) | \ + ((sha2) << ID_AA64ISAR0_EL1_SHA2_SHIFT) | \ + ((sha1) << ID_AA64ISAR0_EL1_SHA1_SHIFT) | \ + ((aes) << ID_AA64ISAR0_EL1_AES_SHIFT)) + +#define ID_AA64ISAR0_EL1_ARM_CORTEX_A53_R3 \ + ID_AA64ISAR0_EL1_MODEL(0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, \ + 1UL, 1UL, 1UL, 2UL) + +#endif // CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ From 80762ce37a722bfddb5b16dbf8e2e44050a1255f Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Tue, 9 Nov 2021 17:50:46 +0200 Subject: [PATCH 11/13] Fix extract bit range and add ID_AA64 tests update message --- include/cpu_features_macros.h | 2 +- include/cpuinfo_aarch64.h | 104 +++++++++--------- include/internal/bit_utils.h | 4 +- include/internal/cputype_aarch64.h | 42 +++++++ src/define_cpuid_aarch64.inl | 2 +- src/impl_aarch64__base_implementation.inl | 17 +-- src/impl_aarch64_macos.c | 25 ----- ...phone.c => impl_aarch64_macos_or_iphone.c} | 4 +- src/impl_aarch64_windows.c | 25 ----- test/CMakeLists.txt | 7 +- test/cpuinfo_aarch64_test.cc | 23 +++- test/cputype_aarch64_for_testing.h | 54 --------- 12 files changed, 130 insertions(+), 179 deletions(-) delete mode 100644 src/impl_aarch64_macos.c rename src/{impl_aarch64_iphone.c => impl_aarch64_macos_or_iphone.c} (84%) delete mode 100644 src/impl_aarch64_windows.c delete mode 100644 test/cputype_aarch64_for_testing.h diff --git a/include/cpu_features_macros.h b/include/cpu_features_macros.h index d8545a54..384f31c5 100644 --- a/include/cpu_features_macros.h +++ b/include/cpu_features_macros.h @@ -39,7 +39,7 @@ #define CPU_FEATURES_ARCH_ARM #endif -#if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)) +#if (defined(__aarch64__) || defined(__arm64__)) #define CPU_FEATURES_ARCH_AARCH64 #endif diff --git a/include/cpuinfo_aarch64.h b/include/cpuinfo_aarch64.h index a587e113..1b57d218 100644 --- a/include/cpuinfo_aarch64.h +++ b/include/cpuinfo_aarch64.h @@ -15,65 +15,63 @@ #ifndef CPU_FEATURES_INCLUDE_CPUINFO_AARCH64_H_ #define CPU_FEATURES_INCLUDE_CPUINFO_AARCH64_H_ -#include - #include "cpu_features_cache_info.h" #include "cpu_features_macros.h" CPU_FEATURES_START_CPP_NAMESPACE typedef struct { - uint8_t fp; // Floating-point. - uint8_t asimd; // Advanced SIMD. - uint8_t evtstrm; // Generic timer generated events. - uint8_t aes; // Hardware-accelerated Advanced Encryption Standard. - uint8_t pmull; // Polynomial multiply long. - uint8_t sha1; // Hardware-accelerated SHA1. - uint8_t sha2; // Hardware-accelerated SHA2-256. - uint8_t crc32; // Hardware-accelerated CRC-32. - uint8_t atomics; // Armv8.1 atomic instructions. - uint8_t fphp; // Half-precision floating point support. - uint8_t asimdhp; // Advanced SIMD half-precision support. - uint8_t cpuid; // Access to certain ID registers. - uint8_t asimdrdm; // Rounding Double Multiply Accumulate/Subtract. - uint8_t jscvt; // Support for JavaScript conversion. - uint8_t fcma; // Floating point complex numbers. - uint8_t lrcpc; // Support for weaker release consistency. - uint8_t dcpop; // Data persistence writeback. - uint8_t sha3; // Hardware-accelerated SHA3. - uint8_t sm3; // Hardware-accelerated SM3. - uint8_t sm4; // Hardware-accelerated SM4. - uint8_t asimddp; // Dot product instruction. - uint8_t sha512; // Hardware-accelerated SHA512. - uint8_t sve; // Scalable Vector Extension. - uint8_t asimdfhm; // Additional half-precision instructions. - uint8_t dit; // Data independent timing. - uint8_t uscat; // Unaligned atomics support. - uint8_t ilrcpc; // Additional support for weaker release consistency. - uint8_t flagm; // Flag manipulation instructions. - uint8_t ssbs; // Speculative Store Bypass Safe PSTATE bit. - uint8_t sb; // Speculation barrier. - uint8_t paca; // Address authentication. - uint8_t pacg; // Generic authentication. - uint8_t dcpodp; // Data cache clean to point of persistence. - uint8_t sve2; // Scalable Vector Extension (version 2). - uint8_t sveaes; // SVE AES instructions. - uint8_t svepmull; // SVE polynomial multiply long instructions. - uint8_t svebitperm; // SVE bit permute instructions. - uint8_t svesha3; // SVE SHA3 instructions. - uint8_t svesm4; // SVE SM4 instructions. - uint8_t flagm2; // Additional flag manipulation instructions. - uint8_t frint; // Floating point to integer rounding. - uint8_t svei8mm; // SVE Int8 matrix multiplication instructions. - uint8_t svef32mm; // SVE FP32 matrix multiplication instruction. - uint8_t svef64mm; // SVE FP64 matrix multiplication instructions. - uint8_t svebf16; // SVE BFloat16 instructions. - uint8_t i8mm; // Int8 matrix multiplication instructions. - uint8_t bf16; // BFloat16 instructions. - uint8_t dgh; // Data Gathering Hint instruction. - uint8_t rng; // True random number generator support. - uint8_t bti; // Branch target identification. - uint8_t mte; // Memory tagging extension. + int fp : 1; // Floating-point. + int asimd : 1; // Advanced SIMD. + int evtstrm : 1; // Generic timer generated events. + int aes : 1; // Hardware-accelerated Advanced Encryption Standard. + int pmull : 1; // Polynomial multiply long. + int sha1 : 1; // Hardware-accelerated SHA1. + int sha2 : 1; // Hardware-accelerated SHA2-256. + int crc32 : 1; // Hardware-accelerated CRC-32. + int atomics : 1; // Armv8.1 atomic instructions. + int fphp : 1; // Half-precision floating point support. + int asimdhp : 1; // Advanced SIMD half-precision support. + int cpuid : 1; // Access to certain ID registers. + int asimdrdm : 1; // Rounding Double Multiply Accumulate/Subtract. + int jscvt : 1; // Support for JavaScript conversion. + int fcma : 1; // Floating point complex numbers. + int lrcpc : 1; // Support for weaker release consistency. + int dcpop : 1; // Data persistence writeback. + int sha3 : 1; // Hardware-accelerated SHA3. + int sm3 : 1; // Hardware-accelerated SM3. + int sm4 : 1; // Hardware-accelerated SM4. + int asimddp : 1; // Dot product instruction. + int sha512 : 1; // Hardware-accelerated SHA512. + int sve : 1; // Scalable Vector Extension. + int asimdfhm : 1; // Additional half-precision instructions. + int dit : 1; // Data independent timing. + int uscat : 1; // Unaligned atomics support. + int ilrcpc : 1; // Additional support for weaker release consistency. + int flagm : 1; // Flag manipulation instructions. + int ssbs : 1; // Speculative Store Bypass Safe PSTATE bit. + int sb : 1; // Speculation barrier. + int paca : 1; // Address authentication. + int pacg : 1; // Generic authentication. + int dcpodp : 1; // Data cache clean to point of persistence. + int sve2 : 1; // Scalable Vector Extension (version 2). + int sveaes : 1; // SVE AES instructions. + int svepmull : 1; // SVE polynomial multiply long instructions. + int svebitperm : 1; // SVE bit permute instructions. + int svesha3 : 1; // SVE SHA3 instructions. + int svesm4 : 1; // SVE SM4 instructions. + int flagm2 : 1; // Additional flag manipulation instructions. + int frint : 1; // Floating point to integer rounding. + int svei8mm : 1; // SVE Int8 matrix multiplication instructions. + int svef32mm : 1; // SVE FP32 matrix multiplication instruction. + int svef64mm : 1; // SVE FP64 matrix multiplication instructions. + int svebf16 : 1; // SVE BFloat16 instructions. + int i8mm : 1; // Int8 matrix multiplication instructions. + int bf16 : 1; // BFloat16 instructions. + int dgh : 1; // Data Gathering Hint instruction. + int rng : 1; // True random number generator support. + int bti : 1; // Branch target identification. + int mte : 1; // Memory tagging extension. // Make sure to update Aarch64FeaturesEnum below if you add a field here. } Aarch64Features; diff --git a/include/internal/bit_utils.h b/include/internal/bit_utils.h index 3467ff93..4b5f19ea 100644 --- a/include/internal/bit_utils.h +++ b/include/internal/bit_utils.h @@ -27,8 +27,8 @@ inline static bool IsBitSet(uint32_t reg, uint32_t bit) { return (reg >> bit) & 0x1; } -inline static uint32_t ExtractBitRange(uint32_t reg, uint32_t msb, - uint32_t lsb) { +inline static uint64_t ExtractBitRange(uint64_t reg, uint64_t msb, + uint64_t lsb) { const uint64_t bits = msb - lsb + 1ULL; const uint64_t mask = (1ULL << bits) - 1ULL; assert(msb >= lsb); diff --git a/include/internal/cputype_aarch64.h b/include/internal/cputype_aarch64.h index 292641f5..31f4d93c 100644 --- a/include/internal/cputype_aarch64.h +++ b/include/internal/cputype_aarch64.h @@ -166,4 +166,46 @@ #define MIDR_APPLE_M1_FIRESTORM_TONGA \ MIDR_CPU_MODEL_IP(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_TONGA) +#define ID_AA64_SCHEME0_SHIFT 0 +#define ID_AA64_SCHEME1_SHIFT 4 +#define ID_AA64_SCHEME2_SHIFT 8 +#define ID_AA64_SCHEME3_SHIFT 12 +#define ID_AA64_SCHEME4_SHIFT 16 +#define ID_AA64_SCHEME5_SHIFT 20 +#define ID_AA64_SCHEME6_SHIFT 24 +#define ID_AA64_SCHEME7_SHIFT 28 +#define ID_AA64_SCHEME8_SHIFT 32 +#define ID_AA64_SCHEME9_SHIFT 36 +#define ID_AA64_SCHEME10_SHIFT 40 +#define ID_AA64_SCHEME11_SHIFT 44 +#define ID_AA64_SCHEME12_SHIFT 48 +#define ID_AA64_SCHEME13_SHIFT 52 +#define ID_AA64_SCHEME14_SHIFT 56 +#define ID_AA64_SCHEME15_SHIFT 60 + +#define ID_AA64_SCHEME_MODEL(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, \ + r12, r13, r14, r15) \ + (((r15) << ID_AA64_SCHEME15_SHIFT) | ((r14) << ID_AA64_SCHEME14_SHIFT) | \ + ((r13) << ID_AA64_SCHEME13_SHIFT) | ((r12) << ID_AA64_SCHEME12_SHIFT) | \ + ((r11) << ID_AA64_SCHEME11_SHIFT) | ((r10) << ID_AA64_SCHEME10_SHIFT) | \ + ((r9) << ID_AA64_SCHEME9_SHIFT) | ((r8) << ID_AA64_SCHEME8_SHIFT) | \ + ((r7) << ID_AA64_SCHEME7_SHIFT) | ((r6) << ID_AA64_SCHEME6_SHIFT) | \ + ((r5) << ID_AA64_SCHEME5_SHIFT) | ((r4) << ID_AA64_SCHEME4_SHIFT) | \ + ((r3) << ID_AA64_SCHEME3_SHIFT) | ((r2) << ID_AA64_SCHEME2_SHIFT) | \ + ((r1) << ID_AA64_SCHEME1_SHIFT) | ((r0) << ID_AA64_SCHEME0_SHIFT)) + +/////////////////////////////////////////////////////////////////////////////// +// ID_AA64ISAR0_EL1 +/////////////////////////////////////////////////////////////////////////////// +#define ID_AA64ISAR0_EL1_ARM_CORTEX_A53_R3 \ + ID_AA64_SCHEME_MODEL(0UL, 2UL, 1UL, 1UL, 1UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, \ + 0UL, 0UL, 0UL, 0UL, 0UL) + +/////////////////////////////////////////////////////////////////////////////// +// ID_AA64PFR0_EL1 +/////////////////////////////////////////////////////////////////////////////// +#define ID_AA64PFR0_EL1_ARM_CORTEX_A53_R3 \ + ID_AA64_SCHEME_MODEL(0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, \ + 0UL, 0UL, 0UL, 0UL, 0UL) + #endif // CPU_FEATURES_CPUTYPE_AARCH64_H_ diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index e8962f8c..6004bca9 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -62,5 +62,5 @@ __val; \ }) #else -#error "Cannot compile cpuinfo_aarch64 on a non supported compiler." +#error "Unsupported compiler, aarch64 cpuid requires either GCC or Clang." #endif diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index c9026c98..3554ff7a 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -64,9 +64,7 @@ static void DetectFeaturesBase(Aarch64Info* info) { // * Single-precision and double-precision floating-point types. // * Conversions between single-precision and half-precision data types, // and double-precision and half-precision data types. - if (fp == 0) { - info->features.fp = 1; - } + if (fp == 0) info->features.fp = 1; // fp_value = 0b0001: // As for 0b0000, and also includes support for half-precision @@ -84,9 +82,7 @@ static void DetectFeaturesBase(Aarch64Info* info) { // * Single-precision and double-precision floating-point arithmetic. // * Conversions between single-precision and half-precision data types, // and double-precision and half-precision data types. - if (asimd == 0) { - info->features.asimd = 1; - } + if (asimd == 0) info->features.asimd = 1; // asimd = 0b0001: // As for 0b0000, and also includes support for half-precision @@ -116,13 +112,8 @@ static void DetectFeaturesBase(Aarch64Info* info) { info->features.asimddp = ExtractBitRange(isa0, 47, 44); info->features.asimdfhm = ExtractBitRange(isa0, 51, 48); const uint64_t ts = ExtractBitRange(isa0, 55, 52); - if (ts == 1) { - info->features.flagm = 1; - } - if (ts == 2) { - info->features.flagm = 1; - info->features.flagm2 = 1; - } + if (ts >= 1) info->features.flagm = 1; + if (ts >= 2) info->features.flagm2 = 1; info->features.rng = ExtractBitRange(isa0, 63, 60); const uint64_t isa1 = GetCpuid_ID_AA64ISAR1_EL1(); diff --git a/src/impl_aarch64_macos.c b/src/impl_aarch64_macos.c deleted file mode 100644 index 411a96f1..00000000 --- a/src/impl_aarch64_macos.c +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cpu_features_macros.h" - -#ifdef CPU_FEATURES_ARCH_AARCH64 -#ifdef CPU_FEATURES_OS_MACOS - -#include "impl_aarch64__base_implementation.inl" - -static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } - -#endif // CPU_FEATURES_OS_MACOS -#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/src/impl_aarch64_iphone.c b/src/impl_aarch64_macos_or_iphone.c similarity index 84% rename from src/impl_aarch64_iphone.c rename to src/impl_aarch64_macos_or_iphone.c index d1bef200..4d0fc593 100644 --- a/src/impl_aarch64_iphone.c +++ b/src/impl_aarch64_macos_or_iphone.c @@ -15,11 +15,11 @@ #include "cpu_features_macros.h" #ifdef CPU_FEATURES_ARCH_AARCH64 -#ifdef CPU_FEATURES_OS_IPHONE +#if defined(CPU_FEATURES_OS_MACOS) || defined(CPU_FEATURES_OS_IPHONE) #include "impl_aarch64__base_implementation.inl" static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } -#endif // CPU_FEATURES_OS_IPHONE +#endif // defined(CPU_FEATURES_OS_MACOS) || defined(CPU_FEATURES_OS_IPHONE) #endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/src/impl_aarch64_windows.c b/src/impl_aarch64_windows.c deleted file mode 100644 index c1b603f5..00000000 --- a/src/impl_aarch64_windows.c +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cpu_features_macros.h" - -#ifdef CPU_FEATURES_ARCH_AARCH64 -#ifdef CPU_FEATURES_OS_WINDOWS - -#include "impl_aarch64__base_implementation.inl" - -static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } - -#endif // CPU_FEATURES_OS_WINDOWS -#endif // CPU_FEATURES_ARCH_AARCH64 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 00c20413..3e5ce264 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -71,7 +71,12 @@ endif() ##------------------------------------------------------------------------------ ## cpuinfo_aarch64_test if(PROCESSOR_IS_AARCH64) - add_executable(cpuinfo_aarch64_test cpuinfo_aarch64_test.cc ../src/impl_aarch64_linux_or_android.c) + add_executable(cpuinfo_aarch64_test + cpuinfo_aarch64_test.cc + ../src/impl_aarch64_linux_or_android.c + ../src/impl_aarch64_macos_or_iphone.c + ../src/impl_aarch64_freebsd.c + ) target_compile_definitions(cpuinfo_aarch64_test PUBLIC CPU_FEATURES_MOCK_CPUID_AARCH64) target_link_libraries(cpuinfo_aarch64_test all_libraries) add_test(NAME cpuinfo_aarch64_test COMMAND cpuinfo_aarch64_test) diff --git a/test/cpuinfo_aarch64_test.cc b/test/cpuinfo_aarch64_test.cc index b64c6dcd..7f44b962 100644 --- a/test/cpuinfo_aarch64_test.cc +++ b/test/cpuinfo_aarch64_test.cc @@ -20,7 +20,6 @@ #include "gtest/gtest.h" #include "hwcaps_for_testing.h" #include "internal/cpuid_aarch64.h" -#include "cputype_aarch64_for_testing.h" namespace cpu_features { namespace { @@ -275,12 +274,32 @@ TEST_F(CpuInfoAarch64Test, ARM_CORTEX_A53_R3_MRS_ID_AA64ISAR0_EL1) { EXPECT_TRUE(info.features.sha1); EXPECT_TRUE(info.features.sha2); EXPECT_TRUE(info.features.crc32); - EXPECT_FALSE(info.features.atomics); EXPECT_TRUE(info.features.cpuid); + EXPECT_FALSE(info.features.atomics); EXPECT_FALSE(info.features.sha3); EXPECT_FALSE(info.features.sm3); EXPECT_FALSE(info.features.sm4); } + +TEST_F(CpuInfoAarch64Test, ARM_CORTEX_A53_R3_MRS_ID_AA64PFR0_EL1) { + cpu().SetCpuid_MIDR_EL1(MIDR_CORTEX_A53_R3); + cpu().SetCpuid_ID_AA64PFR0_EL1(ID_AA64PFR0_EL1_ARM_CORTEX_A53_R3); + const auto info = GetAarch64Info(); + + EXPECT_EQ(info.implementer, 0x41); + EXPECT_EQ(info.variant, 0x0); + EXPECT_EQ(info.part, 0xD03); + EXPECT_EQ(info.revision, 3); + + EXPECT_TRUE(info.features.cpuid); + EXPECT_TRUE(info.features.asimd); + EXPECT_TRUE(info.features.fp); + EXPECT_FALSE(info.features.atomics); + EXPECT_FALSE(info.features.sve); + EXPECT_FALSE(info.features.dit); + EXPECT_FALSE(info.features.fphp); + EXPECT_FALSE(info.features.asimdhp); +} #endif } // namespace diff --git a/test/cputype_aarch64_for_testing.h b/test/cputype_aarch64_for_testing.h deleted file mode 100644 index b6d2f56d..00000000 --- a/test/cputype_aarch64_for_testing.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ -#define CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ - -#define ID_AA64ISAR0_EL1_AES_SHIFT 4 -#define ID_AA64ISAR0_EL1_SHA1_SHIFT 8 -#define ID_AA64ISAR0_EL1_SHA2_SHIFT 12 -#define ID_AA64ISAR0_EL1_CRC32_SHIFT 16 -#define ID_AA64ISAR0_EL1_ATOMIC_SHIFT 20 -#define ID_AA64ISAR0_EL1_RDM_SHIFT 28 -#define ID_AA64ISAR0_EL1_SHA3_SHIFT 32 -#define ID_AA64ISAR0_EL1_SM3_SHIFT 36 -#define ID_AA64ISAR0_EL1_SM4_SHIFT 40 -#define ID_AA64ISAR0_EL1_DP_SHIFT 44 -#define ID_AA64ISAR0_EL1_FHM_SHIFT 48 -#define ID_AA64ISAR0_EL1_TS_SHIFT 52 -#define ID_AA64ISAR0_EL1_TLB_SHIFT 56 -#define ID_AA64ISAR0_EL1_RNDR_SHIFT 60 - -#define ID_AA64ISAR0_EL1_MODEL(rndr, tlb, ts, fhm, dp, sm4, sm3, sha3, rdm, \ - atomic, crc32, sha2, sha1, aes) \ - (((rndr) << ID_AA64ISAR0_EL1_RNDR_SHIFT) | \ - ((tlb) << ID_AA64ISAR0_EL1_TLB_SHIFT) | \ - ((ts) << ID_AA64ISAR0_EL1_TS_SHIFT) | \ - ((fhm) << ID_AA64ISAR0_EL1_FHM_SHIFT) | \ - ((dp) << ID_AA64ISAR0_EL1_DP_SHIFT) | \ - ((sm4) << ID_AA64ISAR0_EL1_SM4_SHIFT) | \ - ((sm3) << ID_AA64ISAR0_EL1_SM3_SHIFT) | \ - ((sha3) << ID_AA64ISAR0_EL1_SHA3_SHIFT) | \ - ((rdm) << ID_AA64ISAR0_EL1_RDM_SHIFT) | \ - ((atomic) << ID_AA64ISAR0_EL1_ATOMIC_SHIFT) | \ - ((crc32) << ID_AA64ISAR0_EL1_CRC32_SHIFT) | \ - ((sha2) << ID_AA64ISAR0_EL1_SHA2_SHIFT) | \ - ((sha1) << ID_AA64ISAR0_EL1_SHA1_SHIFT) | \ - ((aes) << ID_AA64ISAR0_EL1_AES_SHIFT)) - -#define ID_AA64ISAR0_EL1_ARM_CORTEX_A53_R3 \ - ID_AA64ISAR0_EL1_MODEL(0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, \ - 1UL, 1UL, 1UL, 2UL) - -#endif // CPUFEATURES_CPUTYPE_AARCH64_FOR_TESTING_H_ From 6baf78f55afbba77ed9e6f6dbed425c9f03f0a96 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Wed, 10 Nov 2021 00:56:46 +0200 Subject: [PATCH 12/13] Add base introspection table --- src/define_introspection_base_aarch64.inl | 58 +++++++++++++++++++++++ src/impl_aarch64_freebsd.c | 2 +- src/impl_aarch64_linux_or_android.c | 1 - src/impl_aarch64_macos_or_iphone.c | 2 +- 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 src/define_introspection_base_aarch64.inl diff --git a/src/define_introspection_base_aarch64.inl b/src/define_introspection_base_aarch64.inl new file mode 100644 index 00000000..436eaa8a --- /dev/null +++ b/src/define_introspection_base_aarch64.inl @@ -0,0 +1,58 @@ +//////////////////////////////////////////////////////////////////////////////// +// Definitions for introspection. +//////////////////////////////////////////////////////////////////////////////// +#define INTROSPECTION_TABLE \ + LINE(AARCH64_FP, fp, , , ) \ + LINE(AARCH64_ASIMD, asimd, , , ) \ + LINE(AARCH64_EVTSTRM, evtstrm, , , ) \ + LINE(AARCH64_AES, aes, , , ) \ + LINE(AARCH64_PMULL, pmull, , , ) \ + LINE(AARCH64_SHA1, sha1, , , ) \ + LINE(AARCH64_SHA2, sha2, , , ) \ + LINE(AARCH64_CRC32, crc32, , , ) \ + LINE(AARCH64_ATOMICS, atomics, , , ) \ + LINE(AARCH64_FPHP, fphp, , , ) \ + LINE(AARCH64_ASIMDHP, asimdhp, , , ) \ + LINE(AARCH64_CPUID, cpuid, , , ) \ + LINE(AARCH64_ASIMDRDM, asimdrdm, , , ) \ + LINE(AARCH64_JSCVT, jscvt, , , ) \ + LINE(AARCH64_FCMA, fcma, , , ) \ + LINE(AARCH64_LRCPC, lrcpc, , , ) \ + LINE(AARCH64_DCPOP, dcpop, , , ) \ + LINE(AARCH64_SHA3, sha3, , , ) \ + LINE(AARCH64_SM3, sm3, , , ) \ + LINE(AARCH64_SM4, sm4, , , ) \ + LINE(AARCH64_ASIMDDP, asimddp, , , ) \ + LINE(AARCH64_SHA512, sha512, , , ) \ + LINE(AARCH64_SVE, sve, , , ) \ + LINE(AARCH64_ASIMDFHM, asimdfhm, , , ) \ + LINE(AARCH64_DIT, dit, , , ) \ + LINE(AARCH64_USCAT, uscat, , , ) \ + LINE(AARCH64_ILRCPC, ilrcpc, , , ) \ + LINE(AARCH64_FLAGM, flagm, , , ) \ + LINE(AARCH64_SSBS, ssbs, , , ) \ + LINE(AARCH64_SB, sb, , , ) \ + LINE(AARCH64_PACA, paca, , , ) \ + LINE(AARCH64_PACG, pacg, , , ) \ + LINE(AARCH64_DCPODP, dcpodp, , , ) \ + LINE(AARCH64_SVE2, sve2, , , ) \ + LINE(AARCH64_SVEAES, sveaes, , , ) \ + LINE(AARCH64_SVEPMULL, svepmull, , , ) \ + LINE(AARCH64_SVEBITPERM, svebitperm, , , ) \ + LINE(AARCH64_SVESHA3, svesha3, , , ) \ + LINE(AARCH64_SVESM4, svesm4, , , ) \ + LINE(AARCH64_FLAGM2, flagm2, , , ) \ + LINE(AARCH64_FRINT, frint, , , ) \ + LINE(AARCH64_SVEI8MM, svei8mm, , , ) \ + LINE(AARCH64_SVEF32MM, svef32mm, , , ) \ + LINE(AARCH64_SVEF64MM, svef64mm, , , ) \ + LINE(AARCH64_SVEBF16, svebf16, , , ) \ + LINE(AARCH64_I8MM, i8mm, , , ) \ + LINE(AARCH64_BF16, bf16, , , ) \ + LINE(AARCH64_DGH, dgh, , , ) \ + LINE(AARCH64_RNG, rng, , , ) \ + LINE(AARCH64_BTI, bti, , , ) \ + LINE(AARCH64_MTE, mte, , , ) +#define INTROSPECTION_PREFIX Aarch64 +#define INTROSPECTION_ENUM_PREFIX AARCH64 +#include "define_introspection.inl" diff --git a/src/impl_aarch64_freebsd.c b/src/impl_aarch64_freebsd.c index c68ee32c..96d94ad6 100644 --- a/src/impl_aarch64_freebsd.c +++ b/src/impl_aarch64_freebsd.c @@ -16,7 +16,7 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #ifdef CPU_FEATURES_OS_FREEBSD - +#include "define_introspection_base_aarch64.inl" #include "impl_aarch64__base_implementation.inl" static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } diff --git a/src/impl_aarch64_linux_or_android.c b/src/impl_aarch64_linux_or_android.c index d19a4b08..dd6a0e90 100644 --- a/src/impl_aarch64_linux_or_android.c +++ b/src/impl_aarch64_linux_or_android.c @@ -16,7 +16,6 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #if defined(CPU_FEATURES_OS_LINUX) || defined(CPU_FEATURES_OS_ANDROID) - #include "cpuinfo_aarch64.h" //////////////////////////////////////////////////////////////////////////////// diff --git a/src/impl_aarch64_macos_or_iphone.c b/src/impl_aarch64_macos_or_iphone.c index 4d0fc593..76ce2fe7 100644 --- a/src/impl_aarch64_macos_or_iphone.c +++ b/src/impl_aarch64_macos_or_iphone.c @@ -16,7 +16,7 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #if defined(CPU_FEATURES_OS_MACOS) || defined(CPU_FEATURES_OS_IPHONE) - +#include "define_introspection_base_aarch64.inl" #include "impl_aarch64__base_implementation.inl" static void DetectFeatures(Aarch64Info* info) { DetectFeaturesBase(info); } From cb22995626103d8c94a66c3428a7eef09b7c0855 Mon Sep 17 00:00:00 2001 From: Mykola Hohsdze Date: Tue, 16 Nov 2021 04:02:57 +0200 Subject: [PATCH 13/13] Add features --- include/internal/cpuid_aarch64.h | 2 + src/define_cpuid_aarch64.inl | 5 +- src/define_introspection.inl | 2 +- src/impl_aarch64__base_implementation.inl | 102 ++++++++++++++++------ src/impl_aarch64_freebsd.c | 1 + src/impl_aarch64_macos_or_iphone.c | 1 + src/{stringize.h => stringize.inl} | 0 test/cpuinfo_aarch64_test.cc | 20 +++++ 8 files changed, 103 insertions(+), 30 deletions(-) rename src/{stringize.h => stringize.inl} (100%) diff --git a/include/internal/cpuid_aarch64.h b/include/internal/cpuid_aarch64.h index 6d3f667c..9f46f98c 100644 --- a/include/internal/cpuid_aarch64.h +++ b/include/internal/cpuid_aarch64.h @@ -26,6 +26,8 @@ uint64_t GetCpuid_ID_AA64ISAR0_EL1(); uint64_t GetCpuid_ID_AA64ISAR1_EL1(); uint64_t GetCpuid_ID_AA64PFR0_EL1(); uint64_t GetCpuid_ID_AA64ZFR0_EL1(); +uint64_t GetCpuid_ID_AA64MMFR2_EL1(); +uint64_t GetCpuid_ID_AA64PFR1_EL1(); CPU_FEATURES_END_CPP_NAMESPACE diff --git a/src/define_cpuid_aarch64.inl b/src/define_cpuid_aarch64.inl index 6004bca9..55f8bada 100644 --- a/src/define_cpuid_aarch64.inl +++ b/src/define_cpuid_aarch64.inl @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "stringize.h" +#include "stringize.inl" #define OP0_SHIFT 19 #define OP1_SHIFT 16 @@ -27,11 +27,14 @@ #define SYS_MIDR_EL1 SYS_REG(3, 0, 0, 0, 0) #define SYS_ID_AA64PFR0_EL1 SYS_REG(3, 0, 0, 4, 0) +#define SYS_ID_AA64PFR1_EL1 SYS_REG(3, 0, 0, 4, 1) #define SYS_ID_AA64ZFR0_EL1 SYS_REG(3, 0, 0, 4, 4) #define SYS_ID_AA64ISAR0_EL1 SYS_REG(3, 0, 0, 6, 0) #define SYS_ID_AA64ISAR1_EL1 SYS_REG(3, 0, 0, 6, 1) +#define SYS_ID_AA64MMFR2_EL1 SYS_REG(3, 0, 0, 7, 2) + #if (defined(CPU_FEATURES_COMPILER_GCC) || defined(CPU_FEATURES_COMPILER_CLANG)) #define EMIT_INST(x) ".inst " STRINGIZE((x)) "\n\t" diff --git a/src/define_introspection.inl b/src/define_introspection.inl index ac80d6db..e2e2a92e 100644 --- a/src/define_introspection.inl +++ b/src/define_introspection.inl @@ -24,7 +24,7 @@ #include -#include "stringize.h" +#include "stringize.inl" #define FEAT_TYPE_NAME__(X) X##Features #define FEAT_TYPE_NAME_(X) FEAT_TYPE_NAME__(X) diff --git a/src/impl_aarch64__base_implementation.inl b/src/impl_aarch64__base_implementation.inl index 3554ff7a..f6e9f179 100644 --- a/src/impl_aarch64__base_implementation.inl +++ b/src/impl_aarch64__base_implementation.inl @@ -45,17 +45,28 @@ uint64_t GetCpuid_ID_AA64ISAR1_EL1() { uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return READ_SYS_REG_S(SYS_ID_AA64ZFR0_EL1); } + +uint64_t GetCpuid_ID_AA64MMFR2_EL1() { + return READ_SYS_REG_S(SYS_ID_AA64MMFR2_EL1); +} + +uint64_t GetCpuid_ID_AA64PFR1_EL1() { + return READ_SYS_REG_S(SYS_ID_AA64PFR1_EL1); +} #endif +#define ID_SCHEME(reg, msb, lsb) ExtractBitRange((reg), (msb), (lsb)) >= 1 + // This function have to be implemented by the OS or // can use base implementation DetectFeaturesBase. static void DetectFeatures(Aarch64Info* info); static void DetectFeaturesBase(Aarch64Info* info) { + // ID_AA64PFR0_EL1 const uint64_t pfr0 = GetCpuid_ID_AA64PFR0_EL1(); - info->features.sve = ExtractBitRange(pfr0, 35, 32); - info->features.dit = ExtractBitRange(pfr0, 51, 48); + info->features.sve = ID_SCHEME(pfr0, 35, 32); + info->features.dit = ID_SCHEME(pfr0, 51, 48); const uint64_t fp = ExtractBitRange(pfr0, 19, 16); @@ -92,54 +103,89 @@ static void DetectFeaturesBase(Aarch64Info* info) { info->features.asimdhp = 1; } - const uint64_t isa0 = GetCpuid_ID_AA64ISAR0_EL1(); + // ID_AA64PFR1_EL1 + const uint64_t pfr1 = GetCpuid_ID_AA64PFR1_EL1(); + info->features.bti = ID_SCHEME(pfr1, 3, 0); + + const uint64_t ssbs = ExtractBitRange(pfr1, 7, 4); + if (ssbs >= 2) info->features.ssbs = 1; + info->features.bti = ID_SCHEME(pfr1, 11, 8); + + // ID_AA64ISAR0_EL1 + const uint64_t isa0 = GetCpuid_ID_AA64ISAR0_EL1(); const uint64_t aes = ExtractBitRange(isa0, 7, 4); if (aes >= 1) info->features.aes = 1; if (aes >= 2) info->features.pmull = 1; - info->features.sha1 = ExtractBitRange(isa0, 11, 8); + info->features.sha1 = ID_SCHEME(isa0, 11, 8); + const uint64_t sha2 = ExtractBitRange(isa0, 15, 12); if (sha2 >= 1) info->features.sha2 = 1; if (sha2 >= 2) info->features.sha512 = 1; - info->features.crc32 = ExtractBitRange(isa0, 19, 16); - info->features.atomics = ExtractBitRange(isa0, 23, 20); - info->features.asimdrdm = ExtractBitRange(isa0, 31, 28); - info->features.sha3 = ExtractBitRange(isa0, 35, 32); - info->features.sm3 = ExtractBitRange(isa0, 39, 36); - info->features.sm4 = ExtractBitRange(isa0, 43, 40); - info->features.asimddp = ExtractBitRange(isa0, 47, 44); - info->features.asimdfhm = ExtractBitRange(isa0, 51, 48); + info->features.crc32 = ID_SCHEME(isa0, 19, 16); + info->features.atomics = ID_SCHEME(isa0, 23, 20); + info->features.asimdrdm = ID_SCHEME(isa0, 31, 28); + info->features.sha3 = ID_SCHEME(isa0, 35, 32); + info->features.sm3 = ID_SCHEME(isa0, 39, 36); + info->features.sm4 = ID_SCHEME(isa0, 43, 40); + info->features.asimddp = ID_SCHEME(isa0, 47, 44); + info->features.asimdfhm = ID_SCHEME(isa0, 51, 48); + const uint64_t ts = ExtractBitRange(isa0, 55, 52); if (ts >= 1) info->features.flagm = 1; if (ts >= 2) info->features.flagm2 = 1; - info->features.rng = ExtractBitRange(isa0, 63, 60); + info->features.rng = ID_SCHEME(isa0, 63, 60); + + // ID_AA64ISAR1_EL1 const uint64_t isa1 = GetCpuid_ID_AA64ISAR1_EL1(); - info->features.jscvt = ExtractBitRange(isa1, 15, 12); - info->features.fcma = ExtractBitRange(isa1, 19, 16); - info->features.lrcpc = ExtractBitRange(isa1, 23, 20); - info->features.sb = ExtractBitRange(isa1, 39, 36); - info->features.bf16 = ExtractBitRange(isa1, 47, 44); - info->features.dgh = ExtractBitRange(isa1, 51, 48); - info->features.i8mm = ExtractBitRange(isa1, 55, 52); + const uint64_t dpb = ExtractBitRange(isa1, 3, 0); + if (dpb >= 1) info->features.dcpop = 1; + if (dpb >= 2) info->features.dcpodp = 1; + + const uint64_t apa = ExtractBitRange(isa1, 7, 4); + const uint64_t api = ExtractBitRange(isa1, 11, 8); + if (apa >= 1 || api >= 1) info->features.paca = 1; + + info->features.jscvt = ID_SCHEME(isa1, 15, 12); + info->features.fcma = ID_SCHEME(isa1, 19, 16); + const uint64_t lrcpc = ExtractBitRange(isa1, 23, 20); + if (lrcpc >= 1) info->features.lrcpc = 1; + if (lrcpc >= 2) info->features.ilrcpc = 1; + + const uint64_t gpa = ExtractBitRange(isa1, 27, 24); + const uint64_t gpi = ExtractBitRange(isa1, 31, 28); + if (gpa >= 1 || gpi >= 1) info->features.pacg = 1; + + info->features.frint = ID_SCHEME(isa1, 35, 32); + info->features.sb = ID_SCHEME(isa1, 39, 36); + info->features.bf16 = ID_SCHEME(isa1, 47, 44); + info->features.dgh = ID_SCHEME(isa1, 51, 48); + info->features.i8mm = ID_SCHEME(isa1, 55, 52); + + // ID_AA64ZFR0_EL1 if (info->features.sve) { const uint64_t zfr0 = GetCpuid_ID_AA64ZFR0_EL1(); - info->features.sve2 = ExtractBitRange(zfr0, 0, 3); - info->features.svebitperm = ExtractBitRange(zfr0, 19, 16); - info->features.svebf16 = ExtractBitRange(zfr0, 23, 20); - info->features.svesha3 = ExtractBitRange(zfr0, 35, 32); - info->features.svesm4 = ExtractBitRange(zfr0, 43, 40); - info->features.svei8mm = ExtractBitRange(zfr0, 47, 44); - info->features.svef32mm = ExtractBitRange(zfr0, 55, 52); - info->features.svef64mm = ExtractBitRange(zfr0, 59, 56); + info->features.sve2 = ID_SCHEME(zfr0, 0, 3); + info->features.svebitperm = ID_SCHEME(zfr0, 19, 16); + info->features.svebf16 = ID_SCHEME(zfr0, 23, 20); + info->features.svesha3 = ID_SCHEME(zfr0, 35, 32); + info->features.svesm4 = ID_SCHEME(zfr0, 43, 40); + info->features.svei8mm = ID_SCHEME(zfr0, 47, 44); + info->features.svef32mm = ID_SCHEME(zfr0, 55, 52); + info->features.svef64mm = ID_SCHEME(zfr0, 59, 56); const uint64_t sveaes = ExtractBitRange(zfr0, 7, 4); if (sveaes >= 1) info->features.sveaes = 1; if (sveaes >= 2) info->features.svepmull = 1; } + + // ID_AA64MMFR2_EL1 + const uint64_t mmfr2 = GetCpuid_ID_AA64MMFR2_EL1(); + info->features.uscat = ID_SCHEME(mmfr2, 35, 32); } static const Aarch64Info kEmptyAarch64Info; diff --git a/src/impl_aarch64_freebsd.c b/src/impl_aarch64_freebsd.c index 96d94ad6..0507d8f0 100644 --- a/src/impl_aarch64_freebsd.c +++ b/src/impl_aarch64_freebsd.c @@ -16,6 +16,7 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #ifdef CPU_FEATURES_OS_FREEBSD +#include "cpuinfo_aarch64.h" #include "define_introspection_base_aarch64.inl" #include "impl_aarch64__base_implementation.inl" diff --git a/src/impl_aarch64_macos_or_iphone.c b/src/impl_aarch64_macos_or_iphone.c index 76ce2fe7..91e7aa43 100644 --- a/src/impl_aarch64_macos_or_iphone.c +++ b/src/impl_aarch64_macos_or_iphone.c @@ -16,6 +16,7 @@ #ifdef CPU_FEATURES_ARCH_AARCH64 #if defined(CPU_FEATURES_OS_MACOS) || defined(CPU_FEATURES_OS_IPHONE) +#include "cpuinfo_aarch64.h" #include "define_introspection_base_aarch64.inl" #include "impl_aarch64__base_implementation.inl" diff --git a/src/stringize.h b/src/stringize.inl similarity index 100% rename from src/stringize.h rename to src/stringize.inl diff --git a/test/cpuinfo_aarch64_test.cc b/test/cpuinfo_aarch64_test.cc index 7f44b962..f1cf94ee 100644 --- a/test/cpuinfo_aarch64_test.cc +++ b/test/cpuinfo_aarch64_test.cc @@ -32,6 +32,8 @@ class FakeCpuAarch64 { uint64_t GetCpuid_ID_AA64ISAR1_EL1() const { return _id_aa64isar1_el1; } uint64_t GetCpuid_ID_AA64PFR0_EL1() const { return _id_aa64pfr0_el1; } uint64_t GetCpuid_ID_AA64ZFR0_EL1() const { return _id_aa64zfr0_el1; } + uint64_t GetCpuid_ID_AA64PFR1_EL1() const { return _id_aa64pfr1_el1; } + uint64_t GetCpuid_ID_AA64MMFR2_EL1() const { return _id_aa64mmfr2_el1; } void SetCpuid_MIDR_EL1(uint64_t midr_el1) { _midr_el1 = midr_el1; } @@ -51,12 +53,22 @@ class FakeCpuAarch64 { _id_aa64pfr0_el1 = id_aa64zfr0_el1; } + void SetCpuid_ID_AA64PFR1_EL1(uint64_t id_aa64pfr1_el1) { + _id_aa64pfr1_el1 = id_aa64pfr1_el1; + } + + void SetCpuid_ID_AA64MMFR2_EL1(uint64_t id_aa64mmfr2_el1) { + _id_aa64mmfr2_el1 = id_aa64mmfr2_el1; + } + private: uint64_t _midr_el1; uint64_t _id_aa64isar0_el1; uint64_t _id_aa64isar1_el1; uint64_t _id_aa64pfr0_el1; uint64_t _id_aa64zfr0_el1; + uint64_t _id_aa64mmfr2_el1; + uint64_t _id_aa64pfr1_el1; }; FakeCpuAarch64* g_fake_cpu_aarch64; @@ -84,6 +96,14 @@ extern "C" uint64_t GetCpuid_ID_AA64ZFR0_EL1() { return cpu().GetCpuid_ID_AA64ISAR1_EL1(); } +extern "C" uint64_t GetCpuid_ID_AA64PFR1_EL1() { + return cpu().GetCpuid_ID_AA64PFR1_EL1(); +} + +extern "C" uint64_t GetCpuid_ID_AA64MMFR2_EL1() { + return cpu().GetCpuid_ID_AA64MMFR2_EL1(); +} + void DisableHardwareCapabilities() { SetHardwareCapabilities(0, 0); } class CpuInfoAarch64Test : public ::testing::Test {