arm64: Add support for SB barrier and patch in over DSB; ISB sequences

We currently use a DSB; ISB sequence to inhibit speculation in set_fs().
Whilst this works for current CPUs, future CPUs may implement a new SB
barrier instruction which acts as an architected speculation barrier.

On CPUs that support it, patch in an SB; NOP sequence over the DSB; ISB
sequence and advertise the presence of the new instruction to userspace.

Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 6142402c2eb4ea0c72072cb60d0c1aef72b7d592..953208267252d5404d830199577992996f85d014 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -122,6 +122,19 @@
        hint    #20
        .endm
 
+/*
+ * Speculation barrier
+ */
+       .macro  sb
+alternative_if_not ARM64_HAS_SB
+       dsb     nsh
+       isb
+alternative_else
+       SB_BARRIER_INSN
+       nop
+alternative_endif
+       .endm
+
 /*
  * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
  * of bounds.

diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h
index 822a9192c55118ddc624687d3917f086cad1637f..f66bb04fdf2dd3cc21f79a36e3d2d79115c7f670 100644 (file)
--- a/arch/arm64/include/asm/barrier.h
+++ b/arch/arm64/include/asm/barrier.h
@@ -34,6 +34,10 @@
 #define psb_csync()    asm volatile("hint #17" : : : "memory")
 #define csdb()         asm volatile("hint #20" : : : "memory")
 
+#define spec_bar()     asm volatile(ALTERNATIVE("dsb nsh\nisb\n",              \
+                                                SB_BARRIER_INSN"nop\n",        \
+                                                ARM64_HAS_SB))
+
 #define mb()           dsb(sy)
 #define rmb()          dsb(ld)
 #define wmb()          dsb(st)

diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 6e2d254c09ebc1204d7f03e20f042d934e7caf70..b7f0709a21af35dcada2c5414676b797cca340f3 100644 (file)
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -54,7 +54,8 @@
 #define ARM64_HAS_CRC32                                33
 #define ARM64_SSBS                             34
 #define ARM64_WORKAROUND_1188873               35
+#define ARM64_HAS_SB                           36
 
-#define ARM64_NCAPS                            36
+#define ARM64_NCAPS                            37
 
 #endif /* __ASM_CPUCAPS_H */

diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 0c909c4a932ff3da741fbda7c16cf6a3780d6107..78f15858535e8c8d2d637416aae4ee69f8b1d62e 100644 (file)
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -104,6 +104,11 @@
 #define SET_PSTATE_UAO(x)              __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift))
 #define SET_PSTATE_SSBS(x)             __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift))
 
+#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
+       __emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
+
+#define SB_BARRIER_INSN                        __SYS_BARRIER_INSN(0, 7, 31)
+
 #define SYS_DC_ISW                     sys_insn(1, 0, 7, 6, 2)
 #define SYS_DC_CSW                     sys_insn(1, 0, 7, 10, 2)
 #define SYS_DC_CISW                    sys_insn(1, 0, 7, 14, 2)
@@ -528,6 +533,7 @@
 #define ID_AA64ISAR0_AES_SHIFT         4
 
 /* id_aa64isar1 */
+#define ID_AA64ISAR1_SB_SHIFT          36
 #define ID_AA64ISAR1_LRCPC_SHIFT       20
 #define ID_AA64ISAR1_FCMA_SHIFT                16
 #define ID_AA64ISAR1_JSCVT_SHIFT       12

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 07c34087bd5e8c09ad0f09d51b63d8ed04ca947d..fad33f5fde47e41610ac9adbda9ad2a0175294d5 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -45,8 +45,7 @@ static inline void set_fs(mm_segment_t fs)
         * Prevent a mispredicted conditional call to set_fs from forwarding
         * the wrong address limit to access_ok under speculation.
         */
-       dsb(nsh);
-       isb();
+       spec_bar();
 
        /* On user-mode return, check fs is correct */
        set_thread_flag(TIF_FSCHECK);

diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 2bcd6e4f34740337c0d122fef5a2b513dcb75dd1..7784f7cba16cf395140bf726276925a9c825f114 100644 (file)
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -49,5 +49,6 @@
 #define HWCAP_ILRCPC           (1 << 26)
 #define HWCAP_FLAGM            (1 << 27)
 #define HWCAP_SSBS             (1 << 28)
+#define HWCAP_SB               (1 << 29)
 
 #endif /* _UAPI__ASM_HWCAP_H */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 1aeb62500fd4d31d981853ea4394b9627681fbcf..e6467e64ee9120902373d237f20ba910e342bca2 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -142,6 +142,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
+       ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SB_SHIFT, 4, 0),
        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
        ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
@@ -1398,6 +1399,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .cpu_enable = cpu_enable_cnp,
        },
 #endif
+       {
+               .desc = "Speculation barrier (SB)",
+               .capability = ARM64_HAS_SB,
+               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+               .matches = has_cpuid_feature,
+               .sys_reg = SYS_ID_AA64ISAR1_EL1,
+               .field_pos = ID_AA64ISAR1_SB_SHIFT,
+               .sign = FTR_UNSIGNED,
+               .min_field_value = 1,
+       },
        {},
 };
 
@@ -1439,6 +1450,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
        HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
        HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
        HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ILRCPC),
+       HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SB),
        HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_USCAT),
 #ifdef CONFIG_ARM64_SVE
        HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),

diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index bcc2831399cbc6ca0c6e2b8d87002998eefce234..7cb0b08ab0a7fd5b2ae8c5585f9ed62a24baec73 100644 (file)
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -82,6 +82,7 @@ static const char *const hwcap_str[] = {
        "ilrcpc",
        "flagm",
        "ssbs",
+       "sb",
        NULL
 };