arm64: entry: Add vectors that have the bhb mitigation sequences

Some CPUs affected by Spectre-BHB need a sequence of branches, or a
firmware call to be run before any indirect branch. This needs to go
in the vectors. No CPU needs both.

While this can be patched in, it would run on all CPUs as there is a
single set of vectors. If only one part of a big/little combination is
affected, the unaffected CPUs have to run the mitigation too.

Create extra vectors that include the sequence. Subsequent patches will
allow affected CPUs to select this set of vectors. Later patches will
modify the loop count to match what the CPU requires.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index e8bd0af0141c36164170701ca598d8ff6610288b..046c38ee2841a030b7add99192918b3969de5824 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -850,4 +850,28 @@ alternative_endif
 
 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
 
+       .macro __mitigate_spectre_bhb_loop      tmp
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       mov     \tmp, #32
+.Lspectre_bhb_loop\@:
+       b       . + 4
+       subs    \tmp, \tmp, #1
+       b.ne    .Lspectre_bhb_loop\@
+       sb
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+       .endm
+
+       /* Save/restores x0-x3 to the stack */
+       .macro __mitigate_spectre_bhb_fw
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       stp     x0, x1, [sp, #-16]!
+       stp     x2, x3, [sp, #-16]!
+       mov     w0, #ARM_SMCCC_ARCH_WORKAROUND_3
+alternative_cb smccc_patch_fw_mitigation_conduit
+       nop                                     // Patched to SMC/HVC #0
+alternative_cb_end
+       ldp     x2, x3, [sp], #16
+       ldp     x0, x1, [sp], #16
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+       .endm
 #endif /* __ASM_ASSEMBLER_H */

diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h
new file mode 100644 (file)
index 0000000..bac53fa
--- /dev/null
+++ b/arch/arm64/include/asm/vectors.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2022 ARM Ltd.
+ */
+#ifndef __ASM_VECTORS_H
+#define __ASM_VECTORS_H
+
+/*
+ * Note: the order of this enum corresponds to two arrays in entry.S:
+ * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
+ * 'full fat' vectors are used directly.
+ */
+enum arm64_bp_harden_el1_vectors {
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       /*
+        * Perform the BHB loop mitigation, before branching to the canonical
+        * vectors.
+        */
+       EL1_VECTOR_BHB_LOOP,
+
+       /*
+        * Make the SMC call for firmware mitigation, before branching to the
+        * canonical vectors.
+        */
+       EL1_VECTOR_BHB_FW,
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+
+       /*
+        * Remap the kernel before branching to the canonical vectors.
+        */
+       EL1_VECTOR_KPTI,
++};
+
+#endif /* __ASM_VECTORS_H */

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 9c4ff75f983e51a12d048ad7082c663087a32574..2ceb0c3647b40636adc0aa4b92d8e7ad2e9946c6 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -646,13 +646,26 @@ alternative_else_nop_endif
        sub     \dst, \dst, PAGE_SIZE
        .endm
 
-       .macro tramp_ventry, vector_start, regsize, kpti
+
+#define BHB_MITIGATION_NONE    0
+#define BHB_MITIGATION_LOOP    1
+#define BHB_MITIGATION_FW      2
+
+       .macro tramp_ventry, vector_start, regsize, kpti, bhb
        .align  7
 1:
        .if     \regsize == 64
        msr     tpidrro_el0, x30        // Restored in kernel_ventry
        .endif
 
+       .if     \bhb == BHB_MITIGATION_LOOP
+       /*
+        * This sequence must appear before the first indirect branch. i.e. the
+        * ret out of tramp_ventry. It appears here because x30 is free.
+        */
+       __mitigate_spectre_bhb_loop     x30
+       .endif // \bhb == BHB_MITIGATION_LOOP
+
        .if     \kpti == 1
        /*
         * Defend against branch aliasing attacks by pushing a dummy
@@ -680,6 +693,15 @@ alternative_else_nop_endif
        ldr     x30, =vectors
        .endif // \kpti == 1
 
+       .if     \bhb == BHB_MITIGATION_FW
+       /*
+        * The firmware sequence must appear before the first indirect branch.
+        * i.e. the ret out of tramp_ventry. But it also needs the stack to be
+        * mapped to save/restore the registers the SMC clobbers.
+        */
+       __mitigate_spectre_bhb_fw
+       .endif // \bhb == BHB_MITIGATION_FW
+
        add     x30, x30, #(1b - \vector_start + 4)
        ret
 .org 1b + 128  // Did we overflow the ventry slot?
@@ -687,6 +709,9 @@ alternative_else_nop_endif
 
        .macro tramp_exit, regsize = 64
        adr     x30, tramp_vectors
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       add     x30, x30, SZ_4K
+#endif
        msr     vbar_el1, x30
        ldr     lr, [sp, #S_LR]
        tramp_unmap_kernel      x29
@@ -698,26 +723,32 @@ alternative_else_nop_endif
        sb
        .endm
 
-       .macro  generate_tramp_vector,  kpti
+       .macro  generate_tramp_vector,  kpti, bhb
 .Lvector_start\@:
        .space  0x400
 
        .rept   4
-       tramp_ventry    .Lvector_start\@, 64, \kpti
+       tramp_ventry    .Lvector_start\@, 64, \kpti, \bhb
        .endr
        .rept   4
-       tramp_ventry    .Lvector_start\@, 32, \kpti
+       tramp_ventry    .Lvector_start\@, 32, \kpti, \bhb
        .endr
        .endm
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 /*
  * Exception vectors trampoline.
+ * The order must match __bp_harden_el1_vectors and the
+ * arm64_bp_harden_el1_vectors enum.
  */
        .pushsection ".entry.tramp.text", "ax"
        .align  11
 SYM_CODE_START_NOALIGN(tramp_vectors)
-       generate_tramp_vector   kpti=1
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_LOOP
+       generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_FW
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+       generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_NONE
 SYM_CODE_END(tramp_vectors)
 
 SYM_CODE_START(tramp_exit_native)
@@ -744,7 +775,7 @@ SYM_DATA_END(__entry_tramp_data_start)
  * Exception vectors for spectre mitigations on entry from EL1 when
  * kpti is not in use.
  */
-       .macro generate_el1_vector
+       .macro generate_el1_vector, bhb
 .Lvector_start\@:
        kernel_ventry   1, t, 64, sync          // Synchronous EL1t
        kernel_ventry   1, t, 64, irq           // IRQ EL1t
@@ -757,17 +788,21 @@ SYM_DATA_END(__entry_tramp_data_start)
        kernel_ventry   1, h, 64, error         // Error EL1h
 
        .rept   4
-       tramp_ventry    .Lvector_start\@, 64, kpti=0
+       tramp_ventry    .Lvector_start\@, 64, 0, \bhb
        .endr
        .rept 4
-       tramp_ventry    .Lvector_start\@, 32, kpti=0
+       tramp_ventry    .Lvector_start\@, 32, 0, \bhb
        .endr
        .endm
 
+/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
        .pushsection ".entry.text", "ax"
        .align  11
 SYM_CODE_START(__bp_harden_el1_vectors)
-       generate_el1_vector
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+       generate_el1_vector     bhb=BHB_MITIGATION_LOOP
+       generate_el1_vector     bhb=BHB_MITIGATION_FW
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
 SYM_CODE_END(__bp_harden_el1_vectors)
        .popsection
 

diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 9394f21d756635678ed0b1e99276aa86bb50f92b..6a5eeb8beea384a7ed779bfe8357a5886ad85cb3 100644 (file)
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -770,3 +770,19 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
                return -ENODEV;
        }
 }
+
+/* Patched to NOP when enabled */
+void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
+                                                    __le32 *origptr,
+                                                     __le32 *updptr, int nr_inst)
+{
+       BUG_ON(nr_inst != 1);
+}
+
+/* Patched to NOP when enabled */
+void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
+                                                  __le32 *origptr,
+                                                  __le32 *updptr, int nr_inst)
+{
+       BUG_ON(nr_inst != 1);
+}

diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
index 63ccb525219022e17e39771c669de161ed54666a..220c8c60e021a788f7062a35f44ddc7be4e4a332 100644 (file)
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -92,6 +92,11 @@
                           ARM_SMCCC_SMC_32,                            \
                           0, 0x7fff)
 
+#define ARM_SMCCC_ARCH_WORKAROUND_3                                    \
+       ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                         \
+                          ARM_SMCCC_SMC_32,                            \
+                          0, 0x3fff)
+
 #define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID                          \
        ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                         \
                           ARM_SMCCC_SMC_32,                            \