x86/nmi: Protect NMI entry against instrumentation

Mark all functions in the fragile code parts noinstr or force inlining so
they can't be instrumented.

Also make the hardware latency tracer invocation explicit outside of
non-instrumentable section.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Andy Lutomirski <luto@kernel.org>
Link: https://lkml.kernel.org/r/20200505135314.716186134@linutronix.de

diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index 085a2dd312b47d7f304816a4712433cd1085b549..d6c3d346c63af42aeb266493a3173fda844f4303 100644 (file)
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -214,7 +214,7 @@ static inline void native_load_gdt(const struct desc_ptr *dtr)
        asm volatile("lgdt %0"::"m" (*dtr));
 }
 
-static inline void native_load_idt(const struct desc_ptr *dtr)
+static __always_inline void native_load_idt(const struct desc_ptr *dtr)
 {
        asm volatile("lidt %0"::"m" (*dtr));
 }
@@ -392,7 +392,7 @@ extern unsigned long system_vectors[];
 
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(u32, debug_idt_ctr);
-static inline bool is_debug_idt_enabled(void)
+static __always_inline bool is_debug_idt_enabled(void)
 {
        if (this_cpu_read(debug_idt_ctr))
                return true;
@@ -400,7 +400,7 @@ static inline bool is_debug_idt_enabled(void)
        return false;
 }
 
-static inline void load_debug_idt(void)
+static __always_inline void load_debug_idt(void)
 {
        load_idt((const struct desc_ptr *)&debug_idt_descr);
 }
@@ -422,7 +422,7 @@ static inline void load_debug_idt(void)
  * that doesn't need to disable interrupts, as nothing should be
  * bothering the CPU then.
  */
-static inline void load_current_idt(void)
+static __always_inline void load_current_idt(void)
 {
        if (is_debug_idt_enabled())
                load_debug_idt();

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8be042df12c30b3522854818662ce2bf5dc12f2f..f4645f9ff9cb555f0a6a0f64529abc69d3a31d24 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1709,21 +1709,19 @@ void syscall_init(void)
 DEFINE_PER_CPU(int, debug_stack_usage);
 DEFINE_PER_CPU(u32, debug_idt_ctr);
 
-void debug_stack_set_zero(void)
+noinstr void debug_stack_set_zero(void)
 {
        this_cpu_inc(debug_idt_ctr);
        load_current_idt();
 }
-NOKPROBE_SYMBOL(debug_stack_set_zero);
 
-void debug_stack_reset(void)
+noinstr void debug_stack_reset(void)
 {
        if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
                return;
        if (this_cpu_dec_return(debug_idt_ctr) == 0)
                load_current_idt();
 }
-NOKPROBE_SYMBOL(debug_stack_reset);
 
 #else  /* CONFIG_X86_64 */
 

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3b05cc802abbaa7d703a5a5c4b77b19aae320c58..3052c78f03aa27d341daee45be298801e6c64667 100644 (file)
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -303,7 +303,7 @@ NOKPROBE_SYMBOL(unknown_nmi_error);
 static DEFINE_PER_CPU(bool, swallow_nmi);
 static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
 
-static void default_do_nmi(struct pt_regs *regs)
+static noinstr void default_do_nmi(struct pt_regs *regs)
 {
        unsigned char reason = 0;
        int handled;
@@ -329,6 +329,8 @@ static void default_do_nmi(struct pt_regs *regs)
 
        __this_cpu_write(last_nmi_rip, regs->ip);
 
+       instrumentation_begin();
+
        handled = nmi_handle(NMI_LOCAL, regs);
        __this_cpu_add(nmi_stats.normal, handled);
        if (handled) {
@@ -342,7 +344,7 @@ static void default_do_nmi(struct pt_regs *regs)
                 */
                if (handled > 1)
                        __this_cpu_write(swallow_nmi, true);
-               return;
+               goto out;
        }
 
        /*
@@ -374,7 +376,7 @@ static void default_do_nmi(struct pt_regs *regs)
 #endif
                __this_cpu_add(nmi_stats.external, 1);
                raw_spin_unlock(&nmi_reason_lock);
-               return;
+               goto out;
        }
        raw_spin_unlock(&nmi_reason_lock);
 
@@ -412,8 +414,10 @@ static void default_do_nmi(struct pt_regs *regs)
                __this_cpu_add(nmi_stats.swallow, 1);
        else
                unknown_nmi_error(reason, regs);
+
+out:
+       instrumentation_end();
 }
-NOKPROBE_SYMBOL(default_do_nmi);
 
 /*
  * NMIs can page fault or hit breakpoints which will cause it to lose
@@ -485,7 +489,7 @@ static DEFINE_PER_CPU(unsigned long, nmi_cr2);
  */
 static DEFINE_PER_CPU(int, update_debug_stack);
 
-static bool notrace is_debug_stack(unsigned long addr)
+static noinstr bool is_debug_stack(unsigned long addr)
 {
        struct cea_exception_stacks *cs = __this_cpu_read(cea_exception_stacks);
        unsigned long top = CEA_ESTACK_TOP(cs, DB);
@@ -500,7 +504,6 @@ static bool notrace is_debug_stack(unsigned long addr)
         */
        return addr >= bot && addr < top;
 }
-NOKPROBE_SYMBOL(is_debug_stack);
 #endif
 
 DEFINE_IDTENTRY_NMI(exc_nmi)