x86/entry: Convert INT3 exception to IDTENTRY_RAW

Convert #BP to IDTENTRY_RAW:
  - Implement the C entry point with DEFINE_IDTENTRY_RAW
  - Invoke idtentry_enter/exit() from the function body
  - Emit the ASM stub with DECLARE_IDTENTRY_RAW
  - Remove the ASM idtentry in 64bit
  - Remove the open coded ASM entry code in 32bit
  - Fixup the XEN/PV code
  - Remove the old prototypes

No functional change.

This could be a plain IDTENTRY, but as Peter pointed out INT3 is broken
vs. the static key in the context tracking code as this static key might be
in the state of being patched and has an int3 which would recurse forever.
IDTENTRY_RAW is therefore chosen to allow addressing this issue without
lots of code churn.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Andy Lutomirski <luto@kernel.org>
Link: https://lkml.kernel.org/r/20200505135313.938474960@linutronix.de

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index f7a5f1cda058aa031e2abcbd5a89f028cddfe949..b9b0ddb53a08287c49ff5b5626b7a8eb35b84128 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -1649,13 +1649,6 @@ SYM_CODE_START(nmi)
 #endif
 SYM_CODE_END(nmi)
 
-SYM_CODE_START(int3)
-       ASM_CLAC
-       pushl   $0
-       pushl   $do_int3
-       jmp     common_exception
-SYM_CODE_END(int3)
-
 .pushsection .text, "ax"
 SYM_CODE_START(rewind_stack_do_exit)
        /* Prevent any naive code from trying to unwind to our caller. */

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 1bada7b26210723b5a86d5995e35ac761c743a99..69ddd052aef2117ad344060040b3cadec94357ff 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1072,8 +1072,6 @@ apicinterrupt IRQ_WORK_VECTOR                     irq_work_interrupt              smp_irq_work_interrupt
  * Exception entry points.
  */
 
-idtentry       X86_TRAP_BP             int3                    do_int3                         has_error_code=0
-
 idtentry       X86_TRAP_PF             page_fault              do_page_fault                   has_error_code=1
 
 #ifdef CONFIG_X86_MCE

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 2f31d03f3e57c63a4eb1cc92adc4ebd089552d51..3dc4d5b246d315ed1eb4832a5466a9b30c8c7e87 100644 (file)
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -181,4 +181,7 @@ DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS,     exc_stack_segment);
 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP,        exc_general_protection);
 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC,        exc_alignment_check);
 
+/* Raw exception entries which need extra work */
+DECLARE_IDTENTRY_RAW(X86_TRAP_BP,      exc_int3);
+
 #endif

diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 5774d0b6cf77d3b7fb8c7140121f5c65623972a6..698285a2b6608b32d5585bbe997edea3f5228e40 100644 (file)
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -13,7 +13,6 @@
 
 asmlinkage void debug(void);
 asmlinkage void nmi(void);
-asmlinkage void int3(void);
 #ifdef CONFIG_X86_64
 asmlinkage void double_fault(void);
 #endif
@@ -26,7 +25,6 @@ asmlinkage void machine_check(void);
 #if defined(CONFIG_X86_64) && defined(CONFIG_XEN_PV)
 asmlinkage void xen_xennmi(void);
 asmlinkage void xen_xendebug(void);
-asmlinkage void xen_int3(void);
 asmlinkage void xen_double_fault(void);
 asmlinkage void xen_page_fault(void);
 #ifdef CONFIG_X86_MCE
@@ -36,7 +34,6 @@ asmlinkage void xen_machine_check(void);
 
 dotraplinkage void do_debug(struct pt_regs *regs, long error_code);
 dotraplinkage void do_nmi(struct pt_regs *regs, long error_code);
-dotraplinkage void do_int3(struct pt_regs *regs, long error_code);
 dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2);
 dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address);
 dotraplinkage void do_mce(struct pt_regs *regs, long error_code);

diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index 38b565b7e5b893a1a93df083e666d43537ea7d75..9ca8af65a2123f4b4fe4ad93afa74dc0e6cb1128 100644 (file)
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -60,7 +60,7 @@ static bool idt_setup_done __initdata;
  */
 static const __initconst struct idt_data early_idts[] = {
        INTG(X86_TRAP_DB,               debug),
-       SYSG(X86_TRAP_BP,               int3),
+       SYSG(X86_TRAP_BP,               asm_exc_int3),
 #ifdef CONFIG_X86_32
        INTG(X86_TRAP_PF,               page_fault),
 #endif

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 280c290f414f00379f3e8c6acf26e18e6599dc11..0ad12dffde22ce35e10e8b01d789c29bbf1f2418 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -568,7 +568,7 @@ exit:
        cond_local_irq_disable(regs);
 }
 
-dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
+DEFINE_IDTENTRY_RAW(exc_int3)
 {
        /*
         * poke_int3_handler() is completely self contained code; it does (and
@@ -579,16 +579,20 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
                return;
 
        /*
-        * Unlike any other non-IST entry, we can be called from pretty much
-        * any location in the kernel through kprobes -- text_poke() will most
-        * likely be handled by poke_int3_handler() above. This means this
-        * handler is effectively NMI-like.
+        * idtentry_enter() uses static_branch_{,un}likely() and therefore
+        * can trigger INT3, hence poke_int3_handler() must be done
+        * before. If the entry came from kernel mode, then use nmi_enter()
+        * because the INT3 could have been hit in any context including
+        * NMI.
         */
-       if (!user_mode(regs))
+       if (user_mode(regs))
+               idtentry_enter(regs);
+       else
                nmi_enter();
 
+       instrumentation_begin();
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
-       if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
+       if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,
                                SIGTRAP) == NOTIFY_STOP)
                goto exit;
 #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
@@ -598,19 +602,21 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
                goto exit;
 #endif
 
-       if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
+       if (notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,
                        SIGTRAP) == NOTIFY_STOP)
                goto exit;
 
        cond_local_irq_enable(regs);
-       do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, 0, NULL);
+       do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);
        cond_local_irq_disable(regs);
 
 exit:
-       if (!user_mode(regs))
+       instrumentation_end();
+       if (user_mode(regs))
+               idtentry_exit(regs);
+       else
                nmi_exit();
 }
-NOKPROBE_SYMBOL(do_int3);
 
 #ifdef CONFIG_X86_64
 /*

diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 0a30fc0fe0faa7539a57d16e1417506a14b9a43f..5bcd86c4dff045f1a2a3fcde0f84fd3459012d8c 100644 (file)
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -616,7 +616,7 @@ static struct trap_array_entry trap_array[] = {
        { machine_check,               xen_machine_check,               true },
 #endif
        { nmi,                         xen_xennmi,                      true },
-       { int3,                        xen_int3,                        false },
+       TRAP_ENTRY(exc_int3,                            false ),
        TRAP_ENTRY(exc_overflow,                        false ),
 #ifdef CONFIG_IA32_EMULATION
        { entry_INT80_compat,          xen_entry_INT80_compat,          false },

diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 6a91157d5b5ca3187ebc491c471287056ffbc41a..44f55569a37f560f6f8c35222e9aaa688c39e057 100644 (file)
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -31,7 +31,7 @@ _ASM_NOKPROBE(xen_\name)
 xen_pv_trap asm_exc_divide_error
 xen_pv_trap debug
 xen_pv_trap xendebug
-xen_pv_trap int3
+xen_pv_trap asm_exc_int3
 xen_pv_trap xennmi
 xen_pv_trap asm_exc_overflow
 xen_pv_trap asm_exc_bounds