x86/entry: Add IRQENTRY_IRQ macro

Provide a seperate IDTENTRY macro for device interrupts. Similar to
IDTENTRY_ERRORCODE with the addition of invoking irq_enter/exit_rcu() and
providing the errorcode as a 'u8' argument to the C function, which
truncates the sign extended vector number.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Andy Lutomirski <luto@kernel.org>
Link: https://lore.kernel.org/r/20200521202118.984573165@linutronix.de

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 40092c81dcb81f1eb2842a92f3150899ed2f566f..ba2a70d7118a1e3c892151122316a1ad3399f43b 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -751,6 +751,20 @@ SYM_CODE_START(\asmsym)
 SYM_CODE_END(\asmsym)
 .endm
 
+.macro idtentry_irq vector cfunc
+       .p2align CONFIG_X86_L1_CACHE_SHIFT
+SYM_CODE_START_LOCAL(asm_\cfunc)
+       ASM_CLAC
+       SAVE_ALL switch_stacks=1
+       ENCODE_FRAME_POINTER
+       movl    %esp, %eax
+       movl    PT_ORIG_EAX(%esp), %edx         /* get the vector from stack */
+       movl    $-1, PT_ORIG_EAX(%esp)          /* no syscall to restart */
+       call    \cfunc
+       jmp     handle_exception_return
+SYM_CODE_END(asm_\cfunc)
+.endm
+
 /*
  * Include the defines which emit the idt entries which are shared
  * shared between 32 and 64 bit.

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index e7434cda9a381ed4185d2d43830a9ad1b373908e..9162a073e524b12d4050eb45e45838bd0e3b86ce 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -527,6 +527,20 @@ _ASM_NOKPROBE(\asmsym)
 SYM_CODE_END(\asmsym)
 .endm
 
+/*
+ * Interrupt entry/exit.
+ *
+ + The interrupt stubs push (vector) onto the stack, which is the error_code
+ * position of idtentry exceptions, and jump to one of the two idtentry points
+ * (common/spurious).
+ *
+ * common_interrupt is a hotpath, align it to a cache line
+ */
+.macro idtentry_irq vector cfunc
+       .p2align CONFIG_X86_L1_CACHE_SHIFT
+       idtentry \vector asm_\cfunc \cfunc has_error_code=1
+.endm
+
 /*
  * MCE and DB exceptions
  */

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 2fc0dc8af2a4c45e5c6970dc04b1510a3f784a51..eaee48bd6a195873e16de61ad7261a35a58c08bc 100644 (file)
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -165,6 +165,51 @@ __visible noinstr void func(struct pt_regs *regs)
 #define DEFINE_IDTENTRY_RAW_ERRORCODE(func)                            \
 __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
 
+/**
+ * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
+ *                       points (common/spurious)
+ * @vector:    Vector number (ignored for C)
+ * @func:      Function name of the entry point
+ *
+ * Maps to DECLARE_IDTENTRY_ERRORCODE()
+ */
+#define DECLARE_IDTENTRY_IRQ(vector, func)                             \
+       DECLARE_IDTENTRY_ERRORCODE(vector, func)
+
+/**
+ * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
+ * @func:      Function name of the entry point
+ *
+ * The vector number is pushed by the low level entry stub and handed
+ * to the function as error_code argument which needs to be truncated
+ * to an u8 because the push is sign extending.
+ *
+ * On 64-bit idtentry_enter/exit() are invoked in the ASM entry code before
+ * and after switching to the interrupt stack. On 32-bit this happens in C.
+ *
+ * irq_enter/exit_rcu() are invoked before the function body and the
+ * KVM L1D flush request is set.
+ */
+#define DEFINE_IDTENTRY_IRQ(func)                                      \
+static __always_inline void __##func(struct pt_regs *regs, u8 vector); \
+                                                                       \
+__visible noinstr void func(struct pt_regs *regs,                      \
+                           unsigned long error_code)                   \
+{                                                                      \
+       bool rcu_exit = idtentry_enter_cond_rcu(regs);                  \
+                                                                       \
+       instrumentation_begin();                                        \
+       irq_enter_rcu();                                                \
+       kvm_set_cpu_l1tf_flush_l1d();                                   \
+       __##func (regs, (u8)error_code);                                \
+       irq_exit_rcu();                                                 \
+       lockdep_hardirq_exit();                                         \
+       instrumentation_end();                                          \
+       idtentry_exit_cond_rcu(regs, rcu_exit);                         \
+}                                                                      \
+                                                                       \
+static __always_inline void __##func(struct pt_regs *regs, u8 vector)
+
 /**
  * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
  * @vector:    Vector number (ignored for C)
@@ -312,6 +357,10 @@ __visible noinstr void func(struct pt_regs *regs,                  \
 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)                   \
        DECLARE_IDTENTRY_ERRORCODE(vector, func)
 
+/* Entries for common/spurious (device) interrupts */
+#define DECLARE_IDTENTRY_IRQ(vector, func)                             \
+       idtentry_irq vector func
+
 #ifdef CONFIG_X86_64
 # define DECLARE_IDTENTRY_MCE(vector, func)                            \
        idtentry_mce_db vector asm_##func func