x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions

Add cpu feature conditional FSGSBASE access to the relevant helper
functions. That allows to accelerate certain FS/GS base operations in
subsequent changes.

Note, that while possible, the user space entry/exit GSBASE operations are
not going to use the new FSGSBASE instructions. The reason is that it would
require additional storage for the user space value which adds more
complexity to the low level code and experiments have shown marginal
benefit. This may be revisited later but for now the SWAPGS based handling
in the entry code is preserved except for the paranoid entry/exit code.

To preserve the SWAPGS entry mechanism introduce __[rd|wr]gsbase_inactive()
helpers. Note, for Xen PV, paravirt hooks can be added later as they might
allow a very efficient but different implementation.

[ tglx: Massaged changelog, convert it to noinstr and force inline
  	native_swapgs() ]

Signed-off-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/1557309753-24073-7-git-send-email-chang.seok.bae@intel.com
Link: https://lkml.kernel.org/r/20200528201402.1708239-5-sashal@kernel.org

diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h
index fdd1177499b404a548ca7c349bb459af47ce32ef..aefd53767a5d485cc7d7ff4ae0bee29bd30684d0 100644 (file)
--- a/arch/x86/include/asm/fsgsbase.h
+++ b/arch/x86/include/asm/fsgsbase.h
@@ -49,35 +49,32 @@ static __always_inline void wrgsbase(unsigned long gsbase)
        asm volatile("wrgsbase %0" :: "r" (gsbase) : "memory");
 }
 
+#include <asm/cpufeature.h>
+
 /* Helper functions for reading/writing FS/GS base */
 
 static inline unsigned long x86_fsbase_read_cpu(void)
 {
        unsigned long fsbase;
 
-       rdmsrl(MSR_FS_BASE, fsbase);
+       if (static_cpu_has(X86_FEATURE_FSGSBASE))
+               fsbase = rdfsbase();
+       else
+               rdmsrl(MSR_FS_BASE, fsbase);
 
        return fsbase;
 }
 
-static inline unsigned long x86_gsbase_read_cpu_inactive(void)
-{
-       unsigned long gsbase;
-
-       rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
-
-       return gsbase;
-}
-
 static inline void x86_fsbase_write_cpu(unsigned long fsbase)
 {
-       wrmsrl(MSR_FS_BASE, fsbase);
+       if (static_cpu_has(X86_FEATURE_FSGSBASE))
+               wrfsbase(fsbase);
+       else
+               wrmsrl(MSR_FS_BASE, fsbase);
 }
 
-static inline void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
-{
-       wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
-}
+extern unsigned long x86_gsbase_read_cpu_inactive(void);
+extern void x86_gsbase_write_cpu_inactive(unsigned long gsbase);
 
 #endif /* CONFIG_X86_64 */
 

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 42cd333616c487b5730bfd909660784f0f231981..f66202d6121af9c31d53bb846fd6cc7619596049 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -575,7 +575,7 @@ native_load_sp0(unsigned long sp0)
        this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
 }
 
-static inline void native_swapgs(void)
+static __always_inline void native_swapgs(void)
 {
 #ifdef CONFIG_X86_64
        asm volatile("swapgs" ::: "memory");

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 9a97415b21394d24de456930cbcb2ab6f1b3c000..c41e0aae842621ffa0bd413fdb0c2d87514fe67e 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -149,6 +149,44 @@ enum which_selector {
        GS
 };
 
+/*
+ * Out of line to be protected from kprobes and tracing. If this would be
+ * traced or probed than any access to a per CPU variable happens with
+ * the wrong GS.
+ *
+ * It is not used on Xen paravirt. When paravirt support is needed, it
+ * needs to be renamed with native_ prefix.
+ */
+static noinstr unsigned long __rdgsbase_inactive(void)
+{
+       unsigned long gsbase;
+
+       lockdep_assert_irqs_disabled();
+
+       native_swapgs();
+       gsbase = rdgsbase();
+       native_swapgs();
+
+       return gsbase;
+}
+
+/*
+ * Out of line to be protected from kprobes and tracing. If this would be
+ * traced or probed than any access to a per CPU variable happens with
+ * the wrong GS.
+ *
+ * It is not used on Xen paravirt. When paravirt support is needed, it
+ * needs to be renamed with native_ prefix.
+ */
+static noinstr void __wrgsbase_inactive(unsigned long gsbase)
+{
+       lockdep_assert_irqs_disabled();
+
+       native_swapgs();
+       wrgsbase(gsbase);
+       native_swapgs();
+}
+
 /*
  * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are
  * not available.  The goal is to be reasonably fast on non-FSGSBASE systems.
@@ -327,6 +365,36 @@ static unsigned long x86_fsgsbase_read_task(struct task_struct *task,
        return base;
 }
 
+unsigned long x86_gsbase_read_cpu_inactive(void)
+{
+       unsigned long gsbase;
+
+       if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+               unsigned long flags;
+
+               local_irq_save(flags);
+               gsbase = __rdgsbase_inactive();
+               local_irq_restore(flags);
+       } else {
+               rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
+       }
+
+       return gsbase;
+}
+
+void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
+{
+       if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+               unsigned long flags;
+
+               local_irq_save(flags);
+               __wrgsbase_inactive(gsbase);
+               local_irq_restore(flags);
+       } else {
+               wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
+       }
+}
+
 unsigned long x86_fsbase_read_task(struct task_struct *task)
 {
        unsigned long fsbase;