maccess: allow architectures to provide kernel probing directly

Provide alternative versions of probe_kernel_read, probe_kernel_write
and strncpy_from_kernel_unsafe that don't need set_fs magic, but instead
use arch hooks that are modelled after unsafe_{get,put}_user to access
kernel memory in an exception safe way.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20200521152301.2587579-19-hch@lst.de
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/maccess.c b/mm/maccess.c
index 17b26bceb71659f66dacb9717af3727840c767f2..db7cf48d8fedd00920414a173049651fb942ce94 100644 (file)
--- a/mm/maccess.c
+++ b/mm/maccess.c
@@ -11,6 +11,81 @@ bool __weak probe_kernel_read_allowed(const void *unsafe_src, size_t size)
        return true;
 }
 
+#ifdef HAVE_GET_KERNEL_NOFAULT
+
+#define probe_kernel_read_loop(dst, src, len, type, err_label)         \
+       while (len >= sizeof(type)) {                                   \
+               __get_kernel_nofault(dst, src, type, err_label);                \
+               dst += sizeof(type);                                    \
+               src += sizeof(type);                                    \
+               len -= sizeof(type);                                    \
+       }
+
+long probe_kernel_read(void *dst, const void *src, size_t size)
+{
+       if (!probe_kernel_read_allowed(src, size))
+               return -EFAULT;
+
+       pagefault_disable();
+       probe_kernel_read_loop(dst, src, size, u64, Efault);
+       probe_kernel_read_loop(dst, src, size, u32, Efault);
+       probe_kernel_read_loop(dst, src, size, u16, Efault);
+       probe_kernel_read_loop(dst, src, size, u8, Efault);
+       pagefault_enable();
+       return 0;
+Efault:
+       pagefault_enable();
+       return -EFAULT;
+}
+EXPORT_SYMBOL_GPL(probe_kernel_read);
+
+#define probe_kernel_write_loop(dst, src, len, type, err_label)                \
+       while (len >= sizeof(type)) {                                   \
+               __put_kernel_nofault(dst, src, type, err_label);                \
+               dst += sizeof(type);                                    \
+               src += sizeof(type);                                    \
+               len -= sizeof(type);                                    \
+       }
+
+long probe_kernel_write(void *dst, const void *src, size_t size)
+{
+       pagefault_disable();
+       probe_kernel_write_loop(dst, src, size, u64, Efault);
+       probe_kernel_write_loop(dst, src, size, u32, Efault);
+       probe_kernel_write_loop(dst, src, size, u16, Efault);
+       probe_kernel_write_loop(dst, src, size, u8, Efault);
+       pagefault_enable();
+       return 0;
+Efault:
+       pagefault_enable();
+       return -EFAULT;
+}
+
+long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
+{
+       const void *src = unsafe_addr;
+
+       if (unlikely(count <= 0))
+               return 0;
+       if (!probe_kernel_read_allowed(unsafe_addr, count))
+               return -EFAULT;
+
+       pagefault_disable();
+       do {
+               __get_kernel_nofault(dst, src, u8, Efault);
+               dst++;
+               src++;
+       } while (dst[-1] && src - unsafe_addr < count);
+       pagefault_enable();
+
+       dst[-1] = '\0';
+       return src - unsafe_addr;
+Efault:
+       pagefault_enable();
+       dst[-1] = '\0';
+       return -EFAULT;
+}
+#else /* HAVE_GET_KERNEL_NOFAULT */
 /**
  * probe_kernel_read(): safely attempt to read from kernel-space
  * @dst: pointer to the buffer that shall take the data
@@ -113,6 +188,7 @@ long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
 
        return ret ? -EFAULT : src - unsafe_addr;
 }
+#endif /* HAVE_GET_KERNEL_NOFAULT */
 
 /**
  * probe_user_read(): safely attempt to read from a user-space location