um: add support for folded p4d page tables

The UML port uses 4 and 5 level fixups to support higher level page
table directories in the generic VM code.

Implement primitives necessary for the 4th level folding, add walks of
p4d level where appropriate and drop usage of __ARCH_USE_5LEVEL_HACK.

Link: http://lkml.kernel.org/r/1572938135-31886-13-git-send-email-rppt@kernel.org
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
Cc: Anatoly Pugachev <matorola@gmail.com>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Greentime Hu <green.hu@gmail.com>
Cc: Greg Ungerer <gerg@linux-m68k.org>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Mark Salter <msalter@redhat.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Peter Rosin <peda@axentia.se>
Cc: Richard Weinberger <richard@nod.at>
Cc: Rolf Eike Beer <eike-kernel@sf-tec.de>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Russell King <rmk+kernel@armlinux.org.uk>
Cc: Sam Creasey <sammy@sammy.net>
Cc: Vincent Chen <deanbo422@gmail.com>
Cc: Vineet Gupta <Vineet.Gupta1@synopsys.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/um/include/asm/pgtable-2level.h b/arch/um/include/asm/pgtable-2level.h
index 32b3d26a71097a48daa3a7f2b9e7df213df008a0..32106d31e4ab277a1d5fd2c690b8d6e54fc73420 100644 (file)
--- a/arch/um/include/asm/pgtable-2level.h
+++ b/arch/um/include/asm/pgtable-2level.h
@@ -8,7 +8,6 @@
 #ifndef __UM_PGTABLE_2LEVEL_H
 #define __UM_PGTABLE_2LEVEL_H
 
-#define __ARCH_USE_5LEVEL_HACK
 #include <asm-generic/pgtable-nopmd.h>
 
 /* PGDIR_SHIFT determines what a third-level page table entry can map */

diff --git a/arch/um/include/asm/pgtable-3level.h b/arch/um/include/asm/pgtable-3level.h
index 9812269fefc9f1d07773b3d4014a619ae2392599..8a3b689e0f86e188ad4e8d9b56a0a5f08cac5d17 100644 (file)
--- a/arch/um/include/asm/pgtable-3level.h
+++ b/arch/um/include/asm/pgtable-3level.h
@@ -7,7 +7,6 @@
 #ifndef __UM_PGTABLE_3LEVEL_H
 #define __UM_PGTABLE_3LEVEL_H
 
-#define __ARCH_USE_5LEVEL_HACK
 #include <asm-generic/pgtable-nopud.h>
 
 /* PGDIR_SHIFT determines what a third-level page table entry can map */

diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h
index 36a44d58f3739af63e17d43785120ca8aadb78c5..2daa58df2190fb10947f46b3122c7b3192ff8f95 100644 (file)
--- a/arch/um/include/asm/pgtable.h
+++ b/arch/um/include/asm/pgtable.h
@@ -106,6 +106,9 @@ extern unsigned long end_iomem;
 #define pud_newpage(x)  (pud_val(x) & _PAGE_NEWPAGE)
 #define pud_mkuptodate(x) (pud_val(x) &= ~_PAGE_NEWPAGE)
 
+#define p4d_newpage(x)  (p4d_val(x) & _PAGE_NEWPAGE)
+#define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE)
+
 #define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK)
 
 #define pte_page(x) pfn_to_page(pte_pfn(x))

diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 417ff647fb37741a878238a90d22bcb21896aea6..30885d0b94acfe449dcf6d7470b6aad51e53b3cc 100644 (file)
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -96,6 +96,7 @@ static void __init fixrange_init(unsigned long start, unsigned long end,
                                 pgd_t *pgd_base)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        int i, j;
@@ -107,7 +108,8 @@ static void __init fixrange_init(unsigned long start, unsigned long end,
        pgd = pgd_base + i;
 
        for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
-               pud = pud_offset(pgd, vaddr);
+               p4d = p4d_offset(pgd, vaddr);
+               pud = pud_offset(p4d, vaddr);
                if (pud_none(*pud))
                        one_md_table_init(pud);
                pmd = pmd_offset(pud, vaddr);
@@ -124,6 +126,7 @@ static void __init fixaddr_user_init( void)
 #ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA
        long size = FIXADDR_USER_END - FIXADDR_USER_START;
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -144,7 +147,8 @@ static void __init fixaddr_user_init( void)
        for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE,
                      p += PAGE_SIZE) {
                pgd = swapper_pg_dir + pgd_index(vaddr);
-               pud = pud_offset(pgd, vaddr);
+               p4d = p4d_offset(pgd, vaddr);
+               pud = pud_offset(p4d, vaddr);
                pmd = pmd_offset(pud, vaddr);
                pte = pte_offset_kernel(pmd, vaddr);
                pte_set_val(*pte, p, PAGE_READONLY);

diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c
index b5e3d91fc9c28385023d47519ccc25333de3405a..3f0d9a573fd669865052ecac4b70aec7a92d13ce 100644 (file)
--- a/arch/um/kernel/skas/mmu.c
+++ b/arch/um/kernel/skas/mmu.c
@@ -19,15 +19,21 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
                         unsigned long kernel)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
 
        pgd = pgd_offset(mm, proc);
-       pud = pud_alloc(mm, pgd, proc);
-       if (!pud)
+
+       p4d = p4d_alloc(mm, pgd, proc);
+       if (!p4d)
                goto out;
 
+       pud = pud_alloc(mm, p4d, proc);
+       if (!pud)
+               goto out_pud;
+
        pmd = pmd_alloc(mm, pud, proc);
        if (!pmd)
                goto out_pmd;
@@ -44,6 +50,8 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
        pmd_free(mm, pmd);
  out_pmd:
        pud_free(mm, pud);
+ out_pud:
+       p4d_free(mm, p4d);
  out:
        return -ENOMEM;
 }

diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c
index 3236052f20e67bc09732ac4eed0907ef59921329..d617f8dc9c19d67abed02a125e4c5e1ef5e33c2f 100644 (file)
--- a/arch/um/kernel/skas/uaccess.c
+++ b/arch/um/kernel/skas/uaccess.c
@@ -17,6 +17,7 @@
 pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
 
@@ -27,7 +28,11 @@ pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr)
        if (!pgd_present(*pgd))
                return NULL;
 
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (!p4d_present(*p4d))
+               return NULL;
+
+       pud = pud_offset(p4d, addr);
        if (!pud_present(*pud))
                return NULL;
 

diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c
index 8425a22142b786aa33976c6c0634f1d468a075a7..80a358c6d652f7280c0cfac96adf2bbf946e23c0 100644 (file)
--- a/arch/um/kernel/tlb.c
+++ b/arch/um/kernel/tlb.c
@@ -277,7 +277,7 @@ static inline int update_pmd_range(pud_t *pud, unsigned long addr,
        return ret;
 }
 
-static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
+static inline int update_pud_range(p4d_t *p4d, unsigned long addr,
                                   unsigned long end,
                                   struct host_vm_change *hvc)
 {
@@ -285,7 +285,7 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
        unsigned long next;
        int ret = 0;
 
-       pud = pud_offset(pgd, addr);
+       pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (!pud_present(*pud)) {
@@ -299,6 +299,28 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
        return ret;
 }
 
+static inline int update_p4d_range(pgd_t *pgd, unsigned long addr,
+                                  unsigned long end,
+                                  struct host_vm_change *hvc)
+{
+       p4d_t *p4d;
+       unsigned long next;
+       int ret = 0;
+
+       p4d = p4d_offset(pgd, addr);
+       do {
+               next = p4d_addr_end(addr, end);
+               if (!p4d_present(*p4d)) {
+                       if (hvc->force || p4d_newpage(*p4d)) {
+                               ret = add_munmap(addr, next - addr, hvc);
+                               p4d_mkuptodate(*p4d);
+                       }
+               } else
+                       ret = update_pud_range(p4d, addr, next, hvc);
+       } while (p4d++, addr = next, ((addr < end) && !ret));
+       return ret;
+}
+
 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
                      unsigned long end_addr, int force)
 {
@@ -316,8 +338,8 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
                                ret = add_munmap(addr, next - addr, &hvc);
                                pgd_mkuptodate(*pgd);
                        }
-               }
-               else ret = update_pud_range(pgd, addr, next, &hvc);
+               } else
+                       ret = update_p4d_range(pgd, addr, next, &hvc);
        } while (pgd++, addr = next, ((addr < end_addr) && !ret));
 
        if (!ret)
@@ -338,6 +360,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
 {
        struct mm_struct *mm;
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -364,7 +387,23 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
                        continue;
                }
 
-               pud = pud_offset(pgd, addr);
+               p4d = p4d_offset(pgd, addr);
+               if (!p4d_present(*p4d)) {
+                       last = ADD_ROUND(addr, P4D_SIZE);
+                       if (last > end)
+                               last = end;
+                       if (p4d_newpage(*p4d)) {
+                               updated = 1;
+                               err = add_munmap(addr, last - addr, &hvc);
+                               if (err < 0)
+                                       panic("munmap failed, errno = %d\n",
+                                             -err);
+                       }
+                       addr = last;
+                       continue;
+               }
+
+               pud = pud_offset(p4d, addr);
                if (!pud_present(*pud)) {
                        last = ADD_ROUND(addr, PUD_SIZE);
                        if (last > end)
@@ -424,6 +463,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -437,7 +477,11 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
        if (!pgd_present(*pgd))
                goto kill;
 
-       pud = pud_offset(pgd, address);
+       p4d = p4d_offset(pgd, address);
+       if (!p4d_present(*p4d))
+               goto kill;
+
+       pud = pud_offset(p4d, address);
        if (!pud_present(*pud))
                goto kill;
 

diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
index e62296c66c95b2e91796ddbc82e967149725bc67..818553064f04127d06b3f0f79c97039d52f1feda 100644 (file)
--- a/arch/um/kernel/trap.c
+++ b/arch/um/kernel/trap.c
@@ -28,6 +28,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -104,7 +105,8 @@ good_area:
                }
 
                pgd = pgd_offset(mm, address);
-               pud = pud_offset(pgd, address);
+               p4d = p4d_offset(pgd, address);
+               pud = pud_offset(p4d, address);
                pmd = pmd_offset(pud, address);
                pte = pte_offset_kernel(pmd, address);
        } while (!pte_present(*pte));