KVM: arm64: Add guard pages for KVM nVHE hypervisor stack

Map the stack pages in the flexible private VA range and allocate
guard pages below the stack as unbacked VA space. The stack is aligned
so that any valid stack address has PAGE_SHIFT bit as 1 - this is used
for overflow detection (implemented in a subsequent patch in the series).

Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
Tested-by: Fuad Tabba <tabba@google.com>
Reviewed-by: Fuad Tabba <tabba@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220420214317.3303360-4-kaleshsingh@google.com

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index d5b0386ef7653b4cdeacfcd78ec7e19b4cf1b0e2..2e277f2ed6712f7bf4f2ba73a80151beaa503dde 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -169,6 +169,7 @@ struct kvm_nvhe_init_params {
        unsigned long tcr_el2;
        unsigned long tpidr_el2;
        unsigned long stack_hyp_va;
+       unsigned long stack_pa;
        phys_addr_t pgd_pa;
        unsigned long hcr_el2;
        unsigned long vttbr;

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index a50cbb5ba402d28b68e624b5148042c599fba7b1..b208da3bebec82e7b59467f155561ce3a1cc3047 100644 (file)
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -154,6 +154,8 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
 int kvm_share_hyp(void *from, void *to);
 void kvm_unshare_hyp(void *from, void *to);
 int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
+int __create_hyp_mappings(unsigned long start, unsigned long size,
+                         unsigned long phys, enum kvm_pgtable_prot prot);
 int hyp_alloc_private_va_range(size_t size, unsigned long *haddr);
 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
                           void __iomem **kaddr,

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 523bc934fe2f66687b2bb605776f4b239b6114d3..dd257d9f21a2f5fba384f1c2c27e40de2382d42d 100644 (file)
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1483,7 +1483,6 @@ static void cpu_prepare_hyp_mode(int cpu)
        tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
        params->tcr_el2 = tcr;
 
-       params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE);
        params->pgd_pa = kvm_mmu_get_httbr();
        if (is_protected_kvm_enabled())
                params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
@@ -1933,14 +1932,46 @@ static int init_hyp_mode(void)
         * Map the Hyp stack pages
         */
        for_each_possible_cpu(cpu) {
+               struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
                char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
-               err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
-                                         PAGE_HYP);
+               unsigned long hyp_addr;
 
+               /*
+                * Allocate a contiguous HYP private VA range for the stack
+                * and guard page. The allocation is also aligned based on
+                * the order of its size.
+                */
+               err = hyp_alloc_private_va_range(PAGE_SIZE * 2, &hyp_addr);
+               if (err) {
+                       kvm_err("Cannot allocate hyp stack guard page\n");
+                       goto out_err;
+               }
+
+               /*
+                * Since the stack grows downwards, map the stack to the page
+                * at the higher address and leave the lower guard page
+                * unbacked.
+                *
+                * Any valid stack address now has the PAGE_SHIFT bit as 1
+                * and addresses corresponding to the guard page have the
+                * PAGE_SHIFT bit as 0 - this is used for overflow detection.
+                */
+               err = __create_hyp_mappings(hyp_addr + PAGE_SIZE, PAGE_SIZE,
+                                           __pa(stack_page), PAGE_HYP);
                if (err) {
                        kvm_err("Cannot map hyp stack\n");
                        goto out_err;
                }
+
+               /*
+                * Save the stack PA in nvhe_init_params. This will be needed
+                * to recreate the stack mapping in protected nVHE mode.
+                * __hyp_pa() won't do the right thing there, since the stack
+                * has been mapped in the flexible private VA space.
+                */
+               params->stack_pa = __pa(stack_page);
+
+               params->stack_hyp_va = hyp_addr + (2 * PAGE_SIZE);
        }
 
        for_each_possible_cpu(cpu) {

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 7de1e02ebfd11bfcd92311963bcc1523314ec2ac..088e14eae4cf3a4b93b7d1d473618b51f35d2b5c 100644 (file)
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -258,8 +258,8 @@ static bool kvm_host_owns_hyp_mappings(void)
        return true;
 }
 
-static int __create_hyp_mappings(unsigned long start, unsigned long size,
-                                unsigned long phys, enum kvm_pgtable_prot prot)
+int __create_hyp_mappings(unsigned long start, unsigned long size,
+                         unsigned long phys, enum kvm_pgtable_prot prot)
 {
        int err;