KVM: x86/mmu: Do not filter address spaces in for_each_tdp_mmu_root_yield_safe()

commit 441a5dfcd96854cbcb625709e2694a9c60adfaab upstream.

All callers except the MMU notifier want to process all address spaces.
Remove the address space ID argument of for_each_tdp_mmu_root_yield_safe()
and switch the MMU notifier to use __for_each_tdp_mmu_root_yield_safe().

Extracted out of a patch by Sean Christopherson <seanjc@google.com>

Cc: stable@vger.kernel.org
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index e232ddaed3e4b0fa4e5fd0c19b4a79d2197f9fad..2a6fec4e2d1969725e8c83951391dd61b75e18be 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -6079,7 +6079,6 @@ static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_e
 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 {
        bool flush;
-       int i;
 
        if (WARN_ON_ONCE(gfn_end <= gfn_start))
                return;
@@ -6090,11 +6089,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 
        flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
 
-       if (is_tdp_mmu_enabled(kvm)) {
-               for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
-                       flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
-                                                     gfn_end, flush);
-       }
+       if (is_tdp_mmu_enabled(kvm))
+               flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
 
        if (flush)
                kvm_flush_remote_tlbs_with_address(kvm, gfn_start,

diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 2bd4e186a9b5057f3f508508538dea13b17db0e4..9b9fc4e834d098ea84fcbff27a42d746b4d87e85 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -222,8 +222,12 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
 #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared)   \
        __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)
 
-#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id)                  \
-       __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, false, false)
+#define for_each_tdp_mmu_root_yield_safe(_kvm, _root)                  \
+       for (_root = tdp_mmu_next_root(_kvm, NULL, false, false);               \
+            _root;                                                             \
+            _root = tdp_mmu_next_root(_kvm, _root, false, false))              \
+               if (!kvm_lockdep_assert_mmu_lock_held(_kvm, false)) {           \
+               } else
 
 /*
  * Iterate over all TDP MMU roots.  Requires that mmu_lock be held for write,
@@ -955,12 +959,11 @@ static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
  * true if a TLB flush is needed before releasing the MMU lock, i.e. if one or
  * more SPTEs were zapped since the MMU lock was last acquired.
  */
-bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
-                          bool flush)
+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
 {
        struct kvm_mmu_page *root;
 
-       for_each_tdp_mmu_root_yield_safe(kvm, root, as_id)
+       for_each_tdp_mmu_root_yield_safe(kvm, root)
                flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
 
        return flush;
@@ -969,7 +972,6 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
 void kvm_tdp_mmu_zap_all(struct kvm *kvm)
 {
        struct kvm_mmu_page *root;
-       int i;
 
        /*
         * Zap all roots, including invalid roots, as all SPTEs must be dropped
@@ -983,10 +985,8 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm)
         * is being destroyed or the userspace VMM has exited.  In both cases,
         * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.
         */
-       for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
-               for_each_tdp_mmu_root_yield_safe(kvm, root, i)
-                       tdp_mmu_zap_root(kvm, root, false);
-       }
+       for_each_tdp_mmu_root_yield_safe(kvm, root)
+               tdp_mmu_zap_root(kvm, root, false);
 }
 
 /*
@@ -1223,7 +1223,7 @@ bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
 {
        struct kvm_mmu_page *root;
 
-       for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id)
+       __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false)
                flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
                                          range->may_block, flush);
 

diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h
index 1fb4d4e9cde59f9c2d7540da52863ef89e6ff30b..d0a9fe0770fdd885e013c0a7ac67844f6e7bc16d 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.h
+++ b/arch/x86/kvm/mmu/tdp_mmu.h
@@ -15,8 +15,7 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root)
 void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
                          bool shared);
 
-bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
-                          bool flush);
+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
 bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);
 void kvm_tdp_mmu_zap_all(struct kvm *kvm);
 void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);