diff options
| author | Ben Gardon <bgardon@google.com> | 2021-04-01 16:37:29 -0700 |
|---|---|---|
| committer | Paolo Bonzini <pbonzini@redhat.com> | 2021-04-19 09:05:25 -0400 |
| commit | 11cccf5c04721c8a08f9d72a1a5e7281a4041d86 (patch) | |
| tree | 6b947f54a2ef6203a59d051adf6fda5d33d0ea9c | |
| parent | cfc109979b3c879ea9df05e048bb83615964f3e3 (diff) | |
KVM: x86/mmu: Make TDP MMU root refcount atomic
In order to parallelize more operations for the TDP MMU, make the
refcount on TDP MMU roots atomic, so that a future patch can allow
multiple threads to take a reference on the root concurrently, while
holding the MMU lock in read mode.
Signed-off-by: Ben Gardon <bgardon@google.com>
Message-Id: <20210401233736.638171-7-bgardon@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
| -rw-r--r-- | arch/x86/kvm/mmu/mmu_internal.h | 6 | ||||
| -rw-r--r-- | arch/x86/kvm/mmu/tdp_mmu.c | 4 | ||||
| -rw-r--r-- | arch/x86/kvm/mmu/tdp_mmu.h | 10 |
3 files changed, 14 insertions, 6 deletions
diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 8135b6d54d87..f2546d6d390c 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -50,7 +50,11 @@ struct kvm_mmu_page { u64 *spt; /* hold the gfn of each spte inside spt */ gfn_t *gfns; - int root_count; /* Currently serving as active root */ + /* Currently serving as active root */ + union { + int root_count; + refcount_t tdp_mmu_root_count; + }; unsigned int unsync_children; struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */ DECLARE_BITMAP(unsync_child_bitmap, 512); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 81688b41ef5b..f55b415d600a 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -56,7 +56,7 @@ void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) lockdep_assert_held_write(&kvm->mmu_lock); - if (--root->root_count) + if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) return; WARN_ON(!root->tdp_mmu_page); @@ -168,7 +168,7 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) } root = alloc_tdp_mmu_page(vcpu, 0, vcpu->arch.mmu->shadow_root_level); - root->root_count = 1; + refcount_set(&root->tdp_mmu_root_count, 1); list_add(&root->link, &kvm->arch.tdp_mmu_roots); diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index 1b2308c11dd0..0fc9061ff2d8 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -10,10 +10,14 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu); static inline void kvm_tdp_mmu_get_root(struct kvm *kvm, struct kvm_mmu_page *root) { - BUG_ON(!root->root_count); - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); - ++root->root_count; + /* + * This should never fail since roots are removed from the roots + * list under the MMU write lock when their reference count falls + * to zero. + */ + refcount_inc_not_zero(&root->tdp_mmu_root_count); } void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root); |