tdp_mmu.h File Reference

#include <linux/kvm_host.h>
#include "spte.h"

Include dependency graph for tdp_mmu.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
void	kvm_mmu_init_tdp_mmu (struct kvm *kvm)
void	kvm_mmu_uninit_tdp_mmu (struct kvm *kvm)
hpa_t	kvm_tdp_mmu_get_vcpu_root_hpa (struct kvm_vcpu *vcpu)
static __must_check bool	kvm_tdp_mmu_get_root (struct kvm_mmu_page *root)
void	kvm_tdp_mmu_put_root (struct kvm *kvm, struct kvm_mmu_page *root)
bool	kvm_tdp_mmu_zap_leafs (struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
bool	kvm_tdp_mmu_zap_sp (struct kvm *kvm, struct kvm_mmu_page *sp)
void	kvm_tdp_mmu_zap_all (struct kvm *kvm)
void	kvm_tdp_mmu_invalidate_all_roots (struct kvm *kvm)
void	kvm_tdp_mmu_zap_invalidated_roots (struct kvm *kvm)
int	kvm_tdp_mmu_map (struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
bool	kvm_tdp_mmu_unmap_gfn_range (struct kvm *kvm, struct kvm_gfn_range *range, bool flush)
bool	kvm_tdp_mmu_age_gfn_range (struct kvm *kvm, struct kvm_gfn_range *range)
bool	kvm_tdp_mmu_test_age_gfn (struct kvm *kvm, struct kvm_gfn_range *range)
bool	kvm_tdp_mmu_set_spte_gfn (struct kvm *kvm, struct kvm_gfn_range *range)
bool	kvm_tdp_mmu_wrprot_slot (struct kvm *kvm, const struct kvm_memory_slot *slot, int min_level)
bool	kvm_tdp_mmu_clear_dirty_slot (struct kvm *kvm, const struct kvm_memory_slot *slot)
void	kvm_tdp_mmu_clear_dirty_pt_masked (struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, unsigned long mask, bool wrprot)
void	kvm_tdp_mmu_zap_collapsible_sptes (struct kvm *kvm, const struct kvm_memory_slot *slot)
bool	kvm_tdp_mmu_write_protect_gfn (struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, int min_level)
void	kvm_tdp_mmu_try_split_huge_pages (struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t start, gfn_t end, int target_level, bool shared)
static void	kvm_tdp_mmu_walk_lockless_begin (void)
static void	kvm_tdp_mmu_walk_lockless_end (void)
int	kvm_tdp_mmu_get_walk (struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
u64 *	kvm_tdp_mmu_fast_pf_get_last_sptep (struct kvm_vcpu *vcpu, u64 addr, u64 *spte)
static bool	is_tdp_mmu_page (struct kvm_mmu_page *sp)

Function Documentation

is_tdp_mmu_page()

static bool is_tdp_mmu_page ( struct kvm_mmu_page *  sp )

inlinestatic

Definition at line 74 of file tdp_mmu.h.

{ return false; }

Here is the caller graph for this function:

kvm_mmu_init_tdp_mmu()

void kvm_mmu_init_tdp_mmu

(

struct kvm *

kvm

)

Definition at line 15 of file tdp_mmu.c.

{
    INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
    spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
}

Here is the caller graph for this function:

kvm_mmu_uninit_tdp_mmu()

void kvm_mmu_uninit_tdp_mmu

(

struct kvm *

kvm

)

Definition at line 33 of file tdp_mmu.c.

{
    /*
     * Invalidate all roots, which besides the obvious, schedules all roots
     * for zapping and thus puts the TDP MMU's reference to each root, i.e.
     * ultimately frees all roots.
     */
    kvm_tdp_mmu_invalidate_all_roots(kvm);
    kvm_tdp_mmu_zap_invalidated_roots(kvm);
 
    WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));
    WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
 
    /*
     * Ensure that all the outstanding RCU callbacks to free shadow pages
     * can run before the VM is torn down.  Putting the last reference to
     * zapped roots will create new callbacks.
     */
    rcu_barrier();
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_age_gfn_range()

bool kvm_tdp_mmu_age_gfn_range	(	struct kvm *	kvm,
		struct kvm_gfn_range *	range
	)

Definition at line 1195 of file tdp_mmu.c.

{
    return kvm_tdp_mmu_handle_gfn(kvm, range, age_gfn_range);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_clear_dirty_pt_masked()

void kvm_tdp_mmu_clear_dirty_pt_masked	(	struct kvm *	kvm,
		struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		unsigned long	mask,
		bool	wrprot
	)

Definition at line 1629 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    for_each_tdp_mmu_root(kvm, root, slot->as_id)
        clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_clear_dirty_slot()

bool kvm_tdp_mmu_clear_dirty_slot	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot
	)

Definition at line 1560 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
    bool spte_set = false;
 
    lockdep_assert_held_read(&kvm->mmu_lock);
    for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
        spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
                slot->base_gfn + slot->npages);
 
    return spte_set;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_fast_pf_get_last_sptep()

u64* kvm_tdp_mmu_fast_pf_get_last_sptep	(	struct kvm_vcpu *	vcpu,
		u64	addr,
		u64 *	spte
	)

Definition at line 1795 of file tdp_mmu.c.

{
    struct tdp_iter iter;
    struct kvm_mmu *mmu = vcpu->arch.mmu;
    gfn_t gfn = addr >> PAGE_SHIFT;
    tdp_ptep_t sptep = NULL;
 
    tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
        *spte = iter.old_spte;
        sptep = iter.sptep;
    }
 
    /*
     * Perform the rcu_dereference to get the raw spte pointer value since
     * we are passing it up to fast_page_fault, which is shared with the
     * legacy MMU and thus does not retain the TDP MMU-specific __rcu
     * annotation.
     *
     * This is safe since fast_page_fault obeys the contracts of this
     * function as well as all TDP MMU contracts around modifying SPTEs
     * outside of mmu_lock.
     */
    return rcu_dereference(sptep);
}

Here is the caller graph for this function:

kvm_tdp_mmu_get_root()

static __must_check bool kvm_tdp_mmu_get_root ( struct kvm_mmu_page *  root )

inlinestatic

Definition at line 15 of file tdp_mmu.h.

{
    return refcount_inc_not_zero(&root->tdp_mmu_root_count);
}

Here is the caller graph for this function:

kvm_tdp_mmu_get_vcpu_root_hpa()

hpa_t kvm_tdp_mmu_get_vcpu_root_hpa

(

struct kvm_vcpu *

vcpu

)

Definition at line 219 of file tdp_mmu.c.

{
    union kvm_mmu_page_role role = vcpu->arch.mmu->root_role;
    struct kvm *kvm = vcpu->kvm;
    struct kvm_mmu_page *root;
 
    lockdep_assert_held_write(&kvm->mmu_lock);
 
    /*
     * Check for an existing root before allocating a new one.  Note, the
     * role check prevents consuming an invalid root.
     */
    for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
        if (root->role.word == role.word &&
            kvm_tdp_mmu_get_root(root))
            goto out;
    }
 
    root = tdp_mmu_alloc_sp(vcpu);
    tdp_mmu_init_sp(root, NULL, 0, role);
 
    /*
     * TDP MMU roots are kept until they are explicitly invalidated, either
     * by a memslot update or by the destruction of the VM.  Initialize the
     * refcount to two; one reference for the vCPU, and one reference for
     * the TDP MMU itself, which is held until the root is invalidated and
     * is ultimately put by kvm_tdp_mmu_zap_invalidated_roots().
     */
    refcount_set(&root->tdp_mmu_root_count, 2);
 
    spin_lock(&kvm->arch.tdp_mmu_pages_lock);
    list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);
    spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
 
out:
    return __pa(root->spt);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_get_walk()

int kvm_tdp_mmu_get_walk	(	struct kvm_vcpu *	vcpu,
		u64	addr,
		u64 *	sptes,
		int *	root_level
	)

Definition at line 1766 of file tdp_mmu.c.

{
    struct tdp_iter iter;
    struct kvm_mmu *mmu = vcpu->arch.mmu;
    gfn_t gfn = addr >> PAGE_SHIFT;
    int leaf = -1;
 
    *root_level = vcpu->arch.mmu->root_role.level;
 
    tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
        leaf = iter.level;
        sptes[leaf] = iter.old_spte;
    }
 
    return leaf;
}

Here is the caller graph for this function:

kvm_tdp_mmu_invalidate_all_roots()

void kvm_tdp_mmu_invalidate_all_roots

(

struct kvm *

kvm

)

Definition at line 901 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    /*
     * mmu_lock must be held for write to ensure that a root doesn't become
     * invalid while there are active readers (invalidating a root while
     * there are active readers may or may not be problematic in practice,
     * but it's uncharted territory and not supported).
     *
     * Waive the assertion if there are no users of @kvm, i.e. the VM is
     * being destroyed after all references have been put, or if no vCPUs
     * have been created (which means there are no roots), i.e. the VM is
     * being destroyed in an error path of KVM_CREATE_VM.
     */
    if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
        refcount_read(&kvm->users_count) && kvm->created_vcpus)
        lockdep_assert_held_write(&kvm->mmu_lock);
 
    /*
     * As above, mmu_lock isn't held when destroying the VM!  There can't
     * be other references to @kvm, i.e. nothing else can invalidate roots
     * or get/put references to roots.
     */
    list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
        /*
         * Note, invalid roots can outlive a memslot update!  Invalid
         * roots must be *zapped* before the memslot update completes,
         * but a different task can acquire a reference and keep the
         * root alive after its been zapped.
         */
        if (!root->role.invalid) {
            root->tdp_mmu_scheduled_root_to_zap = true;
            root->role.invalid = true;
        }
    }
}

Here is the caller graph for this function:

kvm_tdp_mmu_map()

int kvm_tdp_mmu_map	(	struct kvm_vcpu *	vcpu,
		struct kvm_page_fault *	fault
	)

Definition at line 1032 of file tdp_mmu.c.

{
    struct kvm_mmu *mmu = vcpu->arch.mmu;
    struct kvm *kvm = vcpu->kvm;
    struct tdp_iter iter;
    struct kvm_mmu_page *sp;
    int ret = RET_PF_RETRY;
 
    kvm_mmu_hugepage_adjust(vcpu, fault);
 
    trace_kvm_mmu_spte_requested(fault);
 
    rcu_read_lock();
 
    tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) {
        int r;
 
        if (fault->nx_huge_page_workaround_enabled)
            disallowed_hugepage_adjust(fault, iter.old_spte, iter.level);
 
        /*
         * If SPTE has been frozen by another thread, just give up and
         * retry, avoiding unnecessary page table allocation and free.
         */
        if (is_removed_spte(iter.old_spte))
            goto retry;
 
        if (iter.level == fault->goal_level)
            goto map_target_level;
 
        /* Step down into the lower level page table if it exists. */
        if (is_shadow_present_pte(iter.old_spte) &&
            !is_large_pte(iter.old_spte))
            continue;
 
        /*
         * The SPTE is either non-present or points to a huge page that
         * needs to be split.
         */
        sp = tdp_mmu_alloc_sp(vcpu);
        tdp_mmu_init_child_sp(sp, &iter);
 
        sp->nx_huge_page_disallowed = fault->huge_page_disallowed;
 
        if (is_shadow_present_pte(iter.old_spte))
            r = tdp_mmu_split_huge_page(kvm, &iter, sp, true);
        else
            r = tdp_mmu_link_sp(kvm, &iter, sp, true);
 
        /*
         * Force the guest to retry if installing an upper level SPTE
         * failed, e.g. because a different task modified the SPTE.
         */
        if (r) {
            tdp_mmu_free_sp(sp);
            goto retry;
        }
 
        if (fault->huge_page_disallowed &&
            fault->req_level >= iter.level) {
            spin_lock(&kvm->arch.tdp_mmu_pages_lock);
            if (sp->nx_huge_page_disallowed)
                track_possible_nx_huge_page(kvm, sp);
            spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
        }
    }
 
    /*
     * The walk aborted before reaching the target level, e.g. because the
     * iterator detected an upper level SPTE was frozen during traversal.
     */
    WARN_ON_ONCE(iter.level == fault->goal_level);
    goto retry;
 
map_target_level:
    ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter);
 
retry:
    rcu_read_unlock();
    return ret;
}

Here is the call graph for this function:

kvm_tdp_mmu_put_root()

void kvm_tdp_mmu_put_root	(	struct kvm *	kvm,
		struct kvm_mmu_page *	root
	)

Definition at line 76 of file tdp_mmu.c.

{
    if (!refcount_dec_and_test(&root->tdp_mmu_root_count))
        return;
 
    /*
     * The TDP MMU itself holds a reference to each root until the root is
     * explicitly invalidated, i.e. the final reference should be never be
     * put for a valid root.
     */
    KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);
 
    spin_lock(&kvm->arch.tdp_mmu_pages_lock);
    list_del_rcu(&root->link);
    spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
    call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_set_spte_gfn()

bool kvm_tdp_mmu_set_spte_gfn	(	struct kvm *	kvm,
		struct kvm_gfn_range *	range
	)

Definition at line 1247 of file tdp_mmu.c.

{
    /*
     * No need to handle the remote TLB flush under RCU protection, the
     * target SPTE _must_ be a leaf SPTE, i.e. cannot result in freeing a
     * shadow page. See the WARN on pfn_changed in handle_changed_spte().
     */
    return kvm_tdp_mmu_handle_gfn(kvm, range, set_spte_gfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_test_age_gfn()

bool kvm_tdp_mmu_test_age_gfn	(	struct kvm *	kvm,
		struct kvm_gfn_range *	range
	)

Definition at line 1206 of file tdp_mmu.c.

{
    return kvm_tdp_mmu_handle_gfn(kvm, range, test_age_gfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_try_split_huge_pages()

void kvm_tdp_mmu_try_split_huge_pages	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot,
		gfn_t	start,
		gfn_t	end,
		int	target_level,
		bool	shared
	)

Definition at line 1483 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
    int r = 0;
 
    kvm_lockdep_assert_mmu_lock_held(kvm, shared);
    for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) {
        r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared);
        if (r) {
            kvm_tdp_mmu_put_root(kvm, root);
            break;
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_unmap_gfn_range()

bool kvm_tdp_mmu_unmap_gfn_range	(	struct kvm *	kvm,
		struct kvm_gfn_range *	range,
		bool	flush
	)

Definition at line 1114 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false)
        flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
                      range->may_block, flush);
 
    return flush;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_walk_lockless_begin()

static void kvm_tdp_mmu_walk_lockless_begin ( void  )

inlinestatic

Definition at line 56 of file tdp_mmu.h.

{
    rcu_read_lock();
}

Here is the caller graph for this function:

kvm_tdp_mmu_walk_lockless_end()

static void kvm_tdp_mmu_walk_lockless_end ( void  )

inlinestatic

Definition at line 61 of file tdp_mmu.h.

{
    rcu_read_unlock();
}

Here is the caller graph for this function:

kvm_tdp_mmu_write_protect_gfn()

bool kvm_tdp_mmu_write_protect_gfn	(	struct kvm *	kvm,
		struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		int	min_level
	)

Definition at line 1746 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
    bool spte_set = false;
 
    lockdep_assert_held_write(&kvm->mmu_lock);
    for_each_tdp_mmu_root(kvm, root, slot->as_id)
        spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
 
    return spte_set;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_wrprot_slot()

bool kvm_tdp_mmu_wrprot_slot	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot,
		int	min_level
	)

Definition at line 1300 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
    bool spte_set = false;
 
    lockdep_assert_held_read(&kvm->mmu_lock);
 
    for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
        spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
                 slot->base_gfn + slot->npages, min_level);
 
    return spte_set;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_zap_all()

void kvm_tdp_mmu_zap_all

(

struct kvm *

kvm

)

Definition at line 831 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    /*
     * Zap all roots, including invalid roots, as all SPTEs must be dropped
     * before returning to the caller.  Zap directly even if the root is
     * also being zapped by a worker.  Walking zapped top-level SPTEs isn't
     * all that expensive and mmu_lock is already held, which means the
     * worker has yielded, i.e. flushing the work instead of zapping here
     * isn't guaranteed to be any faster.
     *
     * A TLB flush is unnecessary, KVM zaps everything if and only the VM
     * is being destroyed or the userspace VMM has exited.  In both cases,
     * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.
     */
    lockdep_assert_held_write(&kvm->mmu_lock);
    for_each_tdp_mmu_root_yield_safe(kvm, root)
        tdp_mmu_zap_root(kvm, root, false);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_zap_collapsible_sptes()

void kvm_tdp_mmu_zap_collapsible_sptes	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot
	)

Definition at line 1695 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    lockdep_assert_held_read(&kvm->mmu_lock);
    for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
        zap_collapsible_spte_range(kvm, root, slot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_zap_invalidated_roots()

void kvm_tdp_mmu_zap_invalidated_roots

(

struct kvm *

kvm

)

Definition at line 856 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    read_lock(&kvm->mmu_lock);
 
    for_each_tdp_mmu_root_yield_safe(kvm, root) {
        if (!root->tdp_mmu_scheduled_root_to_zap)
            continue;
 
        root->tdp_mmu_scheduled_root_to_zap = false;
        KVM_BUG_ON(!root->role.invalid, kvm);
 
        /*
         * A TLB flush is not necessary as KVM performs a local TLB
         * flush when allocating a new root (see kvm_mmu_load()), and
         * when migrating a vCPU to a different pCPU.  Note, the local
         * TLB flush on reuse also invalidates paging-structure-cache
         * entries, i.e. TLB entries for intermediate paging structures,
         * that may be zapped, as such entries are associated with the
         * ASID on both VMX and SVM.
         */
        tdp_mmu_zap_root(kvm, root, true);
 
        /*
         * The referenced needs to be put *after* zapping the root, as
         * the root must be reachable by mmu_notifiers while it's being
         * zapped
         */
        kvm_tdp_mmu_put_root(kvm, root);
    }
 
    read_unlock(&kvm->mmu_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_zap_leafs()

bool kvm_tdp_mmu_zap_leafs	(	struct kvm *	kvm,
		gfn_t	start,
		gfn_t	end,
		bool	flush
	)

Definition at line 820 of file tdp_mmu.c.

{
    struct kvm_mmu_page *root;
 
    lockdep_assert_held_write(&kvm->mmu_lock);
    for_each_tdp_mmu_root_yield_safe(kvm, root)
        flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
 
    return flush;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_mmu_zap_sp()

bool kvm_tdp_mmu_zap_sp	(	struct kvm *	kvm,
		struct kvm_mmu_page *	sp
	)

Definition at line 752 of file tdp_mmu.c.

{
    u64 old_spte;
 
    /*
     * This helper intentionally doesn't allow zapping a root shadow page,
     * which doesn't have a parent page table and thus no associated entry.
     */
    if (WARN_ON_ONCE(!sp->ptep))
        return false;
 
    old_spte = kvm_tdp_mmu_read_spte(sp->ptep);
    if (WARN_ON_ONCE(!is_shadow_present_pte(old_spte)))
        return false;
 
    tdp_mmu_set_spte(kvm, kvm_mmu_page_as_id(sp), sp->ptep, old_spte, 0,
             sp->gfn, sp->role.level + 1);
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function: