mmu_internal.h File Reference

#include <linux/types.h>
#include <linux/kvm_host.h>
#include <asm/kvm_host.h>

Include dependency graph for mmu_internal.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	kvm_mmu_page
struct	kvm_page_fault

Macros
#define	KVM_MMU_WARN_ON(x)   BUILD_BUG_ON_INVALID(x)
#define	__PT_BASE_ADDR_MASK   GENMASK_ULL(51, 12)
#define	__PT_LEVEL_SHIFT(level, bits_per_level)    (PAGE_SHIFT + ((level) - 1) * (bits_per_level))
#define	__PT_INDEX(address, level, bits_per_level)    (((address) >> __PT_LEVEL_SHIFT(level, bits_per_level)) & ((1 << (bits_per_level)) - 1))
#define	__PT_LVL_ADDR_MASK(base_addr_mask, level, bits_per_level)    ((base_addr_mask) & ~((1ULL << (PAGE_SHIFT + (((level) - 1) * (bits_per_level)))) - 1))
#define	__PT_LVL_OFFSET_MASK(base_addr_mask, level, bits_per_level)    ((base_addr_mask) & ((1ULL << (PAGE_SHIFT + (((level) - 1) * (bits_per_level)))) - 1))
#define	__PT_ENT_PER_PAGE(bits_per_level)   (1 << (bits_per_level))
#define	INVALID_PAE_ROOT   0
#define	IS_VALID_PAE_ROOT(x)   (!!(x))

Typedefs
typedef u64 __rcu *	tdp_ptep_t

Enumerations
enum	{   RET_PF_CONTINUE = 0 , RET_PF_RETRY , RET_PF_EMULATE , RET_PF_INVALID ,   RET_PF_FIXED , RET_PF_SPURIOUS }

Functions
static hpa_t	kvm_mmu_get_dummy_root (void)
static bool	kvm_mmu_is_dummy_root (hpa_t shadow_page)
static int	kvm_mmu_role_as_id (union kvm_mmu_page_role role)
static int	kvm_mmu_page_as_id (struct kvm_mmu_page *sp)
static bool	kvm_mmu_page_ad_need_write_protect (struct kvm_mmu_page *sp)
static gfn_t	gfn_round_for_level (gfn_t gfn, int level)
int	mmu_try_to_unsync_pages (struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn, bool can_unsync, bool prefetch)
void	kvm_mmu_gfn_disallow_lpage (const struct kvm_memory_slot *slot, gfn_t gfn)
void	kvm_mmu_gfn_allow_lpage (const struct kvm_memory_slot *slot, gfn_t gfn)
bool	kvm_mmu_slot_gfn_write_protect (struct kvm *kvm, struct kvm_memory_slot *slot, u64 gfn, int min_level)
static void	kvm_flush_remote_tlbs_gfn (struct kvm *kvm, gfn_t gfn, int level)
unsigned int	pte_list_count (struct kvm_rmap_head *rmap_head)
static bool	is_nx_huge_page_enabled (struct kvm *kvm)
int	kvm_tdp_page_fault (struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
static int	kvm_mmu_do_page_fault (struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err, bool prefetch, int *emulation_type)
int	kvm_mmu_max_mapping_level (struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn, int max_level)
void	kvm_mmu_hugepage_adjust (struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
void	disallowed_hugepage_adjust (struct kvm_page_fault *fault, u64 spte, int cur_level)
void *	mmu_memory_cache_alloc (struct kvm_mmu_memory_cache *mc)
void	track_possible_nx_huge_page (struct kvm *kvm, struct kvm_mmu_page *sp)
void	untrack_possible_nx_huge_page (struct kvm *kvm, struct kvm_mmu_page *sp)

Variables
struct kmem_cache *	mmu_page_header_cache
int	nx_huge_pages

Macro Definition Documentation

__PT_BASE_ADDR_MASK

#define __PT_BASE_ADDR_MASK   GENMASK_ULL(51, 12)

Definition at line 16 of file mmu_internal.h.

__PT_ENT_PER_PAGE

#define __PT_ENT_PER_PAGE

(

bits_per_level

)

(1 << (bits_per_level))

Definition at line 28 of file mmu_internal.h.

__PT_INDEX

#define __PT_INDEX	(		address,
			level,
			bits_per_level
	)		(((address) >> __PT_LEVEL_SHIFT(level, bits_per_level)) & ((1 << (bits_per_level)) - 1))

Definition at line 19 of file mmu_internal.h.

__PT_LEVEL_SHIFT

#define __PT_LEVEL_SHIFT	(		level,
			bits_per_level
	)		(PAGE_SHIFT + ((level) - 1) * (bits_per_level))

Definition at line 17 of file mmu_internal.h.

__PT_LVL_ADDR_MASK

#define __PT_LVL_ADDR_MASK	(		base_addr_mask,
			level,
			bits_per_level
	)		((base_addr_mask) & ~((1ULL << (PAGE_SHIFT + (((level) - 1) * (bits_per_level)))) - 1))

Definition at line 22 of file mmu_internal.h.

__PT_LVL_OFFSET_MASK

#define __PT_LVL_OFFSET_MASK	(		base_addr_mask,
			level,
			bits_per_level
	)		((base_addr_mask) & ((1ULL << (PAGE_SHIFT + (((level) - 1) * (bits_per_level)))) - 1))

Definition at line 25 of file mmu_internal.h.

INVALID_PAE_ROOT

#define INVALID_PAE_ROOT   0

Definition at line 37 of file mmu_internal.h.

IS_VALID_PAE_ROOT

#define IS_VALID_PAE_ROOT

(

x

)

(!!(x))

Definition at line 38 of file mmu_internal.h.

KVM_MMU_WARN_ON

#define KVM_MMU_WARN_ON

(

x

)

BUILD_BUG_ON_INVALID(x)

Definition at line 12 of file mmu_internal.h.

Typedef Documentation

tdp_ptep_t

typedef u64 __rcu* tdp_ptep_t

Definition at line 50 of file mmu_internal.h.

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
RET_PF_CONTINUE
RET_PF_RETRY
RET_PF_EMULATE
RET_PF_INVALID
RET_PF_FIXED
RET_PF_SPURIOUS

Definition at line 273 of file mmu_internal.h.

     {
    RET_PF_CONTINUE = 0,
    RET_PF_RETRY,
    RET_PF_EMULATE,
    RET_PF_INVALID,
    RET_PF_FIXED,
    RET_PF_SPURIOUS,
};

Function Documentation

disallowed_hugepage_adjust()

void disallowed_hugepage_adjust	(	struct kvm_page_fault *	fault,
		u64	spte,
		int	cur_level
	)

Definition at line 3216 of file mmu.c.

{
    if (cur_level > PG_LEVEL_4K &&
        cur_level == fault->goal_level &&
        is_shadow_present_pte(spte) &&
        !is_large_pte(spte) &&
        spte_to_child_sp(spte)->nx_huge_page_disallowed) {
        /*
         * A small SPTE exists for this pfn, but FNAME(fetch),
         * direct_map(), or kvm_tdp_mmu_map() would like to create a
         * large PTE instead: just force them to go down another level,
         * patching back for them into pfn the next 9 bits of the
         * address.
         */
        u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
                KVM_PAGES_PER_HPAGE(cur_level - 1);
        fault->pfn |= fault->gfn & page_mask;
        fault->goal_level--;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_round_for_level()

static gfn_t gfn_round_for_level ( gfn_t  gfn, int  level  )

inlinestatic

Definition at line 161 of file mmu_internal.h.

{
    return gfn & -KVM_PAGES_PER_HPAGE(level);
}

Here is the caller graph for this function:

is_nx_huge_page_enabled()

static bool is_nx_huge_page_enabled ( struct kvm *  kvm )

inlinestatic

Definition at line 185 of file mmu_internal.h.

{
    return READ_ONCE(nx_huge_pages) && !kvm->arch.disable_nx_huge_pages;
}

Here is the caller graph for this function:

kvm_flush_remote_tlbs_gfn()

static void kvm_flush_remote_tlbs_gfn ( struct kvm *  kvm, gfn_t  gfn, int  level  )

inlinestatic

Definition at line 176 of file mmu_internal.h.

{
    kvm_flush_remote_tlbs_range(kvm, gfn_round_for_level(gfn, level),
                    KVM_PAGES_PER_HPAGE(level));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_do_page_fault()

static int kvm_mmu_do_page_fault ( struct kvm_vcpu *  vcpu, gpa_t  cr2_or_gpa, u32  err, bool  prefetch, int *  emulation_type  )

inlinestatic

Definition at line 282 of file mmu_internal.h.

{
    struct kvm_page_fault fault = {
        .addr = cr2_or_gpa,
        .error_code = err,
        .exec = err & PFERR_FETCH_MASK,
        .write = err & PFERR_WRITE_MASK,
        .present = err & PFERR_PRESENT_MASK,
        .rsvd = err & PFERR_RSVD_MASK,
        .user = err & PFERR_USER_MASK,
        .prefetch = prefetch,
        .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault),
        .nx_huge_page_workaround_enabled =
            is_nx_huge_page_enabled(vcpu->kvm),
 
        .max_level = KVM_MAX_HUGEPAGE_LEVEL,
        .req_level = PG_LEVEL_4K,
        .goal_level = PG_LEVEL_4K,
        .is_private = kvm_mem_is_private(vcpu->kvm, cr2_or_gpa >> PAGE_SHIFT),
    };
    int r;
 
    if (vcpu->arch.mmu->root_role.direct) {
        fault.gfn = fault.addr >> PAGE_SHIFT;
        fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn);
    }
 
    /*
     * Async #PF "faults", a.k.a. prefetch faults, are not faults from the
     * guest perspective and have already been counted at the time of the
     * original fault.
     */
    if (!prefetch)
        vcpu->stat.pf_taken++;
 
    if (IS_ENABLED(CONFIG_RETPOLINE) && fault.is_tdp)
        r = kvm_tdp_page_fault(vcpu, &fault);
    else
        r = vcpu->arch.mmu->page_fault(vcpu, &fault);
 
    if (fault.write_fault_to_shadow_pgtable && emulation_type)
        *emulation_type |= EMULTYPE_WRITE_PF_TO_SP;
 
    /*
     * Similar to above, prefetch faults aren't truly spurious, and the
     * async #PF path doesn't do emulation.  Do count faults that are fixed
     * by the async #PF handler though, otherwise they'll never be counted.
     */
    if (r == RET_PF_FIXED)
        vcpu->stat.pf_fixed++;
    else if (prefetch)
        ;
    else if (r == RET_PF_EMULATE)
        vcpu->stat.pf_emulate++;
    else if (r == RET_PF_SPURIOUS)
        vcpu->stat.pf_spurious++;
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_get_dummy_root()

static hpa_t kvm_mmu_get_dummy_root ( void  )

inlinestatic

Definition at line 40 of file mmu_internal.h.

{
    return my_zero_pfn(0) << PAGE_SHIFT;
}

Here is the caller graph for this function:

kvm_mmu_gfn_allow_lpage()

void kvm_mmu_gfn_allow_lpage	(	const struct kvm_memory_slot *	slot,
		gfn_t	gfn
	)

Definition at line 822 of file mmu.c.

{
    update_gfn_disallow_lpage_count(slot, gfn, -1);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_gfn_disallow_lpage()

void kvm_mmu_gfn_disallow_lpage	(	const struct kvm_memory_slot *	slot,
		gfn_t	gfn
	)

Definition at line 817 of file mmu.c.

{
    update_gfn_disallow_lpage_count(slot, gfn, 1);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_hugepage_adjust()

void kvm_mmu_hugepage_adjust	(	struct kvm_vcpu *	vcpu,
		struct kvm_page_fault *	fault
	)

Definition at line 3180 of file mmu.c.

{
    struct kvm_memory_slot *slot = fault->slot;
    kvm_pfn_t mask;
 
    fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
 
    if (unlikely(fault->max_level == PG_LEVEL_4K))
        return;
 
    if (is_error_noslot_pfn(fault->pfn))
        return;
 
    if (kvm_slot_dirty_track_enabled(slot))
        return;
 
    /*
     * Enforce the iTLB multihit workaround after capturing the requested
     * level, which will be used to do precise, accurate accounting.
     */
    fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot,
                               fault->gfn, fault->max_level,
                               fault->is_private);
    if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
        return;
 
    /*
     * mmu_invalidate_retry() was successful and mmu_lock is held, so
     * the pmd can't be split from under us.
     */
    fault->goal_level = fault->req_level;
    mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
    VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
    fault->pfn &= ~mask;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_is_dummy_root()

static bool kvm_mmu_is_dummy_root ( hpa_t  shadow_page )

inlinestatic

Definition at line 45 of file mmu_internal.h.

{
    return is_zero_pfn(shadow_page >> PAGE_SHIFT);
}

Here is the caller graph for this function:

kvm_mmu_max_mapping_level()

int kvm_mmu_max_mapping_level	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		int	max_level
	)

Definition at line 3170 of file mmu.c.

{
    bool is_private = kvm_slot_can_be_private(slot) &&
              kvm_mem_is_private(kvm, gfn);
 
    return __kvm_mmu_max_mapping_level(kvm, slot, gfn, max_level, is_private);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_page_ad_need_write_protect()

static bool kvm_mmu_page_ad_need_write_protect ( struct kvm_mmu_page *  sp )

inlinestatic

Definition at line 148 of file mmu_internal.h.

{
    /*
     * When using the EPT page-modification log, the GPAs in the CPU dirty
     * log would come from L2 rather than L1.  Therefore, we need to rely
     * on write protection to record dirty pages, which bypasses PML, since
     * writes now result in a vmexit.  Note, the check on CPU dirty logging
     * being enabled is mandatory as the bits used to denote WP-only SPTEs
     * are reserved for PAE paging (32-bit KVM).
     */
    return kvm_x86_ops.cpu_dirty_log_size && sp->role.guest_mode;
}

Here is the caller graph for this function:

kvm_mmu_page_as_id()

static int kvm_mmu_page_as_id ( struct kvm_mmu_page *  sp )

inlinestatic

Definition at line 143 of file mmu_internal.h.

{
    return kvm_mmu_role_as_id(sp->role);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_role_as_id()

static int kvm_mmu_role_as_id ( union kvm_mmu_page_role  role )

inlinestatic

Definition at line 138 of file mmu_internal.h.

{
    return role.smm ? 1 : 0;
}

Here is the caller graph for this function:

kvm_mmu_slot_gfn_write_protect()

bool kvm_mmu_slot_gfn_write_protect	(	struct kvm *	kvm,
		struct kvm_memory_slot *	slot,
		u64	gfn,
		int	min_level
	)

Definition at line 1414 of file mmu.c.

{
    struct kvm_rmap_head *rmap_head;
    int i;
    bool write_protected = false;
 
    if (kvm_memslots_have_rmaps(kvm)) {
        for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
            rmap_head = gfn_to_rmap(gfn, i, slot);
            write_protected |= rmap_write_protect(rmap_head, true);
        }
    }
 
    if (tdp_mmu_enabled)
        write_protected |=
            kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
 
    return write_protected;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_tdp_page_fault()

int kvm_tdp_page_fault	(	struct kvm_vcpu *	vcpu,
		struct kvm_page_fault *	fault
	)

Definition at line 4623 of file mmu.c.

{
    /*
     * If the guest's MTRRs may be used to compute the "real" memtype,
     * restrict the mapping level to ensure KVM uses a consistent memtype
     * across the entire mapping.
     */
    if (kvm_mmu_honors_guest_mtrrs(vcpu->kvm)) {
        for ( ; fault->max_level > PG_LEVEL_4K; --fault->max_level) {
            int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
            gfn_t base = gfn_round_for_level(fault->gfn,
                             fault->max_level);
 
            if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
                break;
        }
    }
 
#ifdef CONFIG_X86_64
    if (tdp_mmu_enabled)
        return kvm_tdp_mmu_page_fault(vcpu, fault);
#endif
 
    return direct_page_fault(vcpu, fault);
}

Here is the call graph for this function:

Here is the caller graph for this function:

mmu_memory_cache_alloc()

void* mmu_memory_cache_alloc

(

struct kvm_mmu_memory_cache *

mc

)

mmu_try_to_unsync_pages()

int mmu_try_to_unsync_pages	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		bool	can_unsync,
		bool	prefetch
	)

Definition at line 2805 of file mmu.c.

{
    struct kvm_mmu_page *sp;
    bool locked = false;
 
    /*
     * Force write-protection if the page is being tracked.  Note, the page
     * track machinery is used to write-protect upper-level shadow pages,
     * i.e. this guards the role.level == 4K assertion below!
     */
    if (kvm_gfn_is_write_tracked(kvm, slot, gfn))
        return -EPERM;
 
    /*
     * The page is not write-tracked, mark existing shadow pages unsync
     * unless KVM is synchronizing an unsync SP (can_unsync = false).  In
     * that case, KVM must complete emulation of the guest TLB flush before
     * allowing shadow pages to become unsync (writable by the guest).
     */
    for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
        if (!can_unsync)
            return -EPERM;
 
        if (sp->unsync)
            continue;
 
        if (prefetch)
            return -EEXIST;
 
        /*
         * TDP MMU page faults require an additional spinlock as they
         * run with mmu_lock held for read, not write, and the unsync
         * logic is not thread safe.  Take the spinklock regardless of
         * the MMU type to avoid extra conditionals/parameters, there's
         * no meaningful penalty if mmu_lock is held for write.
         */
        if (!locked) {
            locked = true;
            spin_lock(&kvm->arch.mmu_unsync_pages_lock);
 
            /*
             * Recheck after taking the spinlock, a different vCPU
             * may have since marked the page unsync.  A false
             * negative on the unprotected check above is not
             * possible as clearing sp->unsync _must_ hold mmu_lock
             * for write, i.e. unsync cannot transition from 1->0
             * while this CPU holds mmu_lock for read (or write).
             */
            if (READ_ONCE(sp->unsync))
                continue;
        }
 
        WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);
        kvm_unsync_page(kvm, sp);
    }
    if (locked)
        spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
 
    /*
     * We need to ensure that the marking of unsync pages is visible
     * before the SPTE is updated to allow writes because
     * kvm_mmu_sync_roots() checks the unsync flags without holding
     * the MMU lock and so can race with this. If the SPTE was updated
     * before the page had been marked as unsync-ed, something like the
     * following could happen:
     *
     * CPU 1                    CPU 2
     * ---------------------------------------------------------------------
     * 1.2 Host updates SPTE
     *     to be writable
     *                      2.1 Guest writes a GPTE for GVA X.
     *                          (GPTE being in the guest page table shadowed
     *                           by the SP from CPU 1.)
     *                          This reads SPTE during the page table walk.
     *                          Since SPTE.W is read as 1, there is no
     *                          fault.
     *
     *                      2.2 Guest issues TLB flush.
     *                          That causes a VM Exit.
     *
     *                      2.3 Walking of unsync pages sees sp->unsync is
     *                          false and skips the page.
     *
     *                      2.4 Guest accesses GVA X.
     *                          Since the mapping in the SP was not updated,
     *                          so the old mapping for GVA X incorrectly
     *                          gets used.
     * 1.1 Host marks SP
     *     as unsync
     *     (sp->unsync = true)
     *
     * The write barrier below ensures that 1.1 happens before 1.2 and thus
     * the situation in 2.4 does not arise.  It pairs with the read barrier
     * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
     */
    smp_wmb();
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

pte_list_count()

unsigned int pte_list_count

(

struct kvm_rmap_head *

rmap_head

)

Definition at line 1073 of file mmu.c.

{
    struct pte_list_desc *desc;
 
    if (!rmap_head->val)
        return 0;
    else if (!(rmap_head->val & 1))
        return 1;
 
    desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
    return desc->tail_count + desc->spte_count;
}

Here is the caller graph for this function:

track_possible_nx_huge_page()

void track_possible_nx_huge_page	(	struct kvm *	kvm,
		struct kvm_mmu_page *	sp
	)

Definition at line 848 of file mmu.c.

{
    /*
     * If it's possible to replace the shadow page with an NX huge page,
     * i.e. if the shadow page is the only thing currently preventing KVM
     * from using a huge page, add the shadow page to the list of "to be
     * zapped for NX recovery" pages.  Note, the shadow page can already be
     * on the list if KVM is reusing an existing shadow page, i.e. if KVM
     * links a shadow page at multiple points.
     */
    if (!list_empty(&sp->possible_nx_huge_page_link))
        return;
 
    ++kvm->stat.nx_lpage_splits;
    list_add_tail(&sp->possible_nx_huge_page_link,
              &kvm->arch.possible_nx_huge_pages);
}

Here is the caller graph for this function:

untrack_possible_nx_huge_page()

void untrack_possible_nx_huge_page	(	struct kvm *	kvm,
		struct kvm_mmu_page *	sp
	)

Definition at line 891 of file mmu.c.

{
    if (list_empty(&sp->possible_nx_huge_page_link))
        return;
 
    --kvm->stat.nx_lpage_splits;
    list_del_init(&sp->possible_nx_huge_page_link);
}

Here is the caller graph for this function:

Variable Documentation

mmu_page_header_cache

struct kmem_cache* mmu_page_header_cache

extern

Definition at line 181 of file mmu.c.

nx_huge_pages

int nx_huge_pages

extern

Definition at line 64 of file mmu.c.