mmu.h File Reference

#include <linux/kvm_host.h>
#include "kvm_cache_regs.h"
#include "cpuid.h"

Include dependency graph for mmu.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	PT_WRITABLE_SHIFT   1
#define	PT_USER_SHIFT   2
#define	PT_PRESENT_MASK   (1ULL << 0)
#define	PT_WRITABLE_MASK   (1ULL << PT_WRITABLE_SHIFT)
#define	PT_USER_MASK   (1ULL << PT_USER_SHIFT)
#define	PT_PWT_MASK   (1ULL << 3)
#define	PT_PCD_MASK   (1ULL << 4)
#define	PT_ACCESSED_SHIFT   5
#define	PT_ACCESSED_MASK   (1ULL << PT_ACCESSED_SHIFT)
#define	PT_DIRTY_SHIFT   6
#define	PT_DIRTY_MASK   (1ULL << PT_DIRTY_SHIFT)
#define	PT_PAGE_SIZE_SHIFT   7
#define	PT_PAGE_SIZE_MASK   (1ULL << PT_PAGE_SIZE_SHIFT)
#define	PT_PAT_MASK   (1ULL << 7)
#define	PT_GLOBAL_MASK   (1ULL << 8)
#define	PT64_NX_SHIFT   63
#define	PT64_NX_MASK   (1ULL << PT64_NX_SHIFT)
#define	PT_PAT_SHIFT   7
#define	PT_DIR_PAT_SHIFT   12
#define	PT_DIR_PAT_MASK   (1ULL << PT_DIR_PAT_SHIFT)
#define	PT64_ROOT_5LEVEL   5
#define	PT64_ROOT_4LEVEL   4
#define	PT32_ROOT_LEVEL   2
#define	PT32E_ROOT_LEVEL   3
#define	KVM_MMU_CR4_ROLE_BITS
#define	KVM_MMU_CR0_ROLE_BITS   (X86_CR0_PG | X86_CR0_WP)
#define	KVM_MMU_EFER_ROLE_BITS   (EFER_LME | EFER_NX)
#define	tdp_mmu_enabled   false

Functions
static __always_inline u64	rsvd_bits (int s, int e)
static gfn_t	kvm_mmu_max_gfn (void)
static u8	kvm_get_shadow_phys_bits (void)
void	kvm_mmu_set_mmio_spte_mask (u64 mmio_value, u64 mmio_mask, u64 access_mask)
void	kvm_mmu_set_me_spte_mask (u64 me_value, u64 me_mask)
void	kvm_mmu_set_ept_masks (bool has_ad_bits, bool has_exec_only)
void	kvm_init_mmu (struct kvm_vcpu *vcpu)
void	kvm_init_shadow_npt_mmu (struct kvm_vcpu *vcpu, unsigned long cr0, unsigned long cr4, u64 efer, gpa_t nested_cr3)
void	kvm_init_shadow_ept_mmu (struct kvm_vcpu *vcpu, bool execonly, int huge_page_level, bool accessed_dirty, gpa_t new_eptp)
bool	kvm_can_do_async_pf (struct kvm_vcpu *vcpu)
int	kvm_handle_page_fault (struct kvm_vcpu *vcpu, u64 error_code, u64 fault_address, char *insn, int insn_len)
void	__kvm_mmu_refresh_passthrough_bits (struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
int	kvm_mmu_load (struct kvm_vcpu *vcpu)
void	kvm_mmu_unload (struct kvm_vcpu *vcpu)
void	kvm_mmu_free_obsolete_roots (struct kvm_vcpu *vcpu)
void	kvm_mmu_sync_roots (struct kvm_vcpu *vcpu)
void	kvm_mmu_sync_prev_roots (struct kvm_vcpu *vcpu)
void	kvm_mmu_track_write (struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes)
static int	kvm_mmu_reload (struct kvm_vcpu *vcpu)
static unsigned long	kvm_get_pcid (struct kvm_vcpu *vcpu, gpa_t cr3)
static unsigned long	kvm_get_active_pcid (struct kvm_vcpu *vcpu)
static unsigned long	kvm_get_active_cr3_lam_bits (struct kvm_vcpu *vcpu)
static void	kvm_mmu_load_pgd (struct kvm_vcpu *vcpu)
static void	kvm_mmu_refresh_passthrough_bits (struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
static u8	permission_fault (struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned pte_access, unsigned pte_pkey, u64 access)
bool	__kvm_mmu_honors_guest_mtrrs (bool vm_has_noncoherent_dma)
static bool	kvm_mmu_honors_guest_mtrrs (struct kvm *kvm)
void	kvm_zap_gfn_range (struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
int	kvm_arch_write_log_dirty (struct kvm_vcpu *vcpu)
int	kvm_mmu_post_init_vm (struct kvm *kvm)
void	kvm_mmu_pre_destroy_vm (struct kvm *kvm)
static bool	kvm_shadow_root_allocated (struct kvm *kvm)
static bool	kvm_memslots_have_rmaps (struct kvm *kvm)
static gfn_t	gfn_to_index (gfn_t gfn, gfn_t base_gfn, int level)
static unsigned long	__kvm_mmu_slot_lpages (struct kvm_memory_slot *slot, unsigned long npages, int level)
static unsigned long	kvm_mmu_slot_lpages (struct kvm_memory_slot *slot, int level)
static void	kvm_update_page_stats (struct kvm *kvm, int level, int count)
gpa_t	translate_nested_gpa (struct kvm_vcpu *vcpu, gpa_t gpa, u64 access, struct x86_exception *exception)
static gpa_t	kvm_translate_gpa (struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gpa_t gpa, u64 access, struct x86_exception *exception)

Variables
bool __read_mostly	enable_mmio_caching
u8 __read_mostly	shadow_phys_bits

Macro Definition Documentation

KVM_MMU_CR0_ROLE_BITS

#define KVM_MMU_CR0_ROLE_BITS   (X86_CR0_PG | X86_CR0_WP)

Definition at line 42 of file mmu.h.

KVM_MMU_CR4_ROLE_BITS

#define KVM_MMU_CR4_ROLE_BITS

Value:

                   (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_LA57 | \
                   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE)

Definition at line 39 of file mmu.h.

KVM_MMU_EFER_ROLE_BITS

#define KVM_MMU_EFER_ROLE_BITS   (EFER_LME | EFER_NX)

Definition at line 43 of file mmu.h.

PT32_ROOT_LEVEL

#define PT32_ROOT_LEVEL   2

Definition at line 36 of file mmu.h.

PT32E_ROOT_LEVEL

#define PT32E_ROOT_LEVEL   3

Definition at line 37 of file mmu.h.

PT64_NX_MASK

#define PT64_NX_MASK   (1ULL << PT64_NX_SHIFT)

Definition at line 28 of file mmu.h.

PT64_NX_SHIFT

#define PT64_NX_SHIFT   63

Definition at line 27 of file mmu.h.

PT64_ROOT_4LEVEL

#define PT64_ROOT_4LEVEL   4

Definition at line 35 of file mmu.h.

PT64_ROOT_5LEVEL

#define PT64_ROOT_5LEVEL   5

Definition at line 34 of file mmu.h.

PT_ACCESSED_MASK

#define PT_ACCESSED_MASK   (1ULL << PT_ACCESSED_SHIFT)

Definition at line 20 of file mmu.h.

PT_ACCESSED_SHIFT

#define PT_ACCESSED_SHIFT   5

Definition at line 19 of file mmu.h.

PT_DIR_PAT_MASK

#define PT_DIR_PAT_MASK   (1ULL << PT_DIR_PAT_SHIFT)

Definition at line 32 of file mmu.h.

PT_DIR_PAT_SHIFT

#define PT_DIR_PAT_SHIFT   12

Definition at line 31 of file mmu.h.

PT_DIRTY_MASK

#define PT_DIRTY_MASK   (1ULL << PT_DIRTY_SHIFT)

Definition at line 22 of file mmu.h.

PT_DIRTY_SHIFT

#define PT_DIRTY_SHIFT   6

Definition at line 21 of file mmu.h.

PT_GLOBAL_MASK

#define PT_GLOBAL_MASK   (1ULL << 8)

Definition at line 26 of file mmu.h.

PT_PAGE_SIZE_MASK

#define PT_PAGE_SIZE_MASK   (1ULL << PT_PAGE_SIZE_SHIFT)

Definition at line 24 of file mmu.h.

PT_PAGE_SIZE_SHIFT

#define PT_PAGE_SIZE_SHIFT   7

Definition at line 23 of file mmu.h.

PT_PAT_MASK

#define PT_PAT_MASK   (1ULL << 7)

Definition at line 25 of file mmu.h.

PT_PAT_SHIFT

#define PT_PAT_SHIFT   7

Definition at line 30 of file mmu.h.

PT_PCD_MASK

#define PT_PCD_MASK   (1ULL << 4)

Definition at line 18 of file mmu.h.

PT_PRESENT_MASK

#define PT_PRESENT_MASK   (1ULL << 0)

Definition at line 14 of file mmu.h.

PT_PWT_MASK

#define PT_PWT_MASK   (1ULL << 3)

Definition at line 17 of file mmu.h.

PT_USER_MASK

#define PT_USER_MASK   (1ULL << PT_USER_SHIFT)

Definition at line 16 of file mmu.h.

PT_USER_SHIFT

#define PT_USER_SHIFT   2

Definition at line 12 of file mmu.h.

PT_WRITABLE_MASK

#define PT_WRITABLE_MASK   (1ULL << PT_WRITABLE_SHIFT)

Definition at line 15 of file mmu.h.

PT_WRITABLE_SHIFT

#define PT_WRITABLE_SHIFT   1

Definition at line 11 of file mmu.h.

tdp_mmu_enabled

#define tdp_mmu_enabled   false

Definition at line 276 of file mmu.h.

Function Documentation

__kvm_mmu_honors_guest_mtrrs()

bool __kvm_mmu_honors_guest_mtrrs

(

bool

vm_has_noncoherent_dma

)

Definition at line 4609 of file mmu.c.

{
    /*
     * If host MTRRs are ignored (shadow_memtype_mask is non-zero), and the
     * VM has non-coherent DMA (DMA doesn't snoop CPU caches), KVM's ABI is
     * to honor the memtype from the guest's MTRRs so that guest accesses
     * to memory that is DMA'd aren't cached against the guest's wishes.
     *
     * Note, KVM may still ultimately ignore guest MTRRs for certain PFNs,
     * e.g. KVM will force UC memtype for host MMIO.
     */
    return vm_has_noncoherent_dma && shadow_memtype_mask;
}

Here is the caller graph for this function:

__kvm_mmu_refresh_passthrough_bits()

void __kvm_mmu_refresh_passthrough_bits	(	struct kvm_vcpu *	vcpu,
		struct kvm_mmu *	mmu
	)

Definition at line 5284 of file mmu.c.

{
    const bool cr0_wp = kvm_is_cr0_bit_set(vcpu, X86_CR0_WP);
 
    BUILD_BUG_ON((KVM_MMU_CR0_ROLE_BITS & KVM_POSSIBLE_CR0_GUEST_BITS) != X86_CR0_WP);
    BUILD_BUG_ON((KVM_MMU_CR4_ROLE_BITS & KVM_POSSIBLE_CR4_GUEST_BITS));
 
    if (is_cr0_wp(mmu) == cr0_wp)
        return;
 
    mmu->cpu_role.base.cr0_wp = cr0_wp;
    reset_guest_paging_metadata(vcpu, mmu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_mmu_slot_lpages()

static unsigned long __kvm_mmu_slot_lpages ( struct kvm_memory_slot *  slot, unsigned long  npages, int  level  )

inlinestatic

Definition at line 292 of file mmu.h.

{
    return gfn_to_index(slot->base_gfn + npages - 1,
                slot->base_gfn, level) + 1;
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_index()

static gfn_t gfn_to_index ( gfn_t  gfn, gfn_t  base_gfn, int  level  )

inlinestatic

Definition at line 284 of file mmu.h.

{
    /* KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K) must be 0. */
    return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
        (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
}

Here is the caller graph for this function:

kvm_arch_write_log_dirty()

int kvm_arch_write_log_dirty

(

struct kvm_vcpu *

vcpu

)

kvm_can_do_async_pf()

bool kvm_can_do_async_pf

(

struct kvm_vcpu *

vcpu

)

Definition at line 13317 of file x86.c.

{
    if (unlikely(!lapic_in_kernel(vcpu) ||
             kvm_event_needs_reinjection(vcpu) ||
             kvm_is_exception_pending(vcpu)))
        return false;
 
    if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
        return false;
 
    /*
     * If interrupts are off we cannot even use an artificial
     * halt state.
     */
    return kvm_arch_interrupt_allowed(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_get_active_cr3_lam_bits()

static unsigned long kvm_get_active_cr3_lam_bits ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 149 of file mmu.h.

{
    if (!guest_can_use(vcpu, X86_FEATURE_LAM))
        return 0;
 
    return kvm_read_cr3(vcpu) & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_get_active_pcid()

static unsigned long kvm_get_active_pcid ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 144 of file mmu.h.

{
    return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_get_pcid()

static unsigned long kvm_get_pcid ( struct kvm_vcpu *  vcpu, gpa_t  cr3  )

inlinestatic

Definition at line 135 of file mmu.h.

{
    BUILD_BUG_ON((X86_CR3_PCID_MASK & PAGE_MASK) != 0);
 
    return kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)
           ? cr3 & X86_CR3_PCID_MASK
           : 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_get_shadow_phys_bits()

static u8 kvm_get_shadow_phys_bits ( void  )

inlinestatic

Definition at line 84 of file mmu.h.

{
    /*
     * boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected
     * in CPU detection code, but the processor treats those reduced bits as
     * 'keyID' thus they are not reserved bits. Therefore KVM needs to look at
     * the physical address bits reported by CPUID.
     */
    if (likely(boot_cpu_data.extended_cpuid_level >= 0x80000008))
        return cpuid_eax(0x80000008) & 0xff;
 
    /*
     * Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with
     * custom CPUID.  Proceed with whatever the kernel found since these features
     * aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008).
     */
    return boot_cpu_data.x86_phys_bits;
}

Here is the caller graph for this function:

kvm_handle_page_fault()

int kvm_handle_page_fault	(	struct kvm_vcpu *	vcpu,
		u64	error_code,
		u64	fault_address,
		char *	insn,
		int	insn_len
	)

Definition at line 4540 of file mmu.c.

{
    int r = 1;
    u32 flags = vcpu->arch.apf.host_apf_flags;
 
#ifndef CONFIG_X86_64
    /* A 64-bit CR2 should be impossible on 32-bit KVM. */
    if (WARN_ON_ONCE(fault_address >> 32))
        return -EFAULT;
#endif
 
    vcpu->arch.l1tf_flush_l1d = true;
    if (!flags) {
        trace_kvm_page_fault(vcpu, fault_address, error_code);
 
        if (kvm_event_needs_reinjection(vcpu))
            kvm_mmu_unprotect_page_virt(vcpu, fault_address);
        r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
                insn_len);
    } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
        vcpu->arch.apf.host_apf_flags = 0;
        local_irq_disable();
        kvm_async_pf_task_wait_schedule(fault_address);
        local_irq_enable();
    } else {
        WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
    }
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_init_mmu()

void kvm_init_mmu

(

struct kvm_vcpu *

vcpu

)

Definition at line 5538 of file mmu.c.

{
    struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
    union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
 
    if (mmu_is_nested(vcpu))
        init_kvm_nested_mmu(vcpu, cpu_role);
    else if (tdp_enabled)
        init_kvm_tdp_mmu(vcpu, cpu_role);
    else
        init_kvm_softmmu(vcpu, cpu_role);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_init_shadow_ept_mmu()

void kvm_init_shadow_ept_mmu	(	struct kvm_vcpu *	vcpu,
		bool	execonly,
		int	huge_page_level,
		bool	accessed_dirty,
		gpa_t	new_eptp
	)

Definition at line 5458 of file mmu.c.

{
    struct kvm_mmu *context = &vcpu->arch.guest_mmu;
    u8 level = vmx_eptp_page_walk_level(new_eptp);
    union kvm_cpu_role new_mode =
        kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
                           execonly, level);
 
    if (new_mode.as_u64 != context->cpu_role.as_u64) {
        /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
        context->cpu_role.as_u64 = new_mode.as_u64;
        context->root_role.word = new_mode.base.word;
 
        context->page_fault = ept_page_fault;
        context->gva_to_gpa = ept_gva_to_gpa;
        context->sync_spte = ept_sync_spte;
 
        update_permission_bitmask(context, true);
        context->pkru_mask = 0;
        reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
        reset_ept_shadow_zero_bits_mask(context, execonly);
    }
 
    kvm_mmu_new_pgd(vcpu, new_eptp);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_init_shadow_npt_mmu()

void kvm_init_shadow_npt_mmu	(	struct kvm_vcpu *	vcpu,
		unsigned long	cr0,
		unsigned long	cr4,
		u64	efer,
		gpa_t	nested_cr3
	)

Definition at line 5407 of file mmu.c.

{
    struct kvm_mmu *context = &vcpu->arch.guest_mmu;
    struct kvm_mmu_role_regs regs = {
        .cr0 = cr0,
        .cr4 = cr4 & ~X86_CR4_PKE,
        .efer = efer,
    };
    union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
    union kvm_mmu_page_role root_role;
 
    /* NPT requires CR0.PG=1. */
    WARN_ON_ONCE(cpu_role.base.direct);
 
    root_role = cpu_role.base;
    root_role.level = kvm_mmu_get_tdp_level(vcpu);
    if (root_role.level == PT64_ROOT_5LEVEL &&
        cpu_role.base.level == PT64_ROOT_4LEVEL)
        root_role.passthrough = 1;
 
    shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
    kvm_mmu_new_pgd(vcpu, nested_cr3);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_memslots_have_rmaps()

static bool kvm_memslots_have_rmaps ( struct kvm *  kvm )

inlinestatic

Definition at line 279 of file mmu.h.

{
    return !tdp_mmu_enabled || kvm_shadow_root_allocated(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_free_obsolete_roots()

void kvm_mmu_free_obsolete_roots

(

struct kvm_vcpu *

vcpu

)

Definition at line 5676 of file mmu.c.

{
    __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
    __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_honors_guest_mtrrs()

static bool kvm_mmu_honors_guest_mtrrs ( struct kvm *  kvm )

inlinestatic

Definition at line 250 of file mmu.h.

{
    return __kvm_mmu_honors_guest_mtrrs(kvm_arch_has_noncoherent_dma(kvm));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_load()

int kvm_mmu_load

(

struct kvm_vcpu *

vcpu

)

Definition at line 5588 of file mmu.c.

{
    int r;
 
    r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);
    if (r)
        goto out;
    r = mmu_alloc_special_roots(vcpu);
    if (r)
        goto out;
    if (vcpu->arch.mmu->root_role.direct)
        r = mmu_alloc_direct_roots(vcpu);
    else
        r = mmu_alloc_shadow_roots(vcpu);
    if (r)
        goto out;
 
    kvm_mmu_sync_roots(vcpu);
 
    kvm_mmu_load_pgd(vcpu);
 
    /*
     * Flush any TLB entries for the new root, the provenance of the root
     * is unknown.  Even if KVM ensures there are no stale TLB entries
     * for a freed root, in theory another hypervisor could have left
     * stale entries.  Flushing on alloc also allows KVM to skip the TLB
     * flush when freeing a root (see kvm_tdp_mmu_put_root()).
     */
    static_call(kvm_x86_flush_tlb_current)(vcpu);
out:
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_load_pgd()

static void kvm_mmu_load_pgd ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 157 of file mmu.h.

{
    u64 root_hpa = vcpu->arch.mmu->root.hpa;
 
    if (!VALID_PAGE(root_hpa))
        return;
 
    static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa,
                      vcpu->arch.mmu->root_role.level);
}

Here is the caller graph for this function:

kvm_mmu_max_gfn()

static gfn_t kvm_mmu_max_gfn ( void  )

inlinestatic

Definition at line 66 of file mmu.h.

{
    /*
     * Note that this uses the host MAXPHYADDR, not the guest's.
     * EPT/NPT cannot support GPAs that would exceed host.MAXPHYADDR;
     * assuming KVM is running on bare metal, guest accesses beyond
     * host.MAXPHYADDR will hit a #PF(RSVD) and never cause a vmexit
     * (either EPT Violation/Misconfig or #NPF), and so KVM will never
     * install a SPTE for such addresses.  If KVM is running as a VM
     * itself, on the other hand, it might see a MAXPHYADDR that is less
     * than hardware's real MAXPHYADDR.  Using the host MAXPHYADDR
     * disallows such SPTEs entirely and simplifies the TDP MMU.
     */
    int max_gpa_bits = likely(tdp_enabled) ? shadow_phys_bits : 52;
 
    return (1ULL << (max_gpa_bits - PAGE_SHIFT)) - 1;
}

Here is the caller graph for this function:

kvm_mmu_post_init_vm()

int kvm_mmu_post_init_vm

(

struct kvm *

kvm

)

Definition at line 7279 of file mmu.c.

{
    int err;
 
    if (nx_hugepage_mitigation_hard_disabled)
        return 0;
 
    err = kvm_vm_create_worker_thread(kvm, kvm_nx_huge_page_recovery_worker, 0,
                      "kvm-nx-lpage-recovery",
                      &kvm->arch.nx_huge_page_recovery_thread);
    if (!err)
        kthread_unpark(kvm->arch.nx_huge_page_recovery_thread);
 
    return err;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_pre_destroy_vm()

void kvm_mmu_pre_destroy_vm

(

struct kvm *

kvm

)

Definition at line 7295 of file mmu.c.

{
    if (kvm->arch.nx_huge_page_recovery_thread)
        kthread_stop(kvm->arch.nx_huge_page_recovery_thread);
}

Here is the caller graph for this function:

kvm_mmu_refresh_passthrough_bits()

static void kvm_mmu_refresh_passthrough_bits ( struct kvm_vcpu *  vcpu, struct kvm_mmu *  mmu  )

inlinestatic

Definition at line 168 of file mmu.h.

{
    /*
     * When EPT is enabled, KVM may passthrough CR0.WP to the guest, i.e.
     * @mmu's snapshot of CR0.WP and thus all related paging metadata may
     * be stale.  Refresh CR0.WP and the metadata on-demand when checking
     * for permission faults.  Exempt nested MMUs, i.e. MMUs for shadowing
     * nEPT and nNPT, as CR0.WP is ignored in both cases.  Note, KVM does
     * need to refresh nested_mmu, a.k.a. the walker used to translate L2
     * GVAs to GPAs, as that "MMU" needs to honor L2's CR0.WP.
     */
    if (!tdp_enabled || mmu == &vcpu->arch.guest_mmu)
        return;
 
    __kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_reload()

static int kvm_mmu_reload ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 127 of file mmu.h.

{
    if (likely(vcpu->arch.mmu->root.hpa != INVALID_PAGE))
        return 0;
 
    return kvm_mmu_load(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_set_ept_masks()

void kvm_mmu_set_ept_masks	(	bool	has_ad_bits,
		bool	has_exec_only
	)

Definition at line 425 of file spte.c.

{
    shadow_user_mask    = VMX_EPT_READABLE_MASK;
    shadow_accessed_mask    = has_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull;
    shadow_dirty_mask   = has_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull;
    shadow_nx_mask      = 0ull;
    shadow_x_mask       = VMX_EPT_EXECUTABLE_MASK;
    shadow_present_mask = has_exec_only ? 0ull : VMX_EPT_READABLE_MASK;
    /*
     * EPT overrides the host MTRRs, and so KVM must program the desired
     * memtype directly into the SPTEs.  Note, this mask is just the mask
     * of all bits that factor into the memtype, the actual memtype must be
     * dynamically calculated, e.g. to ensure host MMIO is mapped UC.
     */
    shadow_memtype_mask = VMX_EPT_MT_MASK | VMX_EPT_IPAT_BIT;
    shadow_acc_track_mask   = VMX_EPT_RWX_MASK;
    shadow_host_writable_mask = EPT_SPTE_HOST_WRITABLE;
    shadow_mmu_writable_mask  = EPT_SPTE_MMU_WRITABLE;
 
    /*
     * EPT Misconfigurations are generated if the value of bits 2:0
     * of an EPT paging-structure entry is 110b (write/execute).
     */
    kvm_mmu_set_mmio_spte_mask(VMX_EPT_MISCONFIG_WX_VALUE,
                   VMX_EPT_RWX_MASK, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_set_me_spte_mask()

void kvm_mmu_set_me_spte_mask	(	u64	me_value,
		u64	me_mask
	)

Definition at line 414 of file spte.c.

{
    /* shadow_me_value must be a subset of shadow_me_mask */
    if (WARN_ON(me_value & ~me_mask))
        me_value = me_mask = 0;
 
    shadow_me_value = me_value;
    shadow_me_mask = me_mask;
}

Here is the caller graph for this function:

kvm_mmu_set_mmio_spte_mask()

void kvm_mmu_set_mmio_spte_mask	(	u64	mmio_value,
		u64	mmio_mask,
		u64	access_mask
	)

Definition at line 362 of file spte.c.

{
    BUG_ON((u64)(unsigned)access_mask != access_mask);
    WARN_ON(mmio_value & shadow_nonpresent_or_rsvd_lower_gfn_mask);
 
    /*
     * Reset to the original module param value to honor userspace's desire
     * to (dis)allow MMIO caching.  Update the param itself so that
     * userspace can see whether or not KVM is actually using MMIO caching.
     */
    enable_mmio_caching = allow_mmio_caching;
    if (!enable_mmio_caching)
        mmio_value = 0;
 
    /*
     * The mask must contain only bits that are carved out specifically for
     * the MMIO SPTE mask, e.g. to ensure there's no overlap with the MMIO
     * generation.
     */
    if (WARN_ON(mmio_mask & ~SPTE_MMIO_ALLOWED_MASK))
        mmio_value = 0;
 
    /*
     * Disable MMIO caching if the MMIO value collides with the bits that
     * are used to hold the relocated GFN when the L1TF mitigation is
     * enabled.  This should never fire as there is no known hardware that
     * can trigger this condition, e.g. SME/SEV CPUs that require a custom
     * MMIO value are not susceptible to L1TF.
     */
    if (WARN_ON(mmio_value & (shadow_nonpresent_or_rsvd_mask <<
                  SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)))
        mmio_value = 0;
 
    /*
     * The masked MMIO value must obviously match itself and a removed SPTE
     * must not get a false positive.  Removed SPTEs and MMIO SPTEs should
     * never collide as MMIO must set some RWX bits, and removed SPTEs must
     * not set any RWX bits.
     */
    if (WARN_ON((mmio_value & mmio_mask) != mmio_value) ||
        WARN_ON(mmio_value && (REMOVED_SPTE & mmio_mask) == mmio_value))
        mmio_value = 0;
 
    if (!mmio_value)
        enable_mmio_caching = false;
 
    shadow_mmio_value = mmio_value;
    shadow_mmio_mask  = mmio_mask;
    shadow_mmio_access_mask = access_mask;
}

Here is the caller graph for this function:

kvm_mmu_slot_lpages()

static unsigned long kvm_mmu_slot_lpages ( struct kvm_memory_slot *  slot, int  level  )

inlinestatic

Definition at line 300 of file mmu.h.

{
    return __kvm_mmu_slot_lpages(slot, slot->npages, level);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_sync_prev_roots()

void kvm_mmu_sync_prev_roots

(

struct kvm_vcpu *

vcpu

)

Definition at line 4062 of file mmu.c.

{
    unsigned long roots_to_free = 0;
    int i;
 
    for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
        if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
            roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
 
    /* sync prev_roots by simply freeing them */
    kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_sync_roots()

void kvm_mmu_sync_roots

(

struct kvm_vcpu *

vcpu

)

Definition at line 4021 of file mmu.c.

{
    int i;
    struct kvm_mmu_page *sp;
 
    if (vcpu->arch.mmu->root_role.direct)
        return;
 
    if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))
        return;
 
    vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
 
    if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
        hpa_t root = vcpu->arch.mmu->root.hpa;
 
        if (!is_unsync_root(root))
            return;
 
        sp = root_to_sp(root);
 
        write_lock(&vcpu->kvm->mmu_lock);
        mmu_sync_children(vcpu, sp, true);
        write_unlock(&vcpu->kvm->mmu_lock);
        return;
    }
 
    write_lock(&vcpu->kvm->mmu_lock);
 
    for (i = 0; i < 4; ++i) {
        hpa_t root = vcpu->arch.mmu->pae_root[i];
 
        if (IS_VALID_PAE_ROOT(root)) {
            sp = spte_to_child_sp(root);
            mmu_sync_children(vcpu, sp, true);
        }
    }
 
    write_unlock(&vcpu->kvm->mmu_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_track_write()

void kvm_mmu_track_write	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		const u8 *	new,
		int	bytes
	)

Definition at line 5781 of file mmu.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    struct kvm_mmu_page *sp;
    LIST_HEAD(invalid_list);
    u64 entry, gentry, *spte;
    int npte;
    bool flush = false;
 
    /*
     * If we don't have indirect shadow pages, it means no page is
     * write-protected, so we can exit simply.
     */
    if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
        return;
 
    write_lock(&vcpu->kvm->mmu_lock);
 
    gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
 
    ++vcpu->kvm->stat.mmu_pte_write;
 
    for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {
        if (detect_write_misaligned(sp, gpa, bytes) ||
              detect_write_flooding(sp)) {
            kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
            ++vcpu->kvm->stat.mmu_flooded;
            continue;
        }
 
        spte = get_written_sptes(sp, gpa, &npte);
        if (!spte)
            continue;
 
        while (npte--) {
            entry = *spte;
            mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
            if (gentry && sp->role.level != PG_LEVEL_4K)
                ++vcpu->kvm->stat.mmu_pde_zapped;
            if (is_shadow_present_pte(entry))
                flush = true;
            ++spte;
        }
    }
    kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
    write_unlock(&vcpu->kvm->mmu_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_unload()

void kvm_mmu_unload

(

struct kvm_vcpu *

vcpu

)

Definition at line 5621 of file mmu.c.

{
    struct kvm *kvm = vcpu->kvm;
 
    kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
    WARN_ON_ONCE(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));
    kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
    WARN_ON_ONCE(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));
    vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_shadow_root_allocated()

static bool kvm_shadow_root_allocated ( struct kvm *  kvm )

inlinestatic

Definition at line 262 of file mmu.h.

{
    /*
     * Read shadow_root_allocated before related pointers. Hence, threads
     * reading shadow_root_allocated in any lock context are guaranteed to
     * see the pointers. Pairs with smp_store_release in
     * mmu_first_shadow_root_alloc.
     */
    return smp_load_acquire(&kvm->arch.shadow_root_allocated);
}

Here is the caller graph for this function:

kvm_translate_gpa()

static gpa_t kvm_translate_gpa ( struct kvm_vcpu *  vcpu, struct kvm_mmu *  mmu, gpa_t  gpa, u64  access, struct x86_exception *  exception  )

inlinestatic

Definition at line 313 of file mmu.h.

{
    if (mmu != &vcpu->arch.nested_mmu)
        return gpa;
    return translate_nested_gpa(vcpu, gpa, access, exception);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_update_page_stats()

static void kvm_update_page_stats ( struct kvm *  kvm, int  level, int  count  )

inlinestatic

Definition at line 305 of file mmu.h.

{
    atomic64_add(count, &kvm->stat.pages[level - 1]);
}

Here is the caller graph for this function:

kvm_zap_gfn_range()

void kvm_zap_gfn_range	(	struct kvm *	kvm,
		gfn_t	gfn_start,
		gfn_t	gfn_end
	)

Definition at line 6373 of file mmu.c.

{
    bool flush;
 
    if (WARN_ON_ONCE(gfn_end <= gfn_start))
        return;
 
    write_lock(&kvm->mmu_lock);
 
    kvm_mmu_invalidate_begin(kvm);
 
    kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
 
    flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
 
    if (tdp_mmu_enabled)
        flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
 
    if (flush)
        kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
 
    kvm_mmu_invalidate_end(kvm);
 
    write_unlock(&kvm->mmu_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

permission_fault()

static u8 permission_fault ( struct kvm_vcpu *  vcpu, struct kvm_mmu *  mmu, unsigned  pte_access, unsigned  pte_pkey, u64  access  )

inlinestatic

Definition at line 194 of file mmu.h.

{
    /* strip nested paging fault error codes */
    unsigned int pfec = access;
    unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
 
    /*
     * For explicit supervisor accesses, SMAP is disabled if EFLAGS.AC = 1.
     * For implicit supervisor accesses, SMAP cannot be overridden.
     *
     * SMAP works on supervisor accesses only, and not_smap can
     * be set or not set when user access with neither has any bearing
     * on the result.
     *
     * We put the SMAP checking bit in place of the PFERR_RSVD_MASK bit;
     * this bit will always be zero in pfec, but it will be one in index
     * if SMAP checks are being disabled.
     */
    u64 implicit_access = access & PFERR_IMPLICIT_ACCESS;
    bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC;
    int index = (pfec + (not_smap << PFERR_RSVD_BIT)) >> 1;
    u32 errcode = PFERR_PRESENT_MASK;
    bool fault;
 
    kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
 
    fault = (mmu->permissions[index] >> pte_access) & 1;
 
    WARN_ON(pfec & (PFERR_PK_MASK | PFERR_RSVD_MASK));
    if (unlikely(mmu->pkru_mask)) {
        u32 pkru_bits, offset;
 
        /*
        * PKRU defines 32 bits, there are 16 domains and 2
        * attribute bits per domain in pkru.  pte_pkey is the
        * index of the protection domain, so pte_pkey * 2 is
        * is the index of the first bit for the domain.
        */
        pkru_bits = (vcpu->arch.pkru >> (pte_pkey * 2)) & 3;
 
        /* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
        offset = (pfec & ~1) +
            ((pte_access & PT_USER_MASK) << (PFERR_RSVD_BIT - PT_USER_SHIFT));
 
        pkru_bits &= mmu->pkru_mask >> offset;
        errcode |= -pkru_bits & PFERR_PK_MASK;
        fault |= (pkru_bits != 0);
    }
 
    return -(u32)fault & errcode;
}

Here is the call graph for this function:

Here is the caller graph for this function:

rsvd_bits()

static __always_inline u64 rsvd_bits ( int  s, int  e  )

static

Definition at line 45 of file mmu.h.

{
    BUILD_BUG_ON(__builtin_constant_p(e) && __builtin_constant_p(s) && e < s);
 
    if (__builtin_constant_p(e))
        BUILD_BUG_ON(e > 63);
    else
        e &= 63;
 
    if (e < s)
        return 0;
 
    return ((2ULL << (e - s)) - 1) << s;
}

Here is the caller graph for this function:

translate_nested_gpa()

gpa_t translate_nested_gpa	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		u64	access,
		struct x86_exception *	exception
	)

Definition at line 7468 of file x86.c.

{
    struct kvm_mmu *mmu = vcpu->arch.mmu;
    gpa_t t_gpa;
 
    BUG_ON(!mmu_is_nested(vcpu));
 
    /* NPT walks are always user-walks */
    access |= PFERR_USER_MASK;
    t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
 
    return t_gpa;
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

enable_mmio_caching

bool __read_mostly enable_mmio_caching

extern

Definition at line 22 of file spte.c.

shadow_phys_bits

u8 __read_mostly shadow_phys_bits

extern

Definition at line 46 of file spte.c.