spte.h File Reference

#include "mmu.h"
#include "mmu_internal.h"

Include dependency graph for spte.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	SPTE_MMU_PRESENT_MASK   BIT_ULL(11)
#define	SPTE_TDP_AD_SHIFT   52
#define	SPTE_TDP_AD_MASK   (3ULL << SPTE_TDP_AD_SHIFT)
#define	SPTE_TDP_AD_ENABLED   (0ULL << SPTE_TDP_AD_SHIFT)
#define	SPTE_TDP_AD_DISABLED   (1ULL << SPTE_TDP_AD_SHIFT)
#define	SPTE_TDP_AD_WRPROT_ONLY   (2ULL << SPTE_TDP_AD_SHIFT)
#define	SPTE_BASE_ADDR_MASK   (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
#define	SPTE_PERM_MASK
#define	ACC_EXEC_MASK   1
#define	ACC_WRITE_MASK   PT_WRITABLE_MASK
#define	ACC_USER_MASK   PT_USER_MASK
#define	ACC_ALL   (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
#define	SPTE_EPT_READABLE_MASK   0x1ull
#define	SPTE_EPT_EXECUTABLE_MASK   0x4ull
#define	SPTE_LEVEL_BITS   9
#define	SPTE_LEVEL_SHIFT(level)   __PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)
#define	SPTE_INDEX(address, level)   __PT_INDEX(address, level, SPTE_LEVEL_BITS)
#define	SPTE_ENT_PER_PAGE   __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS)
#define	SHADOW_ACC_TRACK_SAVED_BITS_MASK
#define	SHADOW_ACC_TRACK_SAVED_BITS_SHIFT   54
#define	SHADOW_ACC_TRACK_SAVED_MASK
#define	DEFAULT_SPTE_HOST_WRITABLE   BIT_ULL(9)
#define	DEFAULT_SPTE_MMU_WRITABLE   BIT_ULL(10)
#define	EPT_SPTE_HOST_WRITABLE   BIT_ULL(57)
#define	EPT_SPTE_MMU_WRITABLE   BIT_ULL(58)
#define	MMIO_SPTE_GEN_LOW_START   3
#define	MMIO_SPTE_GEN_LOW_END   10
#define	MMIO_SPTE_GEN_HIGH_START   52
#define	MMIO_SPTE_GEN_HIGH_END   62
#define	MMIO_SPTE_GEN_LOW_MASK
#define	MMIO_SPTE_GEN_HIGH_MASK
#define	SPTE_MMIO_ALLOWED_MASK   (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))
#define	MMIO_SPTE_GEN_LOW_BITS   (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
#define	MMIO_SPTE_GEN_HIGH_BITS   (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
#define	MMIO_SPTE_GEN_LOW_SHIFT   (MMIO_SPTE_GEN_LOW_START - 0)
#define	MMIO_SPTE_GEN_HIGH_SHIFT   (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
#define	MMIO_SPTE_GEN_MASK   GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
#define	SHADOW_NONPRESENT_OR_RSVD_MASK_LEN   5
#define	REMOVED_SPTE   0x5a0ULL

Functions
static bool	is_removed_spte (u64 spte)
static int	spte_index (u64 *sptep)
static struct kvm_mmu_page *	to_shadow_page (hpa_t shadow_page)
static struct kvm_mmu_page *	spte_to_child_sp (u64 spte)
static struct kvm_mmu_page *	sptep_to_sp (u64 *sptep)
static struct kvm_mmu_page *	root_to_sp (hpa_t root)
static bool	is_mmio_spte (u64 spte)
static bool	is_shadow_present_pte (u64 pte)
static bool	kvm_ad_enabled (void)
static bool	sp_ad_disabled (struct kvm_mmu_page *sp)
static bool	spte_ad_enabled (u64 spte)
static bool	spte_ad_need_write_protect (u64 spte)
static u64	spte_shadow_accessed_mask (u64 spte)
static u64	spte_shadow_dirty_mask (u64 spte)
static bool	is_access_track_spte (u64 spte)
static bool	is_large_pte (u64 pte)
static bool	is_last_spte (u64 pte, int level)
static bool	is_executable_pte (u64 spte)
static kvm_pfn_t	spte_to_pfn (u64 pte)
static bool	is_accessed_spte (u64 spte)
static bool	is_dirty_spte (u64 spte)
static u64	get_rsvd_bits (struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
static bool	__is_rsvd_bits_set (struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
static bool	__is_bad_mt_xwr (struct rsvd_bits_validate *rsvd_check, u64 pte)
static __always_inline bool	is_rsvd_spte (struct rsvd_bits_validate *rsvd_check, u64 spte, int level)
static bool	is_writable_pte (unsigned long pte)
static void	check_spte_writable_invariants (u64 spte)
static bool	is_mmu_writable_spte (u64 spte)
static u64	get_mmio_spte_generation (u64 spte)
bool	spte_has_volatile_bits (u64 spte)
bool	make_spte (struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, const struct kvm_memory_slot *slot, unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool prefetch, bool can_unsync, bool host_writable, u64 *new_spte)
u64	make_huge_page_split_spte (struct kvm *kvm, u64 huge_spte, union kvm_mmu_page_role role, int index)
u64	make_nonleaf_spte (u64 *child_pt, bool ad_disabled)
u64	make_mmio_spte (struct kvm_vcpu *vcpu, u64 gfn, unsigned int access)
u64	mark_spte_for_access_track (u64 spte)
static u64	restore_acc_track_spte (u64 spte)
u64	kvm_mmu_changed_pte_notifier_make_spte (u64 old_spte, kvm_pfn_t new_pfn)
void __init	kvm_mmu_spte_module_init (void)
void	kvm_mmu_reset_all_pte_masks (void)

Variables
u64 __read_mostly	shadow_host_writable_mask
u64 __read_mostly	shadow_mmu_writable_mask
u64 __read_mostly	shadow_nx_mask
u64 __read_mostly	shadow_x_mask
u64 __read_mostly	shadow_user_mask
u64 __read_mostly	shadow_accessed_mask
u64 __read_mostly	shadow_dirty_mask
u64 __read_mostly	shadow_mmio_value
u64 __read_mostly	shadow_mmio_mask
u64 __read_mostly	shadow_mmio_access_mask
u64 __read_mostly	shadow_present_mask
u64 __read_mostly	shadow_memtype_mask
u64 __read_mostly	shadow_me_value
u64 __read_mostly	shadow_me_mask
u64 __read_mostly	shadow_acc_track_mask
u64 __read_mostly	shadow_nonpresent_or_rsvd_mask
u64 __read_mostly	shadow_nonpresent_or_rsvd_lower_gfn_mask

Macro Definition Documentation

ACC_ALL

#define ACC_ALL   (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

Definition at line 49 of file spte.h.

ACC_EXEC_MASK

#define ACC_EXEC_MASK   1

Definition at line 46 of file spte.h.

ACC_USER_MASK

#define ACC_USER_MASK   PT_USER_MASK

Definition at line 48 of file spte.h.

ACC_WRITE_MASK

#define ACC_WRITE_MASK   PT_WRITABLE_MASK

Definition at line 47 of file spte.h.

DEFAULT_SPTE_HOST_WRITABLE

#define DEFAULT_SPTE_HOST_WRITABLE   BIT_ULL(9)

Definition at line 80 of file spte.h.

DEFAULT_SPTE_MMU_WRITABLE

#define DEFAULT_SPTE_MMU_WRITABLE   BIT_ULL(10)

Definition at line 81 of file spte.h.

EPT_SPTE_HOST_WRITABLE

#define EPT_SPTE_HOST_WRITABLE   BIT_ULL(57)

Definition at line 88 of file spte.h.

EPT_SPTE_MMU_WRITABLE

#define EPT_SPTE_MMU_WRITABLE   BIT_ULL(58)

Definition at line 89 of file spte.h.

MMIO_SPTE_GEN_HIGH_BITS

#define MMIO_SPTE_GEN_HIGH_BITS   (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)

Definition at line 142 of file spte.h.

MMIO_SPTE_GEN_HIGH_END

#define MMIO_SPTE_GEN_HIGH_END   62

Definition at line 118 of file spte.h.

MMIO_SPTE_GEN_HIGH_MASK

#define MMIO_SPTE_GEN_HIGH_MASK

Value:

                            GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
                            MMIO_SPTE_GEN_HIGH_START)

Definition at line 122 of file spte.h.

MMIO_SPTE_GEN_HIGH_SHIFT

#define MMIO_SPTE_GEN_HIGH_SHIFT   (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)

Definition at line 148 of file spte.h.

MMIO_SPTE_GEN_HIGH_START

#define MMIO_SPTE_GEN_HIGH_START   52

Definition at line 117 of file spte.h.

MMIO_SPTE_GEN_LOW_BITS

#define MMIO_SPTE_GEN_LOW_BITS   (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)

Definition at line 141 of file spte.h.

MMIO_SPTE_GEN_LOW_END

#define MMIO_SPTE_GEN_LOW_END   10

Definition at line 115 of file spte.h.

MMIO_SPTE_GEN_LOW_MASK

#define MMIO_SPTE_GEN_LOW_MASK

Value:

                            GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
                            MMIO_SPTE_GEN_LOW_START)

Definition at line 120 of file spte.h.

MMIO_SPTE_GEN_LOW_SHIFT

#define MMIO_SPTE_GEN_LOW_SHIFT   (MMIO_SPTE_GEN_LOW_START - 0)

Definition at line 147 of file spte.h.

MMIO_SPTE_GEN_LOW_START

#define MMIO_SPTE_GEN_LOW_START   3

Definition at line 114 of file spte.h.

MMIO_SPTE_GEN_MASK

#define MMIO_SPTE_GEN_MASK   GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)

Definition at line 150 of file spte.h.

REMOVED_SPTE

#define REMOVED_SPTE   0x5a0ULL

Definition at line 197 of file spte.h.

SHADOW_ACC_TRACK_SAVED_BITS_MASK

#define SHADOW_ACC_TRACK_SAVED_BITS_MASK

Value:

                      (SPTE_EPT_READABLE_MASK | \
                      SPTE_EPT_EXECUTABLE_MASK)

Definition at line 67 of file spte.h.

SHADOW_ACC_TRACK_SAVED_BITS_SHIFT

#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT   54

Definition at line 69 of file spte.h.

SHADOW_ACC_TRACK_SAVED_MASK

#define SHADOW_ACC_TRACK_SAVED_MASK

Value:

                     (SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
                     SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)

Definition at line 70 of file spte.h.

SHADOW_NONPRESENT_OR_RSVD_MASK_LEN

#define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN   5

Definition at line 183 of file spte.h.

SPTE_BASE_ADDR_MASK

#define SPTE_BASE_ADDR_MASK   (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))

Definition at line 40 of file spte.h.

SPTE_ENT_PER_PAGE

#define SPTE_ENT_PER_PAGE   __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS)

Definition at line 58 of file spte.h.

SPTE_EPT_EXECUTABLE_MASK

#define SPTE_EPT_EXECUTABLE_MASK   0x4ull

Definition at line 53 of file spte.h.

SPTE_EPT_READABLE_MASK

#define SPTE_EPT_READABLE_MASK   0x1ull

Definition at line 52 of file spte.h.

SPTE_INDEX

#define SPTE_INDEX	(		address,
			level
	)		__PT_INDEX(address, level, SPTE_LEVEL_BITS)

Definition at line 57 of file spte.h.

SPTE_LEVEL_BITS

#define SPTE_LEVEL_BITS   9

Definition at line 55 of file spte.h.

SPTE_LEVEL_SHIFT

#define SPTE_LEVEL_SHIFT

(

level

)

__PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)

Definition at line 56 of file spte.h.

SPTE_MMIO_ALLOWED_MASK

#define SPTE_MMIO_ALLOWED_MASK   (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))

Definition at line 137 of file spte.h.

SPTE_MMU_PRESENT_MASK

#define SPTE_MMU_PRESENT_MASK   BIT_ULL(11)

Definition at line 16 of file spte.h.

SPTE_PERM_MASK

#define SPTE_PERM_MASK

Value:

            (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
            | shadow_x_mask | shadow_nx_mask | shadow_me_mask)

Definition at line 43 of file spte.h.

SPTE_TDP_AD_DISABLED

#define SPTE_TDP_AD_DISABLED   (1ULL << SPTE_TDP_AD_SHIFT)

Definition at line 33 of file spte.h.

SPTE_TDP_AD_ENABLED

#define SPTE_TDP_AD_ENABLED   (0ULL << SPTE_TDP_AD_SHIFT)

Definition at line 32 of file spte.h.

SPTE_TDP_AD_MASK

#define SPTE_TDP_AD_MASK   (3ULL << SPTE_TDP_AD_SHIFT)

Definition at line 31 of file spte.h.

SPTE_TDP_AD_SHIFT

#define SPTE_TDP_AD_SHIFT   52

Definition at line 30 of file spte.h.

SPTE_TDP_AD_WRPROT_ONLY

#define SPTE_TDP_AD_WRPROT_ONLY   (2ULL << SPTE_TDP_AD_SHIFT)

Definition at line 34 of file spte.h.

Function Documentation

__is_bad_mt_xwr()

static bool __is_bad_mt_xwr ( struct rsvd_bits_validate *  rsvd_check, u64  pte  )

inlinestatic

Definition at line 362 of file spte.h.

{
    return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
}

Here is the caller graph for this function:

__is_rsvd_bits_set()

static bool __is_rsvd_bits_set ( struct rsvd_bits_validate *  rsvd_check, u64  pte, int  level  )

inlinestatic

Definition at line 356 of file spte.h.

{
    return pte & get_rsvd_bits(rsvd_check, pte, level);
}

Here is the call graph for this function:

Here is the caller graph for this function:

check_spte_writable_invariants()

static void check_spte_writable_invariants ( u64  spte )

inlinestatic

Definition at line 447 of file spte.h.

{
    if (spte & shadow_mmu_writable_mask)
        WARN_ONCE(!(spte & shadow_host_writable_mask),
              KBUILD_MODNAME ": MMU-writable SPTE is not Host-writable: %llx",
              spte);
    else
        WARN_ONCE(is_writable_pte(spte),
              KBUILD_MODNAME ": Writable SPTE is not MMU-writable: %llx", spte);
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_mmio_spte_generation()

static u64 get_mmio_spte_generation ( u64  spte )

inlinestatic

Definition at line 463 of file spte.h.

{
    u64 gen;
 
    gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
    gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
    return gen;
}

Here is the caller graph for this function:

get_rsvd_bits()

static u64 get_rsvd_bits ( struct rsvd_bits_validate *  rsvd_check, u64  pte, int  level  )

inlinestatic

Definition at line 348 of file spte.h.

{
    int bit7 = (pte >> 7) & 1;
 
    return rsvd_check->rsvd_bits_mask[bit7][level-1];
}

Here is the caller graph for this function:

is_access_track_spte()

static bool is_access_track_spte ( u64  spte )

inlinestatic

Definition at line 308 of file spte.h.

{
    return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_accessed_spte()

static bool is_accessed_spte ( u64  spte )

inlinestatic

Definition at line 333 of file spte.h.

{
    u64 accessed_mask = spte_shadow_accessed_mask(spte);
 
    return accessed_mask ? spte & accessed_mask
                 : !is_access_track_spte(spte);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_dirty_spte()

static bool is_dirty_spte ( u64  spte )

inlinestatic

Definition at line 341 of file spte.h.

{
    u64 dirty_mask = spte_shadow_dirty_mask(spte);
 
    return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_executable_pte()

static bool is_executable_pte ( u64  spte )

inlinestatic

Definition at line 323 of file spte.h.

{
    return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
}

Here is the caller graph for this function:

is_large_pte()

static bool is_large_pte ( u64  pte )

inlinestatic

Definition at line 313 of file spte.h.

{
    return pte & PT_PAGE_SIZE_MASK;
}

Here is the caller graph for this function:

is_last_spte()

static bool is_last_spte ( u64  pte, int  level  )

inlinestatic

Definition at line 318 of file spte.h.

{
    return (level == PG_LEVEL_4K) || is_large_pte(pte);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_mmio_spte()

static bool is_mmio_spte ( u64  spte )

inlinestatic

Definition at line 252 of file spte.h.

{
    return (spte & shadow_mmio_mask) == shadow_mmio_value &&
           likely(enable_mmio_caching);
}

Here is the caller graph for this function:

is_mmu_writable_spte()

static bool is_mmu_writable_spte ( u64  spte )

inlinestatic

Definition at line 458 of file spte.h.

{
    return spte & shadow_mmu_writable_mask;
}

Here is the caller graph for this function:

is_removed_spte()

static bool is_removed_spte ( u64  spte )

inlinestatic

Definition at line 202 of file spte.h.

{
    return spte == REMOVED_SPTE;
}

Here is the caller graph for this function:

is_rsvd_spte()

static __always_inline bool is_rsvd_spte ( struct rsvd_bits_validate *  rsvd_check, u64  spte, int  level  )

static

Definition at line 368 of file spte.h.

{
    return __is_bad_mt_xwr(rsvd_check, spte) ||
           __is_rsvd_bits_set(rsvd_check, spte, level);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_shadow_present_pte()

static bool is_shadow_present_pte ( u64  pte )

inlinestatic

Definition at line 258 of file spte.h.

{
    return !!(pte & SPTE_MMU_PRESENT_MASK);
}

Here is the caller graph for this function:

is_writable_pte()

static bool is_writable_pte ( unsigned long  pte )

inlinestatic

Definition at line 441 of file spte.h.

{
    return pte & PT_WRITABLE_MASK;
}

Here is the caller graph for this function:

kvm_ad_enabled()

static bool kvm_ad_enabled ( void  )

inlinestatic

Definition at line 269 of file spte.h.

{
    return !!shadow_accessed_mask;
}

Here is the caller graph for this function:

kvm_mmu_changed_pte_notifier_make_spte()

u64 kvm_mmu_changed_pte_notifier_make_spte	(	u64	old_spte,
		kvm_pfn_t	new_pfn
	)

Definition at line 325 of file spte.c.

{
    u64 new_spte;
 
    new_spte = old_spte & ~SPTE_BASE_ADDR_MASK;
    new_spte |= (u64)new_pfn << PAGE_SHIFT;
 
    new_spte &= ~PT_WRITABLE_MASK;
    new_spte &= ~shadow_host_writable_mask;
    new_spte &= ~shadow_mmu_writable_mask;
 
    new_spte = mark_spte_for_access_track(new_spte);
 
    return new_spte;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_reset_all_pte_masks()

void kvm_mmu_reset_all_pte_masks

(

void

)

Definition at line 453 of file spte.c.

{
    u8 low_phys_bits;
    u64 mask;
 
    shadow_phys_bits = kvm_get_shadow_phys_bits();
 
    /*
     * If the CPU has 46 or less physical address bits, then set an
     * appropriate mask to guard against L1TF attacks. Otherwise, it is
     * assumed that the CPU is not vulnerable to L1TF.
     *
     * Some Intel CPUs address the L1 cache using more PA bits than are
     * reported by CPUID. Use the PA width of the L1 cache when possible
     * to achieve more effective mitigation, e.g. if system RAM overlaps
     * the most significant bits of legal physical address space.
     */
    shadow_nonpresent_or_rsvd_mask = 0;
    low_phys_bits = boot_cpu_data.x86_phys_bits;
    if (boot_cpu_has_bug(X86_BUG_L1TF) &&
        !WARN_ON_ONCE(boot_cpu_data.x86_cache_bits >=
              52 - SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)) {
        low_phys_bits = boot_cpu_data.x86_cache_bits
            - SHADOW_NONPRESENT_OR_RSVD_MASK_LEN;
        shadow_nonpresent_or_rsvd_mask =
            rsvd_bits(low_phys_bits, boot_cpu_data.x86_cache_bits - 1);
    }
 
    shadow_nonpresent_or_rsvd_lower_gfn_mask =
        GENMASK_ULL(low_phys_bits - 1, PAGE_SHIFT);
 
    shadow_user_mask    = PT_USER_MASK;
    shadow_accessed_mask    = PT_ACCESSED_MASK;
    shadow_dirty_mask   = PT_DIRTY_MASK;
    shadow_nx_mask      = PT64_NX_MASK;
    shadow_x_mask       = 0;
    shadow_present_mask = PT_PRESENT_MASK;
 
    /*
     * For shadow paging and NPT, KVM uses PAT entry '0' to encode WB
     * memtype in the SPTEs, i.e. relies on host MTRRs to provide the
     * correct memtype (WB is the "weakest" memtype).
     */
    shadow_memtype_mask = 0;
    shadow_acc_track_mask   = 0;
    shadow_me_mask      = 0;
    shadow_me_value     = 0;
 
    shadow_host_writable_mask = DEFAULT_SPTE_HOST_WRITABLE;
    shadow_mmu_writable_mask  = DEFAULT_SPTE_MMU_WRITABLE;
 
    /*
     * Set a reserved PA bit in MMIO SPTEs to generate page faults with
     * PFEC.RSVD=1 on MMIO accesses.  64-bit PTEs (PAE, x86-64, and EPT
     * paging) support a maximum of 52 bits of PA, i.e. if the CPU supports
     * 52-bit physical addresses then there are no reserved PA bits in the
     * PTEs and so the reserved PA approach must be disabled.
     */
    if (shadow_phys_bits < 52)
        mask = BIT_ULL(51) | PT_PRESENT_MASK;
    else
        mask = 0;
 
    kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK | ACC_USER_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mmu_spte_module_init()

void __init kvm_mmu_spte_module_init

(

void

)

Definition at line 48 of file spte.c.

{
    /*
     * Snapshot userspace's desire to allow MMIO caching.  Whether or not
     * KVM can actually enable MMIO caching depends on vendor-specific
     * hardware capabilities and other module params that can't be resolved
     * until the vendor module is loaded, i.e. enable_mmio_caching can and
     * will change when the vendor module is (re)loaded.
     */
    allow_mmio_caching = enable_mmio_caching;
}

Here is the caller graph for this function:

make_huge_page_split_spte()

u64 make_huge_page_split_spte	(	struct kvm *	kvm,
		u64	huge_spte,
		union kvm_mmu_page_role	role,
		int	index
	)

Definition at line 274 of file spte.c.

{
    u64 child_spte;
 
    if (WARN_ON_ONCE(!is_shadow_present_pte(huge_spte)))
        return 0;
 
    if (WARN_ON_ONCE(!is_large_pte(huge_spte)))
        return 0;
 
    child_spte = huge_spte;
 
    /*
     * The child_spte already has the base address of the huge page being
     * split. So we just have to OR in the offset to the page at the next
     * lower level for the given index.
     */
    child_spte |= (index * KVM_PAGES_PER_HPAGE(role.level)) << PAGE_SHIFT;
 
    if (role.level == PG_LEVEL_4K) {
        child_spte &= ~PT_PAGE_SIZE_MASK;
 
        /*
         * When splitting to a 4K page where execution is allowed, mark
         * the page executable as the NX hugepage mitigation no longer
         * applies.
         */
        if ((role.access & ACC_EXEC_MASK) && is_nx_huge_page_enabled(kvm))
            child_spte = make_spte_executable(child_spte);
    }
 
    return child_spte;
}

Here is the call graph for this function:

Here is the caller graph for this function:

make_mmio_spte()

u64 make_mmio_spte	(	struct kvm_vcpu *	vcpu,
		u64	gfn,
		unsigned int	access
	)

Definition at line 71 of file spte.c.

{
    u64 gen = kvm_vcpu_memslots(vcpu)->generation & MMIO_SPTE_GEN_MASK;
    u64 spte = generation_mmio_spte_mask(gen);
    u64 gpa = gfn << PAGE_SHIFT;
 
    WARN_ON_ONCE(!shadow_mmio_value);
 
    access &= shadow_mmio_access_mask;
    spte |= shadow_mmio_value | access;
    spte |= gpa | shadow_nonpresent_or_rsvd_mask;
    spte |= (gpa & shadow_nonpresent_or_rsvd_mask)
        << SHADOW_NONPRESENT_OR_RSVD_MASK_LEN;
 
    return spte;
}

Here is the call graph for this function:

Here is the caller graph for this function:

make_nonleaf_spte()

u64 make_nonleaf_spte	(	u64 *	child_pt,
		bool	ad_disabled
	)

Definition at line 310 of file spte.c.

{
    u64 spte = SPTE_MMU_PRESENT_MASK;
 
    spte |= __pa(child_pt) | shadow_present_mask | PT_WRITABLE_MASK |
        shadow_user_mask | shadow_x_mask | shadow_me_value;
 
    if (ad_disabled)
        spte |= SPTE_TDP_AD_DISABLED;
    else
        spte |= shadow_accessed_mask;
 
    return spte;
}

Here is the caller graph for this function:

make_spte()

bool make_spte	(	struct kvm_vcpu *	vcpu,
		struct kvm_mmu_page *	sp,
		const struct kvm_memory_slot *	slot,
		unsigned int	pte_access,
		gfn_t	gfn,
		kvm_pfn_t	pfn,
		u64	old_spte,
		bool	prefetch,
		bool	can_unsync,
		bool	host_writable,
		u64 *	new_spte
	)

Definition at line 137 of file spte.c.

{
    int level = sp->role.level;
    u64 spte = SPTE_MMU_PRESENT_MASK;
    bool wrprot = false;
 
    WARN_ON_ONCE(!pte_access && !shadow_present_mask);
 
    if (sp->role.ad_disabled)
        spte |= SPTE_TDP_AD_DISABLED;
    else if (kvm_mmu_page_ad_need_write_protect(sp))
        spte |= SPTE_TDP_AD_WRPROT_ONLY;
 
    /*
     * For the EPT case, shadow_present_mask is 0 if hardware
     * supports exec-only page table entries.  In that case,
     * ACC_USER_MASK and shadow_user_mask are used to represent
     * read access.  See FNAME(gpte_access) in paging_tmpl.h.
     */
    spte |= shadow_present_mask;
    if (!prefetch)
        spte |= spte_shadow_accessed_mask(spte);
 
    /*
     * For simplicity, enforce the NX huge page mitigation even if not
     * strictly necessary.  KVM could ignore the mitigation if paging is
     * disabled in the guest, as the guest doesn't have any page tables to
     * abuse.  But to safely ignore the mitigation, KVM would have to
     * ensure a new MMU is loaded (or all shadow pages zapped) when CR0.PG
     * is toggled on, and that's a net negative for performance when TDP is
     * enabled.  When TDP is disabled, KVM will always switch to a new MMU
     * when CR0.PG is toggled, but leveraging that to ignore the mitigation
     * would tie make_spte() further to vCPU/MMU state, and add complexity
     * just to optimize a mode that is anything but performance critical.
     */
    if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) &&
        is_nx_huge_page_enabled(vcpu->kvm)) {
        pte_access &= ~ACC_EXEC_MASK;
    }
 
    if (pte_access & ACC_EXEC_MASK)
        spte |= shadow_x_mask;
    else
        spte |= shadow_nx_mask;
 
    if (pte_access & ACC_USER_MASK)
        spte |= shadow_user_mask;
 
    if (level > PG_LEVEL_4K)
        spte |= PT_PAGE_SIZE_MASK;
 
    if (shadow_memtype_mask)
        spte |= static_call(kvm_x86_get_mt_mask)(vcpu, gfn,
                             kvm_is_mmio_pfn(pfn));
    if (host_writable)
        spte |= shadow_host_writable_mask;
    else
        pte_access &= ~ACC_WRITE_MASK;
 
    if (shadow_me_value && !kvm_is_mmio_pfn(pfn))
        spte |= shadow_me_value;
 
    spte |= (u64)pfn << PAGE_SHIFT;
 
    if (pte_access & ACC_WRITE_MASK) {
        spte |= PT_WRITABLE_MASK | shadow_mmu_writable_mask;
 
        /*
         * Optimization: for pte sync, if spte was writable the hash
         * lookup is unnecessary (and expensive). Write protection
         * is responsibility of kvm_mmu_get_page / kvm_mmu_sync_roots.
         * Same reasoning can be applied to dirty page accounting.
         */
        if (is_writable_pte(old_spte))
            goto out;
 
        /*
         * Unsync shadow pages that are reachable by the new, writable
         * SPTE.  Write-protect the SPTE if the page can't be unsync'd,
         * e.g. it's write-tracked (upper-level SPs) or has one or more
         * shadow pages and unsync'ing pages is not allowed.
         */
        if (mmu_try_to_unsync_pages(vcpu->kvm, slot, gfn, can_unsync, prefetch)) {
            wrprot = true;
            pte_access &= ~ACC_WRITE_MASK;
            spte &= ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask);
        }
    }
 
    if (pte_access & ACC_WRITE_MASK)
        spte |= spte_shadow_dirty_mask(spte);
 
out:
    if (prefetch)
        spte = mark_spte_for_access_track(spte);
 
    WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level),
          "spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level,
          get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level));
 
    if ((spte & PT_WRITABLE_MASK) && kvm_slot_dirty_track_enabled(slot)) {
        /* Enforced by kvm_mmu_hugepage_adjust. */
        WARN_ON_ONCE(level > PG_LEVEL_4K);
        mark_page_dirty_in_slot(vcpu->kvm, slot, gfn);
    }
 
    *new_spte = spte;
    return wrprot;
}

Here is the call graph for this function:

Here is the caller graph for this function:

mark_spte_for_access_track()

u64 mark_spte_for_access_track

(

u64

spte

)

Definition at line 341 of file spte.c.

{
    if (spte_ad_enabled(spte))
        return spte & ~shadow_accessed_mask;
 
    if (is_access_track_spte(spte))
        return spte;
 
    check_spte_writable_invariants(spte);
 
    WARN_ONCE(spte & (SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
              SHADOW_ACC_TRACK_SAVED_BITS_SHIFT),
          "Access Tracking saved bit locations are not zero\n");
 
    spte |= (spte & SHADOW_ACC_TRACK_SAVED_BITS_MASK) <<
        SHADOW_ACC_TRACK_SAVED_BITS_SHIFT;
    spte &= ~shadow_acc_track_mask;
 
    return spte;
}

Here is the call graph for this function:

Here is the caller graph for this function:

restore_acc_track_spte()

static u64 restore_acc_track_spte ( u64  spte )

inlinestatic

Definition at line 486 of file spte.h.

{
    u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
             & SHADOW_ACC_TRACK_SAVED_BITS_MASK;
 
    spte &= ~shadow_acc_track_mask;
    spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
          SHADOW_ACC_TRACK_SAVED_BITS_SHIFT);
    spte |= saved_bits;
 
    return spte;
}

Here is the caller graph for this function:

root_to_sp()

static struct kvm_mmu_page* root_to_sp ( hpa_t  root )

inlinestatic

Definition at line 240 of file spte.h.

{
    if (kvm_mmu_is_dummy_root(root))
        return NULL;
 
    /*
     * The "root" may be a special root, e.g. a PAE entry, treat it as a
     * SPTE to ensure any non-PA bits are dropped.
     */
    return spte_to_child_sp(root);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sp_ad_disabled()

static bool sp_ad_disabled ( struct kvm_mmu_page *  sp )

inlinestatic

Definition at line 274 of file spte.h.

{
    return sp->role.ad_disabled;
}

Here is the caller graph for this function:

spte_ad_enabled()

static bool spte_ad_enabled ( u64  spte )

inlinestatic

Definition at line 279 of file spte.h.

{
    KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
    return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED;
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_ad_need_write_protect()

static bool spte_ad_need_write_protect ( u64  spte )

inlinestatic

Definition at line 285 of file spte.h.

{
    KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
    /*
     * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED is '0',
     * and non-TDP SPTEs will never set these bits.  Optimize for 64-bit
     * TDP and do the A/D type check unconditionally.
     */
    return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED;
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_has_volatile_bits()

bool spte_has_volatile_bits

(

u64

spte

)

Definition at line 114 of file spte.c.

{
    /*
     * Always atomically update spte if it can be updated
     * out of mmu-lock, it can ensure dirty bit is not lost,
     * also, it can help us to get a stable is_writable_pte()
     * to ensure tlb flush is not missed.
     */
    if (!is_writable_pte(spte) && is_mmu_writable_spte(spte))
        return true;
 
    if (is_access_track_spte(spte))
        return true;
 
    if (spte_ad_enabled(spte)) {
        if (!(spte & shadow_accessed_mask) ||
            (is_writable_pte(spte) && !(spte & shadow_dirty_mask)))
            return true;
    }
 
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_index()

static int spte_index ( u64 *  sptep )

inlinestatic

Definition at line 208 of file spte.h.

{
    return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1);
}

Here is the caller graph for this function:

spte_shadow_accessed_mask()

static u64 spte_shadow_accessed_mask ( u64  spte )

inlinestatic

Definition at line 296 of file spte.h.

{
    KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
    return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_shadow_dirty_mask()

static u64 spte_shadow_dirty_mask ( u64  spte )

inlinestatic

Definition at line 302 of file spte.h.

{
    KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
    return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_to_child_sp()

static struct kvm_mmu_page* spte_to_child_sp ( u64  spte )

inlinestatic

Definition at line 230 of file spte.h.

{
    return to_shadow_page(spte & SPTE_BASE_ADDR_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

spte_to_pfn()

static kvm_pfn_t spte_to_pfn ( u64  pte )

inlinestatic

Definition at line 328 of file spte.h.

{
    return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
}

Here is the caller graph for this function:

sptep_to_sp()

static struct kvm_mmu_page* sptep_to_sp ( u64 *  sptep )

inlinestatic

Definition at line 235 of file spte.h.

{
    return to_shadow_page(__pa(sptep));
}

Here is the call graph for this function:

Here is the caller graph for this function:

to_shadow_page()

static struct kvm_mmu_page* to_shadow_page ( hpa_t  shadow_page )

inlinestatic

Definition at line 223 of file spte.h.

{
    struct page *page = pfn_to_page((shadow_page) >> PAGE_SHIFT);
 
    return (struct kvm_mmu_page *)page_private(page);
}

Here is the caller graph for this function:

Variable Documentation

shadow_acc_track_mask

u64 __read_mostly shadow_acc_track_mask

extern

Definition at line 41 of file spte.c.

shadow_accessed_mask

u64 __read_mostly shadow_accessed_mask

extern

Definition at line 32 of file spte.c.

shadow_dirty_mask

u64 __read_mostly shadow_dirty_mask

extern

Definition at line 33 of file spte.c.

shadow_host_writable_mask

u64 __read_mostly shadow_host_writable_mask

extern

Definition at line 27 of file spte.c.

shadow_me_mask

u64 __read_mostly shadow_me_mask

extern

Definition at line 40 of file spte.c.

shadow_me_value

u64 __read_mostly shadow_me_value

extern

Definition at line 39 of file spte.c.

shadow_memtype_mask

u64 __read_mostly shadow_memtype_mask

extern

Definition at line 38 of file spte.c.

shadow_mmio_access_mask

u64 __read_mostly shadow_mmio_access_mask

extern

Definition at line 36 of file spte.c.

shadow_mmio_mask

u64 __read_mostly shadow_mmio_mask

extern

Definition at line 35 of file spte.c.

shadow_mmio_value

u64 __read_mostly shadow_mmio_value

extern

Definition at line 34 of file spte.c.

shadow_mmu_writable_mask

u64 __read_mostly shadow_mmu_writable_mask

extern

Definition at line 28 of file spte.c.

shadow_nonpresent_or_rsvd_lower_gfn_mask

u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask

extern

Definition at line 44 of file spte.c.

shadow_nonpresent_or_rsvd_mask

u64 __read_mostly shadow_nonpresent_or_rsvd_mask

extern

Definition at line 43 of file spte.c.

shadow_nx_mask

u64 __read_mostly shadow_nx_mask

extern

Definition at line 29 of file spte.c.

shadow_present_mask

u64 __read_mostly shadow_present_mask

extern

Definition at line 37 of file spte.c.

shadow_user_mask

u64 __read_mostly shadow_user_mask

extern

Definition at line 31 of file spte.c.

shadow_x_mask

u64 __read_mostly shadow_x_mask

extern

Definition at line 30 of file spte.c.