x86.h File Reference

#include <linux/kvm_host.h>
#include <asm/fpu/xstate.h>
#include <asm/mce.h>
#include <asm/pvclock.h>
#include "kvm_cache_regs.h"
#include "kvm_emulate.h"

Include dependency graph for x86.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	kvm_caps

Macros
#define	KVM_NESTED_VMENTER_CONSISTENCY_CHECK(consistency_check)
#define	KVM_FIRST_EMULATED_VMX_MSR   MSR_IA32_VMX_BASIC
#define	KVM_LAST_EMULATED_VMX_MSR   MSR_IA32_VMX_VMFUNC
#define	KVM_DEFAULT_PLE_GAP   128
#define	KVM_VMX_DEFAULT_PLE_WINDOW   4096
#define	KVM_DEFAULT_PLE_WINDOW_GROW   2
#define	KVM_DEFAULT_PLE_WINDOW_SHRINK   0
#define	KVM_VMX_DEFAULT_PLE_WINDOW_MAX   UINT_MAX
#define	KVM_SVM_DEFAULT_PLE_WINDOW_MAX   USHRT_MAX
#define	KVM_SVM_DEFAULT_PLE_WINDOW   3000
#define	MSR_IA32_CR_PAT_DEFAULT   0x0007040600070406ULL
#define	MMIO_GVA_ANY   (~(gva_t)0)
#define	do_shl32_div32(n, base)
#define	KVM_MSR_RET_INVALID   2 /* in-kernel MSR emulation #GP condition */
#define	KVM_MSR_RET_FILTERED   3 /* #GP due to userspace MSR filter */
#define	__cr4_reserved_bits(__cpu_has, __c)

Enumerations
enum	kvm_intr_type { KVM_HANDLING_IRQ = 1 , KVM_HANDLING_NMI }

Functions
void	kvm_spurious_fault (void)
static unsigned int	__grow_ple_window (unsigned int val, unsigned int base, unsigned int modifier, unsigned int max)
static unsigned int	__shrink_ple_window (unsigned int val, unsigned int base, unsigned int modifier, unsigned int min)
void	kvm_service_local_tlb_flush_requests (struct kvm_vcpu *vcpu)
int	kvm_check_nested_events (struct kvm_vcpu *vcpu)
static bool	kvm_vcpu_has_run (struct kvm_vcpu *vcpu)
static bool	kvm_is_exception_pending (struct kvm_vcpu *vcpu)
static void	kvm_clear_exception_queue (struct kvm_vcpu *vcpu)
static void	kvm_queue_interrupt (struct kvm_vcpu *vcpu, u8 vector, bool soft)
static void	kvm_clear_interrupt_queue (struct kvm_vcpu *vcpu)
static bool	kvm_event_needs_reinjection (struct kvm_vcpu *vcpu)
static bool	kvm_exception_is_soft (unsigned int nr)
static bool	is_protmode (struct kvm_vcpu *vcpu)
static bool	is_long_mode (struct kvm_vcpu *vcpu)
static bool	is_64_bit_mode (struct kvm_vcpu *vcpu)
static bool	is_64_bit_hypercall (struct kvm_vcpu *vcpu)
static bool	x86_exception_has_error_code (unsigned int vector)
static bool	mmu_is_nested (struct kvm_vcpu *vcpu)
static bool	is_pae (struct kvm_vcpu *vcpu)
static bool	is_pse (struct kvm_vcpu *vcpu)
static bool	is_paging (struct kvm_vcpu *vcpu)
static bool	is_pae_paging (struct kvm_vcpu *vcpu)
static u8	vcpu_virt_addr_bits (struct kvm_vcpu *vcpu)
static bool	is_noncanonical_address (u64 la, struct kvm_vcpu *vcpu)
static void	vcpu_cache_mmio_info (struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, unsigned access)
static bool	vcpu_match_mmio_gen (struct kvm_vcpu *vcpu)
static void	vcpu_clear_mmio_info (struct kvm_vcpu *vcpu, gva_t gva)
static bool	vcpu_match_mmio_gva (struct kvm_vcpu *vcpu, unsigned long gva)
static bool	vcpu_match_mmio_gpa (struct kvm_vcpu *vcpu, gpa_t gpa)
static unsigned long	kvm_register_read (struct kvm_vcpu *vcpu, int reg)
static void	kvm_register_write (struct kvm_vcpu *vcpu, int reg, unsigned long val)
static bool	kvm_check_has_quirk (struct kvm *kvm, u64 quirk)
void	kvm_inject_realmode_interrupt (struct kvm_vcpu *vcpu, int irq, int inc_eip)
u64	get_kvmclock_ns (struct kvm *kvm)
uint64_t	kvm_get_wall_clock_epoch (struct kvm *kvm)
int	kvm_read_guest_virt (struct kvm_vcpu *vcpu, gva_t addr, void *val, unsigned int bytes, struct x86_exception *exception)
int	kvm_write_guest_virt_system (struct kvm_vcpu *vcpu, gva_t addr, void *val, unsigned int bytes, struct x86_exception *exception)
int	handle_ud (struct kvm_vcpu *vcpu)
void	kvm_deliver_exception_payload (struct kvm_vcpu *vcpu, struct kvm_queued_exception *ex)
void	kvm_vcpu_mtrr_init (struct kvm_vcpu *vcpu)
u8	kvm_mtrr_get_guest_memory_type (struct kvm_vcpu *vcpu, gfn_t gfn)
int	kvm_mtrr_set_msr (struct kvm_vcpu *vcpu, u32 msr, u64 data)
int	kvm_mtrr_get_msr (struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
bool	kvm_mtrr_check_gfn_range_consistency (struct kvm_vcpu *vcpu, gfn_t gfn, int page_num)
bool	kvm_vector_hashing_enabled (void)
void	kvm_fixup_and_inject_pf_error (struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
int	x86_decode_emulated_instruction (struct kvm_vcpu *vcpu, int emulation_type, void *insn, int insn_len)
int	x86_emulate_instruction (struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int emulation_type, void *insn, int insn_len)
fastpath_t	handle_fastpath_set_msr_irqoff (struct kvm_vcpu *vcpu)
static u64	kvm_get_filtered_xcr0 (void)
static bool	kvm_mpx_supported (void)
static void	kvm_pr_unimpl_wrmsr (struct kvm_vcpu *vcpu, u32 msr, u64 data)
static void	kvm_pr_unimpl_rdmsr (struct kvm_vcpu *vcpu, u32 msr)
static u64	nsec_to_cycles (struct kvm_vcpu *vcpu, u64 nsec)
static bool	kvm_mwait_in_guest (struct kvm *kvm)
static bool	kvm_hlt_in_guest (struct kvm *kvm)
static bool	kvm_pause_in_guest (struct kvm *kvm)
static bool	kvm_cstate_in_guest (struct kvm *kvm)
static bool	kvm_notify_vmexit_enabled (struct kvm *kvm)
static __always_inline void	kvm_before_interrupt (struct kvm_vcpu *vcpu, enum kvm_intr_type intr)
static __always_inline void	kvm_after_interrupt (struct kvm_vcpu *vcpu)
static bool	kvm_handling_nmi_from_guest (struct kvm_vcpu *vcpu)
static bool	kvm_pat_valid (u64 data)
static bool	kvm_dr7_valid (u64 data)
static bool	kvm_dr6_valid (u64 data)
static void	kvm_machine_check (void)
void	kvm_load_guest_xsave_state (struct kvm_vcpu *vcpu)
void	kvm_load_host_xsave_state (struct kvm_vcpu *vcpu)
int	kvm_spec_ctrl_test_value (u64 value)
bool	__kvm_is_valid_cr4 (struct kvm_vcpu *vcpu, unsigned long cr4)
int	kvm_handle_memory_failure (struct kvm_vcpu *vcpu, int r, struct x86_exception *e)
int	kvm_handle_invpcid (struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
bool	kvm_msr_allowed (struct kvm_vcpu *vcpu, u32 index, u32 type)
int	kvm_sev_es_mmio_write (struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes, void *dst)
int	kvm_sev_es_mmio_read (struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes, void *dst)
int	kvm_sev_es_string_io (struct kvm_vcpu *vcpu, unsigned int size, unsigned int port, void *data, unsigned int count, int in)

Variables
u64	host_xcr0
u64	host_xss
u64	host_arch_capabilities
struct kvm_caps	kvm_caps
bool	enable_pmu
unsigned int	min_timer_period_us
bool	enable_vmware_backdoor
int	pi_inject_timer
bool	report_ignored_msrs
bool	eager_page_split

Macro Definition Documentation

__cr4_reserved_bits

#define __cr4_reserved_bits	(		__cpu_has,
			__c
	)

Value:

({                                                      \
    u64 __reserved_bits = CR4_RESERVED_BITS;        \
                                                        \
    if (!__cpu_has(__c, X86_FEATURE_XSAVE))         \
        __reserved_bits |= X86_CR4_OSXSAVE;     \
    if (!__cpu_has(__c, X86_FEATURE_SMEP))          \
        __reserved_bits |= X86_CR4_SMEP;        \
    if (!__cpu_has(__c, X86_FEATURE_SMAP))          \
        __reserved_bits |= X86_CR4_SMAP;        \
    if (!__cpu_has(__c, X86_FEATURE_FSGSBASE))      \
        __reserved_bits |= X86_CR4_FSGSBASE;    \
    if (!__cpu_has(__c, X86_FEATURE_PKU))           \
        __reserved_bits |= X86_CR4_PKE;         \
    if (!__cpu_has(__c, X86_FEATURE_LA57))          \
        __reserved_bits |= X86_CR4_LA57;        \
    if (!__cpu_has(__c, X86_FEATURE_UMIP))          \
        __reserved_bits |= X86_CR4_UMIP;        \
    if (!__cpu_has(__c, X86_FEATURE_VMX))           \
        __reserved_bits |= X86_CR4_VMXE;        \
    if (!__cpu_has(__c, X86_FEATURE_PCID))          \
        __reserved_bits |= X86_CR4_PCIDE;       \
    if (!__cpu_has(__c, X86_FEATURE_LAM))           \
        __reserved_bits |= X86_CR4_LAM_SUP;     \
    __reserved_bits;                                \
})

Definition at line 511 of file x86.h.

do_shl32_div32

#define do_shl32_div32	(		n,
			base
	)

Value:

    ({                          \
        u32 __quot, __rem;                  \
        asm("divl %2" : "=a" (__quot), "=d" (__rem)     \
            : "rm" (base), "0" (0), "1" ((u32) n)); \
        n = __quot;                     \
        __rem;                      \
     })

Definition at line 400 of file x86.h.

KVM_DEFAULT_PLE_GAP

#define KVM_DEFAULT_PLE_GAP   128

Definition at line 52 of file x86.h.

KVM_DEFAULT_PLE_WINDOW_GROW

#define KVM_DEFAULT_PLE_WINDOW_GROW   2

Definition at line 54 of file x86.h.

KVM_DEFAULT_PLE_WINDOW_SHRINK

#define KVM_DEFAULT_PLE_WINDOW_SHRINK   0

Definition at line 55 of file x86.h.

KVM_FIRST_EMULATED_VMX_MSR

#define KVM_FIRST_EMULATED_VMX_MSR   MSR_IA32_VMX_BASIC

Definition at line 49 of file x86.h.

KVM_LAST_EMULATED_VMX_MSR

#define KVM_LAST_EMULATED_VMX_MSR   MSR_IA32_VMX_VMFUNC

Definition at line 50 of file x86.h.

KVM_MSR_RET_FILTERED

#define KVM_MSR_RET_FILTERED   3 /* #GP due to userspace MSR filter */

Definition at line 509 of file x86.h.

KVM_MSR_RET_INVALID

#define KVM_MSR_RET_INVALID   2 /* in-kernel MSR emulation #GP condition */

Definition at line 508 of file x86.h.

KVM_NESTED_VMENTER_CONSISTENCY_CHECK

#define KVM_NESTED_VMENTER_CONSISTENCY_CHECK

(

consistency_check

)

Value:

({                                  \
    bool failed = (consistency_check);              \
    if (failed)                         \
        trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
    failed;                             \
})

Definition at line 36 of file x86.h.

KVM_SVM_DEFAULT_PLE_WINDOW

#define KVM_SVM_DEFAULT_PLE_WINDOW   3000

Definition at line 58 of file x86.h.

KVM_SVM_DEFAULT_PLE_WINDOW_MAX

#define KVM_SVM_DEFAULT_PLE_WINDOW_MAX   USHRT_MAX

Definition at line 57 of file x86.h.

KVM_VMX_DEFAULT_PLE_WINDOW

#define KVM_VMX_DEFAULT_PLE_WINDOW   4096

Definition at line 53 of file x86.h.

KVM_VMX_DEFAULT_PLE_WINDOW_MAX

#define KVM_VMX_DEFAULT_PLE_WINDOW_MAX   UINT_MAX

Definition at line 56 of file x86.h.

MMIO_GVA_ANY

#define MMIO_GVA_ANY   (~(gva_t)0)

Definition at line 245 of file x86.h.

MSR_IA32_CR_PAT_DEFAULT

#define MSR_IA32_CR_PAT_DEFAULT   0x0007040600070406ULL

Definition at line 90 of file x86.h.

Enumeration Type Documentation

kvm_intr_type

enum kvm_intr_type

Enumerator
KVM_HANDLING_IRQ
KVM_HANDLING_NMI

Definition at line 434 of file x86.h.

                   {
    /* Values are arbitrary, but must be non-zero. */
    KVM_HANDLING_IRQ = 1,
    KVM_HANDLING_NMI,
};

Function Documentation

__grow_ple_window()

static unsigned int __grow_ple_window ( unsigned int  val, unsigned int  base, unsigned int  modifier, unsigned int  max  )

inlinestatic

Definition at line 60 of file x86.h.

{
    u64 ret = val;
 
    if (modifier < 1)
        return base;
 
    if (modifier < base)
        ret *= modifier;
    else
        ret += modifier;
 
    return min(ret, (u64)max);
}

Here is the caller graph for this function:

__kvm_is_valid_cr4()

bool __kvm_is_valid_cr4	(	struct kvm_vcpu *	vcpu,
		unsigned long	cr4
	)

Definition at line 1132 of file x86.c.

{
    if (cr4 & cr4_reserved_bits)
        return false;
 
    if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
        return false;
 
    return true;
}

Here is the caller graph for this function:

__shrink_ple_window()

static unsigned int __shrink_ple_window ( unsigned int  val, unsigned int  base, unsigned int  modifier, unsigned int  min  )

inlinestatic

Definition at line 76 of file x86.h.

{
    if (modifier < 1)
        return base;
 
    if (modifier < base)
        val /= modifier;
    else
        val -= modifier;
 
    return max(val, min);
}

Here is the caller graph for this function:

get_kvmclock_ns()

u64 get_kvmclock_ns

(

struct kvm *

kvm

)

Definition at line 3105 of file x86.c.

{
    struct kvm_clock_data data;
 
    get_kvmclock(kvm, &data);
    return data.clock;
}

Here is the call graph for this function:

Here is the caller graph for this function:

handle_fastpath_set_msr_irqoff()

fastpath_t handle_fastpath_set_msr_irqoff

(

struct kvm_vcpu *

vcpu

)

Definition at line 2185 of file x86.c.

{
    u32 msr = kvm_rcx_read(vcpu);
    u64 data;
    fastpath_t ret = EXIT_FASTPATH_NONE;
 
    kvm_vcpu_srcu_read_lock(vcpu);
 
    switch (msr) {
    case APIC_BASE_MSR + (APIC_ICR >> 4):
        data = kvm_read_edx_eax(vcpu);
        if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
            kvm_skip_emulated_instruction(vcpu);
            ret = EXIT_FASTPATH_EXIT_HANDLED;
        }
        break;
    case MSR_IA32_TSC_DEADLINE:
        data = kvm_read_edx_eax(vcpu);
        if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
            kvm_skip_emulated_instruction(vcpu);
            ret = EXIT_FASTPATH_REENTER_GUEST;
        }
        break;
    default:
        break;
    }
 
    if (ret != EXIT_FASTPATH_NONE)
        trace_kvm_msr_write(msr, data);
 
    kvm_vcpu_srcu_read_unlock(vcpu);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

handle_ud()

int handle_ud

(

struct kvm_vcpu *

vcpu

)

Definition at line 7669 of file x86.c.

{
    static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
    int fep_flags = READ_ONCE(force_emulation_prefix);
    int emul_type = EMULTYPE_TRAP_UD;
    char sig[5]; /* ud2; .ascii "kvm" */
    struct x86_exception e;
    int r;
 
    r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0);
    if (r != X86EMUL_CONTINUE)
        return 1;
 
    if (fep_flags &&
        kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
                sig, sizeof(sig), &e) == 0 &&
        memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
        if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF)
            kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);
        kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
        emul_type = EMULTYPE_TRAP_UD_FORCED;
    }
 
    return kvm_emulate_instruction(vcpu, emul_type);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_64_bit_hypercall()

static bool is_64_bit_hypercall ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 164 of file x86.h.

{
    /*
     * If running with protected guest state, the CS register is not
     * accessible. The hypercall register values will have had to been
     * provided in 64-bit mode, so assume the guest is in 64-bit.
     */
    return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_64_bit_mode()

static bool is_64_bit_mode ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 152 of file x86.h.

{
    int cs_db, cs_l;
 
    WARN_ON_ONCE(vcpu->arch.guest_state_protected);
 
    if (!is_long_mode(vcpu))
        return false;
    static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
    return cs_l;
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_long_mode()

static bool is_long_mode ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 143 of file x86.h.

{
#ifdef CONFIG_X86_64
    return !!(vcpu->arch.efer & EFER_LMA);
#else
    return false;
#endif
}

Here is the caller graph for this function:

is_noncanonical_address()

static bool is_noncanonical_address ( u64  la, struct kvm_vcpu *  vcpu  )

inlinestatic

Definition at line 213 of file x86.h.

{
    return !__is_canonical_address(la, vcpu_virt_addr_bits(vcpu));
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_pae()

static bool is_pae ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 188 of file x86.h.

{
    return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_pae_paging()

static bool is_pae_paging ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 203 of file x86.h.

{
    return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_paging()

static bool is_paging ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 198 of file x86.h.

{
    return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG));
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_protmode()

static bool is_protmode ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 138 of file x86.h.

{
    return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE);
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_pse()

static bool is_pse ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 193 of file x86.h.

{
    return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_after_interrupt()

static __always_inline void kvm_after_interrupt ( struct kvm_vcpu *  vcpu )

static

Definition at line 446 of file x86.h.

{
    WRITE_ONCE(vcpu->arch.handling_intr_from_guest, 0);
}

Here is the caller graph for this function:

kvm_before_interrupt()

static __always_inline void kvm_before_interrupt ( struct kvm_vcpu *  vcpu, enum kvm_intr_type  intr  )

static

Definition at line 440 of file x86.h.

{
    WRITE_ONCE(vcpu->arch.handling_intr_from_guest, (u8)intr);
}

Here is the caller graph for this function:

kvm_check_has_quirk()

static bool kvm_check_has_quirk ( struct kvm *  kvm, u64  quirk  )

inlinestatic

Definition at line 288 of file x86.h.

{
    return !(kvm->arch.disabled_quirks & quirk);
}

Here is the caller graph for this function:

kvm_check_nested_events()

int kvm_check_nested_events

(

struct kvm_vcpu *

vcpu

)

Definition at line 10197 of file x86.c.

{
    if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
        kvm_x86_ops.nested_ops->triple_fault(vcpu);
        return 1;
    }
 
    return kvm_x86_ops.nested_ops->check_events(vcpu);
}

Here is the caller graph for this function:

kvm_clear_exception_queue()

static void kvm_clear_exception_queue ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 107 of file x86.h.

{
    vcpu->arch.exception.pending = false;
    vcpu->arch.exception.injected = false;
    vcpu->arch.exception_vmexit.pending = false;
}

Here is the caller graph for this function:

kvm_clear_interrupt_queue()

static void kvm_clear_interrupt_queue ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 122 of file x86.h.

{
    vcpu->arch.interrupt.injected = false;
}

Here is the caller graph for this function:

kvm_cstate_in_guest()

static bool kvm_cstate_in_guest ( struct kvm *  kvm )

inlinestatic

Definition at line 424 of file x86.h.

{
    return kvm->arch.cstate_in_guest;
}

Here is the caller graph for this function:

kvm_deliver_exception_payload()

void kvm_deliver_exception_payload	(	struct kvm_vcpu *	vcpu,
		struct kvm_queued_exception *	ex
	)

Definition at line 573 of file x86.c.

{
    if (!ex->has_payload)
        return;
 
    switch (ex->vector) {
    case DB_VECTOR:
        /*
         * "Certain debug exceptions may clear bit 0-3.  The
         * remaining contents of the DR6 register are never
         * cleared by the processor".
         */
        vcpu->arch.dr6 &= ~DR_TRAP_BITS;
        /*
         * In order to reflect the #DB exception payload in guest
         * dr6, three components need to be considered: active low
         * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
         * DR6_BS and DR6_BT)
         * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
         * In the target guest dr6:
         * FIXED_1 bits should always be set.
         * Active low bits should be cleared if 1-setting in payload.
         * Active high bits should be set if 1-setting in payload.
         *
         * Note, the payload is compatible with the pending debug
         * exceptions/exit qualification under VMX, that active_low bits
         * are active high in payload.
         * So they need to be flipped for DR6.
         */
        vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
        vcpu->arch.dr6 |= ex->payload;
        vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;
 
        /*
         * The #DB payload is defined as compatible with the 'pending
         * debug exceptions' field under VMX, not DR6. While bit 12 is
         * defined in the 'pending debug exceptions' field (enabled
         * breakpoint), it is reserved and must be zero in DR6.
         */
        vcpu->arch.dr6 &= ~BIT(12);
        break;
    case PF_VECTOR:
        vcpu->arch.cr2 = ex->payload;
        break;
    }
 
    ex->has_payload = false;
    ex->payload = 0;
}

Here is the caller graph for this function:

kvm_dr6_valid()

static bool kvm_dr6_valid ( u64  data )

inlinestatic

Definition at line 469 of file x86.h.

{
    /* Bits [63:32] are reserved */
    return !(data >> 32);
}

Here is the caller graph for this function:

kvm_dr7_valid()

static bool kvm_dr7_valid ( u64  data )

inlinestatic

Definition at line 464 of file x86.h.

{
    /* Bits [63:32] are reserved */
    return !(data >> 32);
}

Here is the caller graph for this function:

kvm_event_needs_reinjection()

static bool kvm_event_needs_reinjection ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 127 of file x86.h.

{
    return vcpu->arch.exception.injected || vcpu->arch.interrupt.injected ||
        vcpu->arch.nmi_injected;
}

Here is the caller graph for this function:

kvm_exception_is_soft()

static bool kvm_exception_is_soft ( unsigned int  nr )

inlinestatic

Definition at line 133 of file x86.h.

{
    return (nr == BP_VECTOR) || (nr == OF_VECTOR);
}

Here is the caller graph for this function:

kvm_fixup_and_inject_pf_error()

void kvm_fixup_and_inject_pf_error	(	struct kvm_vcpu *	vcpu,
		gva_t	gva,
		u16	error_code
	)

Definition at line 13558 of file x86.c.

{
    struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
    struct x86_exception fault;
    u64 access = error_code &
        (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
 
    if (!(error_code & PFERR_PRESENT_MASK) ||
        mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
        /*
         * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
         * tables probably do not match the TLB.  Just proceed
         * with the error code that the processor gave.
         */
        fault.vector = PF_VECTOR;
        fault.error_code_valid = true;
        fault.error_code = error_code;
        fault.nested_page_fault = false;
        fault.address = gva;
        fault.async_page_fault = false;
    }
    vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
}

Here is the caller graph for this function:

kvm_get_filtered_xcr0()

static u64 kvm_get_filtered_xcr0 ( void  )

inlinestatic

Definition at line 341 of file x86.h.

{
    u64 permitted_xcr0 = kvm_caps.supported_xcr0;
 
    BUILD_BUG_ON(XFEATURE_MASK_USER_DYNAMIC != XFEATURE_MASK_XTILE_DATA);
 
    if (permitted_xcr0 & XFEATURE_MASK_USER_DYNAMIC) {
        permitted_xcr0 &= xstate_get_guest_group_perm();
 
        /*
         * Treat XTILE_CFG as unsupported if the current process isn't
         * allowed to use XTILE_DATA, as attempting to set XTILE_CFG in
         * XCR0 without setting XTILE_DATA is architecturally illegal.
         */
        if (!(permitted_xcr0 & XFEATURE_MASK_XTILE_DATA))
            permitted_xcr0 &= ~XFEATURE_MASK_XTILE_CFG;
    }
    return permitted_xcr0;
}

Here is the caller graph for this function:

kvm_get_wall_clock_epoch()

uint64_t kvm_get_wall_clock_epoch

(

struct kvm *

kvm

)

Definition at line 3298 of file x86.c.

{
#ifdef CONFIG_X86_64
    struct pvclock_vcpu_time_info hv_clock;
    struct kvm_arch *ka = &kvm->arch;
    unsigned long seq, local_tsc_khz;
    struct timespec64 ts;
    uint64_t host_tsc;
 
    do {
        seq = read_seqcount_begin(&ka->pvclock_sc);
 
        local_tsc_khz = 0;
        if (!ka->use_master_clock)
            break;
 
        /*
         * The TSC read and the call to get_cpu_tsc_khz() must happen
         * on the same CPU.
         */
        get_cpu();
 
        local_tsc_khz = get_cpu_tsc_khz();
 
        if (local_tsc_khz &&
            !kvm_get_walltime_and_clockread(&ts, &host_tsc))
            local_tsc_khz = 0; /* Fall back to old method */
 
        put_cpu();
 
        /*
         * These values must be snapshotted within the seqcount loop.
         * After that, it's just mathematics which can happen on any
         * CPU at any time.
         */
        hv_clock.tsc_timestamp = ka->master_cycle_now;
        hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
 
    } while (read_seqcount_retry(&ka->pvclock_sc, seq));
 
    /*
     * If the conditions were right, and obtaining the wallclock+TSC was
     * successful, calculate the KVM clock at the corresponding time and
     * subtract one from the other to get the guest's epoch in nanoseconds
     * since 1970-01-01.
     */
    if (local_tsc_khz) {
        kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
                   &hv_clock.tsc_shift,
                   &hv_clock.tsc_to_system_mul);
        return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
            __pvclock_read_cycles(&hv_clock, host_tsc);
    }
#endif
    return ktime_get_real_ns() - get_kvmclock_ns(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_handle_invpcid()

int kvm_handle_invpcid	(	struct kvm_vcpu *	vcpu,
		unsigned long	type,
		gva_t	gva
	)

Definition at line 13612 of file x86.c.

{
    bool pcid_enabled;
    struct x86_exception e;
    struct {
        u64 pcid;
        u64 gla;
    } operand;
    int r;
 
    r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
    if (r != X86EMUL_CONTINUE)
        return kvm_handle_memory_failure(vcpu, r, &e);
 
    if (operand.pcid >> 12 != 0) {
        kvm_inject_gp(vcpu, 0);
        return 1;
    }
 
    pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);
 
    switch (type) {
    case INVPCID_TYPE_INDIV_ADDR:
        /*
         * LAM doesn't apply to addresses that are inputs to TLB
         * invalidation.
         */
        if ((!pcid_enabled && (operand.pcid != 0)) ||
            is_noncanonical_address(operand.gla, vcpu)) {
            kvm_inject_gp(vcpu, 0);
            return 1;
        }
        kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
        return kvm_skip_emulated_instruction(vcpu);
 
    case INVPCID_TYPE_SINGLE_CTXT:
        if (!pcid_enabled && (operand.pcid != 0)) {
            kvm_inject_gp(vcpu, 0);
            return 1;
        }
 
        kvm_invalidate_pcid(vcpu, operand.pcid);
        return kvm_skip_emulated_instruction(vcpu);
 
    case INVPCID_TYPE_ALL_NON_GLOBAL:
        /*
         * Currently, KVM doesn't mark global entries in the shadow
         * page tables, so a non-global flush just degenerates to a
         * global flush. If needed, we could optimize this later by
         * keeping track of global entries in shadow page tables.
         */
 
        fallthrough;
    case INVPCID_TYPE_ALL_INCL_GLOBAL:
        kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
        return kvm_skip_emulated_instruction(vcpu);
 
    default:
        kvm_inject_gp(vcpu, 0);
        return 1;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_handle_memory_failure()

int kvm_handle_memory_failure	(	struct kvm_vcpu *	vcpu,
		int	r,
		struct x86_exception *	e
	)

Definition at line 13588 of file x86.c.

{
    if (r == X86EMUL_PROPAGATE_FAULT) {
        if (KVM_BUG_ON(!e, vcpu->kvm))
            return -EIO;
 
        kvm_inject_emulated_page_fault(vcpu, e);
        return 1;
    }
 
    /*
     * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
     * while handling a VMX instruction KVM could've handled the request
     * correctly by exiting to userspace and performing I/O but there
     * doesn't seem to be a real use-case behind such requests, just return
     * KVM_EXIT_INTERNAL_ERROR for now.
     */
    kvm_prepare_emulation_failure_exit(vcpu);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_handling_nmi_from_guest()

static bool kvm_handling_nmi_from_guest ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 451 of file x86.h.

{
    return vcpu->arch.handling_intr_from_guest == KVM_HANDLING_NMI;
}

Here is the caller graph for this function:

kvm_hlt_in_guest()

static bool kvm_hlt_in_guest ( struct kvm *  kvm )

inlinestatic

Definition at line 414 of file x86.h.

{
    return kvm->arch.hlt_in_guest;
}

Here is the caller graph for this function:

kvm_inject_realmode_interrupt()

void kvm_inject_realmode_interrupt	(	struct kvm_vcpu *	vcpu,
		int	irq,
		int	inc_eip
	)

Definition at line 8639 of file x86.c.

{
    struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
    int ret;
 
    init_emulate_ctxt(vcpu);
 
    ctxt->op_bytes = 2;
    ctxt->ad_bytes = 2;
    ctxt->_eip = ctxt->eip + inc_eip;
    ret = emulate_int_real(ctxt, irq);
 
    if (ret != X86EMUL_CONTINUE) {
        kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
    } else {
        ctxt->eip = ctxt->_eip;
        kvm_rip_write(vcpu, ctxt->eip);
        kvm_set_rflags(vcpu, ctxt->eflags);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_is_exception_pending()

static bool kvm_is_exception_pending ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 100 of file x86.h.

{
    return vcpu->arch.exception.pending ||
           vcpu->arch.exception_vmexit.pending ||
           kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
}

Here is the caller graph for this function:

kvm_load_guest_xsave_state()

void kvm_load_guest_xsave_state

(

struct kvm_vcpu *

vcpu

)

Definition at line 1018 of file x86.c.

{
    if (vcpu->arch.guest_state_protected)
        return;
 
    if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
 
        if (vcpu->arch.xcr0 != host_xcr0)
            xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
 
        if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
            vcpu->arch.ia32_xss != host_xss)
            wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
    }
 
    if (cpu_feature_enabled(X86_FEATURE_PKU) &&
        vcpu->arch.pkru != vcpu->arch.host_pkru &&
        ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
         kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))
        write_pkru(vcpu->arch.pkru);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_load_host_xsave_state()

void kvm_load_host_xsave_state

(

struct kvm_vcpu *

vcpu

)

Definition at line 1041 of file x86.c.

{
    if (vcpu->arch.guest_state_protected)
        return;
 
    if (cpu_feature_enabled(X86_FEATURE_PKU) &&
        ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
         kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {
        vcpu->arch.pkru = rdpkru();
        if (vcpu->arch.pkru != vcpu->arch.host_pkru)
            write_pkru(vcpu->arch.host_pkru);
    }
 
    if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
 
        if (vcpu->arch.xcr0 != host_xcr0)
            xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
 
        if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
            vcpu->arch.ia32_xss != host_xss)
            wrmsrl(MSR_IA32_XSS, host_xss);
    }
 
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_machine_check()

static void kvm_machine_check ( void  )

inlinestatic

Definition at line 482 of file x86.h.

{
#if defined(CONFIG_X86_MCE)
    struct pt_regs regs = {
        .cs = 3, /* Fake ring 3 no matter what the guest ran on */
        .flags = X86_EFLAGS_IF,
    };
 
    do_machine_check(&regs);
#endif
}

Here is the caller graph for this function:

kvm_mpx_supported()

static bool kvm_mpx_supported ( void  )

inlinestatic

Definition at line 361 of file x86.h.

{
    return (kvm_caps.supported_xcr0 & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR))
        == (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
}

Here is the caller graph for this function:

kvm_msr_allowed()

bool kvm_msr_allowed	(	struct kvm_vcpu *	vcpu,
		u32	index,
		u32	type
	)

Definition at line 1796 of file x86.c.

{
    struct kvm_x86_msr_filter *msr_filter;
    struct msr_bitmap_range *ranges;
    struct kvm *kvm = vcpu->kvm;
    bool allowed;
    int idx;
    u32 i;
 
    /* x2APIC MSRs do not support filtering. */
    if (index >= 0x800 && index <= 0x8ff)
        return true;
 
    idx = srcu_read_lock(&kvm->srcu);
 
    msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
    if (!msr_filter) {
        allowed = true;
        goto out;
    }
 
    allowed = msr_filter->default_allow;
    ranges = msr_filter->ranges;
 
    for (i = 0; i < msr_filter->count; i++) {
        u32 start = ranges[i].base;
        u32 end = start + ranges[i].nmsrs;
        u32 flags = ranges[i].flags;
        unsigned long *bitmap = ranges[i].bitmap;
 
        if ((index >= start) && (index < end) && (flags & type)) {
            allowed = test_bit(index - start, bitmap);
            break;
        }
    }
 
out:
    srcu_read_unlock(&kvm->srcu, idx);
 
    return allowed;
}

Here is the caller graph for this function:

kvm_mtrr_check_gfn_range_consistency()

bool kvm_mtrr_check_gfn_range_consistency	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn,
		int	page_num
	)

Definition at line 690 of file mtrr.c.

{
    struct kvm_mtrr *mtrr_state = &vcpu->arch.mtrr_state;
    struct mtrr_iter iter;
    u64 start, end;
    int type = -1;
 
    start = gfn_to_gpa(gfn);
    end = gfn_to_gpa(gfn + page_num);
    mtrr_for_each_mem_type(&iter, mtrr_state, start, end) {
        if (type == -1) {
            type = iter.mem_type;
            continue;
        }
 
        if (type != iter.mem_type)
            return false;
    }
 
    if (iter.mtrr_disabled)
        return true;
 
    if (!iter.partial_map)
        return true;
 
    if (type == -1)
        return true;
 
    return type == mtrr_default_type(mtrr_state);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mtrr_get_guest_memory_type()

u8 kvm_mtrr_get_guest_memory_type	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 614 of file mtrr.c.

{
    struct kvm_mtrr *mtrr_state = &vcpu->arch.mtrr_state;
    struct mtrr_iter iter;
    u64 start, end;
    int type = -1;
    const int wt_wb_mask = (1 << MTRR_TYPE_WRBACK)
                   | (1 << MTRR_TYPE_WRTHROUGH);
 
    start = gfn_to_gpa(gfn);
    end = start + PAGE_SIZE;
 
    mtrr_for_each_mem_type(&iter, mtrr_state, start, end) {
        int curr_type = iter.mem_type;
 
        /*
         * Please refer to Intel SDM Volume 3: 11.11.4.1 MTRR
         * Precedences.
         */
 
        if (type == -1) {
            type = curr_type;
            continue;
        }
 
        /*
         * If two or more variable memory ranges match and the
         * memory types are identical, then that memory type is
         * used.
         */
        if (type == curr_type)
            continue;
 
        /*
         * If two or more variable memory ranges match and one of
         * the memory types is UC, the UC memory type used.
         */
        if (curr_type == MTRR_TYPE_UNCACHABLE)
            return MTRR_TYPE_UNCACHABLE;
 
        /*
         * If two or more variable memory ranges match and the
         * memory types are WT and WB, the WT memory type is used.
         */
        if (((1 << type) & wt_wb_mask) &&
              ((1 << curr_type) & wt_wb_mask)) {
            type = MTRR_TYPE_WRTHROUGH;
            continue;
        }
 
        /*
         * For overlaps not defined by the above rules, processor
         * behavior is undefined.
         */
 
        /* We use WB for this undefined behavior. :( */
        return MTRR_TYPE_WRBACK;
    }
 
    if (iter.mtrr_disabled)
        return mtrr_disabled_type(vcpu);
 
    /* not contained in any MTRRs. */
    if (type == -1)
        return mtrr_default_type(mtrr_state);
 
    /*
     * We just check one page, partially covered by MTRRs is
     * impossible.
     */
    WARN_ON(iter.partial_map);
 
    return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mtrr_get_msr()

int kvm_mtrr_get_msr	(	struct kvm_vcpu *	vcpu,
		u32	msr,
		u64 *	pdata
	)

Definition at line 397 of file mtrr.c.

{
    int index;
 
    /* MSR_MTRRcap is a readonly MSR. */
    if (msr == MSR_MTRRcap) {
        /*
         * SMRR = 0
         * WC = 1
         * FIX = 1
         * VCNT = KVM_NR_VAR_MTRR
         */
        *pdata = 0x500 | KVM_NR_VAR_MTRR;
        return 0;
    }
 
    if (!msr_mtrr_valid(msr))
        return 1;
 
    index = fixed_msr_to_range_index(msr);
    if (index >= 0) {
        *pdata = *(u64 *)&vcpu->arch.mtrr_state.fixed_ranges[index];
    } else if (msr == MSR_MTRRdefType) {
        *pdata = vcpu->arch.mtrr_state.deftype;
    } else {
        /* Variable MTRRs */
        if (is_mtrr_base_msr(msr))
            *pdata = var_mtrr_msr_to_range(vcpu, msr)->base;
        else
            *pdata = var_mtrr_msr_to_range(vcpu, msr)->mask;
 
        *pdata &= ~kvm_vcpu_reserved_gpa_bits_raw(vcpu);
    }
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mtrr_set_msr()

int kvm_mtrr_set_msr	(	struct kvm_vcpu *	vcpu,
		u32	msr,
		u64	data
	)

Definition at line 378 of file mtrr.c.

{
    int index;
 
    if (!kvm_mtrr_valid(vcpu, msr, data))
        return 1;
 
    index = fixed_msr_to_range_index(msr);
    if (index >= 0)
        *(u64 *)&vcpu->arch.mtrr_state.fixed_ranges[index] = data;
    else if (msr == MSR_MTRRdefType)
        vcpu->arch.mtrr_state.deftype = data;
    else
        set_var_mtrr_msr(vcpu, msr, data);
 
    update_mtrr(vcpu, msr);
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_mwait_in_guest()

static bool kvm_mwait_in_guest ( struct kvm *  kvm )

inlinestatic

Definition at line 409 of file x86.h.

{
    return kvm->arch.mwait_in_guest;
}

Here is the caller graph for this function:

kvm_notify_vmexit_enabled()

static bool kvm_notify_vmexit_enabled ( struct kvm *  kvm )

inlinestatic

Definition at line 429 of file x86.h.

{
    return kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_ENABLED;
}

Here is the caller graph for this function:

kvm_pat_valid()

static bool kvm_pat_valid ( u64  data )

inlinestatic

Definition at line 456 of file x86.h.

{
    if (data & 0xF8F8F8F8F8F8F8F8ull)
        return false;
    /* 0, 1, 4, 5, 6, 7 are valid values.  */
    return (data | ((data & 0x0202020202020202ull) << 1)) == data;
}

Here is the caller graph for this function:

kvm_pause_in_guest()

static bool kvm_pause_in_guest ( struct kvm *  kvm )

inlinestatic

Definition at line 419 of file x86.h.

{
    return kvm->arch.pause_in_guest;
}

Here is the caller graph for this function:

kvm_pr_unimpl_rdmsr()

static void kvm_pr_unimpl_rdmsr ( struct kvm_vcpu *  vcpu, u32  msr  )

inlinestatic

Definition at line 383 of file x86.h.

{
    if (report_ignored_msrs)
        vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr);
}

Here is the caller graph for this function:

kvm_pr_unimpl_wrmsr()

static void kvm_pr_unimpl_wrmsr ( struct kvm_vcpu *  vcpu, u32  msr, u64  data  )

inlinestatic

Definition at line 377 of file x86.h.

{
    if (report_ignored_msrs)
        vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data);
}

Here is the caller graph for this function:

kvm_queue_interrupt()

static void kvm_queue_interrupt ( struct kvm_vcpu *  vcpu, u8  vector, bool  soft  )

inlinestatic

Definition at line 114 of file x86.h.

{
    vcpu->arch.interrupt.injected = true;
    vcpu->arch.interrupt.soft = soft;
    vcpu->arch.interrupt.nr = vector;
}

Here is the caller graph for this function:

kvm_read_guest_virt()

int kvm_read_guest_virt	(	struct kvm_vcpu *	vcpu,
		gva_t	addr,
		void *	val,
		unsigned int	bytes,
		struct x86_exception *	exception
	)

Definition at line 7572 of file x86.c.

{
    u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
 
    /*
     * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
     * is returned, but our callers are not ready for that and they blindly
     * call kvm_inject_page_fault.  Ensure that they at least do not leak
     * uninitialized kernel stack memory into cr2 and error code.
     */
    memset(exception, 0, sizeof(*exception));
    return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
                      exception);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_register_read()

static unsigned long kvm_register_read ( struct kvm_vcpu *  vcpu, int  reg  )

inlinestatic

Definition at line 273 of file x86.h.

{
    unsigned long val = kvm_register_read_raw(vcpu, reg);
 
    return is_64_bit_mode(vcpu) ? val : (u32)val;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_register_write()

static void kvm_register_write ( struct kvm_vcpu *  vcpu, int  reg, unsigned long  val  )

inlinestatic

Definition at line 280 of file x86.h.

{
    if (!is_64_bit_mode(vcpu))
        val = (u32)val;
    return kvm_register_write_raw(vcpu, reg, val);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_service_local_tlb_flush_requests()

void kvm_service_local_tlb_flush_requests

(

struct kvm_vcpu *

vcpu

)

Definition at line 3617 of file x86.c.

{
    if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
        kvm_vcpu_flush_tlb_current(vcpu);
 
    if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
        kvm_vcpu_flush_tlb_guest(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_sev_es_mmio_read()

int kvm_sev_es_mmio_read	(	struct kvm_vcpu *	vcpu,
		gpa_t	src,
		unsigned int	bytes,
		void *	dst
	)

Definition at line 13761 of file x86.c.

{
    int handled;
    struct kvm_mmio_fragment *frag;
 
    if (!data)
        return -EINVAL;
 
    handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
    if (handled == bytes)
        return 1;
 
    bytes -= handled;
    gpa += handled;
    data += handled;
 
    /*TODO: Check if need to increment number of frags */
    frag = vcpu->mmio_fragments;
    vcpu->mmio_nr_fragments = 1;
    frag->len = bytes;
    frag->gpa = gpa;
    frag->data = data;
 
    vcpu->mmio_needed = 1;
    vcpu->mmio_cur_fragment = 0;
 
    vcpu->run->mmio.phys_addr = gpa;
    vcpu->run->mmio.len = min(8u, frag->len);
    vcpu->run->mmio.is_write = 0;
    vcpu->run->exit_reason = KVM_EXIT_MMIO;
 
    vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_sev_es_mmio_write()

int kvm_sev_es_mmio_write	(	struct kvm_vcpu *	vcpu,
		gpa_t	src,
		unsigned int	bytes,
		void *	dst
	)

Definition at line 13722 of file x86.c.

{
    int handled;
    struct kvm_mmio_fragment *frag;
 
    if (!data)
        return -EINVAL;
 
    handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
    if (handled == bytes)
        return 1;
 
    bytes -= handled;
    gpa += handled;
    data += handled;
 
    /*TODO: Check if need to increment number of frags */
    frag = vcpu->mmio_fragments;
    vcpu->mmio_nr_fragments = 1;
    frag->len = bytes;
    frag->gpa = gpa;
    frag->data = data;
 
    vcpu->mmio_needed = 1;
    vcpu->mmio_cur_fragment = 0;
 
    vcpu->run->mmio.phys_addr = gpa;
    vcpu->run->mmio.len = min(8u, frag->len);
    vcpu->run->mmio.is_write = 1;
    memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
    vcpu->run->exit_reason = KVM_EXIT_MMIO;
 
    vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_sev_es_string_io()

int kvm_sev_es_string_io	(	struct kvm_vcpu *	vcpu,
		unsigned int	size,
		unsigned int	port,
		void *	data,
		unsigned int	count,
		int	in
	)

Definition at line 13876 of file x86.c.

{
    vcpu->arch.sev_pio_data = data;
    vcpu->arch.sev_pio_count = count;
    return in ? kvm_sev_es_ins(vcpu, size, port)
          : kvm_sev_es_outs(vcpu, size, port);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_spec_ctrl_test_value()

int kvm_spec_ctrl_test_value

(

u64

value

)

Definition at line 13532 of file x86.c.

{
    /*
     * test that setting IA32_SPEC_CTRL to given value
     * is allowed by the host processor
     */
 
    u64 saved_value;
    unsigned long flags;
    int ret = 0;
 
    local_irq_save(flags);
 
    if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value))
        ret = 1;
    else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value))
        ret = 1;
    else
        wrmsrl(MSR_IA32_SPEC_CTRL, saved_value);
 
    local_irq_restore(flags);
 
    return ret;
}

Here is the caller graph for this function:

kvm_spurious_fault()

void kvm_spurious_fault

(

void

)

Definition at line 513 of file x86.c.

{
    /* Fault while not rebooting.  We want the trace. */
    BUG_ON(!kvm_rebooting);
}

Here is the caller graph for this function:

kvm_vcpu_has_run()

static bool kvm_vcpu_has_run ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 95 of file x86.h.

{
    return vcpu->arch.last_vmentry_cpu != -1;
}

Here is the caller graph for this function:

kvm_vcpu_mtrr_init()

void kvm_vcpu_mtrr_init

(

struct kvm_vcpu *

vcpu

)

Definition at line 434 of file mtrr.c.

{
    INIT_LIST_HEAD(&vcpu->arch.mtrr_state.head);
}

Here is the caller graph for this function:

kvm_vector_hashing_enabled()

bool kvm_vector_hashing_enabled

(

void

)

Definition at line 13520 of file x86.c.

{
    return vector_hashing;
}

Here is the caller graph for this function:

kvm_write_guest_virt_system()

int kvm_write_guest_virt_system	(	struct kvm_vcpu *	vcpu,
		gva_t	addr,
		void *	val,
		unsigned int	bytes,
		struct x86_exception *	exception
	)

Definition at line 7651 of file x86.c.

{
    /* kvm_write_guest_virt_system can pull in tons of pages. */
    vcpu->arch.l1tf_flush_l1d = true;
 
    return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
                       PFERR_WRITE_MASK, exception);
}

Here is the call graph for this function:

Here is the caller graph for this function:

mmu_is_nested()

static bool mmu_is_nested ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 183 of file x86.h.

{
    return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu;
}

Here is the caller graph for this function:

nsec_to_cycles()

static u64 nsec_to_cycles ( struct kvm_vcpu *  vcpu, u64  nsec  )

inlinestatic

Definition at line 389 of file x86.h.

{
    return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
                   vcpu->arch.virtual_tsc_shift);
}

Here is the caller graph for this function:

vcpu_cache_mmio_info()

static void vcpu_cache_mmio_info ( struct kvm_vcpu *  vcpu, gva_t  gva, gfn_t  gfn, unsigned  access  )

inlinestatic

Definition at line 218 of file x86.h.

{
    u64 gen = kvm_memslots(vcpu->kvm)->generation;
 
    if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
        return;
 
    /*
     * If this is a shadow nested page table, the "GVA" is
     * actually a nGPA.
     */
    vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;
    vcpu->arch.mmio_access = access;
    vcpu->arch.mmio_gfn = gfn;
    vcpu->arch.mmio_gen = gen;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_clear_mmio_info()

static void vcpu_clear_mmio_info ( struct kvm_vcpu *  vcpu, gva_t  gva  )

inlinestatic

Definition at line 247 of file x86.h.

{
    if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK))
        return;
 
    vcpu->arch.mmio_gva = 0;
}

Here is the caller graph for this function:

vcpu_match_mmio_gen()

static bool vcpu_match_mmio_gen ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 236 of file x86.h.

{
    return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation;
}

Here is the caller graph for this function:

vcpu_match_mmio_gpa()

static bool vcpu_match_mmio_gpa ( struct kvm_vcpu *  vcpu, gpa_t  gpa  )

inlinestatic

Definition at line 264 of file x86.h.

{
    if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn &&
          vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT)
        return true;
 
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_match_mmio_gva()

static bool vcpu_match_mmio_gva ( struct kvm_vcpu *  vcpu, unsigned long  gva  )

inlinestatic

Definition at line 255 of file x86.h.

{
    if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva &&
          vcpu->arch.mmio_gva == (gva & PAGE_MASK))
        return true;
 
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_virt_addr_bits()

static u8 vcpu_virt_addr_bits ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 208 of file x86.h.

{
    return kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 57 : 48;
}

Here is the call graph for this function:

Here is the caller graph for this function:

x86_decode_emulated_instruction()

int x86_decode_emulated_instruction	(	struct kvm_vcpu *	vcpu,
		int	emulation_type,
		void *	insn,
		int	insn_len
	)

Definition at line 9057 of file x86.c.

{
    struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
    int r;
 
    init_emulate_ctxt(vcpu);
 
    r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);
 
    trace_kvm_emulate_insn_start(vcpu);
    ++vcpu->stat.insn_emulation;
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

x86_emulate_instruction()

int x86_emulate_instruction	(	struct kvm_vcpu *	vcpu,
		gpa_t	cr2_or_gpa,
		int	emulation_type,
		void *	insn,
		int	insn_len
	)

Definition at line 9074 of file x86.c.

{
    int r;
    struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
    bool writeback = true;
 
    r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len);
    if (r != X86EMUL_CONTINUE) {
        if (r == X86EMUL_RETRY_INSTR || r == X86EMUL_PROPAGATE_FAULT)
            return 1;
 
        WARN_ON_ONCE(r != X86EMUL_UNHANDLEABLE);
        return handle_emulation_failure(vcpu, emulation_type);
    }
 
    vcpu->arch.l1tf_flush_l1d = true;
 
    if (!(emulation_type & EMULTYPE_NO_DECODE)) {
        kvm_clear_exception_queue(vcpu);
 
        /*
         * Return immediately if RIP hits a code breakpoint, such #DBs
         * are fault-like and are higher priority than any faults on
         * the code fetch itself.
         */
        if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))
            return r;
 
        r = x86_decode_emulated_instruction(vcpu, emulation_type,
                            insn, insn_len);
        if (r != EMULATION_OK)  {
            if ((emulation_type & EMULTYPE_TRAP_UD) ||
                (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
                kvm_queue_exception(vcpu, UD_VECTOR);
                return 1;
            }
            if (reexecute_instruction(vcpu, cr2_or_gpa,
                          emulation_type))
                return 1;
 
            if (ctxt->have_exception &&
                !(emulation_type & EMULTYPE_SKIP)) {
                /*
                 * #UD should result in just EMULATION_FAILED, and trap-like
                 * exception should not be encountered during decode.
                 */
                WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
                         exception_type(ctxt->exception.vector) == EXCPT_TRAP);
                inject_emulated_exception(vcpu);
                return 1;
            }
            return handle_emulation_failure(vcpu, emulation_type);
        }
    }
 
    if ((emulation_type & EMULTYPE_VMWARE_GP) &&
        !is_vmware_backdoor_opcode(ctxt)) {
        kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
        return 1;
    }
 
    /*
     * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for
     * use *only* by vendor callbacks for kvm_skip_emulated_instruction().
     * The caller is responsible for updating interruptibility state and
     * injecting single-step #DBs.
     */
    if (emulation_type & EMULTYPE_SKIP) {
        if (ctxt->mode != X86EMUL_MODE_PROT64)
            ctxt->eip = (u32)ctxt->_eip;
        else
            ctxt->eip = ctxt->_eip;
 
        if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) {
            r = 1;
            goto writeback;
        }
 
        kvm_rip_write(vcpu, ctxt->eip);
        if (ctxt->eflags & X86_EFLAGS_RF)
            kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
        return 1;
    }
 
    if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
        return 1;
 
    /* this is needed for vmware backdoor interface to work since it
       changes registers values  during IO operation */
    if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
        vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
        emulator_invalidate_register_cache(ctxt);
    }
 
restart:
    if (emulation_type & EMULTYPE_PF) {
        /* Save the faulting GPA (cr2) in the address field */
        ctxt->exception.address = cr2_or_gpa;
 
        /* With shadow page tables, cr2 contains a GVA or nGPA. */
        if (vcpu->arch.mmu->root_role.direct) {
            ctxt->gpa_available = true;
            ctxt->gpa_val = cr2_or_gpa;
        }
    } else {
        /* Sanitize the address out of an abundance of paranoia. */
        ctxt->exception.address = 0;
    }
 
    r = x86_emulate_insn(ctxt);
 
    if (r == EMULATION_INTERCEPTED)
        return 1;
 
    if (r == EMULATION_FAILED) {
        if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))
            return 1;
 
        return handle_emulation_failure(vcpu, emulation_type);
    }
 
    if (ctxt->have_exception) {
        WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);
        vcpu->mmio_needed = false;
        r = 1;
        inject_emulated_exception(vcpu);
    } else if (vcpu->arch.pio.count) {
        if (!vcpu->arch.pio.in) {
            /* FIXME: return into emulator if single-stepping.  */
            vcpu->arch.pio.count = 0;
        } else {
            writeback = false;
            vcpu->arch.complete_userspace_io = complete_emulated_pio;
        }
        r = 0;
    } else if (vcpu->mmio_needed) {
        ++vcpu->stat.mmio_exits;
 
        if (!vcpu->mmio_is_write)
            writeback = false;
        r = 0;
        vcpu->arch.complete_userspace_io = complete_emulated_mmio;
    } else if (vcpu->arch.complete_userspace_io) {
        writeback = false;
        r = 0;
    } else if (r == EMULATION_RESTART)
        goto restart;
    else
        r = 1;
 
writeback:
    if (writeback) {
        unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
        toggle_interruptibility(vcpu, ctxt->interruptibility);
        vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
 
        /*
         * Note, EXCPT_DB is assumed to be fault-like as the emulator
         * only supports code breakpoints and general detect #DB, both
         * of which are fault-like.
         */
        if (!ctxt->have_exception ||
            exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
            kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
            if (ctxt->is_branch)
                kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
            kvm_rip_write(vcpu, ctxt->eip);
            if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
                r = kvm_vcpu_do_singlestep(vcpu);
            static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);
            __kvm_set_rflags(vcpu, ctxt->eflags);
        }
 
        /*
         * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
         * do nothing, and it will be requested again as soon as
         * the shadow expires.  But we still need to check here,
         * because POPF has no interrupt shadow.
         */
        if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
            kvm_make_request(KVM_REQ_EVENT, vcpu);
    } else
        vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

x86_exception_has_error_code()

static bool x86_exception_has_error_code ( unsigned int  vector )

inlinestatic

Definition at line 174 of file x86.h.

{
    static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) |
            BIT(NP_VECTOR) | BIT(SS_VECTOR) | BIT(GP_VECTOR) |
            BIT(PF_VECTOR) | BIT(AC_VECTOR);
 
    return (1U << vector) & exception_has_error_code;
}

Here is the caller graph for this function:

Variable Documentation

eager_page_split

bool eager_page_split

extern

Definition at line 196 of file x86.c.

enable_pmu

bool enable_pmu

extern

Definition at line 192 of file x86.c.

enable_vmware_backdoor

bool enable_vmware_backdoor

extern

Definition at line 176 of file x86.c.

host_arch_capabilities

u64 host_arch_capabilities

extern

Definition at line 241 of file x86.c.

host_xcr0

u64 host_xcr0

extern

Definition at line 317 of file x86.c.

host_xss

u64 host_xss

extern

Definition at line 238 of file x86.c.

kvm_caps

struct kvm_caps kvm_caps

extern

min_timer_period_us

unsigned int min_timer_period_us

extern

Definition at line 154 of file x86.c.

pi_inject_timer

int pi_inject_timer

extern

Definition at line 188 of file x86.c.

report_ignored_msrs

bool report_ignored_msrs

extern

Definition at line 150 of file x86.c.