svm.h File Reference

#include <linux/kvm_types.h>
#include <linux/kvm_host.h>
#include <linux/bits.h>
#include <asm/svm.h>
#include <asm/sev-common.h>
#include "cpuid.h"
#include "kvm_cache_regs.h"

Include dependency graph for svm.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	kvm_sev_info
struct	kvm_svm
struct	kvm_vmcb_info
struct	vmcb_save_area_cached
struct	vmcb_ctrl_area_cached
struct	svm_nested_state
struct	vcpu_sev_es_state
struct	vcpu_svm
struct	svm_cpu_data

Macros
#define	__sme_page_pa(x)   __sme_set(page_to_pfn(x) << PAGE_SHIFT)
#define	IOPM_SIZE   PAGE_SIZE * 3
#define	MSRPM_SIZE   PAGE_SIZE * 2
#define	MAX_DIRECT_ACCESS_MSRS   47
#define	MSRPM_OFFSETS   32
#define	VMCB_ALL_CLEAN_MASK
#define	VMCB_ALWAYS_DIRTY_MASK   ((1U << VMCB_INTR) | (1U << VMCB_CR2))
#define	SVM_REGS_LAZY_LOAD_SET   (1 << VCPU_EXREG_PDPTR)
#define	MSR_INVALID   0xffffffffU
#define	DEBUGCTL_RESERVED_BITS   (~(0x3fULL))
#define	NESTED_EXIT_HOST   0 /* Exit handled on host level */
#define	NESTED_EXIT_DONE   1 /* Exit caused nested vmexit */
#define	NESTED_EXIT_CONTINUE   2 /* Further checks needed */
#define	AVIC_REQUIRED_APICV_INHIBITS
#define	GHCB_VERSION_MAX   1ULL
#define	GHCB_VERSION_MIN   1ULL
#define	DEFINE_KVM_GHCB_ACCESSORS(field)

Enumerations
enum	{   VMCB_INTERCEPTS , VMCB_PERM_MAP , VMCB_ASID , VMCB_INTR ,   VMCB_NPT , VMCB_CR , VMCB_DR , VMCB_DT ,   VMCB_SEG , VMCB_CR2 , VMCB_LBR , VMCB_AVIC ,   VMCB_SW = 31 }

Functions
	DECLARE_PER_CPU (struct svm_cpu_data, svm_data)
void	recalc_intercepts (struct vcpu_svm *svm)
static __always_inline struct kvm_svm *	to_kvm_svm (struct kvm *kvm)
static __always_inline bool	sev_guest (struct kvm *kvm)
static __always_inline bool	sev_es_guest (struct kvm *kvm)
static void	vmcb_mark_all_dirty (struct vmcb *vmcb)
static void	vmcb_mark_all_clean (struct vmcb *vmcb)
static void	vmcb_mark_dirty (struct vmcb *vmcb, int bit)
static bool	vmcb_is_dirty (struct vmcb *vmcb, int bit)
static __always_inline struct vcpu_svm *	to_svm (struct kvm_vcpu *vcpu)
static void	vmcb_set_intercept (struct vmcb_control_area *control, u32 bit)
static void	vmcb_clr_intercept (struct vmcb_control_area *control, u32 bit)
static bool	vmcb_is_intercept (struct vmcb_control_area *control, u32 bit)
static bool	vmcb12_is_intercept (struct vmcb_ctrl_area_cached *control, u32 bit)
static void	set_exception_intercept (struct vcpu_svm *svm, u32 bit)
static void	clr_exception_intercept (struct vcpu_svm *svm, u32 bit)
static void	svm_set_intercept (struct vcpu_svm *svm, int bit)
static void	svm_clr_intercept (struct vcpu_svm *svm, int bit)
static bool	svm_is_intercept (struct vcpu_svm *svm, int bit)
static bool	nested_vgif_enabled (struct vcpu_svm *svm)
static struct vmcb *	get_vgif_vmcb (struct vcpu_svm *svm)
static void	enable_gif (struct vcpu_svm *svm)
static void	disable_gif (struct vcpu_svm *svm)
static bool	gif_set (struct vcpu_svm *svm)
static bool	nested_npt_enabled (struct vcpu_svm *svm)
static bool	nested_vnmi_enabled (struct vcpu_svm *svm)
static bool	is_x2apic_msrpm_offset (u32 offset)
static struct vmcb *	get_vnmi_vmcb_l1 (struct vcpu_svm *svm)
static bool	is_vnmi_enabled (struct vcpu_svm *svm)
u32	svm_msrpm_offset (u32 msr)
u32 *	svm_vcpu_alloc_msrpm (void)
void	svm_vcpu_init_msrpm (struct kvm_vcpu *vcpu, u32 *msrpm)
void	svm_vcpu_free_msrpm (u32 *msrpm)
void	svm_copy_lbrs (struct vmcb *to_vmcb, struct vmcb *from_vmcb)
void	svm_update_lbrv (struct kvm_vcpu *vcpu)
int	svm_set_efer (struct kvm_vcpu *vcpu, u64 efer)
void	svm_set_cr0 (struct kvm_vcpu *vcpu, unsigned long cr0)
void	svm_set_cr4 (struct kvm_vcpu *vcpu, unsigned long cr4)
void	disable_nmi_singlestep (struct vcpu_svm *svm)
bool	svm_smi_blocked (struct kvm_vcpu *vcpu)
bool	svm_nmi_blocked (struct kvm_vcpu *vcpu)
bool	svm_interrupt_blocked (struct kvm_vcpu *vcpu)
void	svm_set_gif (struct vcpu_svm *svm, bool value)
int	svm_invoke_exit_handler (struct kvm_vcpu *vcpu, u64 exit_code)
void	set_msr_interception (struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, int read, int write)
void	svm_set_x2apic_msr_interception (struct vcpu_svm *svm, bool disable)
void	svm_complete_interrupt_delivery (struct kvm_vcpu *vcpu, int delivery_mode, int trig_mode, int vec)
static bool	nested_svm_virtualize_tpr (struct kvm_vcpu *vcpu)
static bool	nested_exit_on_smi (struct vcpu_svm *svm)
static bool	nested_exit_on_intr (struct vcpu_svm *svm)
static bool	nested_exit_on_nmi (struct vcpu_svm *svm)
int	enter_svm_guest_mode (struct kvm_vcpu *vcpu, u64 vmcb_gpa, struct vmcb *vmcb12, bool from_vmrun)
void	svm_leave_nested (struct kvm_vcpu *vcpu)
void	svm_free_nested (struct vcpu_svm *svm)
int	svm_allocate_nested (struct vcpu_svm *svm)
int	nested_svm_vmrun (struct kvm_vcpu *vcpu)
void	svm_copy_vmrun_state (struct vmcb_save_area *to_save, struct vmcb_save_area *from_save)
void	svm_copy_vmloadsave_state (struct vmcb *to_vmcb, struct vmcb *from_vmcb)
int	nested_svm_vmexit (struct vcpu_svm *svm)
static int	nested_svm_simple_vmexit (struct vcpu_svm *svm, u32 exit_code)
int	nested_svm_exit_handled (struct vcpu_svm *svm)
int	nested_svm_check_permissions (struct kvm_vcpu *vcpu)
int	nested_svm_check_exception (struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code)
int	nested_svm_exit_special (struct vcpu_svm *svm)
void	nested_svm_update_tsc_ratio_msr (struct kvm_vcpu *vcpu)
void	svm_write_tsc_multiplier (struct kvm_vcpu *vcpu)
void	nested_copy_vmcb_control_to_cache (struct vcpu_svm *svm, struct vmcb_control_area *control)
void	nested_copy_vmcb_save_to_cache (struct vcpu_svm *svm, struct vmcb_save_area *save)
void	nested_sync_control_from_vmcb02 (struct vcpu_svm *svm)
void	nested_vmcb02_compute_g_pat (struct vcpu_svm *svm)
void	svm_switch_vmcb (struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
bool	avic_hardware_setup (void)
int	avic_ga_log_notifier (u32 ga_tag)
void	avic_vm_destroy (struct kvm *kvm)
int	avic_vm_init (struct kvm *kvm)
void	avic_init_vmcb (struct vcpu_svm *svm, struct vmcb *vmcb)
int	avic_incomplete_ipi_interception (struct kvm_vcpu *vcpu)
int	avic_unaccelerated_access_interception (struct kvm_vcpu *vcpu)
int	avic_init_vcpu (struct vcpu_svm *svm)
void	avic_vcpu_load (struct kvm_vcpu *vcpu, int cpu)
void	avic_vcpu_put (struct kvm_vcpu *vcpu)
void	avic_apicv_post_state_restore (struct kvm_vcpu *vcpu)
void	avic_refresh_apicv_exec_ctrl (struct kvm_vcpu *vcpu)
int	avic_pi_update_irte (struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, bool set)
void	avic_vcpu_blocking (struct kvm_vcpu *vcpu)
void	avic_vcpu_unblocking (struct kvm_vcpu *vcpu)
void	avic_ring_doorbell (struct kvm_vcpu *vcpu)
unsigned long	avic_vcpu_get_apicv_inhibit_reasons (struct kvm_vcpu *vcpu)
void	avic_refresh_virtual_apic_mode (struct kvm_vcpu *vcpu)
void	sev_vm_destroy (struct kvm *kvm)
int	sev_mem_enc_ioctl (struct kvm *kvm, void __user *argp)
int	sev_mem_enc_register_region (struct kvm *kvm, struct kvm_enc_region *range)
int	sev_mem_enc_unregister_region (struct kvm *kvm, struct kvm_enc_region *range)
int	sev_vm_copy_enc_context_from (struct kvm *kvm, unsigned int source_fd)
int	sev_vm_move_enc_context_from (struct kvm *kvm, unsigned int source_fd)
void	sev_guest_memory_reclaimed (struct kvm *kvm)
void	pre_sev_run (struct vcpu_svm *svm, int cpu)
void __init	sev_set_cpu_caps (void)
void __init	sev_hardware_setup (void)
void	sev_hardware_unsetup (void)
int	sev_cpu_init (struct svm_cpu_data *sd)
void	sev_init_vmcb (struct vcpu_svm *svm)
void	sev_vcpu_after_set_cpuid (struct vcpu_svm *svm)
void	sev_free_vcpu (struct kvm_vcpu *vcpu)
int	sev_handle_vmgexit (struct kvm_vcpu *vcpu)
int	sev_es_string_io (struct vcpu_svm *svm, int size, unsigned int port, int in)
void	sev_es_vcpu_reset (struct vcpu_svm *svm)
void	sev_vcpu_deliver_sipi_vector (struct kvm_vcpu *vcpu, u8 vector)
void	sev_es_prepare_switch_to_guest (struct sev_es_save_area *hostsa)
void	sev_es_unmap_ghcb (struct vcpu_svm *svm)
void	__svm_sev_es_vcpu_run (struct vcpu_svm *svm, bool spec_ctrl_intercepted)
void	__svm_vcpu_run (struct vcpu_svm *svm, bool spec_ctrl_intercepted)

Variables
u32 msrpm_offsets[MSRPM_OFFSETS]	__read_mostly
bool	npt_enabled
int	nrips
int	vgif
bool	intercept_smi
bool	x2avic_enabled
bool	vnmi
bool	dump_invalid_vmcb
struct kvm_x86_nested_ops	svm_nested_ops
unsigned int	max_sev_asid

Macro Definition Documentation

__sme_page_pa

#define __sme_page_pa

(

x

)

__sme_set(page_to_pfn(x) << PAGE_SHIFT)

Definition at line 28 of file svm.h.

AVIC_REQUIRED_APICV_INHIBITS

#define AVIC_REQUIRED_APICV_INHIBITS

Value:

(                           \
    BIT(APICV_INHIBIT_REASON_DISABLE) |     \
    BIT(APICV_INHIBIT_REASON_ABSENT) |      \
    BIT(APICV_INHIBIT_REASON_HYPERV) |      \
    BIT(APICV_INHIBIT_REASON_NESTED) |      \
    BIT(APICV_INHIBIT_REASON_IRQWIN) |      \
    BIT(APICV_INHIBIT_REASON_PIT_REINJ) |       \
    BIT(APICV_INHIBIT_REASON_BLOCKIRQ) |        \
    BIT(APICV_INHIBIT_REASON_SEV)      |        \
    BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \
    BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) |    \
    BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED) |  \
    BIT(APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED)    \
)

Definition at line 628 of file svm.h.

DEBUGCTL_RESERVED_BITS

#define DEBUGCTL_RESERVED_BITS   (~(0x3fULL))

Definition at line 537 of file svm.h.

DEFINE_KVM_GHCB_ACCESSORS

#define DEFINE_KVM_GHCB_ACCESSORS

(

field

)

Value:

    static __always_inline bool kvm_ghcb_##field##_is_valid(const struct vcpu_svm *svm) \
    {                                   \
        return test_bit(GHCB_BITMAP_IDX(field),             \
                (unsigned long *)&svm->sev_es.valid_bitmap);    \
    }                                   \
                                        \
    static __always_inline u64 kvm_ghcb_get_##field##_if_valid(struct vcpu_svm *svm, struct ghcb *ghcb) \
    {                                   \
        return kvm_ghcb_##field##_is_valid(svm) ? ghcb->save.field : 0; \
    }                                   \

Definition at line 703 of file svm.h.

GHCB_VERSION_MAX

#define GHCB_VERSION_MAX   1ULL

Definition at line 667 of file svm.h.

GHCB_VERSION_MIN

#define GHCB_VERSION_MIN   1ULL

Definition at line 668 of file svm.h.

IOPM_SIZE

#define IOPM_SIZE   PAGE_SIZE * 3

Definition at line 30 of file svm.h.

MAX_DIRECT_ACCESS_MSRS

#define MAX_DIRECT_ACCESS_MSRS   47

Definition at line 33 of file svm.h.

MSR_INVALID

#define MSR_INVALID   0xffffffffU

Definition at line 535 of file svm.h.

MSRPM_OFFSETS

#define MSRPM_OFFSETS   32

Definition at line 34 of file svm.h.

MSRPM_SIZE

#define MSRPM_SIZE   PAGE_SIZE * 2

Definition at line 31 of file svm.h.

NESTED_EXIT_CONTINUE

#define NESTED_EXIT_CONTINUE   2 /* Further checks needed */

Definition at line 567 of file svm.h.

NESTED_EXIT_DONE

#define NESTED_EXIT_DONE   1 /* Exit caused nested vmexit */

Definition at line 566 of file svm.h.

NESTED_EXIT_HOST

#define NESTED_EXIT_HOST   0 /* Exit handled on host level */

Definition at line 565 of file svm.h.

SVM_REGS_LAZY_LOAD_SET

#define SVM_REGS_LAZY_LOAD_SET   (1 << VCPU_EXREG_PDPTR)

Definition at line 377 of file svm.h.

VMCB_ALL_CLEAN_MASK

#define VMCB_ALL_CLEAN_MASK

Value:

    (                   \
    (1U << VMCB_INTERCEPTS) | (1U << VMCB_PERM_MAP) |   \
    (1U << VMCB_ASID) | (1U << VMCB_INTR) |         \
    (1U << VMCB_NPT) | (1U << VMCB_CR) | (1U << VMCB_DR) |  \
    (1U << VMCB_DT) | (1U << VMCB_SEG) | (1U << VMCB_CR2) | \
    (1U << VMCB_LBR) | (1U << VMCB_AVIC) |          \
    (1U << VMCB_SW))

Definition at line 68 of file svm.h.

VMCB_ALWAYS_DIRTY_MASK

#define VMCB_ALWAYS_DIRTY_MASK   ((1U << VMCB_INTR) | (1U << VMCB_CR2))

Definition at line 77 of file svm.h.

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
VMCB_INTERCEPTS
VMCB_PERM_MAP
VMCB_ASID
VMCB_INTR
VMCB_NPT
VMCB_CR
VMCB_DR
VMCB_DT
VMCB_SEG
VMCB_CR2
VMCB_LBR
VMCB_AVIC
VMCB_SW

Definition at line 48 of file svm.h.

     {
    VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
                pause filter count */
    VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
    VMCB_ASID,   /* ASID */
    VMCB_INTR,   /* int_ctl, int_vector */
    VMCB_NPT,        /* npt_en, nCR3, gPAT */
    VMCB_CR,     /* CR0, CR3, CR4, EFER */
    VMCB_DR,         /* DR6, DR7 */
    VMCB_DT,         /* GDT, IDT */
    VMCB_SEG,        /* CS, DS, SS, ES, CPL */
    VMCB_CR2,        /* CR2 only */
    VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
    VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
              * AVIC PHYSICAL_TABLE pointer,
              * AVIC LOGICAL_TABLE pointer
              */
    VMCB_SW = 31,    /* Reserved for hypervisor/software use */
};

Function Documentation

__svm_sev_es_vcpu_run()

void __svm_sev_es_vcpu_run	(	struct vcpu_svm *	svm,
		bool	spec_ctrl_intercepted
	)

Here is the caller graph for this function:

__svm_vcpu_run()

void __svm_vcpu_run	(	struct vcpu_svm *	svm,
		bool	spec_ctrl_intercepted
	)

Here is the caller graph for this function:

avic_apicv_post_state_restore()

void avic_apicv_post_state_restore

(

struct kvm_vcpu *

vcpu

)

Definition at line 738 of file avic.c.

{
    avic_handle_dfr_update(vcpu);
    avic_handle_ldr_update(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_ga_log_notifier()

int avic_ga_log_notifier

(

u32

ga_tag

)

Definition at line 143 of file avic.c.

{
    unsigned long flags;
    struct kvm_svm *kvm_svm;
    struct kvm_vcpu *vcpu = NULL;
    u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
    u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
 
    pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
    trace_kvm_avic_ga_log(vm_id, vcpu_id);
 
    spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
    hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
        if (kvm_svm->avic_vm_id != vm_id)
            continue;
        vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
        break;
    }
    spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 
    /* Note:
     * At this point, the IOMMU should have already set the pending
     * bit in the vAPIC backing page. So, we just need to schedule
     * in the vcpu.
     */
    if (vcpu)
        kvm_vcpu_wake_up(vcpu);
 
    return 0;
}

Here is the call graph for this function:

avic_hardware_setup()

bool avic_hardware_setup

(

void

)

Definition at line 1188 of file avic.c.

{
    if (!npt_enabled)
        return false;
 
    /* AVIC is a prerequisite for x2AVIC. */
    if (!boot_cpu_has(X86_FEATURE_AVIC) && !force_avic) {
        if (boot_cpu_has(X86_FEATURE_X2AVIC)) {
            pr_warn(FW_BUG "Cannot support x2AVIC due to AVIC is disabled");
            pr_warn(FW_BUG "Try enable AVIC using force_avic option");
        }
        return false;
    }
 
    if (boot_cpu_has(X86_FEATURE_AVIC)) {
        pr_info("AVIC enabled\n");
    } else if (force_avic) {
        /*
         * Some older systems does not advertise AVIC support.
         * See Revision Guide for specific AMD processor for more detail.
         */
        pr_warn("AVIC is not supported in CPUID but force enabled");
        pr_warn("Your system might crash and burn");
    }
 
    /* AVIC is a prerequisite for x2AVIC. */
    x2avic_enabled = boot_cpu_has(X86_FEATURE_X2AVIC);
    if (x2avic_enabled)
        pr_info("x2AVIC enabled\n");
 
    amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
 
    return true;
}

Here is the caller graph for this function:

avic_incomplete_ipi_interception()

int avic_incomplete_ipi_interception

(

struct kvm_vcpu *

vcpu

)

Definition at line 490 of file avic.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
    u32 icrl = svm->vmcb->control.exit_info_1;
    u32 id = svm->vmcb->control.exit_info_2 >> 32;
    u32 index = svm->vmcb->control.exit_info_2 & 0x1FF;
    struct kvm_lapic *apic = vcpu->arch.apic;
 
    trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
 
    switch (id) {
    case AVIC_IPI_FAILURE_INVALID_TARGET:
    case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
        /*
         * Emulate IPIs that are not handled by AVIC hardware, which
         * only virtualizes Fixed, Edge-Triggered INTRs, and falls over
         * if _any_ targets are invalid, e.g. if the logical mode mask
         * is a superset of running vCPUs.
         *
         * The exit is a trap, e.g. ICR holds the correct value and RIP
         * has been advanced, KVM is responsible only for emulating the
         * IPI.  Sadly, hardware may sometimes leave the BUSY flag set,
         * in which case KVM needs to emulate the ICR write as well in
         * order to clear the BUSY flag.
         */
        if (icrl & APIC_ICR_BUSY)
            kvm_apic_write_nodecode(vcpu, APIC_ICR);
        else
            kvm_apic_send_ipi(apic, icrl, icrh);
        break;
    case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
        /*
         * At this point, we expect that the AVIC HW has already
         * set the appropriate IRR bits on the valid target
         * vcpus. So, we just need to kick the appropriate vcpu.
         */
        avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index);
        break;
    case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
        WARN_ONCE(1, "Invalid backing page\n");
        break;
    case AVIC_IPI_FAILURE_INVALID_IPI_VECTOR:
        /* Invalid IPI with vector < 16 */
        break;
    default:
        vcpu_unimpl(vcpu, "Unknown avic incomplete IPI interception\n");
    }
 
    return 1;
}

Here is the call graph for this function:

avic_init_vcpu()

int avic_init_vcpu

(

struct vcpu_svm *

svm

)

Definition at line 719 of file avic.c.

{
    int ret;
    struct kvm_vcpu *vcpu = &svm->vcpu;
 
    if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
        return 0;
 
    ret = avic_init_backing_page(vcpu);
    if (ret)
        return ret;
 
    INIT_LIST_HEAD(&svm->ir_list);
    spin_lock_init(&svm->ir_list_lock);
    svm->dfr_reg = APIC_DFR_FLAT;
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_init_vmcb()

void avic_init_vmcb	(	struct vcpu_svm *	svm,
		struct vmcb *	vmcb
	)

Definition at line 244 of file avic.c.

{
    struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
    phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
    phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
    phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
 
    vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
    vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
    vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
    vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK;
 
    if (kvm_apicv_activated(svm->vcpu.kvm))
        avic_activate_vmcb(svm);
    else
        avic_deactivate_vmcb(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_pi_update_irte()

int avic_pi_update_irte	(	struct kvm *	kvm,
		unsigned int	host_irq,
		uint32_t	guest_irq,
		bool	set
	)

Here, we setup with legacy mode in the following cases:

1. When cannot target interrupt to a specific vcpu.
2. Unsetting posted interrupt.
3. APIC virtualization is disabled for the vcpu.
4. IRQ has incompatible delivery mode (SMI, INIT, etc)

Here, we successfully setting up vcpu affinity in IOMMU guest mode. Now, we need to store the posted interrupt information in a per-vcpu ir_list so that we can reference to them directly when we update vcpu scheduling information in IOMMU irte.

Here, pi is used to:

• Tell IOMMU to use legacy mode for this interrupt.
• Retrieve ga_tag of prior interrupt remapping data.

Check if the posted interrupt was previously setup with the guest_mode by checking if the ga_tag was cached. If so, we need to clean up the per-vcpu ir_list.

Definition at line 894 of file avic.c.

{
    struct kvm_kernel_irq_routing_entry *e;
    struct kvm_irq_routing_table *irq_rt;
    int idx, ret = 0;
 
    if (!kvm_arch_has_assigned_device(kvm) ||
        !irq_remapping_cap(IRQ_POSTING_CAP))
        return 0;
 
    pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
         __func__, host_irq, guest_irq, set);
 
    idx = srcu_read_lock(&kvm->irq_srcu);
    irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
 
    if (guest_irq >= irq_rt->nr_rt_entries ||
        hlist_empty(&irq_rt->map[guest_irq])) {
        pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
                 guest_irq, irq_rt->nr_rt_entries);
        goto out;
    }
 
    hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
        struct vcpu_data vcpu_info;
        struct vcpu_svm *svm = NULL;
 
        if (e->type != KVM_IRQ_ROUTING_MSI)
            continue;
 
        /**
         * Here, we setup with legacy mode in the following cases:
         * 1. When cannot target interrupt to a specific vcpu.
         * 2. Unsetting posted interrupt.
         * 3. APIC virtualization is disabled for the vcpu.
         * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
         */
        if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
            kvm_vcpu_apicv_active(&svm->vcpu)) {
            struct amd_iommu_pi_data pi;
 
            /* Try to enable guest_mode in IRTE */
            pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
                        AVIC_HPA_MASK);
            pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
                             svm->vcpu.vcpu_id);
            pi.is_guest_mode = true;
            pi.vcpu_data = &vcpu_info;
            ret = irq_set_vcpu_affinity(host_irq, &pi);
 
            /**
             * Here, we successfully setting up vcpu affinity in
             * IOMMU guest mode. Now, we need to store the posted
             * interrupt information in a per-vcpu ir_list so that
             * we can reference to them directly when we update vcpu
             * scheduling information in IOMMU irte.
             */
            if (!ret && pi.is_guest_mode)
                svm_ir_list_add(svm, &pi);
        } else {
            /* Use legacy mode in IRTE */
            struct amd_iommu_pi_data pi;
 
            /**
             * Here, pi is used to:
             * - Tell IOMMU to use legacy mode for this interrupt.
             * - Retrieve ga_tag of prior interrupt remapping data.
             */
            pi.prev_ga_tag = 0;
            pi.is_guest_mode = false;
            ret = irq_set_vcpu_affinity(host_irq, &pi);
 
            /**
             * Check if the posted interrupt was previously
             * setup with the guest_mode by checking if the ga_tag
             * was cached. If so, we need to clean up the per-vcpu
             * ir_list.
             */
            if (!ret && pi.prev_ga_tag) {
                int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
                struct kvm_vcpu *vcpu;
 
                vcpu = kvm_get_vcpu_by_id(kvm, id);
                if (vcpu)
                    svm_ir_list_del(to_svm(vcpu), &pi);
            }
        }
 
        if (!ret && svm) {
            trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
                         e->gsi, vcpu_info.vector,
                         vcpu_info.pi_desc_addr, set);
        }
 
        if (ret < 0) {
            pr_err("%s: failed to update PI IRTE\n", __func__);
            goto out;
        }
    }
 
    ret = 0;
out:
    srcu_read_unlock(&kvm->irq_srcu, idx);
    return ret;
}

Here is the call graph for this function:

avic_refresh_apicv_exec_ctrl()

void avic_refresh_apicv_exec_ctrl

(

struct kvm_vcpu *

vcpu

)

Definition at line 1136 of file avic.c.

{
    bool activated = kvm_vcpu_apicv_active(vcpu);
 
    if (!enable_apicv)
        return;
 
    avic_refresh_virtual_apic_mode(vcpu);
 
    if (activated)
        avic_vcpu_load(vcpu, vcpu->cpu);
    else
        avic_vcpu_put(vcpu);
 
    avic_set_pi_irte_mode(vcpu, activated);
}

Here is the call graph for this function:

avic_refresh_virtual_apic_mode()

void avic_refresh_virtual_apic_mode

(

struct kvm_vcpu *

vcpu

)

During AVIC temporary deactivation, guest could update APIC ID, DFR and LDR registers, which would not be trapped by avic_unaccelerated_access_interception(). In this case, we need to check and update the AVIC logical APIC ID table accordingly before re-activating.

Definition at line 1112 of file avic.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    struct vmcb *vmcb = svm->vmcb01.ptr;
 
    if (!lapic_in_kernel(vcpu) || !enable_apicv)
        return;
 
    if (kvm_vcpu_apicv_active(vcpu)) {
        /**
         * During AVIC temporary deactivation, guest could update
         * APIC ID, DFR and LDR registers, which would not be trapped
         * by avic_unaccelerated_access_interception(). In this case,
         * we need to check and update the AVIC logical APIC ID table
         * accordingly before re-activating.
         */
        avic_apicv_post_state_restore(vcpu);
        avic_activate_vmcb(svm);
    } else {
        avic_deactivate_vmcb(svm);
    }
    vmcb_mark_dirty(vmcb, VMCB_AVIC);
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_ring_doorbell()

void avic_ring_doorbell

(

struct kvm_vcpu *

vcpu

)

Definition at line 321 of file avic.c.

{
    /*
     * Note, the vCPU could get migrated to a different pCPU at any point,
     * which could result in signalling the wrong/previous pCPU.  But if
     * that happens the vCPU is guaranteed to do a VMRUN (after being
     * migrated) and thus will process pending interrupts, i.e. a doorbell
     * is not needed (and the spurious one is harmless).
     */
    int cpu = READ_ONCE(vcpu->cpu);
 
    if (cpu != get_cpu()) {
        wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
        trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu));
    }
    put_cpu();
}

Here is the caller graph for this function:

avic_unaccelerated_access_interception()

int avic_unaccelerated_access_interception

(

struct kvm_vcpu *

vcpu

)

Definition at line 693 of file avic.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    int ret = 0;
    u32 offset = svm->vmcb->control.exit_info_1 &
             AVIC_UNACCEL_ACCESS_OFFSET_MASK;
    u32 vector = svm->vmcb->control.exit_info_2 &
             AVIC_UNACCEL_ACCESS_VECTOR_MASK;
    bool write = (svm->vmcb->control.exit_info_1 >> 32) &
             AVIC_UNACCEL_ACCESS_WRITE_MASK;
    bool trap = is_avic_unaccelerated_access_trap(offset);
 
    trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset,
                        trap, write, vector);
    if (trap) {
        /* Handling Trap */
        WARN_ONCE(!write, "svm: Handling trap read.\n");
        ret = avic_unaccel_trap_write(vcpu);
    } else {
        /* Handling Fault */
        ret = kvm_emulate_instruction(vcpu, 0);
    }
 
    return ret;
}

Here is the call graph for this function:

avic_vcpu_blocking()

void avic_vcpu_blocking

(

struct kvm_vcpu *

vcpu

)

Definition at line 1153 of file avic.c.

{
    if (!kvm_vcpu_apicv_active(vcpu))
        return;
 
       /*
        * Unload the AVIC when the vCPU is about to block, _before_
        * the vCPU actually blocks.
        *
        * Any IRQs that arrive before IsRunning=0 will not cause an
        * incomplete IPI vmexit on the source, therefore vIRR will also
        * be checked by kvm_vcpu_check_block() before blocking.  The
        * memory barrier implicit in set_current_state orders writing
        * IsRunning=0 before reading the vIRR.  The processor needs a
        * matching memory barrier on interrupt delivery between writing
        * IRR and reading IsRunning; the lack of this barrier might be
        * the cause of errata #1235).
        */
    avic_vcpu_put(vcpu);
}

Here is the call graph for this function:

avic_vcpu_get_apicv_inhibit_reasons()

unsigned long avic_vcpu_get_apicv_inhibit_reasons

(

struct kvm_vcpu *

vcpu

)

Definition at line 542 of file avic.c.

{
    if (is_guest_mode(vcpu))
        return APICV_INHIBIT_REASON_NESTED;
    return 0;
}

Here is the call graph for this function:

avic_vcpu_load()

void avic_vcpu_load	(	struct kvm_vcpu *	vcpu,
		int	cpu
	)

Definition at line 1028 of file avic.c.

{
    u64 entry;
    int h_physical_id = kvm_cpu_get_apicid(cpu);
    struct vcpu_svm *svm = to_svm(vcpu);
    unsigned long flags;
 
    lockdep_assert_preemption_disabled();
 
    if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
        return;
 
    /*
     * No need to update anything if the vCPU is blocking, i.e. if the vCPU
     * is being scheduled in after being preempted.  The CPU entries in the
     * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'.
     * If the vCPU was migrated, its new CPU value will be stuffed when the
     * vCPU unblocks.
     */
    if (kvm_vcpu_is_blocking(vcpu))
        return;
 
    /*
     * Grab the per-vCPU interrupt remapping lock even if the VM doesn't
     * _currently_ have assigned devices, as that can change.  Holding
     * ir_list_lock ensures that either svm_ir_list_add() will consume
     * up-to-date entry information, or that this task will wait until
     * svm_ir_list_add() completes to set the new target pCPU.
     */
    spin_lock_irqsave(&svm->ir_list_lock, flags);
 
    entry = READ_ONCE(*(svm->avic_physical_id_cache));
    WARN_ON_ONCE(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 
    entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
    entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
    entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
 
    WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
    avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true);
 
    spin_unlock_irqrestore(&svm->ir_list_lock, flags);
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_vcpu_put()

void avic_vcpu_put

(

struct kvm_vcpu *

vcpu

)

Definition at line 1072 of file avic.c.

{
    u64 entry;
    struct vcpu_svm *svm = to_svm(vcpu);
    unsigned long flags;
 
    lockdep_assert_preemption_disabled();
 
    /*
     * Note, reading the Physical ID entry outside of ir_list_lock is safe
     * as only the pCPU that has loaded (or is loading) the vCPU is allowed
     * to modify the entry, and preemption is disabled.  I.e. the vCPU
     * can't be scheduled out and thus avic_vcpu_{put,load}() can't run
     * recursively.
     */
    entry = READ_ONCE(*(svm->avic_physical_id_cache));
 
    /* Nothing to do if IsRunning == '0' due to vCPU blocking. */
    if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK))
        return;
 
    /*
     * Take and hold the per-vCPU interrupt remapping lock while updating
     * the Physical ID entry even though the lock doesn't protect against
     * multiple writers (see above).  Holding ir_list_lock ensures that
     * either svm_ir_list_add() will consume up-to-date entry information,
     * or that this task will wait until svm_ir_list_add() completes to
     * mark the vCPU as not running.
     */
    spin_lock_irqsave(&svm->ir_list_lock, flags);
 
    avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
 
    entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
    WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
 
    spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_vcpu_unblocking()

void avic_vcpu_unblocking

(

struct kvm_vcpu *

vcpu

)

Definition at line 1174 of file avic.c.

{
    if (!kvm_vcpu_apicv_active(vcpu))
        return;
 
    avic_vcpu_load(vcpu, vcpu->cpu);
}

Here is the call graph for this function:

avic_vm_destroy()

void avic_vm_destroy

(

struct kvm *

kvm

)

Definition at line 174 of file avic.c.

{
    unsigned long flags;
    struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 
    if (!enable_apicv)
        return;
 
    if (kvm_svm->avic_logical_id_table_page)
        __free_page(kvm_svm->avic_logical_id_table_page);
    if (kvm_svm->avic_physical_id_table_page)
        __free_page(kvm_svm->avic_physical_id_table_page);
 
    spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
    hash_del(&kvm_svm->hnode);
    spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
}

Here is the call graph for this function:

Here is the caller graph for this function:

avic_vm_init()

int avic_vm_init

(

struct kvm *

kvm

)

Definition at line 192 of file avic.c.

{
    unsigned long flags;
    int err = -ENOMEM;
    struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
    struct kvm_svm *k2;
    struct page *p_page;
    struct page *l_page;
    u32 vm_id;
 
    if (!enable_apicv)
        return 0;
 
    /* Allocating physical APIC ID table (4KB) */
    p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
    if (!p_page)
        goto free_avic;
 
    kvm_svm->avic_physical_id_table_page = p_page;
 
    /* Allocating logical APIC ID table (4KB) */
    l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
    if (!l_page)
        goto free_avic;
 
    kvm_svm->avic_logical_id_table_page = l_page;
 
    spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 again:
    vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
    if (vm_id == 0) { /* id is 1-based, zero is not okay */
        next_vm_id_wrapped = 1;
        goto again;
    }
    /* Is it still in use? Only possible if wrapped at least once */
    if (next_vm_id_wrapped) {
        hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
            if (k2->avic_vm_id == vm_id)
                goto again;
        }
    }
    kvm_svm->avic_vm_id = vm_id;
    hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
    spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 
    return 0;
 
free_avic:
    avic_vm_destroy(kvm);
    return err;
}

Here is the call graph for this function:

clr_exception_intercept()

static void clr_exception_intercept ( struct vcpu_svm *  svm, u32  bit  )

inlinestatic

Definition at line 413 of file svm.h.

{
    struct vmcb *vmcb = svm->vmcb01.ptr;
 
    WARN_ON_ONCE(bit >= 32);
    vmcb_clr_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
 
    recalc_intercepts(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

DECLARE_PER_CPU()

DECLARE_PER_CPU	(	struct svm_cpu_data	,
		svm_data
	)

disable_gif()

static void disable_gif ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 473 of file svm.h.

{
    struct vmcb *vmcb = get_vgif_vmcb(svm);
 
    if (vmcb)
        vmcb->control.int_ctl &= ~V_GIF_MASK;
    else
        svm->guest_gif = false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

disable_nmi_singlestep()

void disable_nmi_singlestep

(

struct vcpu_svm *

svm

)

Definition at line 1054 of file svm.c.

{
    svm->nmi_singlestep = false;
 
    if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
        /* Clear our flags if they were not set by the guest */
        if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
            svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
        if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
            svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
    }
}

Here is the caller graph for this function:

enable_gif()

static void enable_gif ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 463 of file svm.h.

{
    struct vmcb *vmcb = get_vgif_vmcb(svm);
 
    if (vmcb)
        vmcb->control.int_ctl |= V_GIF_MASK;
    else
        svm->guest_gif = true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

enter_svm_guest_mode()

int enter_svm_guest_mode	(	struct kvm_vcpu *	vcpu,
		u64	vmcb_gpa,
		struct vmcb *	vmcb12,
		bool	from_vmrun
	)

Definition at line 785 of file nested.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    int ret;
 
    trace_kvm_nested_vmenter(svm->vmcb->save.rip,
                 vmcb12_gpa,
                 vmcb12->save.rip,
                 vmcb12->control.int_ctl,
                 vmcb12->control.event_inj,
                 vmcb12->control.nested_ctl,
                 vmcb12->control.nested_cr3,
                 vmcb12->save.cr3,
                 KVM_ISA_SVM);
 
    trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff,
                    vmcb12->control.intercepts[INTERCEPT_CR] >> 16,
                    vmcb12->control.intercepts[INTERCEPT_EXCEPTION],
                    vmcb12->control.intercepts[INTERCEPT_WORD3],
                    vmcb12->control.intercepts[INTERCEPT_WORD4],
                    vmcb12->control.intercepts[INTERCEPT_WORD5]);
 
 
    svm->nested.vmcb12_gpa = vmcb12_gpa;
 
    WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr);
 
    nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);
 
    svm_switch_vmcb(svm, &svm->nested.vmcb02);
    nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base);
    nested_vmcb02_prepare_save(svm, vmcb12);
 
    ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3,
                  nested_npt_enabled(svm), from_vmrun);
    if (ret)
        return ret;
 
    if (!from_vmrun)
        kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 
    svm_set_gif(svm, true);
 
    if (kvm_vcpu_apicv_active(vcpu))
        kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
 
    nested_svm_hv_update_vm_vp_ids(vcpu);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_vgif_vmcb()

static struct vmcb* get_vgif_vmcb ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 452 of file svm.h.

{
    if (!vgif)
        return NULL;
 
    if (is_guest_mode(&svm->vcpu) && !nested_vgif_enabled(svm))
        return svm->nested.vmcb02.ptr;
    else
        return svm->vmcb01.ptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_vnmi_vmcb_l1()

static struct vmcb* get_vnmi_vmcb_l1 ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 513 of file svm.h.

{
    if (!vnmi)
        return NULL;
 
    if (is_guest_mode(&svm->vcpu))
        return NULL;
    else
        return svm->vmcb01.ptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

gif_set()

static bool gif_set ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 483 of file svm.h.

{
    struct vmcb *vmcb = get_vgif_vmcb(svm);
 
    if (vmcb)
        return !!(vmcb->control.int_ctl & V_GIF_MASK);
    else
        return svm->guest_gif;
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_vnmi_enabled()

static bool is_vnmi_enabled ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 524 of file svm.h.

{
    struct vmcb *vmcb = get_vnmi_vmcb_l1(svm);
 
    if (vmcb)
        return !!(vmcb->control.int_ctl & V_NMI_ENABLE_MASK);
    else
        return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

is_x2apic_msrpm_offset()

static bool is_x2apic_msrpm_offset ( u32  offset )

inlinestatic

Definition at line 504 of file svm.h.

{
    /* 4 msrs per u8, and 4 u8 in u32 */
    u32 msr = offset * 16;
 
    return (msr >= APIC_BASE_MSR) &&
           (msr < (APIC_BASE_MSR + 0x100));
}

Here is the caller graph for this function:

nested_copy_vmcb_control_to_cache()

void nested_copy_vmcb_control_to_cache	(	struct vcpu_svm *	svm,
		struct vmcb_control_area *	control
	)

Definition at line 382 of file nested.c.

{
    __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_copy_vmcb_save_to_cache()

void nested_copy_vmcb_save_to_cache	(	struct vcpu_svm *	svm,
		struct vmcb_save_area *	save
	)

Definition at line 404 of file nested.c.

{
    __nested_copy_vmcb_save_to_cache(&svm->nested.save, save);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_exit_on_intr()

static bool nested_exit_on_intr ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 581 of file svm.h.

{
    return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INTR);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_exit_on_nmi()

static bool nested_exit_on_nmi ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 586 of file svm.h.

{
    return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_NMI);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_exit_on_smi()

static bool nested_exit_on_smi ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 576 of file svm.h.

{
    return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SMI);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_npt_enabled()

static bool nested_npt_enabled ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 493 of file svm.h.

{
    return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE;
}

Here is the caller graph for this function:

nested_svm_check_exception()

int nested_svm_check_exception	(	struct vcpu_svm *	svm,
		unsigned	nr,
		bool	has_error_code,
		u32	error_code
	)

nested_svm_check_permissions()

int nested_svm_check_permissions

(

struct kvm_vcpu *

vcpu

)

Definition at line 1357 of file nested.c.

{
    if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) {
        kvm_queue_exception(vcpu, UD_VECTOR);
        return 1;
    }
 
    if (to_svm(vcpu)->vmcb->save.cpl) {
        kvm_inject_gp(vcpu, 0);
        return 1;
    }
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_exit_handled()

int nested_svm_exit_handled

(

struct vcpu_svm *

svm

)

Definition at line 1345 of file nested.c.

{
    int vmexit;
 
    vmexit = nested_svm_intercept(svm);
 
    if (vmexit == NESTED_EXIT_DONE)
        nested_svm_vmexit(svm);
 
    return vmexit;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_exit_special()

int nested_svm_exit_special

(

struct vcpu_svm *

svm

)

Definition at line 1496 of file nested.c.

{
    u32 exit_code = svm->vmcb->control.exit_code;
    struct kvm_vcpu *vcpu = &svm->vcpu;
 
    switch (exit_code) {
    case SVM_EXIT_INTR:
    case SVM_EXIT_NMI:
    case SVM_EXIT_NPF:
        return NESTED_EXIT_HOST;
    case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
        u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
 
        if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
            excp_bits)
            return NESTED_EXIT_HOST;
        else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
             svm->vcpu.arch.apf.host_apf_flags)
            /* Trap async PF even if not shadowing */
            return NESTED_EXIT_HOST;
        break;
    }
    case SVM_EXIT_VMMCALL:
        /* Hyper-V L2 TLB flush hypercall is handled by L0 */
        if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
            nested_svm_l2_tlb_flush_enabled(vcpu) &&
            kvm_hv_is_tlb_flush_hcall(vcpu))
            return NESTED_EXIT_HOST;
        break;
    default:
        break;
    }
 
    return NESTED_EXIT_CONTINUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_simple_vmexit()

static int nested_svm_simple_vmexit ( struct vcpu_svm *  svm, u32  exit_code  )

inlinestatic

Definition at line 602 of file svm.h.

{
    svm->vmcb->control.exit_code   = exit_code;
    svm->vmcb->control.exit_info_1 = 0;
    svm->vmcb->control.exit_info_2 = 0;
    return nested_svm_vmexit(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_update_tsc_ratio_msr()

void nested_svm_update_tsc_ratio_msr

(

struct kvm_vcpu *

vcpu

)

Definition at line 1532 of file nested.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
 
    vcpu->arch.tsc_scaling_ratio =
        kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio,
                           svm->tsc_ratio_msr);
    svm_write_tsc_multiplier(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_virtualize_tpr()

static bool nested_svm_virtualize_tpr ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 569 of file svm.h.

{
    struct vcpu_svm *svm = to_svm(vcpu);
 
    return is_guest_mode(vcpu) && (svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_vmexit()

int nested_svm_vmexit

(

struct vcpu_svm *

svm

)

Definition at line 967 of file nested.c.

{
    struct kvm_vcpu *vcpu = &svm->vcpu;
    struct vmcb *vmcb01 = svm->vmcb01.ptr;
    struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
    struct vmcb *vmcb12;
    struct kvm_host_map map;
    int rc;
 
    rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);
    if (rc) {
        if (rc == -EINVAL)
            kvm_inject_gp(vcpu, 0);
        return 1;
    }
 
    vmcb12 = map.hva;
 
    /* Exit Guest-Mode */
    leave_guest_mode(vcpu);
    svm->nested.vmcb12_gpa = 0;
    WARN_ON_ONCE(svm->nested.nested_run_pending);
 
    kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 
    /* in case we halted in L2 */
    svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE;
 
    /* Give the current vmcb to the guest */
 
    vmcb12->save.es     = vmcb02->save.es;
    vmcb12->save.cs     = vmcb02->save.cs;
    vmcb12->save.ss     = vmcb02->save.ss;
    vmcb12->save.ds     = vmcb02->save.ds;
    vmcb12->save.gdtr   = vmcb02->save.gdtr;
    vmcb12->save.idtr   = vmcb02->save.idtr;
    vmcb12->save.efer   = svm->vcpu.arch.efer;
    vmcb12->save.cr0    = kvm_read_cr0(vcpu);
    vmcb12->save.cr3    = kvm_read_cr3(vcpu);
    vmcb12->save.cr2    = vmcb02->save.cr2;
    vmcb12->save.cr4    = svm->vcpu.arch.cr4;
    vmcb12->save.rflags = kvm_get_rflags(vcpu);
    vmcb12->save.rip    = kvm_rip_read(vcpu);
    vmcb12->save.rsp    = kvm_rsp_read(vcpu);
    vmcb12->save.rax    = kvm_rax_read(vcpu);
    vmcb12->save.dr7    = vmcb02->save.dr7;
    vmcb12->save.dr6    = svm->vcpu.arch.dr6;
    vmcb12->save.cpl    = vmcb02->save.cpl;
 
    vmcb12->control.int_state         = vmcb02->control.int_state;
    vmcb12->control.exit_code         = vmcb02->control.exit_code;
    vmcb12->control.exit_code_hi      = vmcb02->control.exit_code_hi;
    vmcb12->control.exit_info_1       = vmcb02->control.exit_info_1;
    vmcb12->control.exit_info_2       = vmcb02->control.exit_info_2;
 
    if (vmcb12->control.exit_code != SVM_EXIT_ERR)
        nested_save_pending_event_to_vmcb12(svm, vmcb12);
 
    if (guest_can_use(vcpu, X86_FEATURE_NRIPS))
        vmcb12->control.next_rip  = vmcb02->control.next_rip;
 
    vmcb12->control.int_ctl           = svm->nested.ctl.int_ctl;
    vmcb12->control.event_inj         = svm->nested.ctl.event_inj;
    vmcb12->control.event_inj_err     = svm->nested.ctl.event_inj_err;
 
    if (!kvm_pause_in_guest(vcpu->kvm)) {
        vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count;
        vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
 
    }
 
    nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);
 
    svm_switch_vmcb(svm, &svm->vmcb01);
 
    /*
     * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01:
     *
     * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR:  If L1 doesn't
     * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related
     * flags) to detect interrupt windows for L1 IRQs (even if L1 uses
     * virtual interrupt masking).  Raise KVM_REQ_EVENT to ensure that
     * KVM re-requests an interrupt window if necessary, which implicitly
     * copies this bits from vmcb02 to vmcb01.
     *
     * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR
     * is stored in vmcb02, but its value doesn't need to be copied from/to
     * vmcb01 because it is copied from/to the virtual APIC's TPR register
     * on each VM entry/exit.
     *
     * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's
     * V_GIF.  However, GIF is architecturally clear on each VM exit, thus
     * there is no need to copy V_GIF from vmcb02 to vmcb01.
     */
    if (!nested_exit_on_intr(svm))
        kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
 
    if (unlikely(guest_can_use(vcpu, X86_FEATURE_LBRV) &&
             (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) {
        svm_copy_lbrs(vmcb12, vmcb02);
        svm_update_lbrv(vcpu);
    } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) {
        svm_copy_lbrs(vmcb01, vmcb02);
        svm_update_lbrv(vcpu);
    }
 
    if (vnmi) {
        if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK)
            vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK;
        else
            vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
 
        if (vcpu->arch.nmi_pending) {
            vcpu->arch.nmi_pending--;
            vmcb01->control.int_ctl |= V_NMI_PENDING_MASK;
        } else {
            vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK;
        }
    }
 
    /*
     * On vmexit the  GIF is set to false and
     * no event can be injected in L1.
     */
    svm_set_gif(svm, false);
    vmcb01->control.exit_int_info = 0;
 
    svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset;
    if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) {
        vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset;
        vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
    }
 
    if (kvm_caps.has_tsc_control &&
        vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) {
        vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
        svm_write_tsc_multiplier(vcpu);
    }
 
    svm->nested.ctl.nested_cr3 = 0;
 
    /*
     * Restore processor state that had been saved in vmcb01
     */
    kvm_set_rflags(vcpu, vmcb01->save.rflags);
    svm_set_efer(vcpu, vmcb01->save.efer);
    svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE);
    svm_set_cr4(vcpu, vmcb01->save.cr4);
    kvm_rax_write(vcpu, vmcb01->save.rax);
    kvm_rsp_write(vcpu, vmcb01->save.rsp);
    kvm_rip_write(vcpu, vmcb01->save.rip);
 
    svm->vcpu.arch.dr7 = DR7_FIXED_1;
    kvm_update_dr7(&svm->vcpu);
 
    trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
                       vmcb12->control.exit_info_1,
                       vmcb12->control.exit_info_2,
                       vmcb12->control.exit_int_info,
                       vmcb12->control.exit_int_info_err,
                       KVM_ISA_SVM);
 
    kvm_vcpu_unmap(vcpu, &map, true);
 
    nested_svm_transition_tlb_flush(vcpu);
 
    nested_svm_uninit_mmu_context(vcpu);
 
    rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true);
    if (rc)
        return 1;
 
    /*
     * Drop what we picked up for L2 via svm_complete_interrupts() so it
     * doesn't end up in L1.
     */
    svm->vcpu.arch.nmi_injected = false;
    kvm_clear_exception_queue(vcpu);
    kvm_clear_interrupt_queue(vcpu);
 
    /*
     * If we are here following the completion of a VMRUN that
     * is being single-stepped, queue the pending #DB intercept
     * right now so that it an be accounted for before we execute
     * L1's next instruction.
     */
    if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF))
        kvm_queue_exception(&(svm->vcpu), DB_VECTOR);
 
    /*
     * Un-inhibit the AVIC right away, so that other vCPUs can start
     * to benefit from it right away.
     */
    if (kvm_apicv_activated(vcpu->kvm))
        __kvm_vcpu_update_apicv(vcpu);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_svm_vmrun()

int nested_svm_vmrun

(

struct kvm_vcpu *

vcpu

)

Definition at line 837 of file nested.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    int ret;
    struct vmcb *vmcb12;
    struct kvm_host_map map;
    u64 vmcb12_gpa;
    struct vmcb *vmcb01 = svm->vmcb01.ptr;
 
    if (!svm->nested.hsave_msr) {
        kvm_inject_gp(vcpu, 0);
        return 1;
    }
 
    if (is_smm(vcpu)) {
        kvm_queue_exception(vcpu, UD_VECTOR);
        return 1;
    }
 
    /* This fails when VP assist page is enabled but the supplied GPA is bogus */
    ret = kvm_hv_verify_vp_assist(vcpu);
    if (ret) {
        kvm_inject_gp(vcpu, 0);
        return ret;
    }
 
    vmcb12_gpa = svm->vmcb->save.rax;
    ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map);
    if (ret == -EINVAL) {
        kvm_inject_gp(vcpu, 0);
        return 1;
    } else if (ret) {
        return kvm_skip_emulated_instruction(vcpu);
    }
 
    ret = kvm_skip_emulated_instruction(vcpu);
 
    vmcb12 = map.hva;
 
    if (WARN_ON_ONCE(!svm->nested.initialized))
        return -EINVAL;
 
    nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
    nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
 
    if (!nested_vmcb_check_save(vcpu) ||
        !nested_vmcb_check_controls(vcpu)) {
        vmcb12->control.exit_code    = SVM_EXIT_ERR;
        vmcb12->control.exit_code_hi = 0;
        vmcb12->control.exit_info_1  = 0;
        vmcb12->control.exit_info_2  = 0;
        goto out;
    }
 
    /*
     * Since vmcb01 is not in use, we can use it to store some of the L1
     * state.
     */
    vmcb01->save.efer   = vcpu->arch.efer;
    vmcb01->save.cr0    = kvm_read_cr0(vcpu);
    vmcb01->save.cr4    = vcpu->arch.cr4;
    vmcb01->save.rflags = kvm_get_rflags(vcpu);
    vmcb01->save.rip    = kvm_rip_read(vcpu);
 
    if (!npt_enabled)
        vmcb01->save.cr3 = kvm_read_cr3(vcpu);
 
    svm->nested.nested_run_pending = 1;
 
    if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true))
        goto out_exit_err;
 
    if (nested_svm_vmrun_msrpm(svm))
        goto out;
 
out_exit_err:
    svm->nested.nested_run_pending = 0;
    svm->nmi_l1_to_l2 = false;
    svm->soft_int_injected = false;
 
    svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
    svm->vmcb->control.exit_code_hi = 0;
    svm->vmcb->control.exit_info_1  = 0;
    svm->vmcb->control.exit_info_2  = 0;
 
    nested_svm_vmexit(svm);
 
out:
    kvm_vcpu_unmap(vcpu, &map, true);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_sync_control_from_vmcb02()

void nested_sync_control_from_vmcb02

(

struct vcpu_svm *

svm

)

Definition at line 414 of file nested.c.

{
    u32 mask;
    svm->nested.ctl.event_inj      = svm->vmcb->control.event_inj;
    svm->nested.ctl.event_inj_err  = svm->vmcb->control.event_inj_err;
 
    /* Only a few fields of int_ctl are written by the processor.  */
    mask = V_IRQ_MASK | V_TPR_MASK;
    /*
     * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting
     * virtual interrupts in order to request an interrupt window, as KVM
     * has usurped vmcb02's int_ctl.  If an interrupt window opens before
     * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl.
     * If no window opens, V_IRQ will be correctly preserved in vmcb12's
     * int_ctl (because it was never recognized while L2 was running).
     */
    if (svm_is_intercept(svm, INTERCEPT_VINTR) &&
        !test_bit(INTERCEPT_VINTR, (unsigned long *)svm->nested.ctl.intercepts))
        mask &= ~V_IRQ_MASK;
 
    if (nested_vgif_enabled(svm))
        mask |= V_GIF_MASK;
 
    if (nested_vnmi_enabled(svm))
        mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK;
 
    svm->nested.ctl.int_ctl        &= ~mask;
    svm->nested.ctl.int_ctl        |= svm->vmcb->control.int_ctl & mask;
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_vgif_enabled()

static bool nested_vgif_enabled ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 446 of file svm.h.

{
    return guest_can_use(&svm->vcpu, X86_FEATURE_VGIF) &&
           (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

nested_vmcb02_compute_g_pat()

void nested_vmcb02_compute_g_pat

(

struct vcpu_svm *

svm

)

Definition at line 528 of file nested.c.

{
    if (!svm->nested.vmcb02.ptr)
        return;
 
    /* FIXME: merge g_pat from vmcb01 and vmcb12.  */
    svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;
}

Here is the caller graph for this function:

nested_vnmi_enabled()

static bool nested_vnmi_enabled ( struct vcpu_svm *  svm )

inlinestatic

Definition at line 498 of file svm.h.

{
    return guest_can_use(&svm->vcpu, X86_FEATURE_VNMI) &&
           (svm->nested.ctl.int_ctl & V_NMI_ENABLE_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

pre_sev_run()

void pre_sev_run	(	struct vcpu_svm *	svm,
		int	cpu
	)

Definition at line 2646 of file sev.c.

{
    struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
    unsigned int asid = sev_get_asid(svm->vcpu.kvm);
 
    /* Assign the asid allocated with this SEV guest */
    svm->asid = asid;
 
    /*
     * Flush guest TLB:
     *
     * 1) when different VMCB for the same ASID is to be run on the same host CPU.
     * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
     */
    if (sd->sev_vmcbs[asid] == svm->vmcb &&
        svm->vcpu.arch.last_vmentry_cpu == cpu)
        return;
 
    sd->sev_vmcbs[asid] = svm->vmcb;
    svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
    vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
}

Here is the call graph for this function:

Here is the caller graph for this function:

recalc_intercepts()

void recalc_intercepts

(

struct vcpu_svm *

svm

)

Definition at line 122 of file nested.c.

{
    struct vmcb_control_area *c, *h;
    struct vmcb_ctrl_area_cached *g;
    unsigned int i;
 
    vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 
    if (!is_guest_mode(&svm->vcpu))
        return;
 
    c = &svm->vmcb->control;
    h = &svm->vmcb01.ptr->control;
    g = &svm->nested.ctl;
 
    for (i = 0; i < MAX_INTERCEPT; i++)
        c->intercepts[i] = h->intercepts[i];
 
    if (g->int_ctl & V_INTR_MASKING_MASK) {
        /*
         * If L2 is active and V_INTR_MASKING is enabled in vmcb12,
         * disable intercept of CR8 writes as L2's CR8 does not affect
         * any interrupt KVM may want to inject.
         *
         * Similarly, disable intercept of virtual interrupts (used to
         * detect interrupt windows) if the saved RFLAGS.IF is '0', as
         * the effective RFLAGS.IF for L1 interrupts will never be set
         * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs).
         */
        vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE);
        if (!(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF))
            vmcb_clr_intercept(c, INTERCEPT_VINTR);
    }
 
    /*
     * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB
     * flush feature is enabled.
     */
    if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu))
        vmcb_clr_intercept(c, INTERCEPT_VMMCALL);
 
    for (i = 0; i < MAX_INTERCEPT; i++)
        c->intercepts[i] |= g->intercepts[i];
 
    /* If SMI is not intercepted, ignore guest SMI intercept as well  */
    if (!intercept_smi)
        vmcb_clr_intercept(c, INTERCEPT_SMI);
 
    if (nested_vmcb_needs_vls_intercept(svm)) {
        /*
         * If the virtual VMLOAD/VMSAVE is not enabled for the L2,
         * we must intercept these instructions to correctly
         * emulate them in case L1 doesn't intercept them.
         */
        vmcb_set_intercept(c, INTERCEPT_VMLOAD);
        vmcb_set_intercept(c, INTERCEPT_VMSAVE);
    } else {
        WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK));
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

set_exception_intercept()

static void set_exception_intercept ( struct vcpu_svm *  svm, u32  bit  )

inlinestatic

Definition at line 403 of file svm.h.

{
    struct vmcb *vmcb = svm->vmcb01.ptr;
 
    WARN_ON_ONCE(bit >= 32);
    vmcb_set_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
 
    recalc_intercepts(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

set_msr_interception()

void set_msr_interception	(	struct kvm_vcpu *	vcpu,
		u32 *	msrpm,
		u32	msr,
		int	read,
		int	write
	)

Definition at line 859 of file svm.c.

{
    set_shadow_msr_intercept(vcpu, msr, read, write);
    set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_cpu_init()

int sev_cpu_init

(

struct svm_cpu_data *

sd

)

Definition at line 2313 of file sev.c.

{
    if (!sev_enabled)
        return 0;
 
    sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL);
    if (!sd->sev_vmcbs)
        return -ENOMEM;
 
    return 0;
}

Here is the caller graph for this function:

sev_es_guest()

static __always_inline bool sev_es_guest ( struct kvm *  kvm )

static

Definition at line 332 of file svm.h.

{
#ifdef CONFIG_KVM_AMD_SEV
    struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 
    return sev->es_active && !WARN_ON_ONCE(!sev->active);
#else
    return false;
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_es_prepare_switch_to_guest()

void sev_es_prepare_switch_to_guest

(

struct sev_es_save_area *

hostsa

)

Definition at line 3119 of file sev.c.

{
    /*
     * All host state for SEV-ES guests is categorized into three swap types
     * based on how it is handled by hardware during a world switch:
     *
     * A: VMRUN:   Host state saved in host save area
     *    VMEXIT:  Host state loaded from host save area
     *
     * B: VMRUN:   Host state _NOT_ saved in host save area
     *    VMEXIT:  Host state loaded from host save area
     *
     * C: VMRUN:   Host state _NOT_ saved in host save area
     *    VMEXIT:  Host state initialized to default(reset) values
     *
     * Manually save type-B state, i.e. state that is loaded by VMEXIT but
     * isn't saved by VMRUN, that isn't already saved by VMSAVE (performed
     * by common SVM code).
     */
    hostsa->xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
    hostsa->pkru = read_pkru();
    hostsa->xss = host_xss;
 
    /*
     * If DebugSwap is enabled, debug registers are loaded but NOT saved by
     * the CPU (Type-B). If DebugSwap is disabled/unsupported, the CPU both
     * saves and loads debug registers (Type-A).
     */
    if (sev_es_debug_swap_enabled) {
        hostsa->dr0 = native_get_debugreg(0);
        hostsa->dr1 = native_get_debugreg(1);
        hostsa->dr2 = native_get_debugreg(2);
        hostsa->dr3 = native_get_debugreg(3);
        hostsa->dr0_addr_mask = amd_get_dr_addr_mask(0);
        hostsa->dr1_addr_mask = amd_get_dr_addr_mask(1);
        hostsa->dr2_addr_mask = amd_get_dr_addr_mask(2);
        hostsa->dr3_addr_mask = amd_get_dr_addr_mask(3);
    }
}

Here is the caller graph for this function:

sev_es_string_io()

int sev_es_string_io	(	struct vcpu_svm *	svm,
		int	size,
		unsigned int	port,
		int	in
	)

Definition at line 2963 of file sev.c.

{
    int count;
    int bytes;
    int r;
 
    if (svm->vmcb->control.exit_info_2 > INT_MAX)
        return -EINVAL;
 
    count = svm->vmcb->control.exit_info_2;
    if (unlikely(check_mul_overflow(count, size, &bytes)))
        return -EINVAL;
 
    r = setup_vmgexit_scratch(svm, in, bytes);
    if (r)
        return r;
 
    return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa,
                    count, in);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_es_unmap_ghcb()

void sev_es_unmap_ghcb

(

struct vcpu_svm *

svm

)

Definition at line 2614 of file sev.c.

{
    if (!svm->sev_es.ghcb)
        return;
 
    if (svm->sev_es.ghcb_sa_free) {
        /*
         * The scratch area lives outside the GHCB, so there is a
         * buffer that, depending on the operation performed, may
         * need to be synced, then freed.
         */
        if (svm->sev_es.ghcb_sa_sync) {
            kvm_write_guest(svm->vcpu.kvm,
                    svm->sev_es.sw_scratch,
                    svm->sev_es.ghcb_sa,
                    svm->sev_es.ghcb_sa_len);
            svm->sev_es.ghcb_sa_sync = false;
        }
 
        kvfree(svm->sev_es.ghcb_sa);
        svm->sev_es.ghcb_sa = NULL;
        svm->sev_es.ghcb_sa_free = false;
    }
 
    trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb);
 
    sev_es_sync_to_ghcb(svm);
 
    kvm_vcpu_unmap(&svm->vcpu, &svm->sev_es.ghcb_map, true);
    svm->sev_es.ghcb = NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_es_vcpu_reset()

void sev_es_vcpu_reset

(

struct vcpu_svm *

svm

)

Definition at line 3108 of file sev.c.

{
    /*
     * Set the GHCB MSR value as per the GHCB specification when emulating
     * vCPU RESET for an SEV-ES guest.
     */
    set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX,
                        GHCB_VERSION_MIN,
                        sev_enc_bit));
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_free_vcpu()

void sev_free_vcpu

(

struct kvm_vcpu *

vcpu

)

Definition at line 2373 of file sev.c.

{
    struct vcpu_svm *svm;
 
    if (!sev_es_guest(vcpu->kvm))
        return;
 
    svm = to_svm(vcpu);
 
    if (vcpu->arch.guest_state_protected)
        sev_flush_encrypted_page(vcpu, svm->sev_es.vmsa);
 
    __free_page(virt_to_page(svm->sev_es.vmsa));
 
    if (svm->sev_es.ghcb_sa_free)
        kvfree(svm->sev_es.ghcb_sa);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_guest()

static __always_inline bool sev_guest ( struct kvm *  kvm )

static

Definition at line 321 of file svm.h.

{
#ifdef CONFIG_KVM_AMD_SEV
    struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 
    return sev->active;
#else
    return false;
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_guest_memory_reclaimed()

void sev_guest_memory_reclaimed

(

struct kvm *

kvm

)

Definition at line 2365 of file sev.c.

{
    if (!sev_guest(kvm))
        return;
 
    wbinvd_on_all_cpus();
}

Here is the call graph for this function:

sev_handle_vmgexit()

int sev_handle_vmgexit

(

struct kvm_vcpu *

vcpu

)

Definition at line 2857 of file sev.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    struct vmcb_control_area *control = &svm->vmcb->control;
    u64 ghcb_gpa, exit_code;
    int ret;
 
    /* Validate the GHCB */
    ghcb_gpa = control->ghcb_gpa;
    if (ghcb_gpa & GHCB_MSR_INFO_MASK)
        return sev_handle_vmgexit_msr_protocol(svm);
 
    if (!ghcb_gpa) {
        vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n");
 
        /* Without a GHCB, just return right back to the guest */
        return 1;
    }
 
    if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->sev_es.ghcb_map)) {
        /* Unable to map GHCB from guest */
        vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n",
                ghcb_gpa);
 
        /* Without a GHCB, just return right back to the guest */
        return 1;
    }
 
    svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva;
 
    trace_kvm_vmgexit_enter(vcpu->vcpu_id, svm->sev_es.ghcb);
 
    sev_es_sync_from_ghcb(svm);
    ret = sev_es_validate_vmgexit(svm);
    if (ret)
        return ret;
 
    ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 0);
    ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 0);
 
    exit_code = kvm_ghcb_get_sw_exit_code(control);
    switch (exit_code) {
    case SVM_VMGEXIT_MMIO_READ:
        ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
        if (ret)
            break;
 
        ret = kvm_sev_es_mmio_read(vcpu,
                       control->exit_info_1,
                       control->exit_info_2,
                       svm->sev_es.ghcb_sa);
        break;
    case SVM_VMGEXIT_MMIO_WRITE:
        ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);
        if (ret)
            break;
 
        ret = kvm_sev_es_mmio_write(vcpu,
                        control->exit_info_1,
                        control->exit_info_2,
                        svm->sev_es.ghcb_sa);
        break;
    case SVM_VMGEXIT_NMI_COMPLETE:
        ++vcpu->stat.nmi_window_exits;
        svm->nmi_masked = false;
        kvm_make_request(KVM_REQ_EVENT, vcpu);
        ret = 1;
        break;
    case SVM_VMGEXIT_AP_HLT_LOOP:
        ret = kvm_emulate_ap_reset_hold(vcpu);
        break;
    case SVM_VMGEXIT_AP_JUMP_TABLE: {
        struct kvm_sev_info *sev = &to_kvm_svm(vcpu->kvm)->sev_info;
 
        switch (control->exit_info_1) {
        case 0:
            /* Set AP jump table address */
            sev->ap_jump_table = control->exit_info_2;
            break;
        case 1:
            /* Get AP jump table address */
            ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, sev->ap_jump_table);
            break;
        default:
            pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n",
                   control->exit_info_1);
            ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2);
            ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, GHCB_ERR_INVALID_INPUT);
        }
 
        ret = 1;
        break;
    }
    case SVM_VMGEXIT_UNSUPPORTED_EVENT:
        vcpu_unimpl(vcpu,
                "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n",
                control->exit_info_1, control->exit_info_2);
        ret = -EINVAL;
        break;
    default:
        ret = svm_invoke_exit_handler(vcpu, exit_code);
    }
 
    return ret;
}

Here is the call graph for this function:

sev_hardware_setup()

void __init sev_hardware_setup

(

void

)

Definition at line 2193 of file sev.c.

{
#ifdef CONFIG_KVM_AMD_SEV
    unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count;
    bool sev_es_supported = false;
    bool sev_supported = false;
 
    if (!sev_enabled || !npt_enabled || !nrips)
        goto out;
 
    /*
     * SEV must obviously be supported in hardware.  Sanity check that the
     * CPU supports decode assists, which is mandatory for SEV guests to
     * support instruction emulation.  Ditto for flushing by ASID, as SEV
     * guests are bound to a single ASID, i.e. KVM can't rotate to a new
     * ASID to effect a TLB flush.
     */
    if (!boot_cpu_has(X86_FEATURE_SEV) ||
        WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) ||
        WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_FLUSHBYASID)))
        goto out;
 
    /* Retrieve SEV CPUID information */
    cpuid(0x8000001f, &eax, &ebx, &ecx, &edx);
 
    /* Set encryption bit location for SEV-ES guests */
    sev_enc_bit = ebx & 0x3f;
 
    /* Maximum number of encrypted guests supported simultaneously */
    max_sev_asid = ecx;
    if (!max_sev_asid)
        goto out;
 
    /* Minimum ASID value that should be used for SEV guest */
    min_sev_asid = edx;
    sev_me_mask = 1UL << (ebx & 0x3f);
 
    /*
     * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap,
     * even though it's never used, so that the bitmap is indexed by the
     * actual ASID.
     */
    nr_asids = max_sev_asid + 1;
    sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
    if (!sev_asid_bitmap)
        goto out;
 
    sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
    if (!sev_reclaim_asid_bitmap) {
        bitmap_free(sev_asid_bitmap);
        sev_asid_bitmap = NULL;
        goto out;
    }
 
    if (min_sev_asid <= max_sev_asid) {
        sev_asid_count = max_sev_asid - min_sev_asid + 1;
        WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count));
    }
    sev_supported = true;
 
    /* SEV-ES support requested? */
    if (!sev_es_enabled)
        goto out;
 
    /*
     * SEV-ES requires MMIO caching as KVM doesn't have access to the guest
     * instruction stream, i.e. can't emulate in response to a #NPF and
     * instead relies on #NPF(RSVD) being reflected into the guest as #VC
     * (the guest can then do a #VMGEXIT to request MMIO emulation).
     */
    if (!enable_mmio_caching)
        goto out;
 
    /* Does the CPU support SEV-ES? */
    if (!boot_cpu_has(X86_FEATURE_SEV_ES))
        goto out;
 
    /* Has the system been allocated ASIDs for SEV-ES? */
    if (min_sev_asid == 1)
        goto out;
 
    sev_es_asid_count = min_sev_asid - 1;
    WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count));
    sev_es_supported = true;
 
out:
    if (boot_cpu_has(X86_FEATURE_SEV))
        pr_info("SEV %s (ASIDs %u - %u)\n",
            sev_supported ? min_sev_asid <= max_sev_asid ? "enabled" :
                                       "unusable" :
                                       "disabled",
            min_sev_asid, max_sev_asid);
    if (boot_cpu_has(X86_FEATURE_SEV_ES))
        pr_info("SEV-ES %s (ASIDs %u - %u)\n",
            sev_es_supported ? "enabled" : "disabled",
            min_sev_asid > 1 ? 1 : 0, min_sev_asid - 1);
 
    sev_enabled = sev_supported;
    sev_es_enabled = sev_es_supported;
    if (!sev_es_enabled || !cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP) ||
        !cpu_feature_enabled(X86_FEATURE_NO_NESTED_DATA_BP))
        sev_es_debug_swap_enabled = false;
#endif
}

Here is the caller graph for this function:

sev_hardware_unsetup()

void sev_hardware_unsetup

(

void

)

Definition at line 2298 of file sev.c.

{
    if (!sev_enabled)
        return;
 
    /* No need to take sev_bitmap_lock, all VMs have been destroyed. */
    sev_flush_asids(1, max_sev_asid);
 
    bitmap_free(sev_asid_bitmap);
    bitmap_free(sev_reclaim_asid_bitmap);
 
    misc_cg_set_capacity(MISC_CG_RES_SEV, 0);
    misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_init_vmcb()

void sev_init_vmcb

(

struct vcpu_svm *

svm

)

Definition at line 3093 of file sev.c.

{
    svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
    clr_exception_intercept(svm, UD_VECTOR);
 
    /*
     * Don't intercept #GP for SEV guests, e.g. for the VMware backdoor, as
     * KVM can't decrypt guest memory to decode the faulting instruction.
     */
    clr_exception_intercept(svm, GP_VECTOR);
 
    if (sev_es_guest(svm->vcpu.kvm))
        sev_es_init_vmcb(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_mem_enc_ioctl()

int sev_mem_enc_ioctl	(	struct kvm *	kvm,
		void __user *	argp
	)

Definition at line 1862 of file sev.c.

{
    struct kvm_sev_cmd sev_cmd;
    int r;
 
    if (!sev_enabled)
        return -ENOTTY;
 
    if (!argp)
        return 0;
 
    if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
        return -EFAULT;
 
    mutex_lock(&kvm->lock);
 
    /* Only the enc_context_owner handles some memory enc operations. */
    if (is_mirroring_enc_context(kvm) &&
        !is_cmd_allowed_from_mirror(sev_cmd.id)) {
        r = -EINVAL;
        goto out;
    }
 
    switch (sev_cmd.id) {
    case KVM_SEV_ES_INIT:
        if (!sev_es_enabled) {
            r = -ENOTTY;
            goto out;
        }
        fallthrough;
    case KVM_SEV_INIT:
        r = sev_guest_init(kvm, &sev_cmd);
        break;
    case KVM_SEV_LAUNCH_START:
        r = sev_launch_start(kvm, &sev_cmd);
        break;
    case KVM_SEV_LAUNCH_UPDATE_DATA:
        r = sev_launch_update_data(kvm, &sev_cmd);
        break;
    case KVM_SEV_LAUNCH_UPDATE_VMSA:
        r = sev_launch_update_vmsa(kvm, &sev_cmd);
        break;
    case KVM_SEV_LAUNCH_MEASURE:
        r = sev_launch_measure(kvm, &sev_cmd);
        break;
    case KVM_SEV_LAUNCH_FINISH:
        r = sev_launch_finish(kvm, &sev_cmd);
        break;
    case KVM_SEV_GUEST_STATUS:
        r = sev_guest_status(kvm, &sev_cmd);
        break;
    case KVM_SEV_DBG_DECRYPT:
        r = sev_dbg_crypt(kvm, &sev_cmd, true);
        break;
    case KVM_SEV_DBG_ENCRYPT:
        r = sev_dbg_crypt(kvm, &sev_cmd, false);
        break;
    case KVM_SEV_LAUNCH_SECRET:
        r = sev_launch_secret(kvm, &sev_cmd);
        break;
    case KVM_SEV_GET_ATTESTATION_REPORT:
        r = sev_get_attestation_report(kvm, &sev_cmd);
        break;
    case KVM_SEV_SEND_START:
        r = sev_send_start(kvm, &sev_cmd);
        break;
    case KVM_SEV_SEND_UPDATE_DATA:
        r = sev_send_update_data(kvm, &sev_cmd);
        break;
    case KVM_SEV_SEND_FINISH:
        r = sev_send_finish(kvm, &sev_cmd);
        break;
    case KVM_SEV_SEND_CANCEL:
        r = sev_send_cancel(kvm, &sev_cmd);
        break;
    case KVM_SEV_RECEIVE_START:
        r = sev_receive_start(kvm, &sev_cmd);
        break;
    case KVM_SEV_RECEIVE_UPDATE_DATA:
        r = sev_receive_update_data(kvm, &sev_cmd);
        break;
    case KVM_SEV_RECEIVE_FINISH:
        r = sev_receive_finish(kvm, &sev_cmd);
        break;
    default:
        r = -EINVAL;
        goto out;
    }
 
    if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
        r = -EFAULT;
 
out:
    mutex_unlock(&kvm->lock);
    return r;
}

Here is the call graph for this function:

sev_mem_enc_register_region()

int sev_mem_enc_register_region	(	struct kvm *	kvm,
		struct kvm_enc_region *	range
	)

Definition at line 1959 of file sev.c.

{
    struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
    struct enc_region *region;
    int ret = 0;
 
    if (!sev_guest(kvm))
        return -ENOTTY;
 
    /* If kvm is mirroring encryption context it isn't responsible for it */
    if (is_mirroring_enc_context(kvm))
        return -EINVAL;
 
    if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
        return -EINVAL;
 
    region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
    if (!region)
        return -ENOMEM;
 
    mutex_lock(&kvm->lock);
    region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
    if (IS_ERR(region->pages)) {
        ret = PTR_ERR(region->pages);
        mutex_unlock(&kvm->lock);
        goto e_free;
    }
 
    /*
     * The guest may change the memory encryption attribute from C=0 -> C=1
     * or vice versa for this memory range. Lets make sure caches are
     * flushed to ensure that guest data gets written into memory with
     * correct C-bit.  Note, this must be done before dropping kvm->lock,
     * as region and its array of pages can be freed by a different task
     * once kvm->lock is released.
     */
    sev_clflush_pages(region->pages, region->npages);
 
    region->uaddr = range->addr;
    region->size = range->size;
 
    list_add_tail(&region->list, &sev->regions_list);
    mutex_unlock(&kvm->lock);
 
    return ret;
 
e_free:
    kfree(region);
    return ret;
}

Here is the call graph for this function:

sev_mem_enc_unregister_region()

int sev_mem_enc_unregister_region	(	struct kvm *	kvm,
		struct kvm_enc_region *	range
	)

Definition at line 2035 of file sev.c.

{
    struct enc_region *region;
    int ret;
 
    /* If kvm is mirroring encryption context it isn't responsible for it */
    if (is_mirroring_enc_context(kvm))
        return -EINVAL;
 
    mutex_lock(&kvm->lock);
 
    if (!sev_guest(kvm)) {
        ret = -ENOTTY;
        goto failed;
    }
 
    region = find_enc_region(kvm, range);
    if (!region) {
        ret = -EINVAL;
        goto failed;
    }
 
    /*
     * Ensure that all guest tagged cache entries are flushed before
     * releasing the pages back to the system for use. CLFLUSH will
     * not do this, so issue a WBINVD.
     */
    wbinvd_on_all_cpus();
 
    __unregister_enc_region_locked(kvm, region);
 
    mutex_unlock(&kvm->lock);
    return 0;
 
failed:
    mutex_unlock(&kvm->lock);
    return ret;
}

Here is the call graph for this function:

sev_set_cpu_caps()

void __init sev_set_cpu_caps

(

void

)

Definition at line 2185 of file sev.c.

{
    if (!sev_enabled)
        kvm_cpu_cap_clear(X86_FEATURE_SEV);
    if (!sev_es_enabled)
        kvm_cpu_cap_clear(X86_FEATURE_SEV_ES);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_vcpu_after_set_cpuid()

void sev_vcpu_after_set_cpuid

(

struct vcpu_svm *

svm

)

Definition at line 3015 of file sev.c.

{
    struct kvm_vcpu *vcpu = &svm->vcpu;
    struct kvm_cpuid_entry2 *best;
 
    /* For sev guests, the memory encryption bit is not reserved in CR3.  */
    best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
    if (best)
        vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
 
    if (sev_es_guest(svm->vcpu.kvm))
        sev_es_vcpu_after_set_cpuid(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_vcpu_deliver_sipi_vector()

void sev_vcpu_deliver_sipi_vector	(	struct kvm_vcpu *	vcpu,
		u8	vector
	)

Definition at line 3159 of file sev.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
 
    /* First SIPI: Use the values as initially set by the VMM */
    if (!svm->sev_es.received_first_sipi) {
        svm->sev_es.received_first_sipi = true;
        return;
    }
 
    /*
     * Subsequent SIPI: Return from an AP Reset Hold VMGEXIT, where
     * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a
     * non-zero value.
     */
    if (!svm->sev_es.ghcb)
        return;
 
    ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 1);
}

Here is the caller graph for this function:

sev_vm_copy_enc_context_from()

int sev_vm_copy_enc_context_from	(	struct kvm *	kvm,
		unsigned int	source_fd
	)

Definition at line 2075 of file sev.c.

{
    struct fd f = fdget(source_fd);
    struct kvm *source_kvm;
    struct kvm_sev_info *source_sev, *mirror_sev;
    int ret;
 
    if (!f.file)
        return -EBADF;
 
    if (!file_is_kvm(f.file)) {
        ret = -EBADF;
        goto e_source_fput;
    }
 
    source_kvm = f.file->private_data;
    ret = sev_lock_two_vms(kvm, source_kvm);
    if (ret)
        goto e_source_fput;
 
    /*
     * Mirrors of mirrors should work, but let's not get silly.  Also
     * disallow out-of-band SEV/SEV-ES init if the target is already an
     * SEV guest, or if vCPUs have been created.  KVM relies on vCPUs being
     * created after SEV/SEV-ES initialization, e.g. to init intercepts.
     */
    if (sev_guest(kvm) || !sev_guest(source_kvm) ||
        is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) {
        ret = -EINVAL;
        goto e_unlock;
    }
 
    /*
     * The mirror kvm holds an enc_context_owner ref so its asid can't
     * disappear until we're done with it
     */
    source_sev = &to_kvm_svm(source_kvm)->sev_info;
    kvm_get_kvm(source_kvm);
    mirror_sev = &to_kvm_svm(kvm)->sev_info;
    list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms);
 
    /* Set enc_context_owner and copy its encryption context over */
    mirror_sev->enc_context_owner = source_kvm;
    mirror_sev->active = true;
    mirror_sev->asid = source_sev->asid;
    mirror_sev->fd = source_sev->fd;
    mirror_sev->es_active = source_sev->es_active;
    mirror_sev->handle = source_sev->handle;
    INIT_LIST_HEAD(&mirror_sev->regions_list);
    INIT_LIST_HEAD(&mirror_sev->mirror_vms);
    ret = 0;
 
    /*
     * Do not copy ap_jump_table. Since the mirror does not share the same
     * KVM contexts as the original, and they may have different
     * memory-views.
     */
 
e_unlock:
    sev_unlock_two_vms(kvm, source_kvm);
e_source_fput:
    fdput(f);
    return ret;
}

Here is the call graph for this function:

sev_vm_destroy()

void sev_vm_destroy

(

struct kvm *

kvm

)

Definition at line 2140 of file sev.c.

{
    struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
    struct list_head *head = &sev->regions_list;
    struct list_head *pos, *q;
 
    if (!sev_guest(kvm))
        return;
 
    WARN_ON(!list_empty(&sev->mirror_vms));
 
    /* If this is a mirror_kvm release the enc_context_owner and skip sev cleanup */
    if (is_mirroring_enc_context(kvm)) {
        struct kvm *owner_kvm = sev->enc_context_owner;
 
        mutex_lock(&owner_kvm->lock);
        list_del(&sev->mirror_entry);
        mutex_unlock(&owner_kvm->lock);
        kvm_put_kvm(owner_kvm);
        return;
    }
 
    /*
     * Ensure that all guest tagged cache entries are flushed before
     * releasing the pages back to the system for use. CLFLUSH will
     * not do this, so issue a WBINVD.
     */
    wbinvd_on_all_cpus();
 
    /*
     * if userspace was terminated before unregistering the memory regions
     * then lets unpin all the registered memory.
     */
    if (!list_empty(head)) {
        list_for_each_safe(pos, q, head) {
            __unregister_enc_region_locked(kvm,
                list_entry(pos, struct enc_region, list));
            cond_resched();
        }
    }
 
    sev_unbind_asid(kvm, sev->handle);
    sev_asid_free(sev);
}

Here is the call graph for this function:

Here is the caller graph for this function:

sev_vm_move_enc_context_from()

int sev_vm_move_enc_context_from	(	struct kvm *	kvm,
		unsigned int	source_fd
	)

Definition at line 1791 of file sev.c.

{
    struct kvm_sev_info *dst_sev = &to_kvm_svm(kvm)->sev_info;
    struct kvm_sev_info *src_sev, *cg_cleanup_sev;
    struct fd f = fdget(source_fd);
    struct kvm *source_kvm;
    bool charged = false;
    int ret;
 
    if (!f.file)
        return -EBADF;
 
    if (!file_is_kvm(f.file)) {
        ret = -EBADF;
        goto out_fput;
    }
 
    source_kvm = f.file->private_data;
    ret = sev_lock_two_vms(kvm, source_kvm);
    if (ret)
        goto out_fput;
 
    if (sev_guest(kvm) || !sev_guest(source_kvm)) {
        ret = -EINVAL;
        goto out_unlock;
    }
 
    src_sev = &to_kvm_svm(source_kvm)->sev_info;
 
    dst_sev->misc_cg = get_current_misc_cg();
    cg_cleanup_sev = dst_sev;
    if (dst_sev->misc_cg != src_sev->misc_cg) {
        ret = sev_misc_cg_try_charge(dst_sev);
        if (ret)
            goto out_dst_cgroup;
        charged = true;
    }
 
    ret = sev_lock_vcpus_for_migration(kvm, SEV_MIGRATION_SOURCE);
    if (ret)
        goto out_dst_cgroup;
    ret = sev_lock_vcpus_for_migration(source_kvm, SEV_MIGRATION_TARGET);
    if (ret)
        goto out_dst_vcpu;
 
    ret = sev_check_source_vcpus(kvm, source_kvm);
    if (ret)
        goto out_source_vcpu;
 
    sev_migrate_from(kvm, source_kvm);
    kvm_vm_dead(source_kvm);
    cg_cleanup_sev = src_sev;
    ret = 0;
 
out_source_vcpu:
    sev_unlock_vcpus_for_migration(source_kvm);
out_dst_vcpu:
    sev_unlock_vcpus_for_migration(kvm);
out_dst_cgroup:
    /* Operates on the source on success, on the destination on failure.  */
    if (charged)
        sev_misc_cg_uncharge(cg_cleanup_sev);
    put_misc_cg(cg_cleanup_sev->misc_cg);
    cg_cleanup_sev->misc_cg = NULL;
out_unlock:
    sev_unlock_two_vms(kvm, source_kvm);
out_fput:
    fdput(f);
    return ret;
}

Here is the call graph for this function:

svm_allocate_nested()

int svm_allocate_nested

(

struct vcpu_svm *

svm

)

Definition at line 1177 of file nested.c.

{
    struct page *vmcb02_page;
 
    if (svm->nested.initialized)
        return 0;
 
    vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
    if (!vmcb02_page)
        return -ENOMEM;
    svm->nested.vmcb02.ptr = page_address(vmcb02_page);
    svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);
 
    svm->nested.msrpm = svm_vcpu_alloc_msrpm();
    if (!svm->nested.msrpm)
        goto err_free_vmcb02;
    svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm);
 
    svm->nested.initialized = true;
    return 0;
 
err_free_vmcb02:
    __free_page(vmcb02_page);
    return -ENOMEM;
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_clr_intercept()

static void svm_clr_intercept ( struct vcpu_svm *  svm, int  bit  )

inlinestatic

Definition at line 432 of file svm.h.

{
    struct vmcb *vmcb = svm->vmcb01.ptr;
 
    vmcb_clr_intercept(&vmcb->control, bit);
 
    recalc_intercepts(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_complete_interrupt_delivery()

void svm_complete_interrupt_delivery	(	struct kvm_vcpu *	vcpu,
		int	delivery_mode,
		int	trig_mode,
		int	vec
	)

Definition at line 3633 of file svm.c.

{
    /*
     * apic->apicv_active must be read after vcpu->mode.
     * Pairs with smp_store_release in vcpu_enter_guest.
     */
    bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
 
    /* Note, this is called iff the local APIC is in-kernel. */
    if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
        /* Process the interrupt via kvm_check_and_inject_events(). */
        kvm_make_request(KVM_REQ_EVENT, vcpu);
        kvm_vcpu_kick(vcpu);
        return;
    }
 
    trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
    if (in_guest_mode) {
        /*
         * Signal the doorbell to tell hardware to inject the IRQ.  If
         * the vCPU exits the guest before the doorbell chimes, hardware
         * will automatically process AVIC interrupts at the next VMRUN.
         */
        avic_ring_doorbell(vcpu);
    } else {
        /*
         * Wake the vCPU if it was blocking.  KVM will then detect the
         * pending IRQ when checking if the vCPU has a wake event.
         */
        kvm_vcpu_wake_up(vcpu);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_copy_lbrs()

void svm_copy_lbrs	(	struct vmcb *	to_vmcb,
		struct vmcb *	from_vmcb
	)

Definition at line 982 of file svm.c.

{
    to_vmcb->save.dbgctl        = from_vmcb->save.dbgctl;
    to_vmcb->save.br_from       = from_vmcb->save.br_from;
    to_vmcb->save.br_to     = from_vmcb->save.br_to;
    to_vmcb->save.last_excp_from    = from_vmcb->save.last_excp_from;
    to_vmcb->save.last_excp_to  = from_vmcb->save.last_excp_to;
 
    vmcb_mark_dirty(to_vmcb, VMCB_LBR);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_copy_vmloadsave_state()

void svm_copy_vmloadsave_state	(	struct vmcb *	to_vmcb,
		struct vmcb *	from_vmcb
	)

Definition at line 951 of file nested.c.

{
    to_vmcb->save.fs = from_vmcb->save.fs;
    to_vmcb->save.gs = from_vmcb->save.gs;
    to_vmcb->save.tr = from_vmcb->save.tr;
    to_vmcb->save.ldtr = from_vmcb->save.ldtr;
    to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
    to_vmcb->save.star = from_vmcb->save.star;
    to_vmcb->save.lstar = from_vmcb->save.lstar;
    to_vmcb->save.cstar = from_vmcb->save.cstar;
    to_vmcb->save.sfmask = from_vmcb->save.sfmask;
    to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
    to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
    to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
}

Here is the caller graph for this function:

svm_copy_vmrun_state()

void svm_copy_vmrun_state	(	struct vmcb_save_area *	to_save,
		struct vmcb_save_area *	from_save
	)

Definition at line 931 of file nested.c.

{
    to_save->es = from_save->es;
    to_save->cs = from_save->cs;
    to_save->ss = from_save->ss;
    to_save->ds = from_save->ds;
    to_save->gdtr = from_save->gdtr;
    to_save->idtr = from_save->idtr;
    to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED;
    to_save->efer = from_save->efer;
    to_save->cr0 = from_save->cr0;
    to_save->cr3 = from_save->cr3;
    to_save->cr4 = from_save->cr4;
    to_save->rax = from_save->rax;
    to_save->rsp = from_save->rsp;
    to_save->rip = from_save->rip;
    to_save->cpl = 0;
}

Here is the caller graph for this function:

svm_free_nested()

void svm_free_nested

(

struct vcpu_svm *

svm

)

Definition at line 1203 of file nested.c.

{
    if (!svm->nested.initialized)
        return;
 
    if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr))
        svm_switch_vmcb(svm, &svm->vmcb01);
 
    svm_vcpu_free_msrpm(svm->nested.msrpm);
    svm->nested.msrpm = NULL;
 
    __free_page(virt_to_page(svm->nested.vmcb02.ptr));
    svm->nested.vmcb02.ptr = NULL;
 
    /*
     * When last_vmcb12_gpa matches the current vmcb12 gpa,
     * some vmcb12 fields are not loaded if they are marked clean
     * in the vmcb12, since in this case they are up to date already.
     *
     * When the vmcb02 is freed, this optimization becomes invalid.
     */
    svm->nested.last_vmcb12_gpa = INVALID_GPA;
 
    svm->nested.initialized = false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_interrupt_blocked()

bool svm_interrupt_blocked

(

struct kvm_vcpu *

vcpu

)

Definition at line 3767 of file svm.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    struct vmcb *vmcb = svm->vmcb;
 
    if (!gif_set(svm))
        return true;
 
    if (is_guest_mode(vcpu)) {
        /* As long as interrupts are being delivered...  */
        if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
            ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
            : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
            return true;
 
        /* ... vmexits aren't blocked by the interrupt shadow  */
        if (nested_exit_on_intr(svm))
            return false;
    } else {
        if (!svm_get_if_flag(vcpu))
            return true;
    }
 
    return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_invoke_exit_handler()

int svm_invoke_exit_handler	(	struct kvm_vcpu *	vcpu,
		u64	exit_code
	)

Definition at line 3453 of file svm.c.

{
    if (!svm_check_exit_valid(exit_code))
        return svm_handle_invalid_exit(vcpu, exit_code);
 
#ifdef CONFIG_RETPOLINE
    if (exit_code == SVM_EXIT_MSR)
        return msr_interception(vcpu);
    else if (exit_code == SVM_EXIT_VINTR)
        return interrupt_window_interception(vcpu);
    else if (exit_code == SVM_EXIT_INTR)
        return intr_interception(vcpu);
    else if (exit_code == SVM_EXIT_HLT)
        return kvm_emulate_halt(vcpu);
    else if (exit_code == SVM_EXIT_NPF)
        return npf_interception(vcpu);
#endif
    return svm_exit_handlers[exit_code](vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_is_intercept()

static bool svm_is_intercept ( struct vcpu_svm *  svm, int  bit  )

inlinestatic

Definition at line 441 of file svm.h.

{
    return vmcb_is_intercept(&svm->vmcb->control, bit);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_leave_nested()

void svm_leave_nested

(

struct kvm_vcpu *

vcpu

)

Definition at line 1229 of file nested.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
 
    if (is_guest_mode(vcpu)) {
        svm->nested.nested_run_pending = 0;
        svm->nested.vmcb12_gpa = INVALID_GPA;
 
        leave_guest_mode(vcpu);
 
        svm_switch_vmcb(svm, &svm->vmcb01);
 
        nested_svm_uninit_mmu_context(vcpu);
        vmcb_mark_all_dirty(svm->vmcb);
 
        if (kvm_apicv_activated(vcpu->kvm))
            kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
    }
 
    kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_msrpm_offset()

u32 svm_msrpm_offset

(

u32

msr

)

Definition at line 264 of file svm.c.

{
    u32 offset;
    int i;
 
    for (i = 0; i < NUM_MSR_MAPS; i++) {
        if (msr < msrpm_ranges[i] ||
            msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
            continue;
 
        offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
        offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 
        /* Now we have the u8 offset - but need the u32 offset */
        return offset / 4;
    }
 
    /* MSR not in any range */
    return MSR_INVALID;
}

Here is the caller graph for this function:

svm_nmi_blocked()

bool svm_nmi_blocked

(

struct kvm_vcpu *

vcpu

)

Definition at line 3735 of file svm.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    struct vmcb *vmcb = svm->vmcb;
 
    if (!gif_set(svm))
        return true;
 
    if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
        return false;
 
    if (svm_get_nmi_mask(vcpu))
        return true;
 
    return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_cr0()

void svm_set_cr0	(	struct kvm_vcpu *	vcpu,
		unsigned long	cr0
	)

Definition at line 1852 of file svm.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    u64 hcr0 = cr0;
    bool old_paging = is_paging(vcpu);
 
#ifdef CONFIG_X86_64
    if (vcpu->arch.efer & EFER_LME) {
        if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
            vcpu->arch.efer |= EFER_LMA;
            if (!vcpu->arch.guest_state_protected)
                svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
        }
 
        if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
            vcpu->arch.efer &= ~EFER_LMA;
            if (!vcpu->arch.guest_state_protected)
                svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
        }
    }
#endif
    vcpu->arch.cr0 = cr0;
 
    if (!npt_enabled) {
        hcr0 |= X86_CR0_PG | X86_CR0_WP;
        if (old_paging != is_paging(vcpu))
            svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
    }
 
    /*
     * re-enable caching here because the QEMU bios
     * does not do it - this results in some delay at
     * reboot
     */
    if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
        hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
 
    svm->vmcb->save.cr0 = hcr0;
    vmcb_mark_dirty(svm->vmcb, VMCB_CR);
 
    /*
     * SEV-ES guests must always keep the CR intercepts cleared. CR
     * tracking is done using the CR write traps.
     */
    if (sev_es_guest(vcpu->kvm))
        return;
 
    if (hcr0 == cr0) {
        /* Selective CR0 write remains on.  */
        svm_clr_intercept(svm, INTERCEPT_CR0_READ);
        svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
    } else {
        svm_set_intercept(svm, INTERCEPT_CR0_READ);
        svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_cr4()

void svm_set_cr4	(	struct kvm_vcpu *	vcpu,
		unsigned long	cr4
	)

Definition at line 1914 of file svm.c.

{
    unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
    unsigned long old_cr4 = vcpu->arch.cr4;
 
    if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
        svm_flush_tlb_current(vcpu);
 
    vcpu->arch.cr4 = cr4;
    if (!npt_enabled) {
        cr4 |= X86_CR4_PAE;
 
        if (!is_paging(vcpu))
            cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
    }
    cr4 |= host_cr4_mce;
    to_svm(vcpu)->vmcb->save.cr4 = cr4;
    vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 
    if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
        kvm_update_cpuid_runtime(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_efer()

int svm_set_efer	(	struct kvm_vcpu *	vcpu,
		u64	efer
	)

Definition at line 296 of file svm.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    u64 old_efer = vcpu->arch.efer;
    vcpu->arch.efer = efer;
 
    if (!npt_enabled) {
        /* Shadow paging assumes NX to be available.  */
        efer |= EFER_NX;
 
        if (!(efer & EFER_LMA))
            efer &= ~EFER_LME;
    }
 
    if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
        if (!(efer & EFER_SVME)) {
            svm_leave_nested(vcpu);
            svm_set_gif(svm, true);
            /* #GP intercept is still needed for vmware backdoor */
            if (!enable_vmware_backdoor)
                clr_exception_intercept(svm, GP_VECTOR);
 
            /*
             * Free the nested guest state, unless we are in SMM.
             * In this case we will return to the nested guest
             * as soon as we leave SMM.
             */
            if (!is_smm(vcpu))
                svm_free_nested(svm);
 
        } else {
            int ret = svm_allocate_nested(svm);
 
            if (ret) {
                vcpu->arch.efer = old_efer;
                return ret;
            }
 
            /*
             * Never intercept #GP for SEV guests, KVM can't
             * decrypt guest memory to workaround the erratum.
             */
            if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
                set_exception_intercept(svm, GP_VECTOR);
        }
    }
 
    svm->vmcb->save.efer = efer | EFER_SVME;
    vmcb_mark_dirty(svm->vmcb, VMCB_CR);
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_gif()

void svm_set_gif	(	struct vcpu_svm *	svm,
		bool	value
	)

Definition at line 2406 of file svm.c.

{
    if (value) {
        /*
         * If VGIF is enabled, the STGI intercept is only added to
         * detect the opening of the SMI/NMI window; remove it now.
         * Likewise, clear the VINTR intercept, we will set it
         * again while processing KVM_REQ_EVENT if needed.
         */
        if (vgif)
            svm_clr_intercept(svm, INTERCEPT_STGI);
        if (svm_is_intercept(svm, INTERCEPT_VINTR))
            svm_clear_vintr(svm);
 
        enable_gif(svm);
        if (svm->vcpu.arch.smi_pending ||
            svm->vcpu.arch.nmi_pending ||
            kvm_cpu_has_injectable_intr(&svm->vcpu) ||
            kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
            kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
    } else {
        disable_gif(svm);
 
        /*
         * After a CLGI no interrupts should come.  But if vGIF is
         * in use, we still rely on the VINTR intercept (rather than
         * STGI) to detect an open interrupt window.
        */
        if (!vgif)
            svm_clear_vintr(svm);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_intercept()

static void svm_set_intercept ( struct vcpu_svm *  svm, int  bit  )

inlinestatic

Definition at line 423 of file svm.h.

{
    struct vmcb *vmcb = svm->vmcb01.ptr;
 
    vmcb_set_intercept(&vmcb->control, bit);
 
    recalc_intercepts(svm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_set_x2apic_msr_interception()

void svm_set_x2apic_msr_interception	(	struct vcpu_svm *	svm,
		bool	disable
	)

Definition at line 892 of file svm.c.

{
    int i;
 
    if (intercept == svm->x2avic_msrs_intercepted)
        return;
 
    if (!x2avic_enabled)
        return;
 
    for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
        int index = direct_access_msrs[i].index;
 
        if ((index < APIC_BASE_MSR) ||
            (index > APIC_BASE_MSR + 0xff))
            continue;
        set_msr_interception(&svm->vcpu, svm->msrpm, index,
                     !intercept, !intercept);
    }
 
    svm->x2avic_msrs_intercepted = intercept;
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_smi_blocked()

bool svm_smi_blocked

(

struct kvm_vcpu *

vcpu

)

Here is the caller graph for this function:

svm_switch_vmcb()

void svm_switch_vmcb	(	struct vcpu_svm *	svm,
		struct kvm_vmcb_info *	target_vmcb
	)

Definition at line 1407 of file svm.c.

{
    svm->current_vmcb = target_vmcb;
    svm->vmcb = target_vmcb->ptr;
}

Here is the caller graph for this function:

svm_update_lbrv()

void svm_update_lbrv

(

struct kvm_vcpu *

vcpu

)

Definition at line 1037 of file svm.c.

{
    struct vcpu_svm *svm = to_svm(vcpu);
    bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
    bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
                (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) &&
                (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
 
    if (enable_lbrv == current_enable_lbrv)
        return;
 
    if (enable_lbrv)
        svm_enable_lbrv(vcpu);
    else
        svm_disable_lbrv(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_vcpu_alloc_msrpm()

u32* svm_vcpu_alloc_msrpm

(

void

)

Definition at line 866 of file svm.c.

{
    unsigned int order = get_order(MSRPM_SIZE);
    struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
    u32 *msrpm;
 
    if (!pages)
        return NULL;
 
    msrpm = page_address(pages);
    memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
 
    return msrpm;
}

Here is the caller graph for this function:

svm_vcpu_free_msrpm()

void svm_vcpu_free_msrpm

(

u32 *

msrpm

)

Definition at line 915 of file svm.c.

{
    __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
}

Here is the caller graph for this function:

svm_vcpu_init_msrpm()

void svm_vcpu_init_msrpm	(	struct kvm_vcpu *	vcpu,
		u32 *	msrpm
	)

Definition at line 881 of file svm.c.

{
    int i;
 
    for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
        if (!direct_access_msrs[i].always)
            continue;
        set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

svm_write_tsc_multiplier()

void svm_write_tsc_multiplier

(

struct kvm_vcpu *

vcpu

)

Definition at line 1163 of file svm.c.

{
    preempt_disable();
    if (to_svm(vcpu)->guest_state_loaded)
        __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
    preempt_enable();
}

Here is the call graph for this function:

Here is the caller graph for this function:

to_kvm_svm()

static __always_inline struct kvm_svm* to_kvm_svm ( struct kvm *  kvm )

static

Definition at line 316 of file svm.h.

{
    return container_of(kvm, struct kvm_svm, kvm);
}

Here is the caller graph for this function:

to_svm()

static __always_inline struct vcpu_svm* to_svm ( struct kvm_vcpu *  vcpu )

static

Definition at line 364 of file svm.h.

{
    return container_of(vcpu, struct vcpu_svm, vcpu);
}

vmcb12_is_intercept()

static bool vmcb12_is_intercept ( struct vmcb_ctrl_area_cached *  control, u32  bit  )

inlinestatic

Definition at line 397 of file svm.h.

{
    WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
    return test_bit(bit, (unsigned long *)&control->intercepts);
}

Here is the caller graph for this function:

vmcb_clr_intercept()

static void vmcb_clr_intercept ( struct vmcb_control_area *  control, u32  bit  )

inlinestatic

Definition at line 385 of file svm.h.

{
    WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
    __clear_bit(bit, (unsigned long *)&control->intercepts);
}

Here is the caller graph for this function:

vmcb_is_dirty()

static bool vmcb_is_dirty ( struct vmcb *  vmcb, int  bit  )

inlinestatic

Definition at line 359 of file svm.h.

{
        return !test_bit(bit, (unsigned long *)&vmcb->control.clean);
}

Here is the caller graph for this function:

vmcb_is_intercept()

static bool vmcb_is_intercept ( struct vmcb_control_area *  control, u32  bit  )

inlinestatic

Definition at line 391 of file svm.h.

{
    WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
    return test_bit(bit, (unsigned long *)&control->intercepts);
}

Here is the caller graph for this function:

vmcb_mark_all_clean()

static void vmcb_mark_all_clean ( struct vmcb *  vmcb )

inlinestatic

Definition at line 348 of file svm.h.

{
    vmcb->control.clean = VMCB_ALL_CLEAN_MASK
                   & ~VMCB_ALWAYS_DIRTY_MASK;
}

Here is the caller graph for this function:

vmcb_mark_all_dirty()

static void vmcb_mark_all_dirty ( struct vmcb *  vmcb )

inlinestatic

Definition at line 343 of file svm.h.

{
    vmcb->control.clean = 0;
}

Here is the caller graph for this function:

vmcb_mark_dirty()

static void vmcb_mark_dirty ( struct vmcb *  vmcb, int  bit  )

inlinestatic

Definition at line 354 of file svm.h.

{
    vmcb->control.clean &= ~(1 << bit);
}

Here is the caller graph for this function:

vmcb_set_intercept()

static void vmcb_set_intercept ( struct vmcb_control_area *  control, u32  bit  )

inlinestatic

Definition at line 379 of file svm.h.

{
    WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT);
    __set_bit(bit, (unsigned long *)&control->intercepts);
}

Here is the caller graph for this function:

Variable Documentation

__read_mostly

u32 msrpm_offsets [MSRPM_OFFSETS] __read_mostly

extern

Definition at line 36 of file cpuid.c.

dump_invalid_vmcb

bool dump_invalid_vmcb

extern

Definition at line 231 of file svm.c.

intercept_smi

bool intercept_smi

extern

Definition at line 235 of file svm.c.

max_sev_asid

unsigned int max_sev_asid

extern

Definition at line 71 of file sev.c.

npt_enabled

bool npt_enabled

extern

Definition at line 198 of file svm.c.

nrips

int nrips

extern

Definition at line 206 of file svm.c.

svm_nested_ops

struct kvm_x86_nested_ops svm_nested_ops

extern

Definition at line 1780 of file nested.c.

vgif

int vgif

extern

Definition at line 214 of file svm.c.

vnmi

bool vnmi

extern

Definition at line 238 of file svm.c.

x2avic_enabled

bool x2avic_enabled

extern

Definition at line 75 of file avic.c.