emulate-nested.c File Reference

#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_nested.h>
#include "hyp/include/hyp/adjust_pc.h"
#include "trace.h"

Include dependency graph for emulate-nested.c:

Go to the source code of this file.

Classes
struct	trap_bits
union	trap_config
struct	encoding_to_trap_config

Macros
#define	MCB(id, ...)
#define	CCC(id, fn)    [id - __COMPLEX_CONDITIONS__] = fn
#define	TC_CGT_BITS   10
#define	TC_FGT_BITS   4
#define	TC_FGF_BITS   5
#define	SR_RANGE_TRAP(sr_start, sr_end, trap_id)
#define	SR_TRAP(sr, trap_id)   SR_RANGE_TRAP(sr, sr, trap_id)
#define	SR_FGF(sr, g, b, p, f)
#define	SR_FGT(sr, g, b, p)   SR_FGF(sr, g, b, p, __NO_FGF__)
#define	sanitised_sys_reg(vcpu, reg)

Typedefs
typedef enum trap_behaviour(*	complex_condition_check) (struct kvm_vcpu *)

Enumerations
enum	trap_behaviour { BEHAVE_HANDLE_LOCALLY = 0 , BEHAVE_FORWARD_READ = BIT(0) , BEHAVE_FORWARD_WRITE = BIT(1) , BEHAVE_FORWARD_ANY = BEHAVE_FORWARD_READ | BEHAVE_FORWARD_WRITE }
enum	cgt_group_id {   __RESERVED__ , CGT_HCR_TID1 , CGT_HCR_TID2 , CGT_HCR_TID3 ,   CGT_HCR_IMO , CGT_HCR_FMO , CGT_HCR_TIDCP , CGT_HCR_TACR ,   CGT_HCR_TSW , CGT_HCR_TPC , CGT_HCR_TPU , CGT_HCR_TTLB ,   CGT_HCR_TVM , CGT_HCR_TDZ , CGT_HCR_TRVM , CGT_HCR_TLOR ,   CGT_HCR_TERR , CGT_HCR_APK , CGT_HCR_NV , CGT_HCR_NV_nNV2 ,   CGT_HCR_NV1_nNV2 , CGT_HCR_AT , CGT_HCR_nFIEN , CGT_HCR_TID4 ,   CGT_HCR_TICAB , CGT_HCR_TOCU , CGT_HCR_ENSCXT , CGT_HCR_TTLBIS ,   CGT_HCR_TTLBOS , CGT_MDCR_TPMCR , CGT_MDCR_TPM , CGT_MDCR_TDE ,   CGT_MDCR_TDA , CGT_MDCR_TDOSA , CGT_MDCR_TDRA , CGT_MDCR_E2PB ,   CGT_MDCR_TPMS , CGT_MDCR_TTRF , CGT_MDCR_E2TB , CGT_MDCR_TDCC ,   __MULTIPLE_CONTROL_BITS__ , CGT_HCR_IMO_FMO = __MULTIPLE_CONTROL_BITS__ , CGT_HCR_TID2_TID4 , CGT_HCR_TTLB_TTLBIS ,   CGT_HCR_TTLB_TTLBOS , CGT_HCR_TVM_TRVM , CGT_HCR_TPU_TICAB , CGT_HCR_TPU_TOCU ,   CGT_HCR_NV1_nNV2_ENSCXT , CGT_MDCR_TPM_TPMCR , CGT_MDCR_TDE_TDA , CGT_MDCR_TDE_TDOSA ,   CGT_MDCR_TDE_TDRA , CGT_MDCR_TDCC_TDE_TDA , __COMPLEX_CONDITIONS__ , CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__ ,   CGT_CNTHCTL_EL1PTEN , __NR_CGT_GROUP_IDS__ }
enum	fgt_group_id {   __NO_FGT_GROUP__ , HFGxTR_GROUP , HDFGRTR_GROUP , HDFGWTR_GROUP ,   HFGITR_GROUP , HAFGRTR_GROUP , __NR_FGT_GROUP_IDS__ }
enum	fg_filter_id { __NO_FGF__ , HCRX_FGTnXS , __NR_FG_FILTER_IDS__ }

Functions
static u64	get_sanitized_cnthctl (struct kvm_vcpu *vcpu)
static enum trap_behaviour	check_cnthctl_el1pcten (struct kvm_vcpu *vcpu)
static enum trap_behaviour	check_cnthctl_el1pten (struct kvm_vcpu *vcpu)
static	DEFINE_XARRAY (sr_forward_xa)
static union trap_config	get_trap_config (u32 sysreg)
static __init void	print_nv_trap_error (const struct encoding_to_trap_config *tc, const char *type, int err)
int __init	populate_nv_trap_config (void)
static enum trap_behaviour	get_behaviour (struct kvm_vcpu *vcpu, const struct trap_bits *tb)
static enum trap_behaviour	__compute_trap_behaviour (struct kvm_vcpu *vcpu, const enum cgt_group_id id, enum trap_behaviour b)
static enum trap_behaviour	compute_trap_behaviour (struct kvm_vcpu *vcpu, const union trap_config tc)
static bool	check_fgt_bit (u64 val, const union trap_config tc)
bool	__check_nv_sr_forward (struct kvm_vcpu *vcpu)
static u64	kvm_check_illegal_exception_return (struct kvm_vcpu *vcpu, u64 spsr)
void	kvm_emulate_nested_eret (struct kvm_vcpu *vcpu)
static void	kvm_inject_el2_exception (struct kvm_vcpu *vcpu, u64 esr_el2, enum exception_type type)
static int	kvm_inject_nested (struct kvm_vcpu *vcpu, u64 esr_el2, enum exception_type type)
int	kvm_inject_nested_sync (struct kvm_vcpu *vcpu, u64 esr_el2)
int	kvm_inject_nested_irq (struct kvm_vcpu *vcpu)

Variables
static const struct trap_bits	coarse_trap_bits []
static enum cgt_group_id *	coarse_control_combo []
static const complex_condition_check	ccc []
static const struct encoding_to_trap_config encoding_to_cgt[]	__initconst

Macro Definition Documentation

CCC

#define CCC	(		id,
			fn
	)		[id - __COMPLEX_CONDITIONS__] = fn

Definition at line 413 of file emulate-nested.c.

MCB

#define MCB	(		id,
			...
	)

Value:

    [id - __MULTIPLE_CONTROL_BITS__]    =       \
        (const enum cgt_group_id[]){            \
        __VA_ARGS__, __RESERVED__           \
        }

Definition at line 350 of file emulate-nested.c.

sanitised_sys_reg

#define sanitised_sys_reg	(		vcpu,
			reg
	)

Value:

    ({                      \
        u64 __val;              \
        __val = __vcpu_sys_reg(vcpu, reg);  \
        __val &= ~__ ## reg ## _RES0;       \
        (__val);                \
    })

Definition at line 1900 of file emulate-nested.c.

SR_FGF

#define SR_FGF	(		sr,
			g,
			b,
			p,
			f
	)

Value:

    {                           \
        .encoding   = sr,               \
        .end        = sr,               \
        .tc     = {             \
            .fgt = g ## _GROUP,         \
            .bit = g ## _EL2_ ## b ## _SHIFT,   \
            .pol = p,               \
            .fgf = f,               \
        },                      \
        .line = __LINE__,               \
    }

Definition at line 1029 of file emulate-nested.c.

SR_FGT

#define SR_FGT	(		sr,
			g,
			b,
			p
	)		SR_FGF(sr, g, b, p, __NO_FGF__)

Definition at line 1042 of file emulate-nested.c.

SR_RANGE_TRAP

#define SR_RANGE_TRAP	(		sr_start,
			sr_end,
			trap_id
	)

Value:

    {                               \
        .encoding   = sr_start,             \
        .end        = sr_end,               \
        .tc     = {                 \
            .cgt        = trap_id,          \
        },                          \
        .line = __LINE__,                   \
    }

Definition at line 457 of file emulate-nested.c.

SR_TRAP

#define SR_TRAP	(		sr,
			trap_id
	)		SR_RANGE_TRAP(sr, sr, trap_id)

Definition at line 467 of file emulate-nested.c.

TC_CGT_BITS

#define TC_CGT_BITS   10

Definition at line 433 of file emulate-nested.c.

TC_FGF_BITS

#define TC_FGF_BITS   5

Definition at line 435 of file emulate-nested.c.

TC_FGT_BITS

#define TC_FGT_BITS   4

Definition at line 434 of file emulate-nested.c.

Typedef Documentation

complex_condition_check

typedef enum trap_behaviour(* complex_condition_check) (struct kvm_vcpu *)

Definition at line 356 of file emulate-nested.c.

Enumeration Type Documentation

cgt_group_id

enum cgt_group_id

Enumerator
__RESERVED__
CGT_HCR_TID1
CGT_HCR_TID2
CGT_HCR_TID3
CGT_HCR_IMO
CGT_HCR_FMO
CGT_HCR_TIDCP
CGT_HCR_TACR
CGT_HCR_TSW
CGT_HCR_TPC
CGT_HCR_TPU
CGT_HCR_TTLB
CGT_HCR_TVM
CGT_HCR_TDZ
CGT_HCR_TRVM
CGT_HCR_TLOR
CGT_HCR_TERR
CGT_HCR_APK
CGT_HCR_NV
CGT_HCR_NV_nNV2
CGT_HCR_NV1_nNV2
CGT_HCR_AT
CGT_HCR_nFIEN
CGT_HCR_TID4
CGT_HCR_TICAB
CGT_HCR_TOCU
CGT_HCR_ENSCXT
CGT_HCR_TTLBIS
CGT_HCR_TTLBOS
CGT_MDCR_TPMCR
CGT_MDCR_TPM
CGT_MDCR_TDE
CGT_MDCR_TDA
CGT_MDCR_TDOSA
CGT_MDCR_TDRA
CGT_MDCR_E2PB
CGT_MDCR_TPMS
CGT_MDCR_TTRF
CGT_MDCR_E2TB
CGT_MDCR_TDCC
__MULTIPLE_CONTROL_BITS__
CGT_HCR_IMO_FMO
CGT_HCR_TID2_TID4
CGT_HCR_TTLB_TTLBIS
CGT_HCR_TTLB_TTLBOS
CGT_HCR_TVM_TRVM
CGT_HCR_TPU_TICAB
CGT_HCR_TPU_TOCU
CGT_HCR_NV1_nNV2_ENSCXT
CGT_MDCR_TPM_TPMCR
CGT_MDCR_TDE_TDA
CGT_MDCR_TDE_TDOSA
CGT_MDCR_TDE_TDRA
CGT_MDCR_TDCC_TDE_TDA
__COMPLEX_CONDITIONS__
CGT_CNTHCTL_EL1PCTEN
CGT_CNTHCTL_EL1PTEN
__NR_CGT_GROUP_IDS__

Definition at line 32 of file emulate-nested.c.

                  {
    /* Indicates no coarse trap control */
    __RESERVED__,
 
    /*
     * The first batch of IDs denote coarse trapping that are used
     * on their own instead of being part of a combination of
     * trap controls.
     */
    CGT_HCR_TID1,
    CGT_HCR_TID2,
    CGT_HCR_TID3,
    CGT_HCR_IMO,
    CGT_HCR_FMO,
    CGT_HCR_TIDCP,
    CGT_HCR_TACR,
    CGT_HCR_TSW,
    CGT_HCR_TPC,
    CGT_HCR_TPU,
    CGT_HCR_TTLB,
    CGT_HCR_TVM,
    CGT_HCR_TDZ,
    CGT_HCR_TRVM,
    CGT_HCR_TLOR,
    CGT_HCR_TERR,
    CGT_HCR_APK,
    CGT_HCR_NV,
    CGT_HCR_NV_nNV2,
    CGT_HCR_NV1_nNV2,
    CGT_HCR_AT,
    CGT_HCR_nFIEN,
    CGT_HCR_TID4,
    CGT_HCR_TICAB,
    CGT_HCR_TOCU,
    CGT_HCR_ENSCXT,
    CGT_HCR_TTLBIS,
    CGT_HCR_TTLBOS,
 
    CGT_MDCR_TPMCR,
    CGT_MDCR_TPM,
    CGT_MDCR_TDE,
    CGT_MDCR_TDA,
    CGT_MDCR_TDOSA,
    CGT_MDCR_TDRA,
    CGT_MDCR_E2PB,
    CGT_MDCR_TPMS,
    CGT_MDCR_TTRF,
    CGT_MDCR_E2TB,
    CGT_MDCR_TDCC,
 
    /*
     * Anything after this point is a combination of coarse trap
     * controls, which must all be evaluated to decide what to do.
     */
    __MULTIPLE_CONTROL_BITS__,
    CGT_HCR_IMO_FMO = __MULTIPLE_CONTROL_BITS__,
    CGT_HCR_TID2_TID4,
    CGT_HCR_TTLB_TTLBIS,
    CGT_HCR_TTLB_TTLBOS,
    CGT_HCR_TVM_TRVM,
    CGT_HCR_TPU_TICAB,
    CGT_HCR_TPU_TOCU,
    CGT_HCR_NV1_nNV2_ENSCXT,
    CGT_MDCR_TPM_TPMCR,
    CGT_MDCR_TDE_TDA,
    CGT_MDCR_TDE_TDOSA,
    CGT_MDCR_TDE_TDRA,
    CGT_MDCR_TDCC_TDE_TDA,
 
    /*
     * Anything after this point requires a callback evaluating a
     * complex trap condition. Ugly stuff.
     */
    __COMPLEX_CONDITIONS__,
    CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,
    CGT_CNTHCTL_EL1PTEN,
 
    /* Must be last */
    __NR_CGT_GROUP_IDS__
};

fg_filter_id

enum fg_filter_id

Enumerator
__NO_FGF__
HCRX_FGTnXS
__NR_FG_FILTER_IDS__

Definition at line 1021 of file emulate-nested.c.

                  {
    __NO_FGF__,
    HCRX_FGTnXS,
 
    /* Must be last */
    __NR_FG_FILTER_IDS__
};

fgt_group_id

enum fgt_group_id

Enumerator
__NO_FGT_GROUP__
HFGxTR_GROUP
HDFGRTR_GROUP
HDFGWTR_GROUP
HFGITR_GROUP
HAFGRTR_GROUP
__NR_FGT_GROUP_IDS__

Definition at line 1009 of file emulate-nested.c.

                  {
    __NO_FGT_GROUP__,
    HFGxTR_GROUP,
    HDFGRTR_GROUP,
    HDFGWTR_GROUP,
    HFGITR_GROUP,
    HAFGRTR_GROUP,
 
    /* Must be last */
    __NR_FGT_GROUP_IDS__
};

trap_behaviour

enum trap_behaviour

Enumerator
BEHAVE_HANDLE_LOCALLY
BEHAVE_FORWARD_READ
BEHAVE_FORWARD_WRITE
BEHAVE_FORWARD_ANY

Definition at line 17 of file emulate-nested.c.

                    {
    BEHAVE_HANDLE_LOCALLY   = 0,
    BEHAVE_FORWARD_READ = BIT(0),
    BEHAVE_FORWARD_WRITE    = BIT(1),
    BEHAVE_FORWARD_ANY  = BEHAVE_FORWARD_READ | BEHAVE_FORWARD_WRITE,
};

Function Documentation

__check_nv_sr_forward()

bool __check_nv_sr_forward

(

struct kvm_vcpu *

vcpu

)

Definition at line 1908 of file emulate-nested.c.

{
    union trap_config tc;
    enum trap_behaviour b;
    bool is_read;
    u32 sysreg;
    u64 esr, val;
 
    if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
        return false;
 
    esr = kvm_vcpu_get_esr(vcpu);
    sysreg = esr_sys64_to_sysreg(esr);
    is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
 
    tc = get_trap_config(sysreg);
 
    /*
     * A value of 0 for the whole entry means that we know nothing
     * for this sysreg, and that it cannot be re-injected into the
     * nested hypervisor. In this situation, let's cut it short.
     *
     * Note that ultimately, we could also make use of the xarray
     * to store the index of the sysreg in the local descriptor
     * array, avoiding another search... Hint, hint...
     */
    if (!tc.val)
        return false;
 
    switch ((enum fgt_group_id)tc.fgt) {
    case __NO_FGT_GROUP__:
        break;
 
    case HFGxTR_GROUP:
        if (is_read)
            val = sanitised_sys_reg(vcpu, HFGRTR_EL2);
        else
            val = sanitised_sys_reg(vcpu, HFGWTR_EL2);
        break;
 
    case HDFGRTR_GROUP:
    case HDFGWTR_GROUP:
        if (is_read)
            val = sanitised_sys_reg(vcpu, HDFGRTR_EL2);
        else
            val = sanitised_sys_reg(vcpu, HDFGWTR_EL2);
        break;
 
    case HAFGRTR_GROUP:
        val = sanitised_sys_reg(vcpu, HAFGRTR_EL2);
        break;
 
    case HFGITR_GROUP:
        val = sanitised_sys_reg(vcpu, HFGITR_EL2);
        switch (tc.fgf) {
            u64 tmp;
 
        case __NO_FGF__:
            break;
 
        case HCRX_FGTnXS:
            tmp = sanitised_sys_reg(vcpu, HCRX_EL2);
            if (tmp & HCRX_EL2_FGTnXS)
                tc.fgt = __NO_FGT_GROUP__;
        }
        break;
 
    case __NR_FGT_GROUP_IDS__:
        /* Something is really wrong, bail out */
        WARN_ONCE(1, "__NR_FGT_GROUP_IDS__");
        return false;
    }
 
    if (tc.fgt != __NO_FGT_GROUP__ && check_fgt_bit(val, tc))
        goto inject;
 
    b = compute_trap_behaviour(vcpu, tc);
 
    if (((b & BEHAVE_FORWARD_READ) && is_read) ||
        ((b & BEHAVE_FORWARD_WRITE) && !is_read))
        goto inject;
 
    return false;
 
inject:
    trace_kvm_forward_sysreg_trap(vcpu, sysreg, is_read);
 
    kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__compute_trap_behaviour()

static enum trap_behaviour __compute_trap_behaviour ( struct kvm_vcpu *  vcpu, const enum cgt_group_id  id, enum trap_behaviour  b  )

static

Definition at line 1760 of file emulate-nested.c.

{
    switch (id) {
        const enum cgt_group_id *cgids;
 
    case __RESERVED__ ... __MULTIPLE_CONTROL_BITS__ - 1:
        if (likely(id != __RESERVED__))
            b |= get_behaviour(vcpu, &coarse_trap_bits[id]);
        break;
    case __MULTIPLE_CONTROL_BITS__ ... __COMPLEX_CONDITIONS__ - 1:
        /* Yes, this is recursive. Don't do anything stupid. */
        cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
        for (int i = 0; cgids[i] != __RESERVED__; i++)
            b |= __compute_trap_behaviour(vcpu, cgids[i], b);
        break;
    default:
        if (ARRAY_SIZE(ccc))
            b |= ccc[id -  __COMPLEX_CONDITIONS__](vcpu);
        break;
    }
 
    return b;
}

Here is the call graph for this function:

check_cnthctl_el1pcten()

static enum trap_behaviour check_cnthctl_el1pcten ( struct kvm_vcpu *  vcpu )

static

Definition at line 387 of file emulate-nested.c.

{
    if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCTEN << 10))
        return BEHAVE_HANDLE_LOCALLY;
 
    return BEHAVE_FORWARD_ANY;
}

check_cnthctl_el1pten()

static enum trap_behaviour check_cnthctl_el1pten ( struct kvm_vcpu *  vcpu )

static

Definition at line 387 of file emulate-nested.c.

{
    if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCEN << 10))
        return BEHAVE_HANDLE_LOCALLY;
 
    return BEHAVE_FORWARD_ANY;
}

check_fgt_bit()

static bool check_fgt_bit ( u64  val, const union trap_config  tc  )

static

Definition at line 1895 of file emulate-nested.c.

{
    return ((val >> tc.bit) & 1) == tc.pol;
}

Here is the caller graph for this function:

compute_trap_behaviour()

static enum trap_behaviour compute_trap_behaviour ( struct kvm_vcpu *  vcpu, const union trap_config  tc  )

static

Definition at line 1760 of file emulate-nested.c.

{
    enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
 
    return __compute_trap_behaviour(vcpu, tc.cgt, b);
}

Here is the caller graph for this function:

DEFINE_XARRAY()

static DEFINE_XARRAY ( sr_forward_xa  )

static

get_behaviour()

static enum trap_behaviour get_behaviour ( struct kvm_vcpu *  vcpu, const struct trap_bits *  tb  )

static

Definition at line 1760 of file emulate-nested.c.

{
    enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
    u64 val;
 
    val = __vcpu_sys_reg(vcpu, tb->index);
    if ((val & tb->mask) == tb->value)
        b |= tb->behaviour;
 
    return b;
}

get_sanitized_cnthctl()

static u64 get_sanitized_cnthctl ( struct kvm_vcpu *  vcpu )

static

Definition at line 387 of file emulate-nested.c.

{
    u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
 
    if (!vcpu_el2_e2h_is_set(vcpu))
        val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
 
    return val & ((CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN) << 10);
}

get_trap_config()

static union trap_config get_trap_config ( u32  sysreg )

static

Definition at line 1044 of file emulate-nested.c.

{
    return (union trap_config) {
        .val = xa_to_value(xa_load(&sr_forward_xa, sysreg)),
    };
}

Here is the caller graph for this function:

kvm_check_illegal_exception_return()

static u64 kvm_check_illegal_exception_return ( struct kvm_vcpu *  vcpu, u64  spsr  )

static

Definition at line 1999 of file emulate-nested.c.

{
    u64 mode = spsr & PSR_MODE_MASK;
 
    /*
     * Possible causes for an Illegal Exception Return from EL2:
     * - trying to return to EL3
     * - trying to return to an illegal M value
     * - trying to return to a 32bit EL
     * - trying to return to EL1 with HCR_EL2.TGE set
     */
    if (mode == PSR_MODE_EL3t   || mode == PSR_MODE_EL3h ||
        mode == 0b00001         || (mode & BIT(1))       ||
        (spsr & PSR_MODE32_BIT) ||
        (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
                       mode == PSR_MODE_EL1h))) {
        /*
         * The guest is playing with our nerves. Preserve EL, SP,
         * masks, flags from the existing PSTATE, and set IL.
         * The HW will then generate an Illegal State Exception
         * immediately after ERET.
         */
        spsr = *vcpu_cpsr(vcpu);
 
        spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
             PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
             PSR_MODE_MASK | PSR_MODE32_BIT);
        spsr |= PSR_IL_BIT;
    }
 
    return spsr;
}

Here is the caller graph for this function:

kvm_emulate_nested_eret()

void kvm_emulate_nested_eret

(

struct kvm_vcpu *

vcpu

)

Definition at line 2032 of file emulate-nested.c.

{
    u64 spsr, elr, mode;
    bool direct_eret;
 
    /*
     * Going through the whole put/load motions is a waste of time
     * if this is a VHE guest hypervisor returning to its own
     * userspace, or the hypervisor performing a local exception
     * return. No need to save/restore registers, no need to
     * switch S2 MMU. Just do the canonical ERET.
     */
    spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
    spsr = kvm_check_illegal_exception_return(vcpu, spsr);
 
    mode = spsr & (PSR_MODE_MASK | PSR_MODE32_BIT);
 
    direct_eret  = (mode == PSR_MODE_EL0t &&
            vcpu_el2_e2h_is_set(vcpu) &&
            vcpu_el2_tge_is_set(vcpu));
    direct_eret |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
 
    if (direct_eret) {
        *vcpu_pc(vcpu) = vcpu_read_sys_reg(vcpu, ELR_EL2);
        *vcpu_cpsr(vcpu) = spsr;
        trace_kvm_nested_eret(vcpu, *vcpu_pc(vcpu), spsr);
        return;
    }
 
    preempt_disable();
    kvm_arch_vcpu_put(vcpu);
 
    elr = __vcpu_sys_reg(vcpu, ELR_EL2);
 
    trace_kvm_nested_eret(vcpu, elr, spsr);
 
    /*
     * Note that the current exception level is always the virtual EL2,
     * since we set HCR_EL2.NV bit only when entering the virtual EL2.
     */
    *vcpu_pc(vcpu) = elr;
    *vcpu_cpsr(vcpu) = spsr;
 
    kvm_arch_vcpu_load(vcpu, smp_processor_id());
    preempt_enable();
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_inject_el2_exception()

static void kvm_inject_el2_exception ( struct kvm_vcpu *  vcpu, u64  esr_el2, enum exception_type  type  )

static

Definition at line 2079 of file emulate-nested.c.

{
    trace_kvm_inject_nested_exception(vcpu, esr_el2, type);
 
    switch (type) {
    case except_type_sync:
        kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
        vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
        break;
    case except_type_irq:
        kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
        break;
    default:
        WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_inject_nested()

static int kvm_inject_nested ( struct kvm_vcpu *  vcpu, u64  esr_el2, enum exception_type  type  )

static

Definition at line 2101 of file emulate-nested.c.

{
    u64 pstate, mode;
    bool direct_inject;
 
    if (!vcpu_has_nv(vcpu)) {
        kvm_err("Unexpected call to %s for the non-nesting configuration\n",
                __func__);
        return -EINVAL;
    }
 
    /*
     * As for ERET, we can avoid doing too much on the injection path by
     * checking that we either took the exception from a VHE host
     * userspace or from vEL2. In these cases, there is no change in
     * translation regime (or anything else), so let's do as little as
     * possible.
     */
    pstate = *vcpu_cpsr(vcpu);
    mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT);
 
    direct_inject  = (mode == PSR_MODE_EL0t &&
              vcpu_el2_e2h_is_set(vcpu) &&
              vcpu_el2_tge_is_set(vcpu));
    direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
 
    if (direct_inject) {
        kvm_inject_el2_exception(vcpu, esr_el2, type);
        return 1;
    }
 
    preempt_disable();
 
    /*
     * We may have an exception or PC update in the EL0/EL1 context.
     * Commit it before entering EL2.
     */
    __kvm_adjust_pc(vcpu);
 
    kvm_arch_vcpu_put(vcpu);
 
    kvm_inject_el2_exception(vcpu, esr_el2, type);
 
    /*
     * A hard requirement is that a switch between EL1 and EL2
     * contexts has to happen between a put/load, so that we can
     * pick the correct timer and interrupt configuration, among
     * other things.
     *
     * Make sure the exception actually took place before we load
     * the new context.
     */
    __kvm_adjust_pc(vcpu);
 
    kvm_arch_vcpu_load(vcpu, smp_processor_id());
    preempt_enable();
 
    return 1;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_inject_nested_irq()

int kvm_inject_nested_irq

(

struct kvm_vcpu *

vcpu

)

Definition at line 2167 of file emulate-nested.c.

{
    /*
     * Do not inject an irq if the:
     *  - Current exception level is EL2, and
     *  - virtual HCR_EL2.TGE == 0
     *  - virtual HCR_EL2.IMO == 0
     *
     * See Table D1-17 "Physical interrupt target and masking when EL3 is
     * not implemented and EL2 is implemented" in ARM DDI 0487C.a.
     */
 
    if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
        !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
        return 1;
 
    /* esr_el2 value doesn't matter for exits due to irqs. */
    return kvm_inject_nested(vcpu, 0, except_type_irq);
}

kvm_inject_nested_sync()

int kvm_inject_nested_sync	(	struct kvm_vcpu *	vcpu,
		u64	esr_el2
	)

Definition at line 2162 of file emulate-nested.c.

{
    return kvm_inject_nested(vcpu, esr_el2, except_type_sync);
}

Here is the call graph for this function:

Here is the caller graph for this function:

populate_nv_trap_config()

int __init populate_nv_trap_config

(

void

)

Definition at line 1760 of file emulate-nested.c.

{
    int ret = 0;
 
    BUILD_BUG_ON(sizeof(union trap_config) != sizeof(void *));
    BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS));
    BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS));
    BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS));
 
    for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) {
        const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i];
        void *prev;
 
        if (cgt->tc.val & BIT(63)) {
            kvm_err("CGT[%d] has MBZ bit set\n", i);
            ret = -EINVAL;
        }
 
        if (cgt->encoding != cgt->end) {
            prev = xa_store_range(&sr_forward_xa,
                          cgt->encoding, cgt->end,
                          xa_mk_value(cgt->tc.val),
                          GFP_KERNEL);
        } else {
            prev = xa_store(&sr_forward_xa, cgt->encoding,
                    xa_mk_value(cgt->tc.val), GFP_KERNEL);
            if (prev && !xa_is_err(prev)) {
                ret = -EINVAL;
                print_nv_trap_error(cgt, "Duplicate CGT", ret);
            }
        }
 
        if (xa_is_err(prev)) {
            ret = xa_err(prev);
            print_nv_trap_error(cgt, "Failed CGT insertion", ret);
        }
    }
 
    kvm_info("nv: %ld coarse grained trap handlers\n",
         ARRAY_SIZE(encoding_to_cgt));
 
    if (!cpus_have_final_cap(ARM64_HAS_FGT))
        goto check_mcb;
 
    for (int i = 0; i < ARRAY_SIZE(encoding_to_fgt); i++) {
        const struct encoding_to_trap_config *fgt = &encoding_to_fgt[i];
        union trap_config tc;
 
        if (fgt->tc.fgt >= __NR_FGT_GROUP_IDS__) {
            ret = -EINVAL;
            print_nv_trap_error(fgt, "Invalid FGT", ret);
        }
 
        tc = get_trap_config(fgt->encoding);
 
        if (tc.fgt) {
            ret = -EINVAL;
            print_nv_trap_error(fgt, "Duplicate FGT", ret);
        }
 
        tc.val |= fgt->tc.val;
        xa_store(&sr_forward_xa, fgt->encoding,
             xa_mk_value(tc.val), GFP_KERNEL);
    }
 
    kvm_info("nv: %ld fine grained trap handlers\n",
         ARRAY_SIZE(encoding_to_fgt));
 
check_mcb:
    for (int id = __MULTIPLE_CONTROL_BITS__; id < __COMPLEX_CONDITIONS__; id++) {
        const enum cgt_group_id *cgids;
 
        cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
 
        for (int i = 0; cgids[i] != __RESERVED__; i++) {
            if (cgids[i] >= __MULTIPLE_CONTROL_BITS__) {
                kvm_err("Recursive MCB %d/%d\n", id, cgids[i]);
                ret = -EINVAL;
            }
        }
    }
 
    if (ret)
        xa_destroy(&sr_forward_xa);
 
    return ret;
}

Here is the caller graph for this function:

print_nv_trap_error()

static __init void print_nv_trap_error ( const struct encoding_to_trap_config *  tc, const char *  type, int  err  )

static

Definition at line 1745 of file emulate-nested.c.

{
    kvm_err("%s line %d encoding range "
        "(%d, %d, %d, %d, %d) - (%d, %d, %d, %d, %d) (err=%d)\n",
        type, tc->line,
        sys_reg_Op0(tc->encoding), sys_reg_Op1(tc->encoding),
        sys_reg_CRn(tc->encoding), sys_reg_CRm(tc->encoding),
        sys_reg_Op2(tc->encoding),
        sys_reg_Op0(tc->end), sys_reg_Op1(tc->end),
        sys_reg_CRn(tc->end), sys_reg_CRm(tc->end),
        sys_reg_Op2(tc->end),
        err);
}

Here is the caller graph for this function:

Variable Documentation

__initconst

const struct encoding_to_trap_config encoding_to_fgt [] __initconst

static

Definition at line 476 of file emulate-nested.c.

ccc

const complex_condition_check ccc[]

static

Initial value:

= {
    CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),
    CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),
}

Definition at line 416 of file emulate-nested.c.

coarse_control_combo

enum cgt_group_id* coarse_control_combo[]

static

Initial value:

= {
    MCB(CGT_HCR_IMO_FMO,        CGT_HCR_IMO, CGT_HCR_FMO),
    MCB(CGT_HCR_TID2_TID4,      CGT_HCR_TID2, CGT_HCR_TID4),
    MCB(CGT_HCR_TTLB_TTLBIS,    CGT_HCR_TTLB, CGT_HCR_TTLBIS),
    MCB(CGT_HCR_TTLB_TTLBOS,    CGT_HCR_TTLB, CGT_HCR_TTLBOS),
    MCB(CGT_HCR_TVM_TRVM,       CGT_HCR_TVM, CGT_HCR_TRVM),
    MCB(CGT_HCR_TPU_TICAB,      CGT_HCR_TPU, CGT_HCR_TICAB),
    MCB(CGT_HCR_TPU_TOCU,       CGT_HCR_TPU, CGT_HCR_TOCU),
    MCB(CGT_HCR_NV1_nNV2_ENSCXT,    CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
    MCB(CGT_MDCR_TPM_TPMCR,     CGT_MDCR_TPM, CGT_MDCR_TPMCR),
    MCB(CGT_MDCR_TDE_TDA,       CGT_MDCR_TDE, CGT_MDCR_TDA),
    MCB(CGT_MDCR_TDE_TDOSA,     CGT_MDCR_TDE, CGT_MDCR_TDOSA),
    MCB(CGT_MDCR_TDE_TDRA,      CGT_MDCR_TDE, CGT_MDCR_TDRA),
    MCB(CGT_MDCR_TDCC_TDE_TDA,  CGT_MDCR_TDCC, CGT_MDCR_TDE, CGT_MDCR_TDA),
}

Definition at line 356 of file emulate-nested.c.

coarse_trap_bits

const struct trap_bits coarse_trap_bits[]

static

Definition at line 1 of file emulate-nested.c.