exception.c File Reference

#include <hyp/adjust_pc.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nested.h>

Include dependency graph for exception.c:

Go to the source code of this file.

Functions
static u64	__vcpu_read_sys_reg (const struct kvm_vcpu *vcpu, int reg)
static void	__vcpu_write_sys_reg (struct kvm_vcpu *vcpu, u64 val, int reg)
static void	__vcpu_write_spsr (struct kvm_vcpu *vcpu, unsigned long target_mode, u64 val)
static void	__vcpu_write_spsr_abt (struct kvm_vcpu *vcpu, u64 val)
static void	__vcpu_write_spsr_und (struct kvm_vcpu *vcpu, u64 val)
static void	enter_exception64 (struct kvm_vcpu *vcpu, unsigned long target_mode, enum exception_type type)
static unsigned long	get_except32_cpsr (struct kvm_vcpu *vcpu, u32 mode)
static void	enter_exception32 (struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
static void	kvm_inject_exception (struct kvm_vcpu *vcpu)
void	__kvm_adjust_pc (struct kvm_vcpu *vcpu)

Variables
static const u8	return_offsets [8][2]

Function Documentation

__kvm_adjust_pc()

void __kvm_adjust_pc

(

struct kvm_vcpu *

vcpu

)

Definition at line 365 of file exception.c.

{
    if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
        kvm_inject_exception(vcpu);
        vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
        vcpu_clear_flag(vcpu, EXCEPT_MASK);
    } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
        kvm_skip_instr(vcpu);
        vcpu_clear_flag(vcpu, INCREMENT_PC);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

__vcpu_read_sys_reg()

static u64 __vcpu_read_sys_reg ( const struct kvm_vcpu *  vcpu, int  reg  )

inlinestatic

Definition at line 23 of file exception.c.

{
    u64 val;
 
    if (unlikely(vcpu_has_nv(vcpu)))
        return vcpu_read_sys_reg(vcpu, reg);
    else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
        return val;
 
    return __vcpu_sys_reg(vcpu, reg);
}

Here is the call graph for this function:

Here is the caller graph for this function:

__vcpu_write_spsr()

static void __vcpu_write_spsr ( struct kvm_vcpu *  vcpu, unsigned long  target_mode, u64  val  )

static

Definition at line 43 of file exception.c.

{
    if (unlikely(vcpu_has_nv(vcpu))) {
        if (target_mode == PSR_MODE_EL1h)
            vcpu_write_sys_reg(vcpu, val, SPSR_EL1);
        else
            vcpu_write_sys_reg(vcpu, val, SPSR_EL2);
    } else if (has_vhe()) {
        write_sysreg_el1(val, SYS_SPSR);
    } else {
        __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

__vcpu_write_spsr_abt()

static void __vcpu_write_spsr_abt ( struct kvm_vcpu *  vcpu, u64  val  )

static

Definition at line 58 of file exception.c.

{
    if (has_vhe())
        write_sysreg(val, spsr_abt);
    else
        vcpu->arch.ctxt.spsr_abt = val;
}

Here is the caller graph for this function:

__vcpu_write_spsr_und()

static void __vcpu_write_spsr_und ( struct kvm_vcpu *  vcpu, u64  val  )

static

Definition at line 66 of file exception.c.

{
    if (has_vhe())
        write_sysreg(val, spsr_und);
    else
        vcpu->arch.ctxt.spsr_und = val;
}

Here is the caller graph for this function:

__vcpu_write_sys_reg()

static void __vcpu_write_sys_reg ( struct kvm_vcpu *  vcpu, u64  val, int  reg  )

inlinestatic

Definition at line 35 of file exception.c.

{
    if (unlikely(vcpu_has_nv(vcpu)))
        vcpu_write_sys_reg(vcpu, val, reg);
    else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
        __vcpu_sys_reg(vcpu, reg) = val;
}

Here is the call graph for this function:

Here is the caller graph for this function:

enter_exception32()

static void enter_exception32 ( struct kvm_vcpu *  vcpu, u32  mode, u32  vect_offset  )

static

Definition at line 286 of file exception.c.

{
    unsigned long spsr = *vcpu_cpsr(vcpu);
    bool is_thumb = (spsr & PSR_AA32_T_BIT);
    u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
    u32 return_address;
 
    *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
    return_address   = *vcpu_pc(vcpu);
    return_address  += return_offsets[vect_offset >> 2][is_thumb];
 
    /* KVM only enters the ABT and UND modes, so only deal with those */
    switch(mode) {
    case PSR_AA32_MODE_ABT:
        __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
        vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
        break;
 
    case PSR_AA32_MODE_UND:
        __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
        vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
        break;
    }
 
    /* Branch to exception vector */
    if (sctlr & (1 << 13))
        vect_offset += 0xffff0000;
    else /* always have security exceptions */
        vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
 
    *vcpu_pc(vcpu) = vect_offset;
}

Here is the call graph for this function:

Here is the caller graph for this function:

enter_exception64()

static void enter_exception64 ( struct kvm_vcpu *  vcpu, unsigned long  target_mode, enum exception_type  type  )

static

Definition at line 91 of file exception.c.

{
    unsigned long sctlr, vbar, old, new, mode;
    u64 exc_offset;
 
    mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
 
    if      (mode == target_mode)
        exc_offset = CURRENT_EL_SP_ELx_VECTOR;
    else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
        exc_offset = CURRENT_EL_SP_EL0_VECTOR;
    else if (!(mode & PSR_MODE32_BIT))
        exc_offset = LOWER_EL_AArch64_VECTOR;
    else
        exc_offset = LOWER_EL_AArch32_VECTOR;
 
    switch (target_mode) {
    case PSR_MODE_EL1h:
        vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
        sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
        __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
        break;
    case PSR_MODE_EL2h:
        vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2);
        sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2);
        __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2);
        break;
    default:
        /* Don't do that */
        BUG();
    }
 
    *vcpu_pc(vcpu) = vbar + exc_offset + type;
 
    old = *vcpu_cpsr(vcpu);
    new = 0;
 
    new |= (old & PSR_N_BIT);
    new |= (old & PSR_Z_BIT);
    new |= (old & PSR_C_BIT);
    new |= (old & PSR_V_BIT);
 
    if (kvm_has_mte(kern_hyp_va(vcpu->kvm)))
        new |= PSR_TCO_BIT;
 
    new |= (old & PSR_DIT_BIT);
 
    // PSTATE.UAO is set to zero upon any exception to AArch64
    // See ARM DDI 0487E.a, page D5-2579.
 
    // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
    // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
    // See ARM DDI 0487E.a, page D5-2578.
    new |= (old & PSR_PAN_BIT);
    if (!(sctlr & SCTLR_EL1_SPAN))
        new |= PSR_PAN_BIT;
 
    // PSTATE.SS is set to zero upon any exception to AArch64
    // See ARM DDI 0487E.a, page D2-2452.
 
    // PSTATE.IL is set to zero upon any exception to AArch64
    // See ARM DDI 0487E.a, page D1-2306.
 
    // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
    // See ARM DDI 0487E.a, page D13-3258
    if (sctlr & SCTLR_ELx_DSSBS)
        new |= PSR_SSBS_BIT;
 
    // PSTATE.BTYPE is set to zero upon any exception to AArch64
    // See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
 
    new |= PSR_D_BIT;
    new |= PSR_A_BIT;
    new |= PSR_I_BIT;
    new |= PSR_F_BIT;
 
    new |= target_mode;
 
    *vcpu_cpsr(vcpu) = new;
    __vcpu_write_spsr(vcpu, target_mode, old);
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_except32_cpsr()

static unsigned long get_except32_cpsr ( struct kvm_vcpu *  vcpu, u32  mode  )

static

Definition at line 192 of file exception.c.

{
    u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
    unsigned long old, new;
 
    old = *vcpu_cpsr(vcpu);
    new = 0;
 
    new |= (old & PSR_AA32_N_BIT);
    new |= (old & PSR_AA32_Z_BIT);
    new |= (old & PSR_AA32_C_BIT);
    new |= (old & PSR_AA32_V_BIT);
    new |= (old & PSR_AA32_Q_BIT);
 
    // CPSR.IT[7:0] are set to zero upon any exception
    // See ARM DDI 0487E.a, section G1.12.3
    // See ARM DDI 0406C.d, section B1.8.3
 
    new |= (old & PSR_AA32_DIT_BIT);
 
    // CPSR.SSBS is set to SCTLR.DSSBS upon any exception
    // See ARM DDI 0487E.a, page G8-6244
    if (sctlr & BIT(31))
        new |= PSR_AA32_SSBS_BIT;
 
    // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
    // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
    // See ARM DDI 0487E.a, page G8-6246
    new |= (old & PSR_AA32_PAN_BIT);
    if (!(sctlr & BIT(23)))
        new |= PSR_AA32_PAN_BIT;
 
    // SS does not exist in AArch32, so ignore
 
    // CPSR.IL is set to zero upon any exception
    // See ARM DDI 0487E.a, page G1-5527
 
    new |= (old & PSR_AA32_GE_MASK);
 
    // CPSR.IT[7:0] are set to zero upon any exception
    // See prior comment above
 
    // CPSR.E is set to SCTLR.EE upon any exception
    // See ARM DDI 0487E.a, page G8-6245
    // See ARM DDI 0406C.d, page B4-1701
    if (sctlr & BIT(25))
        new |= PSR_AA32_E_BIT;
 
    // CPSR.A is unchanged upon an exception to Undefined, Supervisor
    // CPSR.A is set upon an exception to other modes
    // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
    // See ARM DDI 0406C.d, page B1-1182
    new |= (old & PSR_AA32_A_BIT);
    if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
        new |= PSR_AA32_A_BIT;
 
    // CPSR.I is set upon any exception
    // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
    // See ARM DDI 0406C.d, page B1-1182
    new |= PSR_AA32_I_BIT;
 
    // CPSR.F is set upon an exception to FIQ
    // CPSR.F is unchanged upon an exception to other modes
    // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
    // See ARM DDI 0406C.d, page B1-1182
    new |= (old & PSR_AA32_F_BIT);
    if (mode == PSR_AA32_MODE_FIQ)
        new |= PSR_AA32_F_BIT;
 
    // CPSR.T is set to SCTLR.TE upon any exception
    // See ARM DDI 0487E.a, page G8-5514
    // See ARM DDI 0406C.d, page B1-1181
    if (sctlr & BIT(30))
        new |= PSR_AA32_T_BIT;
 
    new |= mode;
 
    return new;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_inject_exception()

static void kvm_inject_exception ( struct kvm_vcpu *  vcpu )

static

Definition at line 319 of file exception.c.

{
    if (vcpu_el1_is_32bit(vcpu)) {
        switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
        case unpack_vcpu_flag(EXCEPT_AA32_UND):
            enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
            break;
        case unpack_vcpu_flag(EXCEPT_AA32_IABT):
            enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
            break;
        case unpack_vcpu_flag(EXCEPT_AA32_DABT):
            enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
            break;
        default:
            /* Err... */
            break;
        }
    } else {
        switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
        case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
            enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
            break;
 
        case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
            enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
            break;
 
        case unpack_vcpu_flag(EXCEPT_AA64_EL2_IRQ):
            enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
            break;
 
        default:
            /*
             * Only EL1_SYNC and EL2_{SYNC,IRQ} makes
             * sense so far. Everything else gets silently
             * ignored.
             */
            break;
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

return_offsets

const u8 return_offsets[8][2]

static

Initial value:

= {
    [0] = { 0, 0 },     
    [1] = { 4, 2 },     
    [2] = { 0, 0 },     
    [3] = { 4, 4 },     
    [4] = { 8, 8 },     
    [5] = { 0, 0 },     
    [6] = { 4, 4 },     
    [7] = { 4, 4 },     
}

Definition at line 275 of file exception.c.