va_layout.c File Reference

#include <linux/kvm_host.h>
#include <linux/random.h>
#include <linux/memblock.h>
#include <asm/alternative.h>
#include <asm/debug-monitors.h>
#include <asm/insn.h>
#include <asm/kvm_mmu.h>
#include <asm/memory.h>

Include dependency graph for va_layout.c:

Go to the source code of this file.

Functions
static u64	__early_kern_hyp_va (u64 addr)
static void	init_hyp_physvirt_offset (void)
__init void	kvm_compute_layout (void)
__init void	kvm_apply_hyp_relocations (void)
static u32	compute_instruction (int n, u32 rd, u32 rn)
void __init	kvm_update_va_mask (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
void	kvm_patch_vector_branch (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
static void	generate_mov_q (u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
void	kvm_get_kimage_voffset (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
void	kvm_compute_final_ctr_el0 (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)

Variables
static u8	tag_lsb
static u64	tag_val
static u64	va_mask

Function Documentation

__early_kern_hyp_va()

static u64 __early_kern_hyp_va ( u64  addr )

static

Definition at line 29 of file va_layout.c.

{
    addr &= va_mask;
    addr |= tag_val << tag_lsb;
    return addr;
}

Here is the caller graph for this function:

compute_instruction()

static u32 compute_instruction ( int  n, u32  rd, u32  rn  )

static

Definition at line 112 of file va_layout.c.

{
    u32 insn = AARCH64_BREAK_FAULT;
 
    switch (n) {
    case 0:
        insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
                              AARCH64_INSN_VARIANT_64BIT,
                              rn, rd, va_mask);
        break;
 
    case 1:
        /* ROR is a variant of EXTR with Rm = Rn */
        insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
                         rn, rn, rd,
                         tag_lsb);
        break;
 
    case 2:
        insn = aarch64_insn_gen_add_sub_imm(rd, rn,
                            tag_val & GENMASK(11, 0),
                            AARCH64_INSN_VARIANT_64BIT,
                            AARCH64_INSN_ADSB_ADD);
        break;
 
    case 3:
        insn = aarch64_insn_gen_add_sub_imm(rd, rn,
                            tag_val & GENMASK(23, 12),
                            AARCH64_INSN_VARIANT_64BIT,
                            AARCH64_INSN_ADSB_ADD);
        break;
 
    case 4:
        /* ROR is a variant of EXTR with Rm = Rn */
        insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
                         rn, rn, rd, 64 - tag_lsb);
        break;
    }
 
    return insn;
}

Here is the caller graph for this function:

generate_mov_q()

static void generate_mov_q ( u64  val, __le32 *  origptr, __le32 *  updptr, int  nr_inst  )

static

Definition at line 244 of file va_layout.c.

{
    u32 insn, oinsn, rd;
 
    BUG_ON(nr_inst != 4);
 
    /* Compute target register */
    oinsn = le32_to_cpu(*origptr);
    rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
 
    /* movz rd, #(val & 0xffff) */
    insn = aarch64_insn_gen_movewide(rd,
                     (u16)val,
                     0,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_ZERO);
    *updptr++ = cpu_to_le32(insn);
 
    /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
    insn = aarch64_insn_gen_movewide(rd,
                     (u16)(val >> 16),
                     16,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_KEEP);
    *updptr++ = cpu_to_le32(insn);
 
    /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
    insn = aarch64_insn_gen_movewide(rd,
                     (u16)(val >> 32),
                     32,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_KEEP);
    *updptr++ = cpu_to_le32(insn);
 
    /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
    insn = aarch64_insn_gen_movewide(rd,
                     (u16)(val >> 48),
                     48,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_KEEP);
    *updptr++ = cpu_to_le32(insn);
}

Here is the caller graph for this function:

init_hyp_physvirt_offset()

static void init_hyp_physvirt_offset ( void  )

static

Definition at line 39 of file va_layout.c.

{
    u64 kern_va, hyp_va;
 
    /* Compute the offset from the hyp VA and PA of a random symbol. */
    kern_va = (u64)lm_alias(__hyp_text_start);
    hyp_va = __early_kern_hyp_va(kern_va);
    hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_apply_hyp_relocations()

__init void kvm_apply_hyp_relocations

(

void

)

Definition at line 89 of file va_layout.c.

{
    int32_t *rel;
    int32_t *begin = (int32_t *)__hyp_reloc_begin;
    int32_t *end = (int32_t *)__hyp_reloc_end;
 
    for (rel = begin; rel < end; ++rel) {
        uintptr_t *ptr, kimg_va;
 
        /*
         * Each entry contains a 32-bit relative offset from itself
         * to a kimg VA position.
         */
        ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
 
        /* Read the kimg VA value at the relocation address. */
        kimg_va = *ptr;
 
        /* Convert to hyp VA and store back to the relocation address. */
        *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
    }
}

Here is the call graph for this function:

kvm_compute_final_ctr_el0()

void kvm_compute_final_ctr_el0	(	struct alt_instr *	alt,
		__le32 *	origptr,
		__le32 *	updptr,
		int	nr_inst
	)

Definition at line 293 of file va_layout.c.

{
    generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0),
               origptr, updptr, nr_inst);
}

Here is the call graph for this function:

kvm_compute_layout()

__init void kvm_compute_layout

(

void

)

Definition at line 60 of file va_layout.c.

{
    phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
    u64 hyp_va_msb;
 
    /* Where is my RAM region? */
    hyp_va_msb  = idmap_addr & BIT(vabits_actual - 1);
    hyp_va_msb ^= BIT(vabits_actual - 1);
 
    tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
            (u64)(high_memory - 1));
 
    va_mask = GENMASK_ULL(tag_lsb - 1, 0);
    tag_val = hyp_va_msb;
 
    if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
        /* We have some free bits to insert a random tag. */
        tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
    }
    tag_val >>= tag_lsb;
 
    init_hyp_physvirt_offset();
}

Here is the call graph for this function:

kvm_get_kimage_voffset()

void kvm_get_kimage_voffset	(	struct alt_instr *	alt,
		__le32 *	origptr,
		__le32 *	updptr,
		int	nr_inst
	)

Definition at line 287 of file va_layout.c.

{
    generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
}

Here is the call graph for this function:

kvm_patch_vector_branch()

void kvm_patch_vector_branch	(	struct alt_instr *	alt,
		__le32 *	origptr,
		__le32 *	updptr,
		int	nr_inst
	)

Definition at line 188 of file va_layout.c.

{
    u64 addr;
    u32 insn;
 
    BUG_ON(nr_inst != 4);
 
    if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
        WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
        return;
 
    /*
     * Compute HYP VA by using the same computation as kern_hyp_va()
     */
    addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
 
    /* Use PC[10:7] to branch to the same vector in KVM */
    addr |= ((u64)origptr & GENMASK_ULL(10, 7));
 
    /*
     * Branch over the preamble in order to avoid the initial store on
     * the stack (which we already perform in the hardening vectors).
     */
    addr += KVM_VECTOR_PREAMBLE;
 
    /* movz x0, #(addr & 0xffff) */
    insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
                     (u16)addr,
                     0,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_ZERO);
    *updptr++ = cpu_to_le32(insn);
 
    /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
    insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
                     (u16)(addr >> 16),
                     16,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_KEEP);
    *updptr++ = cpu_to_le32(insn);
 
    /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
    insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
                     (u16)(addr >> 32),
                     32,
                     AARCH64_INSN_VARIANT_64BIT,
                     AARCH64_INSN_MOVEWIDE_KEEP);
    *updptr++ = cpu_to_le32(insn);
 
    /* br x0 */
    insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
                       AARCH64_INSN_BRANCH_NOLINK);
    *updptr++ = cpu_to_le32(insn);
}

Here is the call graph for this function:

kvm_update_va_mask()

void __init kvm_update_va_mask	(	struct alt_instr *	alt,
		__le32 *	origptr,
		__le32 *	updptr,
		int	nr_inst
	)

Definition at line 154 of file va_layout.c.

{
    int i;
 
    BUG_ON(nr_inst != 5);
 
    for (i = 0; i < nr_inst; i++) {
        u32 rd, rn, insn, oinsn;
 
        /*
         * VHE doesn't need any address translation, let's NOP
         * everything.
         *
         * Alternatively, if the tag is zero (because the layout
         * dictates it and we don't have any spare bits in the
         * address), NOP everything after masking the kernel VA.
         */
        if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
            updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
            continue;
        }
 
        oinsn = le32_to_cpu(origptr[i]);
        rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
        rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
 
        insn = compute_instruction(i, rd, rn);
        BUG_ON(insn == AARCH64_BREAK_FAULT);
 
        updptr[i] = cpu_to_le32(insn);
    }
}

Here is the call graph for this function:

Variable Documentation

tag_lsb

u8 tag_lsb

static

Definition at line 19 of file va_layout.c.

tag_val

u64 tag_val

static

Definition at line 23 of file va_layout.c.

va_mask

u64 va_mask

static

Definition at line 24 of file va_layout.c.