KVM
Functions | Variables
va_layout.c File Reference
#include <linux/kvm_host.h>
#include <linux/random.h>
#include <linux/memblock.h>
#include <asm/alternative.h>
#include <asm/debug-monitors.h>
#include <asm/insn.h>
#include <asm/kvm_mmu.h>
#include <asm/memory.h>
Include dependency graph for va_layout.c:

Go to the source code of this file.

Functions

static u64 __early_kern_hyp_va (u64 addr)
 
static void init_hyp_physvirt_offset (void)
 
__init void kvm_compute_layout (void)
 
__init void kvm_apply_hyp_relocations (void)
 
static u32 compute_instruction (int n, u32 rd, u32 rn)
 
void __init kvm_update_va_mask (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
 
void kvm_patch_vector_branch (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
 
static void generate_mov_q (u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
 
void kvm_get_kimage_voffset (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
 
void kvm_compute_final_ctr_el0 (struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst)
 

Variables

static u8 tag_lsb
 
static u64 tag_val
 
static u64 va_mask
 

Function Documentation

◆ __early_kern_hyp_va()

static u64 __early_kern_hyp_va ( u64  addr)
static

Definition at line 29 of file va_layout.c.

30 {
31  addr &= va_mask;
32  addr |= tag_val << tag_lsb;
33  return addr;
34 }
tag_lsb
static u8 tag_lsb
Definition: va_layout.c:19
tag_val
static u64 tag_val
Definition: va_layout.c:23
va_mask
static u64 va_mask
Definition: va_layout.c:24
Here is the caller graph for this function:

◆ compute_instruction()

static u32 compute_instruction ( int  n,
u32  rd,
u32  rn 
)
static

Definition at line 112 of file va_layout.c.

113 {
114  u32 insn = AARCH64_BREAK_FAULT;
115 
116  switch (n) {
117  case 0:
118  insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
119  AARCH64_INSN_VARIANT_64BIT,
120  rn, rd, va_mask);
121  break;
122 
123  case 1:
124  /* ROR is a variant of EXTR with Rm = Rn */
125  insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
126  rn, rn, rd,
127  tag_lsb);
128  break;
129 
130  case 2:
131  insn = aarch64_insn_gen_add_sub_imm(rd, rn,
132  tag_val & GENMASK(11, 0),
133  AARCH64_INSN_VARIANT_64BIT,
134  AARCH64_INSN_ADSB_ADD);
135  break;
136 
137  case 3:
138  insn = aarch64_insn_gen_add_sub_imm(rd, rn,
139  tag_val & GENMASK(23, 12),
140  AARCH64_INSN_VARIANT_64BIT,
141  AARCH64_INSN_ADSB_ADD);
142  break;
143 
144  case 4:
145  /* ROR is a variant of EXTR with Rm = Rn */
146  insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
147  rn, rn, rd, 64 - tag_lsb);
148  break;
149  }
150 
151  return insn;
152 }
Here is the caller graph for this function:

◆ generate_mov_q()

static void generate_mov_q ( u64  val,
__le32 *  origptr,
__le32 *  updptr,
int  nr_inst 
)
static

Definition at line 244 of file va_layout.c.

245 {
246  u32 insn, oinsn, rd;
247 
248  BUG_ON(nr_inst != 4);
249 
250  /* Compute target register */
251  oinsn = le32_to_cpu(*origptr);
252  rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
253 
254  /* movz rd, #(val & 0xffff) */
255  insn = aarch64_insn_gen_movewide(rd,
256  (u16)val,
257  0,
258  AARCH64_INSN_VARIANT_64BIT,
259  AARCH64_INSN_MOVEWIDE_ZERO);
260  *updptr++ = cpu_to_le32(insn);
261 
262  /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
263  insn = aarch64_insn_gen_movewide(rd,
264  (u16)(val >> 16),
265  16,
266  AARCH64_INSN_VARIANT_64BIT,
267  AARCH64_INSN_MOVEWIDE_KEEP);
268  *updptr++ = cpu_to_le32(insn);
269 
270  /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
271  insn = aarch64_insn_gen_movewide(rd,
272  (u16)(val >> 32),
273  32,
274  AARCH64_INSN_VARIANT_64BIT,
275  AARCH64_INSN_MOVEWIDE_KEEP);
276  *updptr++ = cpu_to_le32(insn);
277 
278  /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
279  insn = aarch64_insn_gen_movewide(rd,
280  (u16)(val >> 48),
281  48,
282  AARCH64_INSN_VARIANT_64BIT,
283  AARCH64_INSN_MOVEWIDE_KEEP);
284  *updptr++ = cpu_to_le32(insn);
285 }
Here is the caller graph for this function:

◆ init_hyp_physvirt_offset()

static void init_hyp_physvirt_offset ( void  )
static

Definition at line 39 of file va_layout.c.

40 {
41  u64 kern_va, hyp_va;
42 
43  /* Compute the offset from the hyp VA and PA of a random symbol. */
44  kern_va = (u64)lm_alias(__hyp_text_start);
45  hyp_va = __early_kern_hyp_va(kern_va);
46  hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
47 }
hyp_physvirt_offset
s64 __ro_after_init hyp_physvirt_offset
Definition: early_alloc.c:13
__early_kern_hyp_va
static u64 __early_kern_hyp_va(u64 addr)
Definition: va_layout.c:29
Here is the call graph for this function:
Here is the caller graph for this function:

◆ kvm_apply_hyp_relocations()

__init void kvm_apply_hyp_relocations ( void  )

Definition at line 89 of file va_layout.c.

90 {
91  int32_t *rel;
92  int32_t *begin = (int32_t *)__hyp_reloc_begin;
93  int32_t *end = (int32_t *)__hyp_reloc_end;
94 
95  for (rel = begin; rel < end; ++rel) {
96  uintptr_t *ptr, kimg_va;
97 
98  /*
99  * Each entry contains a 32-bit relative offset from itself
100  * to a kimg VA position.
101  */
102  ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
103 
104  /* Read the kimg VA value at the relocation address. */
105  kimg_va = *ptr;
106 
107  /* Convert to hyp VA and store back to the relocation address. */
108  *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
109  }
110 }
end
static unsigned long end
Definition: early_alloc.c:16
begin
char * begin
Definition: gen-hyprel.c:132
Here is the call graph for this function:

◆ kvm_compute_final_ctr_el0()

void kvm_compute_final_ctr_el0 ( struct alt_instr *  alt,
__le32 *  origptr,
__le32 *  updptr,
int  nr_inst 
)

Definition at line 293 of file va_layout.c.

295 {
296  generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0),
297  origptr, updptr, nr_inst);
298 }
generate_mov_q
static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
Definition: va_layout.c:244
Here is the call graph for this function:

◆ kvm_compute_layout()

__init void kvm_compute_layout ( void  )

Definition at line 60 of file va_layout.c.

61 {
62  phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
63  u64 hyp_va_msb;
64 
65  /* Where is my RAM region? */
66  hyp_va_msb = idmap_addr & BIT(vabits_actual - 1);
67  hyp_va_msb ^= BIT(vabits_actual - 1);
68 
69  tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
70  (u64)(high_memory - 1));
71 
72  va_mask = GENMASK_ULL(tag_lsb - 1, 0);
73  tag_val = hyp_va_msb;
74 
75  if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
76  /* We have some free bits to insert a random tag. */
77  tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
78  }
79  tag_val >>= tag_lsb;
80 
81  init_hyp_physvirt_offset();
82 }
init_hyp_physvirt_offset
static void init_hyp_physvirt_offset(void)
Definition: va_layout.c:39
Here is the call graph for this function:

◆ kvm_get_kimage_voffset()

void kvm_get_kimage_voffset ( struct alt_instr *  alt,
__le32 *  origptr,
__le32 *  updptr,
int  nr_inst 
)

Definition at line 287 of file va_layout.c.

289 {
290  generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
291 }
Here is the call graph for this function:

◆ kvm_patch_vector_branch()

void kvm_patch_vector_branch ( struct alt_instr *  alt,
__le32 *  origptr,
__le32 *  updptr,
int  nr_inst 
)

Definition at line 188 of file va_layout.c.

190 {
191  u64 addr;
192  u32 insn;
193 
194  BUG_ON(nr_inst != 4);
195 
196  if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
197  WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
198  return;
199 
200  /*
201  * Compute HYP VA by using the same computation as kern_hyp_va()
202  */
203  addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
204 
205  /* Use PC[10:7] to branch to the same vector in KVM */
206  addr |= ((u64)origptr & GENMASK_ULL(10, 7));
207 
208  /*
209  * Branch over the preamble in order to avoid the initial store on
210  * the stack (which we already perform in the hardening vectors).
211  */
212  addr += KVM_VECTOR_PREAMBLE;
213 
214  /* movz x0, #(addr & 0xffff) */
215  insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
216  (u16)addr,
217  0,
218  AARCH64_INSN_VARIANT_64BIT,
219  AARCH64_INSN_MOVEWIDE_ZERO);
220  *updptr++ = cpu_to_le32(insn);
221 
222  /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
223  insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
224  (u16)(addr >> 16),
225  16,
226  AARCH64_INSN_VARIANT_64BIT,
227  AARCH64_INSN_MOVEWIDE_KEEP);
228  *updptr++ = cpu_to_le32(insn);
229 
230  /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
231  insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
232  (u16)(addr >> 32),
233  32,
234  AARCH64_INSN_VARIANT_64BIT,
235  AARCH64_INSN_MOVEWIDE_KEEP);
236  *updptr++ = cpu_to_le32(insn);
237 
238  /* br x0 */
239  insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
240  AARCH64_INSN_BRANCH_NOLINK);
241  *updptr++ = cpu_to_le32(insn);
242 }
Here is the call graph for this function:

◆ kvm_update_va_mask()

void __init kvm_update_va_mask ( struct alt_instr *  alt,
__le32 *  origptr,
__le32 *  updptr,
int  nr_inst 
)

Definition at line 154 of file va_layout.c.

156 {
157  int i;
158 
159  BUG_ON(nr_inst != 5);
160 
161  for (i = 0; i < nr_inst; i++) {
162  u32 rd, rn, insn, oinsn;
163 
164  /*
165  * VHE doesn't need any address translation, let's NOP
166  * everything.
167  *
168  * Alternatively, if the tag is zero (because the layout
169  * dictates it and we don't have any spare bits in the
170  * address), NOP everything after masking the kernel VA.
171  */
172  if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
173  updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
174  continue;
175  }
176 
177  oinsn = le32_to_cpu(origptr[i]);
178  rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
179  rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
180 
181  insn = compute_instruction(i, rd, rn);
182  BUG_ON(insn == AARCH64_BREAK_FAULT);
183 
184  updptr[i] = cpu_to_le32(insn);
185  }
186 }
compute_instruction
static u32 compute_instruction(int n, u32 rd, u32 rn)
Definition: va_layout.c:112
Here is the call graph for this function:

Variable Documentation

◆ tag_lsb

u8 tag_lsb
static

Definition at line 19 of file va_layout.c.

◆ tag_val

u64 tag_val
static

Definition at line 23 of file va_layout.c.

◆ va_mask

u64 va_mask
static

Definition at line 24 of file va_layout.c.

Generated by doxygen 1.9.1