KVM
Functions
fpsimd.c File Reference
#include <linux/irqflags.h>
#include <linux/sched.h>
#include <linux/kvm_host.h>
#include <asm/fpsimd.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/sysreg.h>
Include dependency graph for fpsimd.c:

Go to the source code of this file.

Functions

void kvm_vcpu_unshare_task_fp (struct kvm_vcpu *vcpu)
 
int kvm_arch_vcpu_run_map_fp (struct kvm_vcpu *vcpu)
 
void kvm_arch_vcpu_load_fp (struct kvm_vcpu *vcpu)
 
void kvm_arch_vcpu_ctxflush_fp (struct kvm_vcpu *vcpu)
 
void kvm_arch_vcpu_ctxsync_fp (struct kvm_vcpu *vcpu)
 
void kvm_arch_vcpu_put_fp (struct kvm_vcpu *vcpu)
 

Function Documentation

◆ kvm_arch_vcpu_ctxflush_fp()

void kvm_arch_vcpu_ctxflush_fp ( struct kvm_vcpu *  vcpu)

Definition at line 126 of file fpsimd.c.

127 {
128  if (test_thread_flag(TIF_FOREIGN_FPSTATE))
129  vcpu->arch.fp_state = FP_STATE_FREE;
130 }
Here is the caller graph for this function:

◆ kvm_arch_vcpu_ctxsync_fp()

void kvm_arch_vcpu_ctxsync_fp ( struct kvm_vcpu *  vcpu)

Definition at line 139 of file fpsimd.c.

140 {
141  struct cpu_fp_state fp_state;
142 
143  WARN_ON_ONCE(!irqs_disabled());
144 
145  if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {
146 
147  /*
148  * Currently we do not support SME guests so SVCR is
149  * always 0 and we just need a variable to point to.
150  */
151  fp_state.st = &vcpu->arch.ctxt.fp_regs;
152  fp_state.sve_state = vcpu->arch.sve_state;
153  fp_state.sve_vl = vcpu->arch.sve_max_vl;
154  fp_state.sme_state = NULL;
155  fp_state.svcr = &vcpu->arch.svcr;
156  fp_state.fp_type = &vcpu->arch.fp_type;
157 
158  if (vcpu_has_sve(vcpu))
159  fp_state.to_save = FP_STATE_SVE;
160  else
161  fp_state.to_save = FP_STATE_FPSIMD;
162 
163  fpsimd_bind_state_to_cpu(&fp_state);
164 
165  clear_thread_flag(TIF_FOREIGN_FPSTATE);
166  }
167 }
Here is the caller graph for this function:

◆ kvm_arch_vcpu_load_fp()

void kvm_arch_vcpu_load_fp ( struct kvm_vcpu *  vcpu)

Definition at line 75 of file fpsimd.c.

76 {
77  BUG_ON(!current->mm);
78 
79  if (!system_supports_fpsimd())
80  return;
81 
82  fpsimd_kvm_prepare();
83 
84  /*
85  * We will check TIF_FOREIGN_FPSTATE just before entering the
86  * guest in kvm_arch_vcpu_ctxflush_fp() and override this to
87  * FP_STATE_FREE if the flag set.
88  */
89  vcpu->arch.fp_state = FP_STATE_HOST_OWNED;
90 
91  vcpu_clear_flag(vcpu, HOST_SVE_ENABLED);
92  if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
93  vcpu_set_flag(vcpu, HOST_SVE_ENABLED);
94 
95  if (system_supports_sme()) {
96  vcpu_clear_flag(vcpu, HOST_SME_ENABLED);
97  if (read_sysreg(cpacr_el1) & CPACR_EL1_SMEN_EL0EN)
98  vcpu_set_flag(vcpu, HOST_SME_ENABLED);
99 
100  /*
101  * If PSTATE.SM is enabled then save any pending FP
102  * state and disable PSTATE.SM. If we leave PSTATE.SM
103  * enabled and the guest does not enable SME via
104  * CPACR_EL1.SMEN then operations that should be valid
105  * may generate SME traps from EL1 to EL1 which we
106  * can't intercept and which would confuse the guest.
107  *
108  * Do the same for PSTATE.ZA in the case where there
109  * is state in the registers which has not already
110  * been saved, this is very unlikely to happen.
111  */
112  if (read_sysreg_s(SYS_SVCR) & (SVCR_SM_MASK | SVCR_ZA_MASK)) {
113  vcpu->arch.fp_state = FP_STATE_FREE;
114  fpsimd_save_and_flush_cpu_state();
115  }
116  }
117 }
Here is the caller graph for this function:

◆ kvm_arch_vcpu_put_fp()

void kvm_arch_vcpu_put_fp ( struct kvm_vcpu *  vcpu)

Definition at line 175 of file fpsimd.c.

176 {
177  unsigned long flags;
178 
179  local_irq_save(flags);
180 
181  /*
182  * If we have VHE then the Hyp code will reset CPACR_EL1 to
183  * the default value and we need to reenable SME.
184  */
185  if (has_vhe() && system_supports_sme()) {
186  /* Also restore EL0 state seen on entry */
187  if (vcpu_get_flag(vcpu, HOST_SME_ENABLED))
188  sysreg_clear_set(CPACR_EL1, 0,
189  CPACR_EL1_SMEN_EL0EN |
190  CPACR_EL1_SMEN_EL1EN);
191  else
192  sysreg_clear_set(CPACR_EL1,
193  CPACR_EL1_SMEN_EL0EN,
194  CPACR_EL1_SMEN_EL1EN);
195  isb();
196  }
197 
198  if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {
199  if (vcpu_has_sve(vcpu)) {
200  __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
201 
202  /* Restore the VL that was saved when bound to the CPU */
203  if (!has_vhe())
204  sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
205  SYS_ZCR_EL1);
206  }
207 
208  fpsimd_save_and_flush_cpu_state();
209  } else if (has_vhe() && system_supports_sve()) {
210  /*
211  * The FPSIMD/SVE state in the CPU has not been touched, and we
212  * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
213  * reset by kvm_reset_cptr_el2() in the Hyp code, disabling SVE
214  * for EL0. To avoid spurious traps, restore the trap state
215  * seen by kvm_arch_vcpu_load_fp():
216  */
217  if (vcpu_get_flag(vcpu, HOST_SVE_ENABLED))
218  sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN);
219  else
220  sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
221  }
222 
223  local_irq_restore(flags);
224 }
Here is the caller graph for this function:

◆ kvm_arch_vcpu_run_map_fp()

int kvm_arch_vcpu_run_map_fp ( struct kvm_vcpu *  vcpu)

Definition at line 39 of file fpsimd.c.

40 {
41  int ret;
42 
43  struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;
44 
45  kvm_vcpu_unshare_task_fp(vcpu);
46 
47  /* Make sure the host task fpsimd state is visible to hyp: */
48  ret = kvm_share_hyp(fpsimd, fpsimd + 1);
49  if (ret)
50  return ret;
51 
52  vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
53 
54  /*
55  * We need to keep current's task_struct pinned until its data has been
56  * unshared with the hypervisor to make sure it is not re-used by the
57  * kernel and donated to someone else while already shared -- see
58  * kvm_vcpu_unshare_task_fp() for the matching put_task_struct().
59  */
60  if (is_protected_kvm_enabled()) {
61  get_task_struct(current);
62  vcpu->arch.parent_task = current;
63  }
64 
65  return 0;
66 }
kvm_vcpu_unshare_task_fp
void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu)
Definition: fpsimd.c:17
kvm_share_hyp
int kvm_share_hyp(void *from, void *to)
Definition: mmu.c:516
Here is the call graph for this function:
Here is the caller graph for this function:

◆ kvm_vcpu_unshare_task_fp()

void kvm_vcpu_unshare_task_fp ( struct kvm_vcpu *  vcpu)

Definition at line 17 of file fpsimd.c.

18 {
19  struct task_struct *p = vcpu->arch.parent_task;
20  struct user_fpsimd_state *fpsimd;
21 
22  if (!is_protected_kvm_enabled() || !p)
23  return;
24 
25  fpsimd = &p->thread.uw.fpsimd_state;
26  kvm_unshare_hyp(fpsimd, fpsimd + 1);
27  put_task_struct(p);
28 }
kvm_unshare_hyp
void kvm_unshare_hyp(void *from, void *to)
Definition: mmu.c:548
Here is the call graph for this function:
Here is the caller graph for this function:
Generated by doxygen 1.9.1