kvm_main.c File Reference

#include <kvm/iodev.h>
#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
#include <linux/syscore_ops.h>
#include <linux/cpu.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/sched/stat.h>
#include <linux/cpumask.h>
#include <linux/smp.h>
#include <linux/anon_inodes.h>
#include <linux/profile.h>
#include <linux/kvm_para.h>
#include <linux/pagemap.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/bitops.h>
#include <linux/spinlock.h>
#include <linux/compat.h>
#include <linux/srcu.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/bsearch.h>
#include <linux/io.h>
#include <linux/lockdep.h>
#include <linux/kthread.h>
#include <linux/suspend.h>
#include <asm/processor.h>
#include <asm/ioctl.h>
#include <linux/uaccess.h>
#include "coalesced_mmio.h"
#include "async_pf.h"
#include "kvm_mm.h"
#include "vfio.h"
#include <trace/events/ipi.h>
#include <trace/events/kvm.h>
#include <linux/kvm_dirty_ring.h>

Include dependency graph for kvm_main.c:

Go to the source code of this file.

Classes
struct	kvm_vm_worker_thread_context

Macros
#define	CREATE_TRACE_POINTS
#define	ITOA_MAX_LEN   12
#define	KVM_COMPAT(c)
#define	KVM_EVENT_CREATE_VM   0
#define	KVM_EVENT_DESTROY_VM   1
#define	KVM_SET_USER_MEMORY_REGION_V1_FLAGS    (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)
#define	SANITY_CHECK_MEM_REGION_FIELD(field)

Functions
	MODULE_AUTHOR ("Qumranet")
	MODULE_LICENSE ("GPL")
	module_param (halt_poll_ns, uint, 0644)
	EXPORT_SYMBOL_GPL (halt_poll_ns)
	module_param (halt_poll_ns_grow, uint, 0644)
	EXPORT_SYMBOL_GPL (halt_poll_ns_grow)
	module_param (halt_poll_ns_grow_start, uint, 0644)
	EXPORT_SYMBOL_GPL (halt_poll_ns_grow_start)
	module_param (halt_poll_ns_shrink, uint, 0644)
	EXPORT_SYMBOL_GPL (halt_poll_ns_shrink)
	DEFINE_MUTEX (kvm_lock)
	LIST_HEAD (vm_list)
static	DEFINE_PER_CPU (struct kvm_vcpu *, kvm_running_vcpu)
	EXPORT_SYMBOL_GPL (kvm_debugfs_dir)
static long	kvm_vcpu_ioctl (struct file *file, unsigned int ioctl, unsigned long arg)
static long	kvm_no_compat_ioctl (struct file *file, unsigned int ioctl, unsigned long arg)
static int	kvm_no_compat_open (struct inode *inode, struct file *file)
static int	hardware_enable_all (void)
static void	hardware_disable_all (void)
static void	kvm_io_bus_destroy (struct kvm_io_bus *bus)
static void	kvm_uevent_notify_change (unsigned int type, struct kvm *kvm)
static	DEFINE_PER_CPU (cpumask_var_t, cpu_kick_mask)
__weak void	kvm_arch_guest_memory_reclaimed (struct kvm *kvm)
bool	kvm_is_zone_device_page (struct page *page)
struct page *	kvm_pfn_to_refcounted_page (kvm_pfn_t pfn)
void	vcpu_load (struct kvm_vcpu *vcpu)
	EXPORT_SYMBOL_GPL (vcpu_load)
void	vcpu_put (struct kvm_vcpu *vcpu)
	EXPORT_SYMBOL_GPL (vcpu_put)
static bool	kvm_request_needs_ipi (struct kvm_vcpu *vcpu, unsigned req)
static void	ack_kick (void *_completed)
static bool	kvm_kick_many_cpus (struct cpumask *cpus, bool wait)
static void	kvm_make_vcpu_request (struct kvm_vcpu *vcpu, unsigned int req, struct cpumask *tmp, int current_cpu)
bool	kvm_make_vcpus_request_mask (struct kvm *kvm, unsigned int req, unsigned long *vcpu_bitmap)
bool	kvm_make_all_cpus_request_except (struct kvm *kvm, unsigned int req, struct kvm_vcpu *except)
bool	kvm_make_all_cpus_request (struct kvm *kvm, unsigned int req)
	EXPORT_SYMBOL_GPL (kvm_make_all_cpus_request)
void	kvm_flush_remote_tlbs (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_flush_remote_tlbs)
void	kvm_flush_remote_tlbs_range (struct kvm *kvm, gfn_t gfn, u64 nr_pages)
void	kvm_flush_remote_tlbs_memslot (struct kvm *kvm, const struct kvm_memory_slot *memslot)
static void	kvm_flush_shadow_all (struct kvm *kvm)
static void	kvm_vcpu_init (struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
static void	kvm_vcpu_destroy (struct kvm_vcpu *vcpu)
void	kvm_destroy_vcpus (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_destroy_vcpus)
static int	kvm_init_mmu_notifier (struct kvm *kvm)
static void	kvm_init_pm_notifier (struct kvm *kvm)
static void	kvm_destroy_pm_notifier (struct kvm *kvm)
static void	kvm_destroy_dirty_bitmap (struct kvm_memory_slot *memslot)
static void	kvm_free_memslot (struct kvm *kvm, struct kvm_memory_slot *slot)
static void	kvm_free_memslots (struct kvm *kvm, struct kvm_memslots *slots)
static umode_t	kvm_stats_debugfs_mode (const struct _kvm_stats_desc *pdesc)
static void	kvm_destroy_vm_debugfs (struct kvm *kvm)
static int	kvm_create_vm_debugfs (struct kvm *kvm, const char *fdname)
int __weak	kvm_arch_post_init_vm (struct kvm *kvm)
void __weak	kvm_arch_pre_destroy_vm (struct kvm *kvm)
int __weak	kvm_arch_create_vm_debugfs (struct kvm *kvm)
static struct kvm *	kvm_create_vm (unsigned long type, const char *fdname)
static void	kvm_destroy_devices (struct kvm *kvm)
static void	kvm_destroy_vm (struct kvm *kvm)
void	kvm_get_kvm (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_get_kvm)
bool	kvm_get_kvm_safe (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_get_kvm_safe)
void	kvm_put_kvm (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_put_kvm)
void	kvm_put_kvm_no_destroy (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_put_kvm_no_destroy)
static int	kvm_vm_release (struct inode *inode, struct file *filp)
static int	kvm_alloc_dirty_bitmap (struct kvm_memory_slot *memslot)
static struct kvm_memslots *	kvm_get_inactive_memslots (struct kvm *kvm, int as_id)
static int	kvm_memslots_get_as_id (struct kvm_memory_slot *a, struct kvm_memory_slot *b)
static void	kvm_insert_gfn_node (struct kvm_memslots *slots, struct kvm_memory_slot *slot)
static void	kvm_erase_gfn_node (struct kvm_memslots *slots, struct kvm_memory_slot *slot)
static void	kvm_replace_gfn_node (struct kvm_memslots *slots, struct kvm_memory_slot *old, struct kvm_memory_slot *new)
static void	kvm_replace_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new)
static int	check_memory_region_flags (struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem)
static void	kvm_swap_active_memslots (struct kvm *kvm, int as_id)
static int	kvm_prepare_memory_region (struct kvm *kvm, const struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change)
static void	kvm_commit_memory_region (struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change)
static void	kvm_activate_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new)
static void	kvm_copy_memslot (struct kvm_memory_slot *dest, const struct kvm_memory_slot *src)
static void	kvm_invalidate_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *invalid_slot)
static void	kvm_create_memslot (struct kvm *kvm, struct kvm_memory_slot *new)
static void	kvm_delete_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *invalid_slot)
static void	kvm_move_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new, struct kvm_memory_slot *invalid_slot)
static void	kvm_update_flags_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new)
static int	kvm_set_memslot (struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change)
static bool	kvm_check_memslot_overlap (struct kvm_memslots *slots, int id, gfn_t start, gfn_t end)
int	__kvm_set_memory_region (struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem)
	EXPORT_SYMBOL_GPL (__kvm_set_memory_region)
int	kvm_set_memory_region (struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem)
	EXPORT_SYMBOL_GPL (kvm_set_memory_region)
static int	kvm_vm_ioctl_set_memory_region (struct kvm *kvm, struct kvm_userspace_memory_region2 *mem)
int	kvm_get_dirty_log (struct kvm *kvm, struct kvm_dirty_log *log, int *is_dirty, struct kvm_memory_slot **memslot)
	EXPORT_SYMBOL_GPL (kvm_get_dirty_log)
struct kvm_memory_slot *	gfn_to_memslot (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_memslot)
struct kvm_memory_slot *	kvm_vcpu_gfn_to_memslot (struct kvm_vcpu *vcpu, gfn_t gfn)
bool	kvm_is_visible_gfn (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_is_visible_gfn)
bool	kvm_vcpu_is_visible_gfn (struct kvm_vcpu *vcpu, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_vcpu_is_visible_gfn)
unsigned long	kvm_host_page_size (struct kvm_vcpu *vcpu, gfn_t gfn)
static bool	memslot_is_readonly (const struct kvm_memory_slot *slot)
static unsigned long	__gfn_to_hva_many (const struct kvm_memory_slot *slot, gfn_t gfn, gfn_t *nr_pages, bool write)
static unsigned long	gfn_to_hva_many (struct kvm_memory_slot *slot, gfn_t gfn, gfn_t *nr_pages)
unsigned long	gfn_to_hva_memslot (struct kvm_memory_slot *slot, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_hva_memslot)
unsigned long	gfn_to_hva (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_hva)
unsigned long	kvm_vcpu_gfn_to_hva (struct kvm_vcpu *vcpu, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_vcpu_gfn_to_hva)
unsigned long	gfn_to_hva_memslot_prot (struct kvm_memory_slot *slot, gfn_t gfn, bool *writable)
unsigned long	gfn_to_hva_prot (struct kvm *kvm, gfn_t gfn, bool *writable)
unsigned long	kvm_vcpu_gfn_to_hva_prot (struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
static int	check_user_page_hwpoison (unsigned long addr)
static bool	hva_to_pfn_fast (unsigned long addr, bool write_fault, bool *writable, kvm_pfn_t *pfn)
static int	hva_to_pfn_slow (unsigned long addr, bool *async, bool write_fault, bool interruptible, bool *writable, kvm_pfn_t *pfn)
static bool	vma_is_valid (struct vm_area_struct *vma, bool write_fault)
static int	kvm_try_get_pfn (kvm_pfn_t pfn)
static int	hva_to_pfn_remapped (struct vm_area_struct *vma, unsigned long addr, bool write_fault, bool *writable, kvm_pfn_t *p_pfn)
kvm_pfn_t	hva_to_pfn (unsigned long addr, bool atomic, bool interruptible, bool *async, bool write_fault, bool *writable)
kvm_pfn_t	__gfn_to_pfn_memslot (const struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, bool interruptible, bool *async, bool write_fault, bool *writable, hva_t *hva)
	EXPORT_SYMBOL_GPL (__gfn_to_pfn_memslot)
kvm_pfn_t	gfn_to_pfn_prot (struct kvm *kvm, gfn_t gfn, bool write_fault, bool *writable)
	EXPORT_SYMBOL_GPL (gfn_to_pfn_prot)
kvm_pfn_t	gfn_to_pfn_memslot (const struct kvm_memory_slot *slot, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_pfn_memslot)
kvm_pfn_t	gfn_to_pfn_memslot_atomic (const struct kvm_memory_slot *slot, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_pfn_memslot_atomic)
kvm_pfn_t	kvm_vcpu_gfn_to_pfn_atomic (struct kvm_vcpu *vcpu, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_vcpu_gfn_to_pfn_atomic)
kvm_pfn_t	gfn_to_pfn (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_pfn)
kvm_pfn_t	kvm_vcpu_gfn_to_pfn (struct kvm_vcpu *vcpu, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_vcpu_gfn_to_pfn)
int	gfn_to_page_many_atomic (struct kvm_memory_slot *slot, gfn_t gfn, struct page **pages, int nr_pages)
	EXPORT_SYMBOL_GPL (gfn_to_page_many_atomic)
struct page *	gfn_to_page (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (gfn_to_page)
void	kvm_release_pfn (kvm_pfn_t pfn, bool dirty)
int	kvm_vcpu_map (struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
	EXPORT_SYMBOL_GPL (kvm_vcpu_map)
void	kvm_vcpu_unmap (struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty)
	EXPORT_SYMBOL_GPL (kvm_vcpu_unmap)
static bool	kvm_is_ad_tracked_page (struct page *page)
static void	kvm_set_page_dirty (struct page *page)
static void	kvm_set_page_accessed (struct page *page)
void	kvm_release_page_clean (struct page *page)
	EXPORT_SYMBOL_GPL (kvm_release_page_clean)
void	kvm_release_pfn_clean (kvm_pfn_t pfn)
	EXPORT_SYMBOL_GPL (kvm_release_pfn_clean)
void	kvm_release_page_dirty (struct page *page)
	EXPORT_SYMBOL_GPL (kvm_release_page_dirty)
void	kvm_release_pfn_dirty (kvm_pfn_t pfn)
	EXPORT_SYMBOL_GPL (kvm_release_pfn_dirty)
void	kvm_set_pfn_dirty (kvm_pfn_t pfn)
	EXPORT_SYMBOL_GPL (kvm_set_pfn_dirty)
void	kvm_set_pfn_accessed (kvm_pfn_t pfn)
	EXPORT_SYMBOL_GPL (kvm_set_pfn_accessed)
static int	next_segment (unsigned long len, int offset)
static int	__kvm_read_guest_page (struct kvm_memory_slot *slot, gfn_t gfn, void *data, int offset, int len)
int	kvm_read_guest_page (struct kvm *kvm, gfn_t gfn, void *data, int offset, int len)
	EXPORT_SYMBOL_GPL (kvm_read_guest_page)
int	kvm_vcpu_read_guest_page (struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, int len)
	EXPORT_SYMBOL_GPL (kvm_vcpu_read_guest_page)
int	kvm_read_guest (struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_read_guest)
int	kvm_vcpu_read_guest (struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_vcpu_read_guest)
static int	__kvm_read_guest_atomic (struct kvm_memory_slot *slot, gfn_t gfn, void *data, int offset, unsigned long len)
int	kvm_vcpu_read_guest_atomic (struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_vcpu_read_guest_atomic)
static int	__kvm_write_guest_page (struct kvm *kvm, struct kvm_memory_slot *memslot, gfn_t gfn, const void *data, int offset, int len)
int	kvm_write_guest_page (struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len)
	EXPORT_SYMBOL_GPL (kvm_write_guest_page)
int	kvm_vcpu_write_guest_page (struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, int offset, int len)
	EXPORT_SYMBOL_GPL (kvm_vcpu_write_guest_page)
int	kvm_write_guest (struct kvm *kvm, gpa_t gpa, const void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_write_guest)
int	kvm_vcpu_write_guest (struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_vcpu_write_guest)
static int	__kvm_gfn_to_hva_cache_init (struct kvm_memslots *slots, struct gfn_to_hva_cache *ghc, gpa_t gpa, unsigned long len)
int	kvm_gfn_to_hva_cache_init (struct kvm *kvm, struct gfn_to_hva_cache *ghc, gpa_t gpa, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_gfn_to_hva_cache_init)
int	kvm_write_guest_offset_cached (struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned int offset, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_write_guest_offset_cached)
int	kvm_write_guest_cached (struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_write_guest_cached)
int	kvm_read_guest_offset_cached (struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned int offset, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_read_guest_offset_cached)
int	kvm_read_guest_cached (struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_read_guest_cached)
int	kvm_clear_guest (struct kvm *kvm, gpa_t gpa, unsigned long len)
	EXPORT_SYMBOL_GPL (kvm_clear_guest)
void	mark_page_dirty_in_slot (struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn)
	EXPORT_SYMBOL_GPL (mark_page_dirty_in_slot)
void	mark_page_dirty (struct kvm *kvm, gfn_t gfn)
	EXPORT_SYMBOL_GPL (mark_page_dirty)
void	kvm_vcpu_mark_page_dirty (struct kvm_vcpu *vcpu, gfn_t gfn)
	EXPORT_SYMBOL_GPL (kvm_vcpu_mark_page_dirty)
void	kvm_sigset_activate (struct kvm_vcpu *vcpu)
void	kvm_sigset_deactivate (struct kvm_vcpu *vcpu)
static void	grow_halt_poll_ns (struct kvm_vcpu *vcpu)
static void	shrink_halt_poll_ns (struct kvm_vcpu *vcpu)
static int	kvm_vcpu_check_block (struct kvm_vcpu *vcpu)
bool	kvm_vcpu_block (struct kvm_vcpu *vcpu)
static void	update_halt_poll_stats (struct kvm_vcpu *vcpu, ktime_t start, ktime_t end, bool success)
static unsigned int	kvm_vcpu_max_halt_poll_ns (struct kvm_vcpu *vcpu)
void	kvm_vcpu_halt (struct kvm_vcpu *vcpu)
	EXPORT_SYMBOL_GPL (kvm_vcpu_halt)
bool	kvm_vcpu_wake_up (struct kvm_vcpu *vcpu)
	EXPORT_SYMBOL_GPL (kvm_vcpu_wake_up)
void	kvm_vcpu_kick (struct kvm_vcpu *vcpu)
	EXPORT_SYMBOL_GPL (kvm_vcpu_kick)
int	kvm_vcpu_yield_to (struct kvm_vcpu *target)
	EXPORT_SYMBOL_GPL (kvm_vcpu_yield_to)
static bool	kvm_vcpu_eligible_for_directed_yield (struct kvm_vcpu *vcpu)
bool __weak	kvm_arch_dy_runnable (struct kvm_vcpu *vcpu)
static bool	vcpu_dy_runnable (struct kvm_vcpu *vcpu)
bool __weak	kvm_arch_dy_has_pending_interrupt (struct kvm_vcpu *vcpu)
void	kvm_vcpu_on_spin (struct kvm_vcpu *me, bool yield_to_kernel_mode)
	EXPORT_SYMBOL_GPL (kvm_vcpu_on_spin)
static bool	kvm_page_in_dirty_ring (struct kvm *kvm, unsigned long pgoff)
static vm_fault_t	kvm_vcpu_fault (struct vm_fault *vmf)
static int	kvm_vcpu_mmap (struct file *file, struct vm_area_struct *vma)
static int	kvm_vcpu_release (struct inode *inode, struct file *filp)
static int	create_vcpu_fd (struct kvm_vcpu *vcpu)
static int	kvm_vm_ioctl_create_vcpu (struct kvm *kvm, u32 id)
static int	kvm_vcpu_ioctl_set_sigmask (struct kvm_vcpu *vcpu, sigset_t *sigset)
static ssize_t	kvm_vcpu_stats_read (struct file *file, char __user *user_buffer, size_t size, loff_t *offset)
static int	kvm_vcpu_stats_release (struct inode *inode, struct file *file)
static int	kvm_vcpu_ioctl_get_stats_fd (struct kvm_vcpu *vcpu)
static int	kvm_device_mmap (struct file *filp, struct vm_area_struct *vma)
static int	kvm_device_ioctl_attr (struct kvm_device *dev, int(*accessor)(struct kvm_device *dev, struct kvm_device_attr *attr), unsigned long arg)
static long	kvm_device_ioctl (struct file *filp, unsigned int ioctl, unsigned long arg)
static int	kvm_device_release (struct inode *inode, struct file *filp)
struct kvm_device *	kvm_device_from_filp (struct file *filp)
int	kvm_register_device_ops (const struct kvm_device_ops *ops, u32 type)
void	kvm_unregister_device_ops (u32 type)
static int	kvm_ioctl_create_device (struct kvm *kvm, struct kvm_create_device *cd)
static int	kvm_vm_ioctl_check_extension_generic (struct kvm *kvm, long arg)
static int	kvm_vm_ioctl_enable_dirty_log_ring (struct kvm *kvm, u32 size)
static int	kvm_vm_ioctl_reset_dirty_pages (struct kvm *kvm)
int	__attribute__ ((weak))
bool	kvm_are_all_memslots_empty (struct kvm *kvm)
	EXPORT_SYMBOL_GPL (kvm_are_all_memslots_empty)
static int	kvm_vm_ioctl_enable_cap_generic (struct kvm *kvm, struct kvm_enable_cap *cap)
static ssize_t	kvm_vm_stats_read (struct file *file, char __user *user_buffer, size_t size, loff_t *offset)
static int	kvm_vm_stats_release (struct inode *inode, struct file *file)
static int	kvm_vm_ioctl_get_stats_fd (struct kvm *kvm)
static long	kvm_vm_ioctl (struct file *filp, unsigned int ioctl, unsigned long arg)
bool	file_is_kvm (struct file *file)
	EXPORT_SYMBOL_GPL (file_is_kvm)
static int	kvm_dev_ioctl_create_vm (unsigned long type)
static long	kvm_dev_ioctl (struct file *filp, unsigned int ioctl, unsigned long arg)
static void	kvm_iodevice_destructor (struct kvm_io_device *dev)
static int	kvm_io_bus_cmp (const struct kvm_io_range *r1, const struct kvm_io_range *r2)
static int	kvm_io_bus_sort_cmp (const void *p1, const void *p2)
static int	kvm_io_bus_get_first_dev (struct kvm_io_bus *bus, gpa_t addr, int len)
static int	__kvm_io_bus_write (struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, struct kvm_io_range *range, const void *val)
int	kvm_io_bus_write (struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, int len, const void *val)
	EXPORT_SYMBOL_GPL (kvm_io_bus_write)
int	kvm_io_bus_write_cookie (struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, int len, const void *val, long cookie)
static int	__kvm_io_bus_read (struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, struct kvm_io_range *range, void *val)
int	kvm_io_bus_read (struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, int len, void *val)
int	kvm_io_bus_register_dev (struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, struct kvm_io_device *dev)
int	kvm_io_bus_unregister_dev (struct kvm *kvm, enum kvm_bus bus_idx, struct kvm_io_device *dev)
struct kvm_io_device *	kvm_io_bus_get_dev (struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr)
	EXPORT_SYMBOL_GPL (kvm_io_bus_get_dev)
static int	kvm_debugfs_open (struct inode *inode, struct file *file, int(*get)(void *, u64 *), int(*set)(void *, u64), const char *fmt)
static int	kvm_debugfs_release (struct inode *inode, struct file *file)
static int	kvm_get_stat_per_vm (struct kvm *kvm, size_t offset, u64 *val)
static int	kvm_clear_stat_per_vm (struct kvm *kvm, size_t offset)
static int	kvm_get_stat_per_vcpu (struct kvm *kvm, size_t offset, u64 *val)
static int	kvm_clear_stat_per_vcpu (struct kvm *kvm, size_t offset)
static int	kvm_stat_data_get (void *data, u64 *val)
static int	kvm_stat_data_clear (void *data, u64 val)
static int	kvm_stat_data_open (struct inode *inode, struct file *file)
static int	vm_stat_get (void *_offset, u64 *val)
static int	vm_stat_clear (void *_offset, u64 val)
	DEFINE_SIMPLE_ATTRIBUTE (vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n")
	DEFINE_SIMPLE_ATTRIBUTE (vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n")
static int	vcpu_stat_get (void *_offset, u64 *val)
static int	vcpu_stat_clear (void *_offset, u64 val)
	DEFINE_SIMPLE_ATTRIBUTE (vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, "%llu\n")
	DEFINE_SIMPLE_ATTRIBUTE (vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n")
static void	kvm_init_debug (void)
static struct kvm_vcpu *	preempt_notifier_to_vcpu (struct preempt_notifier *pn)
static void	kvm_sched_in (struct preempt_notifier *pn, int cpu)
static void	kvm_sched_out (struct preempt_notifier *pn, struct task_struct *next)
struct kvm_vcpu *	kvm_get_running_vcpu (void)
	EXPORT_SYMBOL_GPL (kvm_get_running_vcpu)
struct kvm_vcpu *__percpu *	kvm_get_running_vcpus (void)
int	kvm_init (unsigned vcpu_size, unsigned vcpu_align, struct module *module)
	EXPORT_SYMBOL_GPL (kvm_init)
void	kvm_exit (void)
	EXPORT_SYMBOL_GPL (kvm_exit)
static int	kvm_vm_worker_thread (void *context)
int	kvm_vm_create_worker_thread (struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, uintptr_t data, const char *name, struct task_struct **thread_ptr)

Variables
unsigned int	halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT
unsigned int	halt_poll_ns_grow = 2
unsigned int	halt_poll_ns_grow_start = 10000
unsigned int	halt_poll_ns_shrink
static struct kmem_cache *	kvm_vcpu_cache
static __read_mostly struct preempt_ops	kvm_preempt_ops
struct dentry *	kvm_debugfs_dir
static const struct file_operations	stat_fops_per_vm
static unsigned long long	kvm_createvm_count
static unsigned long long	kvm_active_vms
static const struct vm_operations_struct	kvm_vcpu_vm_ops
static struct file_operations	kvm_vcpu_fops
static const struct file_operations	kvm_vcpu_stats_fops
static struct file_operations	kvm_device_fops
static const struct kvm_device_ops *	kvm_device_ops_table [KVM_DEV_TYPE_MAX]
static const struct file_operations	kvm_vm_stats_fops
static struct file_operations	kvm_vm_fops
static struct file_operations	kvm_chardev_ops
static struct miscdevice	kvm_dev

Macro Definition Documentation

CREATE_TRACE_POINTS

#define CREATE_TRACE_POINTS

Definition at line 67 of file kvm_main.c.

ITOA_MAX_LEN

#define ITOA_MAX_LEN   12

Definition at line 74 of file kvm_main.c.

KVM_COMPAT

#define KVM_COMPAT

(

c

)

Value:

            .compat_ioctl   = kvm_no_compat_ioctl,  \
            .open       = kvm_no_compat_open

Definition at line 139 of file kvm_main.c.

KVM_EVENT_CREATE_VM

#define KVM_EVENT_CREATE_VM   0

Definition at line 147 of file kvm_main.c.

KVM_EVENT_DESTROY_VM

#define KVM_EVENT_DESTROY_VM   1

Definition at line 148 of file kvm_main.c.

KVM_SET_USER_MEMORY_REGION_V1_FLAGS

#define KVM_SET_USER_MEMORY_REGION_V1_FLAGS    (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)

Definition at line 1602 of file kvm_main.c.

SANITY_CHECK_MEM_REGION_FIELD

#define SANITY_CHECK_MEM_REGION_FIELD

(

field

)

Value:

do {                                        \
    BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) !=     \
             offsetof(struct kvm_userspace_memory_region2, field)); \
    BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) !=     \
             sizeof_field(struct kvm_userspace_memory_region2, field)); \
} while (0)

Definition at line 5090 of file kvm_main.c.

Function Documentation

__attribute__()

int __attribute__

(

(weak)

)

Definition at line 4952 of file kvm_main.c.

{
    return -EINVAL;
}

__gfn_to_hva_many()

static unsigned long __gfn_to_hva_many ( const struct kvm_memory_slot *  slot, gfn_t  gfn, gfn_t *  nr_pages, bool  write  )

static

Definition at line 2714 of file kvm_main.c.

{
    if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
        return KVM_HVA_ERR_BAD;
 
    if (memslot_is_readonly(slot) && write)
        return KVM_HVA_ERR_RO_BAD;
 
    if (nr_pages)
        *nr_pages = slot->npages - (gfn - slot->base_gfn);
 
    return __gfn_to_hva_memslot(slot, gfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

__gfn_to_pfn_memslot()

kvm_pfn_t __gfn_to_pfn_memslot	(	const struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		bool	atomic,
		bool	interruptible,
		bool *	async,
		bool	write_fault,
		bool *	writable,
		hva_t *	hva
	)

Definition at line 3031 of file kvm_main.c.

{
    unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
 
    if (hva)
        *hva = addr;
 
    if (addr == KVM_HVA_ERR_RO_BAD) {
        if (writable)
            *writable = false;
        return KVM_PFN_ERR_RO_FAULT;
    }
 
    if (kvm_is_error_hva(addr)) {
        if (writable)
            *writable = false;
        return KVM_PFN_NOSLOT;
    }
 
    /* Do not map writable pfn in the readonly memslot. */
    if (writable && memslot_is_readonly(slot)) {
        *writable = false;
        writable = NULL;
    }
 
    return hva_to_pfn(addr, atomic, interruptible, async, write_fault,
              writable);
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_gfn_to_hva_cache_init()

static int __kvm_gfn_to_hva_cache_init ( struct kvm_memslots *  slots, struct gfn_to_hva_cache *  ghc, gpa_t  gpa, unsigned long  len  )

static

Definition at line 3491 of file kvm_main.c.

{
    int offset = offset_in_page(gpa);
    gfn_t start_gfn = gpa >> PAGE_SHIFT;
    gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
    gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
    gfn_t nr_pages_avail;
 
    /* Update ghc->generation before performing any error checks. */
    ghc->generation = slots->generation;
 
    if (start_gfn > end_gfn) {
        ghc->hva = KVM_HVA_ERR_BAD;
        return -EINVAL;
    }
 
    /*
     * If the requested region crosses two memslots, we still
     * verify that the entire region is valid here.
     */
    for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
        ghc->memslot = __gfn_to_memslot(slots, start_gfn);
        ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
                       &nr_pages_avail);
        if (kvm_is_error_hva(ghc->hva))
            return -EFAULT;
    }
 
    /* Use the slow path for cross page reads and writes. */
    if (nr_pages_needed == 1)
        ghc->hva += offset;
    else
        ghc->memslot = NULL;
 
    ghc->gpa = gpa;
    ghc->len = len;
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_io_bus_read()

static int __kvm_io_bus_read ( struct kvm_vcpu *  vcpu, struct kvm_io_bus *  bus, struct kvm_io_range *  range, void *  val  )

static

Definition at line 5857 of file kvm_main.c.

{
    int idx;
 
    idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
    if (idx < 0)
        return -EOPNOTSUPP;
 
    while (idx < bus->dev_count &&
        kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
        if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
                       range->len, val))
            return idx;
        idx++;
    }
 
    return -EOPNOTSUPP;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_io_bus_write()

static int __kvm_io_bus_write ( struct kvm_vcpu *  vcpu, struct kvm_io_bus *  bus, struct kvm_io_range *  range, const void *  val  )

static

Definition at line 5786 of file kvm_main.c.

{
    int idx;
 
    idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
    if (idx < 0)
        return -EOPNOTSUPP;
 
    while (idx < bus->dev_count &&
        kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
        if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
                    range->len, val))
            return idx;
        idx++;
    }
 
    return -EOPNOTSUPP;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_read_guest_atomic()

static int __kvm_read_guest_atomic ( struct kvm_memory_slot *  slot, gfn_t  gfn, void *  data, int  offset, unsigned long  len  )

static

Definition at line 3386 of file kvm_main.c.

{
    int r;
    unsigned long addr;
 
    addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
    if (kvm_is_error_hva(addr))
        return -EFAULT;
    pagefault_disable();
    r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
    pagefault_enable();
    if (r)
        return -EFAULT;
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_read_guest_page()

static int __kvm_read_guest_page ( struct kvm_memory_slot *  slot, gfn_t  gfn, void *  data, int  offset, int  len  )

static

Definition at line 3313 of file kvm_main.c.

{
    int r;
    unsigned long addr;
 
    addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
    if (kvm_is_error_hva(addr))
        return -EFAULT;
    r = __copy_from_user(data, (void __user *)addr + offset, len);
    if (r)
        return -EFAULT;
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_set_memory_region()

int __kvm_set_memory_region	(	struct kvm *	kvm,
		const struct kvm_userspace_memory_region2 *	mem
	)

Definition at line 2020 of file kvm_main.c.

{
    struct kvm_memory_slot *old, *new;
    struct kvm_memslots *slots;
    enum kvm_mr_change change;
    unsigned long npages;
    gfn_t base_gfn;
    int as_id, id;
    int r;
 
    r = check_memory_region_flags(kvm, mem);
    if (r)
        return r;
 
    as_id = mem->slot >> 16;
    id = (u16)mem->slot;
 
    /* General sanity checks */
    if ((mem->memory_size & (PAGE_SIZE - 1)) ||
        (mem->memory_size != (unsigned long)mem->memory_size))
        return -EINVAL;
    if (mem->guest_phys_addr & (PAGE_SIZE - 1))
        return -EINVAL;
    /* We can read the guest memory with __xxx_user() later on. */
    if ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
        (mem->userspace_addr != untagged_addr(mem->userspace_addr)) ||
         !access_ok((void __user *)(unsigned long)mem->userspace_addr,
            mem->memory_size))
        return -EINVAL;
    if (mem->flags & KVM_MEM_GUEST_MEMFD &&
        (mem->guest_memfd_offset & (PAGE_SIZE - 1) ||
         mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset))
        return -EINVAL;
    if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
        return -EINVAL;
    if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
        return -EINVAL;
    if ((mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES)
        return -EINVAL;
 
    slots = __kvm_memslots(kvm, as_id);
 
    /*
     * Note, the old memslot (and the pointer itself!) may be invalidated
     * and/or destroyed by kvm_set_memslot().
     */
    old = id_to_memslot(slots, id);
 
    if (!mem->memory_size) {
        if (!old || !old->npages)
            return -EINVAL;
 
        if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
            return -EIO;
 
        return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);
    }
 
    base_gfn = (mem->guest_phys_addr >> PAGE_SHIFT);
    npages = (mem->memory_size >> PAGE_SHIFT);
 
    if (!old || !old->npages) {
        change = KVM_MR_CREATE;
 
        /*
         * To simplify KVM internals, the total number of pages across
         * all memslots must fit in an unsigned long.
         */
        if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
            return -EINVAL;
    } else { /* Modify an existing slot. */
        /* Private memslots are immutable, they can only be deleted. */
        if (mem->flags & KVM_MEM_GUEST_MEMFD)
            return -EINVAL;
        if ((mem->userspace_addr != old->userspace_addr) ||
            (npages != old->npages) ||
            ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
            return -EINVAL;
 
        if (base_gfn != old->base_gfn)
            change = KVM_MR_MOVE;
        else if (mem->flags != old->flags)
            change = KVM_MR_FLAGS_ONLY;
        else /* Nothing to change. */
            return 0;
    }
 
    if ((change == KVM_MR_CREATE || change == KVM_MR_MOVE) &&
        kvm_check_memslot_overlap(slots, id, base_gfn, base_gfn + npages))
        return -EEXIST;
 
    /* Allocate a slot that will persist in the memslot. */
    new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
    if (!new)
        return -ENOMEM;
 
    new->as_id = as_id;
    new->id = id;
    new->base_gfn = base_gfn;
    new->npages = npages;
    new->flags = mem->flags;
    new->userspace_addr = mem->userspace_addr;
    if (mem->flags & KVM_MEM_GUEST_MEMFD) {
        r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
        if (r)
            goto out;
    }
 
    r = kvm_set_memslot(kvm, old, new, change);
    if (r)
        goto out_unbind;
 
    return 0;
 
out_unbind:
    if (mem->flags & KVM_MEM_GUEST_MEMFD)
        kvm_gmem_unbind(new);
out:
    kfree(new);
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__kvm_write_guest_page()

static int __kvm_write_guest_page ( struct kvm *  kvm, struct kvm_memory_slot *  memslot, gfn_t  gfn, const void *  data, int  offset, int  len  )

static

Definition at line 3414 of file kvm_main.c.

{
    int r;
    unsigned long addr;
 
    addr = gfn_to_hva_memslot(memslot, gfn);
    if (kvm_is_error_hva(addr))
        return -EFAULT;
    r = __copy_to_user((void __user *)addr + offset, data, len);
    if (r)
        return -EFAULT;
    mark_page_dirty_in_slot(kvm, memslot, gfn);
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

ack_kick()

static void ack_kick ( void *  _completed )

static

Definition at line 247 of file kvm_main.c.

{
}

Here is the caller graph for this function:

check_memory_region_flags()

static int check_memory_region_flags ( struct kvm *  kvm, const struct kvm_userspace_memory_region2 *  mem  )

static

Definition at line 1605 of file kvm_main.c.

{
    u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
 
    if (kvm_arch_has_private_mem(kvm))
        valid_flags |= KVM_MEM_GUEST_MEMFD;
 
    /* Dirty logging private memory is not currently supported. */
    if (mem->flags & KVM_MEM_GUEST_MEMFD)
        valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 
#ifdef __KVM_HAVE_READONLY_MEM
    /*
     * GUEST_MEMFD is incompatible with read-only memslots, as writes to
     * read-only memslots have emulated MMIO, not page fault, semantics,
     * and KVM doesn't allow emulated MMIO for private memory.
     */
    if (!(mem->flags & KVM_MEM_GUEST_MEMFD))
        valid_flags |= KVM_MEM_READONLY;
#endif
 
    if (mem->flags & ~valid_flags)
        return -EINVAL;
 
    return 0;
}

Here is the caller graph for this function:

check_user_page_hwpoison()

static int check_user_page_hwpoison ( unsigned long  addr )

inlinestatic

Definition at line 2787 of file kvm_main.c.

{
    int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
 
    rc = get_user_pages(addr, 1, flags, NULL);
    return rc == -EHWPOISON;
}

Here is the caller graph for this function:

create_vcpu_fd()

static int create_vcpu_fd ( struct kvm_vcpu *  vcpu )

static

Definition at line 4186 of file kvm_main.c.

{
    char name[8 + 1 + ITOA_MAX_LEN + 1];
 
    snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
    return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
}

Here is the caller graph for this function:

DEFINE_MUTEX()

DEFINE_MUTEX

(

kvm_lock

)

Here is the caller graph for this function:

DEFINE_PER_CPU() [1/2]

static DEFINE_PER_CPU ( cpumask_var_t  , cpu_kick_mask    )

static

DEFINE_PER_CPU() [2/2]

static DEFINE_PER_CPU ( struct kvm_vcpu *  , kvm_running_vcpu    )

static

DEFINE_SIMPLE_ATTRIBUTE() [1/4]

DEFINE_SIMPLE_ATTRIBUTE	(	vcpu_stat_fops	,
		vcpu_stat_get	,
		vcpu_stat_clear	,
		"%llu\n"
	)

DEFINE_SIMPLE_ATTRIBUTE() [2/4]

DEFINE_SIMPLE_ATTRIBUTE	(	vcpu_stat_readonly_fops	,
		vcpu_stat_get	,
		NULL	,
		"%llu\n"
	)

DEFINE_SIMPLE_ATTRIBUTE() [3/4]

DEFINE_SIMPLE_ATTRIBUTE	(	vm_stat_fops	,
		vm_stat_get	,
		vm_stat_clear	,
		"%llu\n"
	)

DEFINE_SIMPLE_ATTRIBUTE() [4/4]

DEFINE_SIMPLE_ATTRIBUTE	(	vm_stat_readonly_fops	,
		vm_stat_get	,
		NULL	,
		"%llu\n"
	)

EXPORT_SYMBOL_GPL() [1/70]

EXPORT_SYMBOL_GPL

(

__gfn_to_pfn_memslot

)

EXPORT_SYMBOL_GPL() [2/70]

EXPORT_SYMBOL_GPL

(

__kvm_set_memory_region

)

EXPORT_SYMBOL_GPL() [3/70]

EXPORT_SYMBOL_GPL

(

file_is_kvm

)

EXPORT_SYMBOL_GPL() [4/70]

EXPORT_SYMBOL_GPL

(

gfn_to_hva

)

EXPORT_SYMBOL_GPL() [5/70]

EXPORT_SYMBOL_GPL

(

gfn_to_hva_memslot

)

EXPORT_SYMBOL_GPL() [6/70]

EXPORT_SYMBOL_GPL

(

gfn_to_memslot

)

EXPORT_SYMBOL_GPL() [7/70]

EXPORT_SYMBOL_GPL

(

gfn_to_page

)

EXPORT_SYMBOL_GPL() [8/70]

EXPORT_SYMBOL_GPL

(

gfn_to_page_many_atomic

)

EXPORT_SYMBOL_GPL() [9/70]

EXPORT_SYMBOL_GPL

(

gfn_to_pfn

)

EXPORT_SYMBOL_GPL() [10/70]

EXPORT_SYMBOL_GPL

(

gfn_to_pfn_memslot

)

EXPORT_SYMBOL_GPL() [11/70]

EXPORT_SYMBOL_GPL

(

gfn_to_pfn_memslot_atomic

)

EXPORT_SYMBOL_GPL() [12/70]

EXPORT_SYMBOL_GPL

(

gfn_to_pfn_prot

)

EXPORT_SYMBOL_GPL() [13/70]

EXPORT_SYMBOL_GPL

(

halt_poll_ns

)

EXPORT_SYMBOL_GPL() [14/70]

EXPORT_SYMBOL_GPL

(

halt_poll_ns_grow

)

EXPORT_SYMBOL_GPL() [15/70]

EXPORT_SYMBOL_GPL

(

halt_poll_ns_grow_start

)

EXPORT_SYMBOL_GPL() [16/70]

EXPORT_SYMBOL_GPL

(

halt_poll_ns_shrink

)

EXPORT_SYMBOL_GPL() [17/70]

EXPORT_SYMBOL_GPL

(

kvm_are_all_memslots_empty

)

EXPORT_SYMBOL_GPL() [18/70]

EXPORT_SYMBOL_GPL

(

kvm_clear_guest

)

EXPORT_SYMBOL_GPL() [19/70]

EXPORT_SYMBOL_GPL

(

kvm_debugfs_dir

)

EXPORT_SYMBOL_GPL() [20/70]

EXPORT_SYMBOL_GPL

(

kvm_destroy_vcpus

)

EXPORT_SYMBOL_GPL() [21/70]

EXPORT_SYMBOL_GPL

(

kvm_exit

)

EXPORT_SYMBOL_GPL() [22/70]

EXPORT_SYMBOL_GPL

(

kvm_flush_remote_tlbs

)

EXPORT_SYMBOL_GPL() [23/70]

EXPORT_SYMBOL_GPL

(

kvm_get_dirty_log

)

EXPORT_SYMBOL_GPL() [24/70]

EXPORT_SYMBOL_GPL

(

kvm_get_kvm

)

EXPORT_SYMBOL_GPL() [25/70]

EXPORT_SYMBOL_GPL

(

kvm_get_kvm_safe

)

EXPORT_SYMBOL_GPL() [26/70]

EXPORT_SYMBOL_GPL

(

kvm_get_running_vcpu

)

EXPORT_SYMBOL_GPL() [27/70]

EXPORT_SYMBOL_GPL

(

kvm_gfn_to_hva_cache_init

)

EXPORT_SYMBOL_GPL() [28/70]

EXPORT_SYMBOL_GPL

(

kvm_init

)

EXPORT_SYMBOL_GPL() [29/70]

EXPORT_SYMBOL_GPL

(

kvm_io_bus_get_dev

)

EXPORT_SYMBOL_GPL() [30/70]

EXPORT_SYMBOL_GPL

(

kvm_io_bus_write

)

EXPORT_SYMBOL_GPL() [31/70]

EXPORT_SYMBOL_GPL

(

kvm_is_visible_gfn

)

EXPORT_SYMBOL_GPL() [32/70]

EXPORT_SYMBOL_GPL

(

kvm_make_all_cpus_request

)

EXPORT_SYMBOL_GPL() [33/70]

EXPORT_SYMBOL_GPL

(

kvm_put_kvm

)

EXPORT_SYMBOL_GPL() [34/70]

EXPORT_SYMBOL_GPL

(

kvm_put_kvm_no_destroy

)

EXPORT_SYMBOL_GPL() [35/70]

EXPORT_SYMBOL_GPL

(

kvm_read_guest

)

EXPORT_SYMBOL_GPL() [36/70]

EXPORT_SYMBOL_GPL

(

kvm_read_guest_cached

)

EXPORT_SYMBOL_GPL() [37/70]

EXPORT_SYMBOL_GPL

(

kvm_read_guest_offset_cached

)

EXPORT_SYMBOL_GPL() [38/70]

EXPORT_SYMBOL_GPL

(

kvm_read_guest_page

)

EXPORT_SYMBOL_GPL() [39/70]

EXPORT_SYMBOL_GPL

(

kvm_release_page_clean

)

EXPORT_SYMBOL_GPL() [40/70]

EXPORT_SYMBOL_GPL

(

kvm_release_page_dirty

)

EXPORT_SYMBOL_GPL() [41/70]

EXPORT_SYMBOL_GPL

(

kvm_release_pfn_clean

)

EXPORT_SYMBOL_GPL() [42/70]

EXPORT_SYMBOL_GPL

(

kvm_release_pfn_dirty

)

EXPORT_SYMBOL_GPL() [43/70]

EXPORT_SYMBOL_GPL

(

kvm_set_memory_region

)

EXPORT_SYMBOL_GPL() [44/70]

EXPORT_SYMBOL_GPL

(

kvm_set_pfn_accessed

)

EXPORT_SYMBOL_GPL() [45/70]

EXPORT_SYMBOL_GPL

(

kvm_set_pfn_dirty

)

EXPORT_SYMBOL_GPL() [46/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_gfn_to_hva

)

EXPORT_SYMBOL_GPL() [47/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_gfn_to_pfn

)

EXPORT_SYMBOL_GPL() [48/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_gfn_to_pfn_atomic

)

EXPORT_SYMBOL_GPL() [49/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_halt

)

EXPORT_SYMBOL_GPL() [50/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_is_visible_gfn

)

EXPORT_SYMBOL_GPL() [51/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_kick

)

EXPORT_SYMBOL_GPL() [52/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_map

)

EXPORT_SYMBOL_GPL() [53/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_mark_page_dirty

)

EXPORT_SYMBOL_GPL() [54/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_on_spin

)

EXPORT_SYMBOL_GPL() [55/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_read_guest

)

EXPORT_SYMBOL_GPL() [56/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_read_guest_atomic

)

EXPORT_SYMBOL_GPL() [57/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_read_guest_page

)

EXPORT_SYMBOL_GPL() [58/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_unmap

)

EXPORT_SYMBOL_GPL() [59/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_wake_up

)

EXPORT_SYMBOL_GPL() [60/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_write_guest

)

EXPORT_SYMBOL_GPL() [61/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_write_guest_page

)

EXPORT_SYMBOL_GPL() [62/70]

EXPORT_SYMBOL_GPL

(

kvm_vcpu_yield_to

)

EXPORT_SYMBOL_GPL() [63/70]

EXPORT_SYMBOL_GPL

(

kvm_write_guest

)

EXPORT_SYMBOL_GPL() [64/70]

EXPORT_SYMBOL_GPL

(

kvm_write_guest_cached

)

EXPORT_SYMBOL_GPL() [65/70]

EXPORT_SYMBOL_GPL

(

kvm_write_guest_offset_cached

)

EXPORT_SYMBOL_GPL() [66/70]

EXPORT_SYMBOL_GPL

(

kvm_write_guest_page

)

EXPORT_SYMBOL_GPL() [67/70]

EXPORT_SYMBOL_GPL

(

mark_page_dirty

)

EXPORT_SYMBOL_GPL() [68/70]

EXPORT_SYMBOL_GPL

(

mark_page_dirty_in_slot

)

EXPORT_SYMBOL_GPL() [69/70]

EXPORT_SYMBOL_GPL

(

vcpu_load

)

EXPORT_SYMBOL_GPL() [70/70]

EXPORT_SYMBOL_GPL

(

vcpu_put

)

file_is_kvm()

bool file_is_kvm

(

struct file *

file

)

Definition at line 5424 of file kvm_main.c.

{
    return file && file->f_op == &kvm_vm_fops;
}

gfn_to_hva()

unsigned long gfn_to_hva	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 2742 of file kvm_main.c.

{
    return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_hva_many()

static unsigned long gfn_to_hva_many ( struct kvm_memory_slot *  slot, gfn_t  gfn, gfn_t *  nr_pages  )

static

Definition at line 2729 of file kvm_main.c.

{
    return __gfn_to_hva_many(slot, gfn, nr_pages, true);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_hva_memslot()

unsigned long gfn_to_hva_memslot	(	struct kvm_memory_slot *	slot,
		gfn_t	gfn
	)

Definition at line 2735 of file kvm_main.c.

{
    return gfn_to_hva_many(slot, gfn, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_hva_memslot_prot()

unsigned long gfn_to_hva_memslot_prot	(	struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		bool *	writable
	)

Definition at line 2762 of file kvm_main.c.

{
    unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
 
    if (!kvm_is_error_hva(hva) && writable)
        *writable = !memslot_is_readonly(slot);
 
    return hva;
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_hva_prot()

unsigned long gfn_to_hva_prot	(	struct kvm *	kvm,
		gfn_t	gfn,
		bool *	writable
	)

Definition at line 2773 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
 
    return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

Here is the call graph for this function:

gfn_to_memslot()

struct kvm_memory_slot* gfn_to_memslot	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 2630 of file kvm_main.c.

{
    return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}

Here is the caller graph for this function:

gfn_to_page()

struct page* gfn_to_page	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 3126 of file kvm_main.c.

{
    struct page *page;
    kvm_pfn_t pfn;
 
    pfn = gfn_to_pfn(kvm, gfn);
 
    if (is_error_noslot_pfn(pfn))
        return KVM_ERR_PTR_BAD_PAGE;
 
    page = kvm_pfn_to_refcounted_page(pfn);
    if (!page)
        return KVM_ERR_PTR_BAD_PAGE;
 
    return page;
}

Here is the call graph for this function:

gfn_to_page_many_atomic()

int gfn_to_page_many_atomic	(	struct kvm_memory_slot *	slot,
		gfn_t	gfn,
		struct page **	pages,
		int	nr_pages
	)

Definition at line 3103 of file kvm_main.c.

{
    unsigned long addr;
    gfn_t entry = 0;
 
    addr = gfn_to_hva_many(slot, gfn, &entry);
    if (kvm_is_error_hva(addr))
        return -1;
 
    if (entry < nr_pages)
        return 0;
 
    return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages);
}

Here is the call graph for this function:

gfn_to_pfn()

kvm_pfn_t gfn_to_pfn	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 3091 of file kvm_main.c.

{
    return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_pfn_memslot()

kvm_pfn_t gfn_to_pfn_memslot	(	const struct kvm_memory_slot *	slot,
		gfn_t	gfn
	)

Definition at line 3071 of file kvm_main.c.

{
    return __gfn_to_pfn_memslot(slot, gfn, false, false, NULL, true,
                    NULL, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_pfn_memslot_atomic()

kvm_pfn_t gfn_to_pfn_memslot_atomic	(	const struct kvm_memory_slot *	slot,
		gfn_t	gfn
	)

Definition at line 3078 of file kvm_main.c.

{
    return __gfn_to_pfn_memslot(slot, gfn, true, false, NULL, true,
                    NULL, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

gfn_to_pfn_prot()

kvm_pfn_t gfn_to_pfn_prot	(	struct kvm *	kvm,
		gfn_t	gfn,
		bool	write_fault,
		bool *	writable
	)

Definition at line 3063 of file kvm_main.c.

{
    return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, false,
                    NULL, write_fault, writable, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

grow_halt_poll_ns()

static void grow_halt_poll_ns ( struct kvm_vcpu *  vcpu )

static

Definition at line 3701 of file kvm_main.c.

{
    unsigned int old, val, grow, grow_start;
 
    old = val = vcpu->halt_poll_ns;
    grow_start = READ_ONCE(halt_poll_ns_grow_start);
    grow = READ_ONCE(halt_poll_ns_grow);
    if (!grow)
        goto out;
 
    val *= grow;
    if (val < grow_start)
        val = grow_start;
 
    vcpu->halt_poll_ns = val;
out:
    trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
}

Here is the caller graph for this function:

hardware_disable_all()

static void hardware_disable_all ( void  )

static

Definition at line 5708 of file kvm_main.c.

{
 
}

Here is the caller graph for this function:

hardware_enable_all()

static int hardware_enable_all ( void  )

static

Definition at line 5703 of file kvm_main.c.

{
    return 0;
}

Here is the caller graph for this function:

hva_to_pfn()

kvm_pfn_t hva_to_pfn	(	unsigned long	addr,
		bool	atomic,
		bool	interruptible,
		bool *	async,
		bool	write_fault,
		bool *	writable
	)

Definition at line 2980 of file kvm_main.c.

{
    struct vm_area_struct *vma;
    kvm_pfn_t pfn;
    int npages, r;
 
    /* we can do it either atomically or asynchronously, not both */
    BUG_ON(atomic && async);
 
    if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
        return pfn;
 
    if (atomic)
        return KVM_PFN_ERR_FAULT;
 
    npages = hva_to_pfn_slow(addr, async, write_fault, interruptible,
                 writable, &pfn);
    if (npages == 1)
        return pfn;
    if (npages == -EINTR)
        return KVM_PFN_ERR_SIGPENDING;
 
    mmap_read_lock(current->mm);
    if (npages == -EHWPOISON ||
          (!async && check_user_page_hwpoison(addr))) {
        pfn = KVM_PFN_ERR_HWPOISON;
        goto exit;
    }
 
retry:
    vma = vma_lookup(current->mm, addr);
 
    if (vma == NULL)
        pfn = KVM_PFN_ERR_FAULT;
    else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
        r = hva_to_pfn_remapped(vma, addr, write_fault, writable, &pfn);
        if (r == -EAGAIN)
            goto retry;
        if (r < 0)
            pfn = KVM_PFN_ERR_FAULT;
    } else {
        if (async && vma_is_valid(vma, write_fault))
            *async = true;
        pfn = KVM_PFN_ERR_FAULT;
    }
exit:
    mmap_read_unlock(current->mm);
    return pfn;
}

Here is the call graph for this function:

Here is the caller graph for this function:

hva_to_pfn_fast()

static bool hva_to_pfn_fast ( unsigned long  addr, bool  write_fault, bool *  writable, kvm_pfn_t *  pfn  )

static

Definition at line 2800 of file kvm_main.c.

{
    struct page *page[1];
 
    /*
     * Fast pin a writable pfn only if it is a write fault request
     * or the caller allows to map a writable pfn for a read fault
     * request.
     */
    if (!(write_fault || writable))
        return false;
 
    if (get_user_page_fast_only(addr, FOLL_WRITE, page)) {
        *pfn = page_to_pfn(page[0]);
 
        if (writable)
            *writable = true;
        return true;
    }
 
    return false;
}

Here is the caller graph for this function:

hva_to_pfn_remapped()

static int hva_to_pfn_remapped ( struct vm_area_struct *  vma, unsigned long  addr, bool  write_fault, bool *  writable, kvm_pfn_t *  p_pfn  )

static

Definition at line 2897 of file kvm_main.c.

{
    kvm_pfn_t pfn;
    pte_t *ptep;
    pte_t pte;
    spinlock_t *ptl;
    int r;
 
    r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
    if (r) {
        /*
         * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
         * not call the fault handler, so do it here.
         */
        bool unlocked = false;
        r = fixup_user_fault(current->mm, addr,
                     (write_fault ? FAULT_FLAG_WRITE : 0),
                     &unlocked);
        if (unlocked)
            return -EAGAIN;
        if (r)
            return r;
 
        r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
        if (r)
            return r;
    }
 
    pte = ptep_get(ptep);
 
    if (write_fault && !pte_write(pte)) {
        pfn = KVM_PFN_ERR_RO_FAULT;
        goto out;
    }
 
    if (writable)
        *writable = pte_write(pte);
    pfn = pte_pfn(pte);
 
    /*
     * Get a reference here because callers of *hva_to_pfn* and
     * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
     * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
     * set, but the kvm_try_get_pfn/kvm_release_pfn_clean pair will
     * simply do nothing for reserved pfns.
     *
     * Whoever called remap_pfn_range is also going to call e.g.
     * unmap_mapping_range before the underlying pages are freed,
     * causing a call to our MMU notifier.
     *
     * Certain IO or PFNMAP mappings can be backed with valid
     * struct pages, but be allocated without refcounting e.g.,
     * tail pages of non-compound higher order allocations, which
     * would then underflow the refcount when the caller does the
     * required put_page. Don't allow those pages here.
     */
    if (!kvm_try_get_pfn(pfn))
        r = -EFAULT;
 
out:
    pte_unmap_unlock(ptep, ptl);
    *p_pfn = pfn;
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

hva_to_pfn_slow()

static int hva_to_pfn_slow ( unsigned long  addr, bool *  async, bool  write_fault, bool  interruptible, bool *  writable, kvm_pfn_t *  pfn  )

static

Definition at line 2828 of file kvm_main.c.

{
    /*
     * When a VCPU accesses a page that is not mapped into the secondary
     * MMU, we lookup the page using GUP to map it, so the guest VCPU can
     * make progress. We always want to honor NUMA hinting faults in that
     * case, because GUP usage corresponds to memory accesses from the VCPU.
     * Otherwise, we'd not trigger NUMA hinting faults once a page is
     * mapped into the secondary MMU and gets accessed by a VCPU.
     *
     * Note that get_user_page_fast_only() and FOLL_WRITE for now
     * implicitly honor NUMA hinting faults and don't need this flag.
     */
    unsigned int flags = FOLL_HWPOISON | FOLL_HONOR_NUMA_FAULT;
    struct page *page;
    int npages;
 
    might_sleep();
 
    if (writable)
        *writable = write_fault;
 
    if (write_fault)
        flags |= FOLL_WRITE;
    if (async)
        flags |= FOLL_NOWAIT;
    if (interruptible)
        flags |= FOLL_INTERRUPTIBLE;
 
    npages = get_user_pages_unlocked(addr, 1, &page, flags);
    if (npages != 1)
        return npages;
 
    /* map read fault as writable if possible */
    if (unlikely(!write_fault) && writable) {
        struct page *wpage;
 
        if (get_user_page_fast_only(addr, FOLL_WRITE, &wpage)) {
            *writable = true;
            put_page(page);
            page = wpage;
        }
    }
    *pfn = page_to_pfn(page);
    return npages;
}

Here is the caller graph for this function:

kvm_activate_memslot()

static void kvm_activate_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new  )

static

Definition at line 1791 of file kvm_main.c.

{
    int as_id = kvm_memslots_get_as_id(old, new);
 
    kvm_swap_active_memslots(kvm, as_id);
 
    /* Propagate the new memslot to the now inactive memslots. */
    kvm_replace_memslot(kvm, old, new);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_alloc_dirty_bitmap()

static int kvm_alloc_dirty_bitmap ( struct kvm_memory_slot *  memslot )

static

Definition at line 1453 of file kvm_main.c.

{
    unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot);
 
    memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT);
    if (!memslot->dirty_bitmap)
        return -ENOMEM;
 
    return 0;
}

Here is the caller graph for this function:

kvm_arch_create_vm_debugfs()

int __weak kvm_arch_create_vm_debugfs

(

struct kvm *

kvm

)

Definition at line 1186 of file kvm_main.c.

{
    return 0;
}

Here is the caller graph for this function:

kvm_arch_dy_has_pending_interrupt()

bool __weak kvm_arch_dy_has_pending_interrupt

(

struct kvm_vcpu *

vcpu

)

Definition at line 4051 of file kvm_main.c.

{
    return false;
}

Here is the caller graph for this function:

kvm_arch_dy_runnable()

bool __weak kvm_arch_dy_runnable

(

struct kvm_vcpu *

vcpu

)

Definition at line 4033 of file kvm_main.c.

{
    return kvm_arch_vcpu_runnable(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_arch_guest_memory_reclaimed()

__weak void kvm_arch_guest_memory_reclaimed

(

struct kvm *

kvm

)

Definition at line 155 of file kvm_main.c.

{
}

Here is the caller graph for this function:

kvm_arch_post_init_vm()

int __weak kvm_arch_post_init_vm

(

struct kvm *

kvm

)

Definition at line 1167 of file kvm_main.c.

{
    return 0;
}

Here is the caller graph for this function:

kvm_arch_pre_destroy_vm()

void __weak kvm_arch_pre_destroy_vm

(

struct kvm *

kvm

)

Definition at line 1176 of file kvm_main.c.

{
}

Here is the caller graph for this function:

kvm_are_all_memslots_empty()

bool kvm_are_all_memslots_empty

(

struct kvm *

kvm

)

Definition at line 4958 of file kvm_main.c.

{
    int i;
 
    lockdep_assert_held(&kvm->slots_lock);
 
    for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
        if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
            return false;
    }
 
    return true;
}

Here is the caller graph for this function:

kvm_check_memslot_overlap()

static bool kvm_check_memslot_overlap ( struct kvm_memslots *  slots, int  id, gfn_t  start, gfn_t  end  )

static

Definition at line 1999 of file kvm_main.c.

{
    struct kvm_memslot_iter iter;
 
    kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) {
        if (iter.slot->id != id)
            return true;
    }
 
    return false;
}

Here is the caller graph for this function:

kvm_clear_guest()

int kvm_clear_guest	(	struct kvm *	kvm,
		gpa_t	gpa,
		unsigned long	len
	)

Definition at line 3615 of file kvm_main.c.

{
    const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
    gfn_t gfn = gpa >> PAGE_SHIFT;
    int seg;
    int offset = offset_in_page(gpa);
    int ret;
 
    while ((seg = next_segment(len, offset)) != 0) {
        ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
        if (ret < 0)
            return ret;
        offset = 0;
        len -= seg;
        ++gfn;
    }
    return 0;
}

Here is the call graph for this function:

kvm_clear_stat_per_vcpu()

static int kvm_clear_stat_per_vcpu ( struct kvm *  kvm, size_t  offset  )

static

Definition at line 6076 of file kvm_main.c.

{
    unsigned long i;
    struct kvm_vcpu *vcpu;
 
    kvm_for_each_vcpu(i, vcpu, kvm)
        *(u64 *)((void *)(&vcpu->stat) + offset) = 0;
 
    return 0;
}

Here is the caller graph for this function:

kvm_clear_stat_per_vm()

static int kvm_clear_stat_per_vm ( struct kvm *  kvm, size_t  offset  )

static

Definition at line 6056 of file kvm_main.c.

{
    *(u64 *)((void *)(&kvm->stat) + offset) = 0;
 
    return 0;
}

Here is the caller graph for this function:

kvm_commit_memory_region()

static void kvm_commit_memory_region ( struct kvm *  kvm, struct kvm_memory_slot *  old, const struct kvm_memory_slot *  new, enum kvm_mr_change  change  )

static

Definition at line 1729 of file kvm_main.c.

{
    int old_flags = old ? old->flags : 0;
    int new_flags = new ? new->flags : 0;
    /*
     * Update the total number of memslot pages before calling the arch
     * hook so that architectures can consume the result directly.
     */
    if (change == KVM_MR_DELETE)
        kvm->nr_memslot_pages -= old->npages;
    else if (change == KVM_MR_CREATE)
        kvm->nr_memslot_pages += new->npages;
 
    if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) {
        int change = (new_flags & KVM_MEM_LOG_DIRTY_PAGES) ? 1 : -1;
        atomic_set(&kvm->nr_memslots_dirty_logging,
               atomic_read(&kvm->nr_memslots_dirty_logging) + change);
    }
 
    kvm_arch_commit_memory_region(kvm, old, new, change);
 
    switch (change) {
    case KVM_MR_CREATE:
        /* Nothing more to do. */
        break;
    case KVM_MR_DELETE:
        /* Free the old memslot and all its metadata. */
        kvm_free_memslot(kvm, old);
        break;
    case KVM_MR_MOVE:
    case KVM_MR_FLAGS_ONLY:
        /*
         * Free the dirty bitmap as needed; the below check encompasses
         * both the flags and whether a ring buffer is being used)
         */
        if (old->dirty_bitmap && !new->dirty_bitmap)
            kvm_destroy_dirty_bitmap(old);
 
        /*
         * The final quirk.  Free the detached, old slot, but only its
         * memory, not any metadata.  Metadata, including arch specific
         * data, may be reused by @new.
         */
        kfree(old);
        break;
    default:
        BUG();
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_copy_memslot()

static void kvm_copy_memslot ( struct kvm_memory_slot *  dest, const struct kvm_memory_slot *  src  )

static

Definition at line 1803 of file kvm_main.c.

{
    dest->base_gfn = src->base_gfn;
    dest->npages = src->npages;
    dest->dirty_bitmap = src->dirty_bitmap;
    dest->arch = src->arch;
    dest->userspace_addr = src->userspace_addr;
    dest->flags = src->flags;
    dest->id = src->id;
    dest->as_id = src->as_id;
}

Here is the caller graph for this function:

kvm_create_memslot()

static void kvm_create_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  new  )

static

Definition at line 1858 of file kvm_main.c.

{
    /* Add the new memslot to the inactive set and activate. */
    kvm_replace_memslot(kvm, NULL, new);
    kvm_activate_memslot(kvm, NULL, new);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_create_vm()

static struct kvm* kvm_create_vm ( unsigned long  type, const char *  fdname  )

static

Definition at line 1191 of file kvm_main.c.

{
    struct kvm *kvm = kvm_arch_alloc_vm();
    struct kvm_memslots *slots;
    int r = -ENOMEM;
    int i, j;
 
    if (!kvm)
        return ERR_PTR(-ENOMEM);
 
    KVM_MMU_LOCK_INIT(kvm);
    mmgrab(current->mm);
    kvm->mm = current->mm;
    kvm_eventfd_init(kvm);
    mutex_init(&kvm->lock);
    mutex_init(&kvm->irq_lock);
    mutex_init(&kvm->slots_lock);
    mutex_init(&kvm->slots_arch_lock);
    spin_lock_init(&kvm->mn_invalidate_lock);
    rcuwait_init(&kvm->mn_memslots_update_rcuwait);
    xa_init(&kvm->vcpu_array);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
    xa_init(&kvm->mem_attr_array);
#endif
 
    INIT_LIST_HEAD(&kvm->gpc_list);
    spin_lock_init(&kvm->gpc_lock);
 
    INIT_LIST_HEAD(&kvm->devices);
    kvm->max_vcpus = KVM_MAX_VCPUS;
 
    BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
 
    /*
     * Force subsequent debugfs file creations to fail if the VM directory
     * is not created (by kvm_create_vm_debugfs()).
     */
    kvm->debugfs_dentry = ERR_PTR(-ENOENT);
 
    snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",
         task_pid_nr(current));
 
    if (init_srcu_struct(&kvm->srcu))
        goto out_err_no_srcu;
    if (init_srcu_struct(&kvm->irq_srcu))
        goto out_err_no_irq_srcu;
 
    refcount_set(&kvm->users_count, 1);
    for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
        for (j = 0; j < 2; j++) {
            slots = &kvm->__memslots[i][j];
 
            atomic_long_set(&slots->last_used_slot, (unsigned long)NULL);
            slots->hva_tree = RB_ROOT_CACHED;
            slots->gfn_tree = RB_ROOT;
            hash_init(slots->id_hash);
            slots->node_idx = j;
 
            /* Generations must be different for each address space. */
            slots->generation = i;
        }
 
        rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
    }
 
    for (i = 0; i < KVM_NR_BUSES; i++) {
        rcu_assign_pointer(kvm->buses[i],
            kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
        if (!kvm->buses[i])
            goto out_err_no_arch_destroy_vm;
    }
 
    r = kvm_arch_init_vm(kvm, type);
    if (r)
        goto out_err_no_arch_destroy_vm;
 
    r = hardware_enable_all();
    if (r)
        goto out_err_no_disable;
 
#ifdef CONFIG_HAVE_KVM_IRQCHIP
    INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
#endif
 
    r = kvm_init_mmu_notifier(kvm);
    if (r)
        goto out_err_no_mmu_notifier;
 
    r = kvm_coalesced_mmio_init(kvm);
    if (r < 0)
        goto out_no_coalesced_mmio;
 
    r = kvm_create_vm_debugfs(kvm, fdname);
    if (r)
        goto out_err_no_debugfs;
 
    r = kvm_arch_post_init_vm(kvm);
    if (r)
        goto out_err;
 
    mutex_lock(&kvm_lock);
    list_add(&kvm->vm_list, &vm_list);
    mutex_unlock(&kvm_lock);
 
    preempt_notifier_inc();
    kvm_init_pm_notifier(kvm);
 
    return kvm;
 
out_err:
    kvm_destroy_vm_debugfs(kvm);
out_err_no_debugfs:
    kvm_coalesced_mmio_free(kvm);
out_no_coalesced_mmio:
#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
    if (kvm->mmu_notifier.ops)
        mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
#endif
out_err_no_mmu_notifier:
    hardware_disable_all();
out_err_no_disable:
    kvm_arch_destroy_vm(kvm);
out_err_no_arch_destroy_vm:
    WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
    for (i = 0; i < KVM_NR_BUSES; i++)
        kfree(kvm_get_bus(kvm, i));
    cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
    cleanup_srcu_struct(&kvm->srcu);
out_err_no_srcu:
    kvm_arch_free_vm(kvm);
    mmdrop(current->mm);
    return ERR_PTR(r);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_create_vm_debugfs()

static int kvm_create_vm_debugfs ( struct kvm *  kvm, const char *  fdname  )

static

Definition at line 1088 of file kvm_main.c.

{
    static DEFINE_MUTEX(kvm_debugfs_lock);
    struct dentry *dent;
    char dir_name[ITOA_MAX_LEN * 2];
    struct kvm_stat_data *stat_data;
    const struct _kvm_stats_desc *pdesc;
    int i, ret = -ENOMEM;
    int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
                      kvm_vcpu_stats_header.num_desc;
 
    if (!debugfs_initialized())
        return 0;
 
    snprintf(dir_name, sizeof(dir_name), "%d-%s", task_pid_nr(current), fdname);
    mutex_lock(&kvm_debugfs_lock);
    dent = debugfs_lookup(dir_name, kvm_debugfs_dir);
    if (dent) {
        pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name);
        dput(dent);
        mutex_unlock(&kvm_debugfs_lock);
        return 0;
    }
    dent = debugfs_create_dir(dir_name, kvm_debugfs_dir);
    mutex_unlock(&kvm_debugfs_lock);
    if (IS_ERR(dent))
        return 0;
 
    kvm->debugfs_dentry = dent;
    kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
                     sizeof(*kvm->debugfs_stat_data),
                     GFP_KERNEL_ACCOUNT);
    if (!kvm->debugfs_stat_data)
        goto out_err;
 
    for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
        pdesc = &kvm_vm_stats_desc[i];
        stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
        if (!stat_data)
            goto out_err;
 
        stat_data->kvm = kvm;
        stat_data->desc = pdesc;
        stat_data->kind = KVM_STAT_VM;
        kvm->debugfs_stat_data[i] = stat_data;
        debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                    kvm->debugfs_dentry, stat_data,
                    &stat_fops_per_vm);
    }
 
    for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
        pdesc = &kvm_vcpu_stats_desc[i];
        stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
        if (!stat_data)
            goto out_err;
 
        stat_data->kvm = kvm;
        stat_data->desc = pdesc;
        stat_data->kind = KVM_STAT_VCPU;
        kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;
        debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                    kvm->debugfs_dentry, stat_data,
                    &stat_fops_per_vm);
    }
 
    ret = kvm_arch_create_vm_debugfs(kvm);
    if (ret)
        goto out_err;
 
    return 0;
out_err:
    kvm_destroy_vm_debugfs(kvm);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_debugfs_open()

static int kvm_debugfs_open ( struct inode *  inode, struct file *  file, int(*)(void *, u64 *)  get, int(*)(void *, u64)  set, const char *  fmt  )

static

Definition at line 6015 of file kvm_main.c.

{
    int ret;
    struct kvm_stat_data *stat_data = inode->i_private;
 
    /*
     * The debugfs files are a reference to the kvm struct which
        * is still valid when kvm_destroy_vm is called.  kvm_get_kvm_safe
        * avoids the race between open and the removal of the debugfs directory.
     */
    if (!kvm_get_kvm_safe(stat_data->kvm))
        return -ENOENT;
 
    ret = simple_attr_open(inode, file, get,
                   kvm_stats_debugfs_mode(stat_data->desc) & 0222
                   ? set : NULL, fmt);
    if (ret)
        kvm_put_kvm(stat_data->kvm);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_debugfs_release()

static int kvm_debugfs_release ( struct inode *  inode, struct file *  file  )

static

Definition at line 6039 of file kvm_main.c.

{
    struct kvm_stat_data *stat_data = inode->i_private;
 
    simple_attr_release(inode, file);
    kvm_put_kvm(stat_data->kvm);
 
    return 0;
}

Here is the call graph for this function:

kvm_delete_memslot()

static void kvm_delete_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  invalid_slot  )

static

Definition at line 1866 of file kvm_main.c.

{
    /*
     * Remove the old memslot (in the inactive memslots) by passing NULL as
     * the "new" slot, and for the invalid version in the active slots.
     */
    kvm_replace_memslot(kvm, old, NULL);
    kvm_activate_memslot(kvm, invalid_slot, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_destroy_devices()

static void kvm_destroy_devices ( struct kvm *  kvm )

static

Definition at line 1326 of file kvm_main.c.

{
    struct kvm_device *dev, *tmp;
 
    /*
     * We do not need to take the kvm->lock here, because nobody else
     * has a reference to the struct kvm at this point and therefore
     * cannot access the devices list anyhow.
     */
    list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
        list_del(&dev->vm_node);
        dev->ops->destroy(dev);
    }
}

Here is the caller graph for this function:

kvm_destroy_dirty_bitmap()

static void kvm_destroy_dirty_bitmap ( struct kvm_memory_slot *  memslot )

static

Definition at line 1016 of file kvm_main.c.

{
    if (!memslot->dirty_bitmap)
        return;
 
    kvfree(memslot->dirty_bitmap);
    memslot->dirty_bitmap = NULL;
}

Here is the caller graph for this function:

kvm_destroy_pm_notifier()

static void kvm_destroy_pm_notifier ( struct kvm *  kvm )

static

Definition at line 1011 of file kvm_main.c.

{
}

Here is the caller graph for this function:

kvm_destroy_vcpus()

void kvm_destroy_vcpus

(

struct kvm *

kvm

)

Definition at line 522 of file kvm_main.c.

{
    unsigned long i;
    struct kvm_vcpu *vcpu;
 
    kvm_for_each_vcpu(i, vcpu, kvm) {
        kvm_vcpu_destroy(vcpu);
        xa_erase(&kvm->vcpu_array, i);
    }
 
    atomic_set(&kvm->online_vcpus, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_destroy_vm()

static void kvm_destroy_vm ( struct kvm *  kvm )

static

Definition at line 1341 of file kvm_main.c.

{
    int i;
    struct mm_struct *mm = kvm->mm;
 
    kvm_destroy_pm_notifier(kvm);
    kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
    kvm_destroy_vm_debugfs(kvm);
    kvm_arch_sync_events(kvm);
    mutex_lock(&kvm_lock);
    list_del(&kvm->vm_list);
    mutex_unlock(&kvm_lock);
    kvm_arch_pre_destroy_vm(kvm);
 
    kvm_free_irq_routing(kvm);
    for (i = 0; i < KVM_NR_BUSES; i++) {
        struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
 
        if (bus)
            kvm_io_bus_destroy(bus);
        kvm->buses[i] = NULL;
    }
    kvm_coalesced_mmio_free(kvm);
#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
    mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
    /*
     * At this point, pending calls to invalidate_range_start()
     * have completed but no more MMU notifiers will run, so
     * mn_active_invalidate_count may remain unbalanced.
     * No threads can be waiting in kvm_swap_active_memslots() as the
     * last reference on KVM has been dropped, but freeing
     * memslots would deadlock without this manual intervention.
     *
     * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU
     * notifier between a start() and end(), then there shouldn't be any
     * in-progress invalidations.
     */
    WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
    if (kvm->mn_active_invalidate_count)
        kvm->mn_active_invalidate_count = 0;
    else
        WARN_ON(kvm->mmu_invalidate_in_progress);
#else
    kvm_flush_shadow_all(kvm);
#endif
    kvm_arch_destroy_vm(kvm);
    kvm_destroy_devices(kvm);
    for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
        kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
        kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
    }
    cleanup_srcu_struct(&kvm->irq_srcu);
    cleanup_srcu_struct(&kvm->srcu);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
    xa_destroy(&kvm->mem_attr_array);
#endif
    kvm_arch_free_vm(kvm);
    preempt_notifier_dec();
    hardware_disable_all();
    mmdrop(mm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_destroy_vm_debugfs()

static void kvm_destroy_vm_debugfs ( struct kvm *  kvm )

static

Definition at line 1070 of file kvm_main.c.

{
    int i;
    int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
                      kvm_vcpu_stats_header.num_desc;
 
    if (IS_ERR(kvm->debugfs_dentry))
        return;
 
    debugfs_remove_recursive(kvm->debugfs_dentry);
 
    if (kvm->debugfs_stat_data) {
        for (i = 0; i < kvm_debugfs_num_entries; i++)
            kfree(kvm->debugfs_stat_data[i]);
        kfree(kvm->debugfs_stat_data);
    }
}

Here is the caller graph for this function:

kvm_dev_ioctl()

static long kvm_dev_ioctl ( struct file *  filp, unsigned int  ioctl, unsigned long  arg  )

static

Definition at line 5473 of file kvm_main.c.

{
    int r = -EINVAL;
 
    switch (ioctl) {
    case KVM_GET_API_VERSION:
        if (arg)
            goto out;
        r = KVM_API_VERSION;
        break;
    case KVM_CREATE_VM:
        r = kvm_dev_ioctl_create_vm(arg);
        break;
    case KVM_CHECK_EXTENSION:
        r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
        break;
    case KVM_GET_VCPU_MMAP_SIZE:
        if (arg)
            goto out;
        r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
        r += PAGE_SIZE;    /* pio data page */
#endif
#ifdef CONFIG_KVM_MMIO
        r += PAGE_SIZE;    /* coalesced mmio ring page */
#endif
        break;
    default:
        return kvm_arch_dev_ioctl(filp, ioctl, arg);
    }
out:
    return r;
}

kvm_dev_ioctl_create_vm()

static int kvm_dev_ioctl_create_vm ( unsigned long  type )

static

Definition at line 5430 of file kvm_main.c.

{
    char fdname[ITOA_MAX_LEN + 1];
    int r, fd;
    struct kvm *kvm;
    struct file *file;
 
    fd = get_unused_fd_flags(O_CLOEXEC);
    if (fd < 0)
        return fd;
 
    snprintf(fdname, sizeof(fdname), "%d", fd);
 
    kvm = kvm_create_vm(type, fdname);
    if (IS_ERR(kvm)) {
        r = PTR_ERR(kvm);
        goto put_fd;
    }
 
    file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
    if (IS_ERR(file)) {
        r = PTR_ERR(file);
        goto put_kvm;
    }
 
    /*
     * Don't call kvm_put_kvm anymore at this point; file->f_op is
     * already set, with ->release() being kvm_vm_release().  In error
     * cases it will be called by the final fput(file) and will take
     * care of doing kvm_put_kvm(kvm).
     */
    kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
 
    fd_install(fd, file);
    return fd;
 
put_kvm:
    kvm_put_kvm(kvm);
put_fd:
    put_unused_fd(fd);
    return r;
}

Here is the call graph for this function:

kvm_device_from_filp()

struct kvm_device* kvm_device_from_filp

(

struct file *

filp

)

Definition at line 4727 of file kvm_main.c.

{
    if (filp->f_op != &kvm_device_fops)
        return NULL;
 
    return filp->private_data;
}

kvm_device_ioctl()

static long kvm_device_ioctl ( struct file *  filp, unsigned int  ioctl, unsigned long  arg  )

static

Definition at line 4681 of file kvm_main.c.

{
    struct kvm_device *dev = filp->private_data;
 
    if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)
        return -EIO;
 
    switch (ioctl) {
    case KVM_SET_DEVICE_ATTR:
        return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
    case KVM_GET_DEVICE_ATTR:
        return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
    case KVM_HAS_DEVICE_ATTR:
        return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
    default:
        if (dev->ops->ioctl)
            return dev->ops->ioctl(dev, ioctl, arg);
 
        return -ENOTTY;
    }
}

Here is the call graph for this function:

kvm_device_ioctl_attr()

static int kvm_device_ioctl_attr ( struct kvm_device *  dev, int(*)(struct kvm_device *dev, struct kvm_device_attr *attr)  accessor, unsigned long  arg  )

static

Definition at line 4665 of file kvm_main.c.

{
    struct kvm_device_attr attr;
 
    if (!accessor)
        return -EPERM;
 
    if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
        return -EFAULT;
 
    return accessor(dev, &attr);
}

Here is the caller graph for this function:

kvm_device_mmap()

static int kvm_device_mmap ( struct file *  filp, struct vm_area_struct *  vma  )

static

Definition at line 4655 of file kvm_main.c.

{
    struct kvm_device *dev = filp->private_data;
 
    if (dev->ops->mmap)
        return dev->ops->mmap(dev, vma);
 
    return -ENODEV;
}

kvm_device_release()

static int kvm_device_release ( struct inode *  inode, struct file *  filp  )

static

Definition at line 4704 of file kvm_main.c.

{
    struct kvm_device *dev = filp->private_data;
    struct kvm *kvm = dev->kvm;
 
    if (dev->ops->release) {
        mutex_lock(&kvm->lock);
        list_del(&dev->vm_node);
        dev->ops->release(dev);
        mutex_unlock(&kvm->lock);
    }
 
    kvm_put_kvm(kvm);
    return 0;
}

kvm_erase_gfn_node()

static void kvm_erase_gfn_node ( struct kvm_memslots *  slots, struct kvm_memory_slot *  slot  )

static

Definition at line 1517 of file kvm_main.c.

{
    rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree);
}

Here is the caller graph for this function:

kvm_exit()

void kvm_exit

(

void

)

Definition at line 6495 of file kvm_main.c.

{
    int cpu;
 
    /*
     * Note, unregistering /dev/kvm doesn't strictly need to come first,
     * fops_get(), a.k.a. try_module_get(), prevents acquiring references
     * to KVM while the module is being stopped.
     */
    misc_deregister(&kvm_dev);
 
    debugfs_remove_recursive(kvm_debugfs_dir);
    for_each_possible_cpu(cpu)
        free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
    kmem_cache_destroy(kvm_vcpu_cache);
    kvm_vfio_ops_exit();
    kvm_async_pf_deinit();
#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
    unregister_syscore_ops(&kvm_syscore_ops);
    cpuhp_remove_state_nocalls(CPUHP_AP_KVM_ONLINE);
#endif
    kvm_irqfd_exit();
}

Here is the call graph for this function:

kvm_flush_remote_tlbs()

void kvm_flush_remote_tlbs

(

struct kvm *

kvm

)

Definition at line 346 of file kvm_main.c.

{
    ++kvm->stat.generic.remote_tlb_flush_requests;
 
    /*
     * We want to publish modifications to the page tables before reading
     * mode. Pairs with a memory barrier in arch-specific code.
     * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
     * and smp_mb in walk_shadow_page_lockless_begin/end.
     * - powerpc: smp_mb in kvmppc_prepare_to_enter.
     *
     * There is already an smp_mb__after_atomic() before
     * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
     * barrier here.
     */
    if (!kvm_arch_flush_remote_tlbs(kvm)
        || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
        ++kvm->stat.generic.remote_tlb_flush;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_flush_remote_tlbs_memslot()

void kvm_flush_remote_tlbs_memslot	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	memslot
	)

Definition at line 380 of file kvm_main.c.

{
    /*
     * All current use cases for flushing the TLBs for a specific memslot
     * are related to dirty logging, and many do the TLB flush out of
     * mmu_lock. The interaction between the various operations on memslot
     * must be serialized by slots_locks to ensure the TLB flush from one
     * operation is observed by any other operation on the same memslot.
     */
    lockdep_assert_held(&kvm->slots_lock);
    kvm_flush_remote_tlbs_range(kvm, memslot->base_gfn, memslot->npages);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_flush_remote_tlbs_range()

void kvm_flush_remote_tlbs_range	(	struct kvm *	kvm,
		gfn_t	gfn,
		u64	nr_pages
	)

Definition at line 367 of file kvm_main.c.

{
    if (!kvm_arch_flush_remote_tlbs_range(kvm, gfn, nr_pages))
        return;
 
    /*
     * Fall back to a flushing entire TLBs if the architecture range-based
     * TLB invalidation is unsupported or can't be performed for whatever
     * reason.
     */
    kvm_flush_remote_tlbs(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_flush_shadow_all()

static void kvm_flush_shadow_all ( struct kvm *  kvm )

static

Definition at line 394 of file kvm_main.c.

{
    kvm_arch_flush_shadow_all(kvm);
    kvm_arch_guest_memory_reclaimed(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_free_memslot()

static void kvm_free_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  slot  )

static

Definition at line 1026 of file kvm_main.c.

{
    if (slot->flags & KVM_MEM_GUEST_MEMFD)
        kvm_gmem_unbind(slot);
 
    kvm_destroy_dirty_bitmap(slot);
 
    kvm_arch_free_memslot(kvm, slot);
 
    kfree(slot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_free_memslots()

static void kvm_free_memslots ( struct kvm *  kvm, struct kvm_memslots *  slots  )

static

Definition at line 1038 of file kvm_main.c.

{
    struct hlist_node *idnode;
    struct kvm_memory_slot *memslot;
    int bkt;
 
    /*
     * The same memslot objects live in both active and inactive sets,
     * arbitrarily free using index '1' so the second invocation of this
     * function isn't operating over a structure with dangling pointers
     * (even though this function isn't actually touching them).
     */
    if (!slots->node_idx)
        return;
 
    hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1])
        kvm_free_memslot(kvm, memslot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_get_dirty_log()

int kvm_get_dirty_log	(	struct kvm *	kvm,
		struct kvm_dirty_log *	log,
		int *	is_dirty,
		struct kvm_memory_slot **	memslot
	)

kvm_get_dirty_log - get a snapshot of dirty pages @kvm: pointer to kvm instance @log: slot id and address to which we copy the log @is_dirty: set to '1' if any dirty pages were found @memslot: set to the associated memslot, always valid on success

Definition at line 2173 of file kvm_main.c.

{
    struct kvm_memslots *slots;
    int i, as_id, id;
    unsigned long n;
    unsigned long any = 0;
 
    /* Dirty ring tracking may be exclusive to dirty log tracking */
    if (!kvm_use_dirty_bitmap(kvm))
        return -ENXIO;
 
    *memslot = NULL;
    *is_dirty = 0;
 
    as_id = log->slot >> 16;
    id = (u16)log->slot;
    if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
        return -EINVAL;
 
    slots = __kvm_memslots(kvm, as_id);
    *memslot = id_to_memslot(slots, id);
    if (!(*memslot) || !(*memslot)->dirty_bitmap)
        return -ENOENT;
 
    kvm_arch_sync_dirty_log(kvm, *memslot);
 
    n = kvm_dirty_bitmap_bytes(*memslot);
 
    for (i = 0; !any && i < n/sizeof(long); ++i)
        any = (*memslot)->dirty_bitmap[i];
 
    if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))
        return -EFAULT;
 
    if (any)
        *is_dirty = 1;
    return 0;
}

Here is the call graph for this function:

kvm_get_inactive_memslots()

static struct kvm_memslots* kvm_get_inactive_memslots ( struct kvm *  kvm, int  as_id  )

static

Definition at line 1464 of file kvm_main.c.

{
    struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
    int node_idx_inactive = active->node_idx ^ 1;
 
    return &kvm->__memslots[as_id][node_idx_inactive];
}

Here is the caller graph for this function:

kvm_get_kvm()

void kvm_get_kvm

(

struct kvm *

kvm

)

Definition at line 1403 of file kvm_main.c.

{
    refcount_inc(&kvm->users_count);
}

Here is the caller graph for this function:

kvm_get_kvm_safe()

bool kvm_get_kvm_safe

(

struct kvm *

kvm

)

Definition at line 1413 of file kvm_main.c.

{
    return refcount_inc_not_zero(&kvm->users_count);
}

Here is the caller graph for this function:

kvm_get_running_vcpu()

struct kvm_vcpu* kvm_get_running_vcpu

(

void

)

kvm_get_running_vcpu - get the vcpu running on the current CPU.

We can disable preemption locally around accessing the per-CPU variable, and use the resolved vcpu pointer after enabling preemption again, because even if the current thread is migrated to another CPU, reading the per-CPU value later will give us the same value as we update the per-CPU variable in the preempt notifier handlers.

Definition at line 6338 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu;
 
    preempt_disable();
    vcpu = __this_cpu_read(kvm_running_vcpu);
    preempt_enable();
 
    return vcpu;
}

Here is the caller graph for this function:

kvm_get_running_vcpus()

struct kvm_vcpu* __percpu* kvm_get_running_vcpus

(

void

)

kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.

Definition at line 6353 of file kvm_main.c.

{
        return &kvm_running_vcpu;
}

Here is the caller graph for this function:

kvm_get_stat_per_vcpu()

static int kvm_get_stat_per_vcpu ( struct kvm *  kvm, size_t  offset, u64 *  val  )

static

Definition at line 6063 of file kvm_main.c.

{
    unsigned long i;
    struct kvm_vcpu *vcpu;
 
    *val = 0;
 
    kvm_for_each_vcpu(i, vcpu, kvm)
        *val += *(u64 *)((void *)(&vcpu->stat) + offset);
 
    return 0;
}

Here is the caller graph for this function:

kvm_get_stat_per_vm()

static int kvm_get_stat_per_vm ( struct kvm *  kvm, size_t  offset, u64 *  val  )

static

Definition at line 6049 of file kvm_main.c.

{
    *val = *(u64 *)((void *)(&kvm->stat) + offset);
 
    return 0;
}

Here is the caller graph for this function:

kvm_gfn_to_hva_cache_init()

int kvm_gfn_to_hva_cache_init	(	struct kvm *	kvm,
		struct gfn_to_hva_cache *	ghc,
		gpa_t	gpa,
		unsigned long	len
	)

Definition at line 3532 of file kvm_main.c.

{
    struct kvm_memslots *slots = kvm_memslots(kvm);
    return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
}

Here is the call graph for this function:

kvm_host_page_size()

unsigned long kvm_host_page_size	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 2685 of file kvm_main.c.

{
    struct vm_area_struct *vma;
    unsigned long addr, size;
 
    size = PAGE_SIZE;
 
    addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
    if (kvm_is_error_hva(addr))
        return PAGE_SIZE;
 
    mmap_read_lock(current->mm);
    vma = find_vma(current->mm, addr);
    if (!vma)
        goto out;
 
    size = vma_kernel_pagesize(vma);
 
out:
    mmap_read_unlock(current->mm);
 
    return size;
}

Here is the call graph for this function:

kvm_init()

int kvm_init	(	unsigned	vcpu_size,
		unsigned	vcpu_align,
		struct module *	module
	)

Definition at line 6402 of file kvm_main.c.

{
    int r;
    int cpu;
 
#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
    r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_ONLINE, "kvm/cpu:online",
                      kvm_online_cpu, kvm_offline_cpu);
    if (r)
        return r;
 
    register_syscore_ops(&kvm_syscore_ops);
#endif
 
    /* A kmem cache lets us meet the alignment requirements of fx_save. */
    if (!vcpu_align)
        vcpu_align = __alignof__(struct kvm_vcpu);
    kvm_vcpu_cache =
        kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
                       SLAB_ACCOUNT,
                       offsetof(struct kvm_vcpu, arch),
                       offsetofend(struct kvm_vcpu, stats_id)
                       - offsetof(struct kvm_vcpu, arch),
                       NULL);
    if (!kvm_vcpu_cache) {
        r = -ENOMEM;
        goto err_vcpu_cache;
    }
 
    for_each_possible_cpu(cpu) {
        if (!alloc_cpumask_var_node(&per_cpu(cpu_kick_mask, cpu),
                        GFP_KERNEL, cpu_to_node(cpu))) {
            r = -ENOMEM;
            goto err_cpu_kick_mask;
        }
    }
 
    r = kvm_irqfd_init();
    if (r)
        goto err_irqfd;
 
    r = kvm_async_pf_init();
    if (r)
        goto err_async_pf;
 
    kvm_chardev_ops.owner = module;
    kvm_vm_fops.owner = module;
    kvm_vcpu_fops.owner = module;
    kvm_device_fops.owner = module;
 
    kvm_preempt_ops.sched_in = kvm_sched_in;
    kvm_preempt_ops.sched_out = kvm_sched_out;
 
    kvm_init_debug();
 
    r = kvm_vfio_ops_init();
    if (WARN_ON_ONCE(r))
        goto err_vfio;
 
    kvm_gmem_init(module);
 
    /*
     * Registration _must_ be the very last thing done, as this exposes
     * /dev/kvm to userspace, i.e. all infrastructure must be setup!
     */
    r = misc_register(&kvm_dev);
    if (r) {
        pr_err("kvm: misc device register failed\n");
        goto err_register;
    }
 
    return 0;
 
err_register:
    kvm_vfio_ops_exit();
err_vfio:
    kvm_async_pf_deinit();
err_async_pf:
    kvm_irqfd_exit();
err_irqfd:
err_cpu_kick_mask:
    for_each_possible_cpu(cpu)
        free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
    kmem_cache_destroy(kvm_vcpu_cache);
err_vcpu_cache:
#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
    unregister_syscore_ops(&kvm_syscore_ops);
    cpuhp_remove_state_nocalls(CPUHP_AP_KVM_ONLINE);
#endif
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_init_debug()

static void kvm_init_debug ( void  )

static

Definition at line 6267 of file kvm_main.c.

{
    const struct file_operations *fops;
    const struct _kvm_stats_desc *pdesc;
    int i;
 
    kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
 
    for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
        pdesc = &kvm_vm_stats_desc[i];
        if (kvm_stats_debugfs_mode(pdesc) & 0222)
            fops = &vm_stat_fops;
        else
            fops = &vm_stat_readonly_fops;
        debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                kvm_debugfs_dir,
                (void *)(long)pdesc->desc.offset, fops);
    }
 
    for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
        pdesc = &kvm_vcpu_stats_desc[i];
        if (kvm_stats_debugfs_mode(pdesc) & 0222)
            fops = &vcpu_stat_fops;
        else
            fops = &vcpu_stat_readonly_fops;
        debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                kvm_debugfs_dir,
                (void *)(long)pdesc->desc.offset, fops);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_init_mmu_notifier()

static int kvm_init_mmu_notifier ( struct kvm *  kvm )

static

Definition at line 977 of file kvm_main.c.

{
    return 0;
}

Here is the caller graph for this function:

kvm_init_pm_notifier()

static void kvm_init_pm_notifier ( struct kvm *  kvm )

static

Definition at line 1007 of file kvm_main.c.

{
}

Here is the caller graph for this function:

kvm_insert_gfn_node()

static void kvm_insert_gfn_node ( struct kvm_memslots *  slots, struct kvm_memory_slot *  slot  )

static

Definition at line 1492 of file kvm_main.c.

{
    struct rb_root *gfn_tree = &slots->gfn_tree;
    struct rb_node **node, *parent;
    int idx = slots->node_idx;
 
    parent = NULL;
    for (node = &gfn_tree->rb_node; *node; ) {
        struct kvm_memory_slot *tmp;
 
        tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
        parent = *node;
        if (slot->base_gfn < tmp->base_gfn)
            node = &(*node)->rb_left;
        else if (slot->base_gfn > tmp->base_gfn)
            node = &(*node)->rb_right;
        else
            BUG();
    }
 
    rb_link_node(&slot->gfn_node[idx], parent, node);
    rb_insert_color(&slot->gfn_node[idx], gfn_tree);
}

Here is the caller graph for this function:

kvm_invalidate_memslot()

static void kvm_invalidate_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  invalid_slot  )

static

Definition at line 1816 of file kvm_main.c.

{
    /*
     * Mark the current slot INVALID.  As with all memslot modifications,
     * this must be done on an unreachable slot to avoid modifying the
     * current slot in the active tree.
     */
    kvm_copy_memslot(invalid_slot, old);
    invalid_slot->flags |= KVM_MEMSLOT_INVALID;
    kvm_replace_memslot(kvm, old, invalid_slot);
 
    /*
     * Activate the slot that is now marked INVALID, but don't propagate
     * the slot to the now inactive slots. The slot is either going to be
     * deleted or recreated as a new slot.
     */
    kvm_swap_active_memslots(kvm, old->as_id);
 
    /*
     * From this point no new shadow pages pointing to a deleted, or moved,
     * memslot will be created.  Validation of sp->gfn happens in:
     *  - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
     *  - kvm_is_visible_gfn (mmu_check_root)
     */
    kvm_arch_flush_shadow_memslot(kvm, old);
    kvm_arch_guest_memory_reclaimed(kvm);
 
    /* Was released by kvm_swap_active_memslots(), reacquire. */
    mutex_lock(&kvm->slots_arch_lock);
 
    /*
     * Copy the arch-specific field of the newly-installed slot back to the
     * old slot as the arch data could have changed between releasing
     * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock
     * above.  Writers are required to retrieve memslots *after* acquiring
     * slots_arch_lock, thus the active slot's data is guaranteed to be fresh.
     */
    old->arch = invalid_slot->arch;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_cmp()

static int kvm_io_bus_cmp ( const struct kvm_io_range *  r1, const struct kvm_io_range *  r2  )

inlinestatic

Definition at line 5732 of file kvm_main.c.

{
    gpa_t addr1 = r1->addr;
    gpa_t addr2 = r2->addr;
 
    if (addr1 < addr2)
        return -1;
 
    /* If r2->len == 0, match the exact address.  If r2->len != 0,
     * accept any overlapping write.  Any order is acceptable for
     * overlapping ranges, because kvm_io_bus_get_first_dev ensures
     * we process all of them.
     */
    if (r2->len) {
        addr1 += r1->len;
        addr2 += r2->len;
    }
 
    if (addr1 > addr2)
        return 1;
 
    return 0;
}

Here is the caller graph for this function:

kvm_io_bus_destroy()

static void kvm_io_bus_destroy ( struct kvm_io_bus *  bus )

static

Definition at line 5720 of file kvm_main.c.

{
    int i;
 
    for (i = 0; i < bus->dev_count; i++) {
        struct kvm_io_device *pos = bus->range[i].dev;
 
        kvm_iodevice_destructor(pos);
    }
    kfree(bus);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_get_dev()

struct kvm_io_device* kvm_io_bus_get_dev	(	struct kvm *	kvm,
		enum kvm_bus	bus_idx,
		gpa_t	addr
	)

Definition at line 5989 of file kvm_main.c.

{
    struct kvm_io_bus *bus;
    int dev_idx, srcu_idx;
    struct kvm_io_device *iodev = NULL;
 
    srcu_idx = srcu_read_lock(&kvm->srcu);
 
    bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
    if (!bus)
        goto out_unlock;
 
    dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
    if (dev_idx < 0)
        goto out_unlock;
 
    iodev = bus->range[dev_idx].dev;
 
out_unlock:
    srcu_read_unlock(&kvm->srcu, srcu_idx);
 
    return iodev;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_get_first_dev()

static int kvm_io_bus_get_first_dev ( struct kvm_io_bus *  bus, gpa_t  addr, int  len  )

static

Definition at line 5762 of file kvm_main.c.

{
    struct kvm_io_range *range, key;
    int off;
 
    key = (struct kvm_io_range) {
        .addr = addr,
        .len = len,
    };
 
    range = bsearch(&key, bus->range, bus->dev_count,
            sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
    if (range == NULL)
        return -ENOENT;
 
    off = range - bus->range;
 
    while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
        off--;
 
    return off;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_read()

int kvm_io_bus_read	(	struct kvm_vcpu *	vcpu,
		enum kvm_bus	bus_idx,
		gpa_t	addr,
		int	len,
		void *	val
	)

Definition at line 5878 of file kvm_main.c.

{
    struct kvm_io_bus *bus;
    struct kvm_io_range range;
    int r;
 
    range = (struct kvm_io_range) {
        .addr = addr,
        .len = len,
    };
 
    bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
    if (!bus)
        return -ENOMEM;
    r = __kvm_io_bus_read(vcpu, bus, &range, val);
    return r < 0 ? r : 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_register_dev()

int kvm_io_bus_register_dev	(	struct kvm *	kvm,
		enum kvm_bus	bus_idx,
		gpa_t	addr,
		int	len,
		struct kvm_io_device *	dev
	)

Definition at line 5897 of file kvm_main.c.

{
    int i;
    struct kvm_io_bus *new_bus, *bus;
    struct kvm_io_range range;
 
    lockdep_assert_held(&kvm->slots_lock);
 
    bus = kvm_get_bus(kvm, bus_idx);
    if (!bus)
        return -ENOMEM;
 
    /* exclude ioeventfd which is limited by maximum fd */
    if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
        return -ENOSPC;
 
    new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
              GFP_KERNEL_ACCOUNT);
    if (!new_bus)
        return -ENOMEM;
 
    range = (struct kvm_io_range) {
        .addr = addr,
        .len = len,
        .dev = dev,
    };
 
    for (i = 0; i < bus->dev_count; i++)
        if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
            break;
 
    memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
    new_bus->dev_count++;
    new_bus->range[i] = range;
    memcpy(new_bus->range + i + 1, bus->range + i,
        (bus->dev_count - i) * sizeof(struct kvm_io_range));
    rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
    synchronize_srcu_expedited(&kvm->srcu);
    kfree(bus);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_sort_cmp()

static int kvm_io_bus_sort_cmp ( const void *  p1, const void *  p2  )

static

Definition at line 5757 of file kvm_main.c.

{
    return kvm_io_bus_cmp(p1, p2);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_unregister_dev()

int kvm_io_bus_unregister_dev	(	struct kvm *	kvm,
		enum kvm_bus	bus_idx,
		struct kvm_io_device *	dev
	)

Definition at line 5941 of file kvm_main.c.

{
    int i;
    struct kvm_io_bus *new_bus, *bus;
 
    lockdep_assert_held(&kvm->slots_lock);
 
    bus = kvm_get_bus(kvm, bus_idx);
    if (!bus)
        return 0;
 
    for (i = 0; i < bus->dev_count; i++) {
        if (bus->range[i].dev == dev) {
            break;
        }
    }
 
    if (i == bus->dev_count)
        return 0;
 
    new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
              GFP_KERNEL_ACCOUNT);
    if (new_bus) {
        memcpy(new_bus, bus, struct_size(bus, range, i));
        new_bus->dev_count--;
        memcpy(new_bus->range + i, bus->range + i + 1,
                flex_array_size(new_bus, range, new_bus->dev_count - i));
    }
 
    rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
    synchronize_srcu_expedited(&kvm->srcu);
 
    /*
     * If NULL bus is installed, destroy the old bus, including all the
     * attached devices. Otherwise, destroy the caller's device only.
     */
    if (!new_bus) {
        pr_err("kvm: failed to shrink bus, removing it completely\n");
        kvm_io_bus_destroy(bus);
        return -ENOMEM;
    }
 
    kvm_iodevice_destructor(dev);
    kfree(bus);
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_write()

int kvm_io_bus_write	(	struct kvm_vcpu *	vcpu,
		enum kvm_bus	bus_idx,
		gpa_t	addr,
		int	len,
		const void *	val
	)

Definition at line 5807 of file kvm_main.c.

{
    struct kvm_io_bus *bus;
    struct kvm_io_range range;
    int r;
 
    range = (struct kvm_io_range) {
        .addr = addr,
        .len = len,
    };
 
    bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
    if (!bus)
        return -ENOMEM;
    r = __kvm_io_bus_write(vcpu, bus, &range, val);
    return r < 0 ? r : 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_io_bus_write_cookie()

int kvm_io_bus_write_cookie	(	struct kvm_vcpu *	vcpu,
		enum kvm_bus	bus_idx,
		gpa_t	addr,
		int	len,
		const void *	val,
		long	cookie
	)

Definition at line 5828 of file kvm_main.c.

{
    struct kvm_io_bus *bus;
    struct kvm_io_range range;
 
    range = (struct kvm_io_range) {
        .addr = addr,
        .len = len,
    };
 
    bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
    if (!bus)
        return -ENOMEM;
 
    /* First try the device referenced by cookie. */
    if ((cookie >= 0) && (cookie < bus->dev_count) &&
        (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
        if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
                    val))
            return cookie;
 
    /*
     * cookie contained garbage; fall back to search and return the
     * correct cookie value.
     */
    return __kvm_io_bus_write(vcpu, bus, &range, val);
}

Here is the call graph for this function:

kvm_ioctl_create_device()

static int kvm_ioctl_create_device ( struct kvm *  kvm, struct kvm_create_device *  cd  )

static

Definition at line 4760 of file kvm_main.c.

{
    const struct kvm_device_ops *ops;
    struct kvm_device *dev;
    bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
    int type;
    int ret;
 
    if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
        return -ENODEV;
 
    type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
    ops = kvm_device_ops_table[type];
    if (ops == NULL)
        return -ENODEV;
 
    if (test)
        return 0;
 
    dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
    if (!dev)
        return -ENOMEM;
 
    dev->ops = ops;
    dev->kvm = kvm;
 
    mutex_lock(&kvm->lock);
    ret = ops->create(dev, type);
    if (ret < 0) {
        mutex_unlock(&kvm->lock);
        kfree(dev);
        return ret;
    }
    list_add(&dev->vm_node, &kvm->devices);
    mutex_unlock(&kvm->lock);
 
    if (ops->init)
        ops->init(dev);
 
    kvm_get_kvm(kvm);
    ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
    if (ret < 0) {
        kvm_put_kvm_no_destroy(kvm);
        mutex_lock(&kvm->lock);
        list_del(&dev->vm_node);
        if (ops->release)
            ops->release(dev);
        mutex_unlock(&kvm->lock);
        if (ops->destroy)
            ops->destroy(dev);
        return ret;
    }
 
    cd->fd = ret;
    return 0;
}

Here is the call graph for this function:

kvm_iodevice_destructor()

static void kvm_iodevice_destructor ( struct kvm_io_device *  dev )

static

Definition at line 5714 of file kvm_main.c.

{
    if (dev->ops->destructor)
        dev->ops->destructor(dev);
}

Here is the caller graph for this function:

kvm_is_ad_tracked_page()

static bool kvm_is_ad_tracked_page ( struct page *  page )

static

Definition at line 3211 of file kvm_main.c.

{
    /*
     * Per page-flags.h, pages tagged PG_reserved "should in general not be
     * touched (e.g. set dirty) except by its owner".
     */
    return !PageReserved(page);
}

Here is the caller graph for this function:

kvm_is_visible_gfn()

bool kvm_is_visible_gfn	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 2669 of file kvm_main.c.

{
    struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
 
    return kvm_is_visible_memslot(memslot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_is_zone_device_page()

bool kvm_is_zone_device_page

(

struct page *

page

)

Definition at line 159 of file kvm_main.c.

{
    /*
     * The metadata used by is_zone_device_page() to determine whether or
     * not a page is ZONE_DEVICE is guaranteed to be valid if and only if
     * the device has been pinned, e.g. by get_user_pages().  WARN if the
     * page_count() is zero to help detect bad usage of this helper.
     */
    if (WARN_ON_ONCE(!page_count(page)))
        return false;
 
    return is_zone_device_page(page);
}

Here is the caller graph for this function:

kvm_kick_many_cpus()

static bool kvm_kick_many_cpus ( struct cpumask *  cpus, bool  wait  )

inlinestatic

Definition at line 251 of file kvm_main.c.

{
    if (cpumask_empty(cpus))
        return false;
 
    smp_call_function_many(cpus, ack_kick, NULL, wait);
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_make_all_cpus_request()

bool kvm_make_all_cpus_request	(	struct kvm *	kvm,
		unsigned int	req
	)

Definition at line 340 of file kvm_main.c.

{
    return kvm_make_all_cpus_request_except(kvm, req, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_make_all_cpus_request_except()

bool kvm_make_all_cpus_request_except	(	struct kvm *	kvm,
		unsigned int	req,
		struct kvm_vcpu *	except
	)

Definition at line 314 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu;
    struct cpumask *cpus;
    unsigned long i;
    bool called;
    int me;
 
    me = get_cpu();
 
    cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
    cpumask_clear(cpus);
 
    kvm_for_each_vcpu(i, vcpu, kvm) {
        if (vcpu == except)
            continue;
        kvm_make_vcpu_request(vcpu, req, cpus, me);
    }
 
    called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
    put_cpu();
 
    return called;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_make_vcpu_request()

static void kvm_make_vcpu_request ( struct kvm_vcpu *  vcpu, unsigned int  req, struct cpumask *  tmp, int  current_cpu  )

static

Definition at line 260 of file kvm_main.c.

{
    int cpu;
 
    if (likely(!(req & KVM_REQUEST_NO_ACTION)))
        __kvm_make_request(req, vcpu);
 
    if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
        return;
 
    /*
     * Note, the vCPU could get migrated to a different pCPU at any point
     * after kvm_request_needs_ipi(), which could result in sending an IPI
     * to the previous pCPU.  But, that's OK because the purpose of the IPI
     * is to ensure the vCPU returns to OUTSIDE_GUEST_MODE, which is
     * satisfied if the vCPU migrates. Entering READING_SHADOW_PAGE_TABLES
     * after this point is also OK, as the requirement is only that KVM wait
     * for vCPUs that were reading SPTEs _before_ any changes were
     * finalized. See kvm_vcpu_kick() for more details on handling requests.
     */
    if (kvm_request_needs_ipi(vcpu, req)) {
        cpu = READ_ONCE(vcpu->cpu);
        if (cpu != -1 && cpu != current_cpu)
            __cpumask_set_cpu(cpu, tmp);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_make_vcpus_request_mask()

bool kvm_make_vcpus_request_mask	(	struct kvm *	kvm,
		unsigned int	req,
		unsigned long *	vcpu_bitmap
	)

Definition at line 288 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu;
    struct cpumask *cpus;
    int i, me;
    bool called;
 
    me = get_cpu();
 
    cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
    cpumask_clear(cpus);
 
    for_each_set_bit(i, vcpu_bitmap, KVM_MAX_VCPUS) {
        vcpu = kvm_get_vcpu(kvm, i);
        if (!vcpu)
            continue;
        kvm_make_vcpu_request(vcpu, req, cpus, me);
    }
 
    called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
    put_cpu();
 
    return called;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_memslots_get_as_id()

static int kvm_memslots_get_as_id ( struct kvm_memory_slot *  a, struct kvm_memory_slot *  b  )

static

Definition at line 1477 of file kvm_main.c.

{
    if (WARN_ON_ONCE(!a && !b))
        return 0;
 
    if (!a)
        return b->as_id;
    if (!b)
        return a->as_id;
 
    WARN_ON_ONCE(a->as_id != b->as_id);
    return a->as_id;
}

Here is the caller graph for this function:

kvm_move_memslot()

static void kvm_move_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new, struct kvm_memory_slot *  invalid_slot  )

static

Definition at line 1878 of file kvm_main.c.

{
    /*
     * Replace the old memslot in the inactive slots, and then swap slots
     * and replace the current INVALID with the new as well.
     */
    kvm_replace_memslot(kvm, old, new);
    kvm_activate_memslot(kvm, invalid_slot, new);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_no_compat_ioctl()

static long kvm_no_compat_ioctl ( struct file *  file, unsigned int  ioctl, unsigned long  arg  )

static

Definition at line 132 of file kvm_main.c.

                                   { return -EINVAL; }

kvm_no_compat_open()

static int kvm_no_compat_open ( struct inode *  inode, struct file *  file  )

static

Definition at line 135 of file kvm_main.c.

{
    return is_compat_task() ? -ENODEV : 0;
}

kvm_page_in_dirty_ring()

static bool kvm_page_in_dirty_ring ( struct kvm *  kvm, unsigned long  pgoff  )

static

Definition at line 4112 of file kvm_main.c.

{
#ifdef CONFIG_HAVE_KVM_DIRTY_RING
    return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) &&
        (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET +
         kvm->dirty_ring_size / PAGE_SIZE);
#else
    return false;
#endif
}

Here is the caller graph for this function:

kvm_pfn_to_refcounted_page()

struct page* kvm_pfn_to_refcounted_page

(

kvm_pfn_t

pfn

)

Definition at line 179 of file kvm_main.c.

{
    struct page *page;
 
    if (!pfn_valid(pfn))
        return NULL;
 
    page = pfn_to_page(pfn);
    if (!PageReserved(page))
        return page;
 
    /* The ZERO_PAGE(s) is marked PG_reserved, but is refcounted. */
    if (is_zero_pfn(pfn))
        return page;
 
    /*
     * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting
     * perspective they are "normal" pages, albeit with slightly different
     * usage rules.
     */
    if (kvm_is_zone_device_page(page))
        return page;
 
    return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_prepare_memory_region()

static int kvm_prepare_memory_region ( struct kvm *  kvm, const struct kvm_memory_slot *  old, struct kvm_memory_slot *  new, enum kvm_mr_change  change  )

static

Definition at line 1691 of file kvm_main.c.

{
    int r;
 
    /*
     * If dirty logging is disabled, nullify the bitmap; the old bitmap
     * will be freed on "commit".  If logging is enabled in both old and
     * new, reuse the existing bitmap.  If logging is enabled only in the
     * new and KVM isn't using a ring buffer, allocate and initialize a
     * new bitmap.
     */
    if (change != KVM_MR_DELETE) {
        if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
            new->dirty_bitmap = NULL;
        else if (old && old->dirty_bitmap)
            new->dirty_bitmap = old->dirty_bitmap;
        else if (kvm_use_dirty_bitmap(kvm)) {
            r = kvm_alloc_dirty_bitmap(new);
            if (r)
                return r;
 
            if (kvm_dirty_log_manual_protect_and_init_set(kvm))
                bitmap_set(new->dirty_bitmap, 0, new->npages);
        }
    }
 
    r = kvm_arch_prepare_memory_region(kvm, old, new, change);
 
    /* Free the bitmap on failure if it was allocated above. */
    if (r && new && new->dirty_bitmap && (!old || !old->dirty_bitmap))
        kvm_destroy_dirty_bitmap(new);
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_put_kvm()

void kvm_put_kvm

(

struct kvm *

kvm

)

Definition at line 1419 of file kvm_main.c.

{
    if (refcount_dec_and_test(&kvm->users_count))
        kvm_destroy_vm(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_put_kvm_no_destroy()

void kvm_put_kvm_no_destroy

(

struct kvm *

kvm

)

Definition at line 1433 of file kvm_main.c.

{
    WARN_ON(refcount_dec_and_test(&kvm->users_count));
}

Here is the caller graph for this function:

kvm_read_guest()

int kvm_read_guest	(	struct kvm *	kvm,
		gpa_t	gpa,
		void *	data,
		unsigned long	len
	)

Definition at line 3346 of file kvm_main.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    int seg;
    int offset = offset_in_page(gpa);
    int ret;
 
    while ((seg = next_segment(len, offset)) != 0) {
        ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
        if (ret < 0)
            return ret;
        offset = 0;
        len -= seg;
        data += seg;
        ++gfn;
    }
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_read_guest_cached()

int kvm_read_guest_cached	(	struct kvm *	kvm,
		struct gfn_to_hva_cache *	ghc,
		void *	data,
		unsigned long	len
	)

Definition at line 3608 of file kvm_main.c.

{
    return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
}

Here is the call graph for this function:

kvm_read_guest_offset_cached()

int kvm_read_guest_offset_cached	(	struct kvm *	kvm,
		struct gfn_to_hva_cache *	ghc,
		void *	data,
		unsigned int	offset,
		unsigned long	len
	)

Definition at line 3578 of file kvm_main.c.

{
    struct kvm_memslots *slots = kvm_memslots(kvm);
    int r;
    gpa_t gpa = ghc->gpa + offset;
 
    if (WARN_ON_ONCE(len + offset > ghc->len))
        return -EINVAL;
 
    if (slots->generation != ghc->generation) {
        if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
            return -EFAULT;
    }
 
    if (kvm_is_error_hva(ghc->hva))
        return -EFAULT;
 
    if (unlikely(!ghc->memslot))
        return kvm_read_guest(kvm, gpa, data, len);
 
    r = __copy_from_user(data, (void __user *)ghc->hva + offset, len);
    if (r)
        return -EFAULT;
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_read_guest_page()

int kvm_read_guest_page	(	struct kvm *	kvm,
		gfn_t	gfn,
		void *	data,
		int	offset,
		int	len
	)

Definition at line 3328 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
 
    return __kvm_read_guest_page(slot, gfn, data, offset, len);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_register_device_ops()

int kvm_register_device_ops	(	const struct kvm_device_ops *	ops,
		u32	type
	)

Definition at line 4742 of file kvm_main.c.

{
    if (type >= ARRAY_SIZE(kvm_device_ops_table))
        return -ENOSPC;
 
    if (kvm_device_ops_table[type] != NULL)
        return -EEXIST;
 
    kvm_device_ops_table[type] = ops;
    return 0;
}

Here is the caller graph for this function:

kvm_release_page_clean()

void kvm_release_page_clean

(

struct page *

page

)

Definition at line 3232 of file kvm_main.c.

{
    WARN_ON(is_error_page(page));
 
    kvm_set_page_accessed(page);
    put_page(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_release_page_dirty()

void kvm_release_page_dirty

(

struct page *

page

)

Definition at line 3256 of file kvm_main.c.

{
    WARN_ON(is_error_page(page));
 
    kvm_set_page_dirty(page);
    kvm_release_page_clean(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_release_pfn()

void kvm_release_pfn	(	kvm_pfn_t	pfn,
		bool	dirty
	)

Definition at line 3144 of file kvm_main.c.

{
    if (dirty)
        kvm_release_pfn_dirty(pfn);
    else
        kvm_release_pfn_clean(pfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_release_pfn_clean()

void kvm_release_pfn_clean

(

kvm_pfn_t

pfn

)

Definition at line 3241 of file kvm_main.c.

{
    struct page *page;
 
    if (is_error_noslot_pfn(pfn))
        return;
 
    page = kvm_pfn_to_refcounted_page(pfn);
    if (!page)
        return;
 
    kvm_release_page_clean(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_release_pfn_dirty()

void kvm_release_pfn_dirty

(

kvm_pfn_t

pfn

)

Definition at line 3265 of file kvm_main.c.

{
    struct page *page;
 
    if (is_error_noslot_pfn(pfn))
        return;
 
    page = kvm_pfn_to_refcounted_page(pfn);
    if (!page)
        return;
 
    kvm_release_page_dirty(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_replace_gfn_node()

static void kvm_replace_gfn_node ( struct kvm_memslots *  slots, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new  )

static

Definition at line 1523 of file kvm_main.c.

{
    int idx = slots->node_idx;
 
    WARN_ON_ONCE(old->base_gfn != new->base_gfn);
 
    rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx],
            &slots->gfn_tree);
}

Here is the caller graph for this function:

kvm_replace_memslot()

static void kvm_replace_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new  )

static

Definition at line 1544 of file kvm_main.c.

{
    int as_id = kvm_memslots_get_as_id(old, new);
    struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
    int idx = slots->node_idx;
 
    if (old) {
        hash_del(&old->id_node[idx]);
        interval_tree_remove(&old->hva_node[idx], &slots->hva_tree);
 
        if ((long)old == atomic_long_read(&slots->last_used_slot))
            atomic_long_set(&slots->last_used_slot, (long)new);
 
        if (!new) {
            kvm_erase_gfn_node(slots, old);
            return;
        }
    }
 
    /*
     * Initialize @new's hva range.  Do this even when replacing an @old
     * slot, kvm_copy_memslot() deliberately does not touch node data.
     */
    new->hva_node[idx].start = new->userspace_addr;
    new->hva_node[idx].last = new->userspace_addr +
                  (new->npages << PAGE_SHIFT) - 1;
 
    /*
     * (Re)Add the new memslot.  There is no O(1) interval_tree_replace(),
     * hva_node needs to be swapped with remove+insert even though hva can't
     * change when replacing an existing slot.
     */
    hash_add(slots->id_hash, &new->id_node[idx], new->id);
    interval_tree_insert(&new->hva_node[idx], &slots->hva_tree);
 
    /*
     * If the memslot gfn is unchanged, rb_replace_node() can be used to
     * switch the node in the gfn tree instead of removing the old and
     * inserting the new as two separate operations. Replacement is a
     * single O(1) operation versus two O(log(n)) operations for
     * remove+insert.
     */
    if (old && old->base_gfn == new->base_gfn) {
        kvm_replace_gfn_node(slots, old, new);
    } else {
        if (old)
            kvm_erase_gfn_node(slots, old);
        kvm_insert_gfn_node(slots, new);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_request_needs_ipi()

static bool kvm_request_needs_ipi ( struct kvm_vcpu *  vcpu, unsigned  req  )

static

Definition at line 230 of file kvm_main.c.

{
    int mode = kvm_vcpu_exiting_guest_mode(vcpu);
 
    /*
     * We need to wait for the VCPU to reenable interrupts and get out of
     * READING_SHADOW_PAGE_TABLES mode.
     */
    if (req & KVM_REQUEST_WAIT)
        return mode != OUTSIDE_GUEST_MODE;
 
    /*
     * Need to kick a running VCPU, but otherwise there is nothing to do.
     */
    return mode == IN_GUEST_MODE;
}

Here is the caller graph for this function:

kvm_sched_in()

static void kvm_sched_in ( struct preempt_notifier *  pn, int  cpu  )

static

Definition at line 6304 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 
    WRITE_ONCE(vcpu->preempted, false);
    WRITE_ONCE(vcpu->ready, false);
 
    __this_cpu_write(kvm_running_vcpu, vcpu);
    kvm_arch_sched_in(vcpu, cpu);
    kvm_arch_vcpu_load(vcpu, cpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_sched_out()

static void kvm_sched_out ( struct preempt_notifier *  pn, struct task_struct *  next  )

static

Definition at line 6316 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 
    if (current->on_rq) {
        WRITE_ONCE(vcpu->preempted, true);
        WRITE_ONCE(vcpu->ready, true);
    }
    kvm_arch_vcpu_put(vcpu);
    __this_cpu_write(kvm_running_vcpu, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_memory_region()

int kvm_set_memory_region	(	struct kvm *	kvm,
		const struct kvm_userspace_memory_region2 *	mem
	)

Definition at line 2144 of file kvm_main.c.

{
    int r;
 
    mutex_lock(&kvm->slots_lock);
    r = __kvm_set_memory_region(kvm, mem);
    mutex_unlock(&kvm->slots_lock);
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_memslot()

static int kvm_set_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new, enum kvm_mr_change  change  )

static

Definition at line 1904 of file kvm_main.c.

{
    struct kvm_memory_slot *invalid_slot;
    int r;
 
    /*
     * Released in kvm_swap_active_memslots().
     *
     * Must be held from before the current memslots are copied until after
     * the new memslots are installed with rcu_assign_pointer, then
     * released before the synchronize srcu in kvm_swap_active_memslots().
     *
     * When modifying memslots outside of the slots_lock, must be held
     * before reading the pointer to the current memslots until after all
     * changes to those memslots are complete.
     *
     * These rules ensure that installing new memslots does not lose
     * changes made to the previous memslots.
     */
    mutex_lock(&kvm->slots_arch_lock);
 
    /*
     * Invalidate the old slot if it's being deleted or moved.  This is
     * done prior to actually deleting/moving the memslot to allow vCPUs to
     * continue running by ensuring there are no mappings or shadow pages
     * for the memslot when it is deleted/moved.  Without pre-invalidation
     * (and without a lock), a window would exist between effecting the
     * delete/move and committing the changes in arch code where KVM or a
     * guest could access a non-existent memslot.
     *
     * Modifications are done on a temporary, unreachable slot.  The old
     * slot needs to be preserved in case a later step fails and the
     * invalidation needs to be reverted.
     */
    if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
        invalid_slot = kzalloc(sizeof(*invalid_slot), GFP_KERNEL_ACCOUNT);
        if (!invalid_slot) {
            mutex_unlock(&kvm->slots_arch_lock);
            return -ENOMEM;
        }
        kvm_invalidate_memslot(kvm, old, invalid_slot);
    }
 
    r = kvm_prepare_memory_region(kvm, old, new, change);
    if (r) {
        /*
         * For DELETE/MOVE, revert the above INVALID change.  No
         * modifications required since the original slot was preserved
         * in the inactive slots.  Changing the active memslots also
         * release slots_arch_lock.
         */
        if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
            kvm_activate_memslot(kvm, invalid_slot, old);
            kfree(invalid_slot);
        } else {
            mutex_unlock(&kvm->slots_arch_lock);
        }
        return r;
    }
 
    /*
     * For DELETE and MOVE, the working slot is now active as the INVALID
     * version of the old slot.  MOVE is particularly special as it reuses
     * the old slot and returns a copy of the old slot (in working_slot).
     * For CREATE, there is no old slot.  For DELETE and FLAGS_ONLY, the
     * old slot is detached but otherwise preserved.
     */
    if (change == KVM_MR_CREATE)
        kvm_create_memslot(kvm, new);
    else if (change == KVM_MR_DELETE)
        kvm_delete_memslot(kvm, old, invalid_slot);
    else if (change == KVM_MR_MOVE)
        kvm_move_memslot(kvm, old, new, invalid_slot);
    else if (change == KVM_MR_FLAGS_ONLY)
        kvm_update_flags_memslot(kvm, old, new);
    else
        BUG();
 
    /* Free the temporary INVALID slot used for DELETE and MOVE. */
    if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
        kfree(invalid_slot);
 
    /*
     * No need to refresh new->arch, changes after dropping slots_arch_lock
     * will directly hit the final, active memslot.  Architectures are
     * responsible for knowing that new->arch may be stale.
     */
    kvm_commit_memory_region(kvm, old, new, change);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_page_accessed()

static void kvm_set_page_accessed ( struct page *  page )

static

Definition at line 3226 of file kvm_main.c.

{
    if (kvm_is_ad_tracked_page(page))
        mark_page_accessed(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_page_dirty()

static void kvm_set_page_dirty ( struct page *  page )

static

Definition at line 3220 of file kvm_main.c.

{
    if (kvm_is_ad_tracked_page(page))
        SetPageDirty(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_pfn_accessed()

void kvm_set_pfn_accessed

(

kvm_pfn_t

pfn

)

Definition at line 3295 of file kvm_main.c.

{
    if (WARN_ON(is_error_noslot_pfn(pfn)))
        return;
 
    if (pfn_valid(pfn))
        kvm_set_page_accessed(pfn_to_page(pfn));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_set_pfn_dirty()

void kvm_set_pfn_dirty

(

kvm_pfn_t

pfn

)

Definition at line 3285 of file kvm_main.c.

{
    if (WARN_ON(is_error_noslot_pfn(pfn)))
        return;
 
    if (pfn_valid(pfn))
        kvm_set_page_dirty(pfn_to_page(pfn));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_sigset_activate()

void kvm_sigset_activate

(

struct kvm_vcpu *

vcpu

)

Definition at line 3678 of file kvm_main.c.

{
    if (!vcpu->sigset_active)
        return;
 
    /*
     * This does a lockless modification of ->real_blocked, which is fine
     * because, only current can change ->real_blocked and all readers of
     * ->real_blocked don't care as long ->real_blocked is always a subset
     * of ->blocked.
     */
    sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
}

Here is the caller graph for this function:

kvm_sigset_deactivate()

void kvm_sigset_deactivate

(

struct kvm_vcpu *

vcpu

)

Definition at line 3692 of file kvm_main.c.

{
    if (!vcpu->sigset_active)
        return;
 
    sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
    sigemptyset(&current->real_blocked);
}

Here is the caller graph for this function:

kvm_stat_data_clear()

static int kvm_stat_data_clear ( void *  data, u64  val  )

static

Definition at line 6106 of file kvm_main.c.

{
    int r = -EFAULT;
    struct kvm_stat_data *stat_data = data;
 
    if (val)
        return -EINVAL;
 
    switch (stat_data->kind) {
    case KVM_STAT_VM:
        r = kvm_clear_stat_per_vm(stat_data->kvm,
                      stat_data->desc->desc.offset);
        break;
    case KVM_STAT_VCPU:
        r = kvm_clear_stat_per_vcpu(stat_data->kvm,
                        stat_data->desc->desc.offset);
        break;
    }
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_stat_data_get()

static int kvm_stat_data_get ( void *  data, u64 *  val  )

static

Definition at line 6087 of file kvm_main.c.

{
    int r = -EFAULT;
    struct kvm_stat_data *stat_data = data;
 
    switch (stat_data->kind) {
    case KVM_STAT_VM:
        r = kvm_get_stat_per_vm(stat_data->kvm,
                    stat_data->desc->desc.offset, val);
        break;
    case KVM_STAT_VCPU:
        r = kvm_get_stat_per_vcpu(stat_data->kvm,
                      stat_data->desc->desc.offset, val);
        break;
    }
 
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_stat_data_open()

static int kvm_stat_data_open ( struct inode *  inode, struct file *  file  )

static

Definition at line 6128 of file kvm_main.c.

{
    __simple_attr_check_format("%llu\n", 0ull);
    return kvm_debugfs_open(inode, file, kvm_stat_data_get,
                kvm_stat_data_clear, "%llu\n");
}

Here is the call graph for this function:

kvm_stats_debugfs_mode()

static umode_t kvm_stats_debugfs_mode ( const struct _kvm_stats_desc *  pdesc )

static

Definition at line 1057 of file kvm_main.c.

{
    switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
    case KVM_STATS_TYPE_INSTANT:
        return 0444;
    case KVM_STATS_TYPE_CUMULATIVE:
    case KVM_STATS_TYPE_PEAK:
    default:
        return 0644;
    }
}

Here is the caller graph for this function:

kvm_swap_active_memslots()

static void kvm_swap_active_memslots ( struct kvm *  kvm, int  as_id  )

static

Definition at line 1633 of file kvm_main.c.

{
    struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
 
    /* Grab the generation from the activate memslots. */
    u64 gen = __kvm_memslots(kvm, as_id)->generation;
 
    WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
    slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
 
    /*
     * Do not store the new memslots while there are invalidations in
     * progress, otherwise the locking in invalidate_range_start and
     * invalidate_range_end will be unbalanced.
     */
    spin_lock(&kvm->mn_invalidate_lock);
    prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);
    while (kvm->mn_active_invalidate_count) {
        set_current_state(TASK_UNINTERRUPTIBLE);
        spin_unlock(&kvm->mn_invalidate_lock);
        schedule();
        spin_lock(&kvm->mn_invalidate_lock);
    }
    finish_rcuwait(&kvm->mn_memslots_update_rcuwait);
    rcu_assign_pointer(kvm->memslots[as_id], slots);
    spin_unlock(&kvm->mn_invalidate_lock);
 
    /*
     * Acquired in kvm_set_memslot. Must be released before synchronize
     * SRCU below in order to avoid deadlock with another thread
     * acquiring the slots_arch_lock in an srcu critical section.
     */
    mutex_unlock(&kvm->slots_arch_lock);
 
    synchronize_srcu_expedited(&kvm->srcu);
 
    /*
     * Increment the new memslot generation a second time, dropping the
     * update in-progress flag and incrementing the generation based on
     * the number of address spaces.  This provides a unique and easily
     * identifiable generation number while the memslots are in flux.
     */
    gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
 
    /*
     * Generations must be unique even across address spaces.  We do not need
     * a global counter for that, instead the generation space is evenly split
     * across address spaces.  For example, with two address spaces, address
     * space 0 will use generations 0, 2, 4, ... while address space 1 will
     * use generations 1, 3, 5, ...
     */
    gen += kvm_arch_nr_memslot_as_ids(kvm);
 
    kvm_arch_memslots_updated(kvm, gen);
 
    slots->generation = gen;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_try_get_pfn()

static int kvm_try_get_pfn ( kvm_pfn_t  pfn )

static

Definition at line 2887 of file kvm_main.c.

{
    struct page *page = kvm_pfn_to_refcounted_page(pfn);
 
    if (!page)
        return 1;
 
    return get_page_unless_zero(page);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_uevent_notify_change()

static void kvm_uevent_notify_change ( unsigned int  type, struct kvm *  kvm  )

static

Definition at line 6217 of file kvm_main.c.

{
    struct kobj_uevent_env *env;
    unsigned long long created, active;
 
    if (!kvm_dev.this_device || !kvm)
        return;
 
    mutex_lock(&kvm_lock);
    if (type == KVM_EVENT_CREATE_VM) {
        kvm_createvm_count++;
        kvm_active_vms++;
    } else if (type == KVM_EVENT_DESTROY_VM) {
        kvm_active_vms--;
    }
    created = kvm_createvm_count;
    active = kvm_active_vms;
    mutex_unlock(&kvm_lock);
 
    env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
    if (!env)
        return;
 
    add_uevent_var(env, "CREATED=%llu", created);
    add_uevent_var(env, "COUNT=%llu", active);
 
    if (type == KVM_EVENT_CREATE_VM) {
        add_uevent_var(env, "EVENT=create");
        kvm->userspace_pid = task_pid_nr(current);
    } else if (type == KVM_EVENT_DESTROY_VM) {
        add_uevent_var(env, "EVENT=destroy");
    }
    add_uevent_var(env, "PID=%d", kvm->userspace_pid);
 
    if (!IS_ERR(kvm->debugfs_dentry)) {
        char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
 
        if (p) {
            tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
            if (!IS_ERR(tmp))
                add_uevent_var(env, "STATS_PATH=%s", tmp);
            kfree(p);
        }
    }
    /* no need for checks, since we are adding at most only 5 keys */
    env->envp[env->envp_idx++] = NULL;
    kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
    kfree(env);
}

Here is the caller graph for this function:

kvm_unregister_device_ops()

void kvm_unregister_device_ops

(

u32

type

)

Definition at line 4754 of file kvm_main.c.

{
    if (kvm_device_ops_table[type] != NULL)
        kvm_device_ops_table[type] = NULL;
}

Here is the caller graph for this function:

kvm_update_flags_memslot()

static void kvm_update_flags_memslot ( struct kvm *  kvm, struct kvm_memory_slot *  old, struct kvm_memory_slot *  new  )

static

Definition at line 1891 of file kvm_main.c.

{
    /*
     * Similar to the MOVE case, but the slot doesn't need to be zapped as
     * an intermediate step. Instead, the old memslot is simply replaced
     * with a new, updated copy in both memslot sets.
     */
    kvm_replace_memslot(kvm, old, new);
    kvm_activate_memslot(kvm, old, new);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_block()

bool kvm_vcpu_block

(

struct kvm_vcpu *

vcpu

)

Definition at line 3764 of file kvm_main.c.

{
    struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
    bool waited = false;
 
    vcpu->stat.generic.blocking = 1;
 
    preempt_disable();
    kvm_arch_vcpu_blocking(vcpu);
    prepare_to_rcuwait(wait);
    preempt_enable();
 
    for (;;) {
        set_current_state(TASK_INTERRUPTIBLE);
 
        if (kvm_vcpu_check_block(vcpu) < 0)
            break;
 
        waited = true;
        schedule();
    }
 
    preempt_disable();
    finish_rcuwait(wait);
    kvm_arch_vcpu_unblocking(vcpu);
    preempt_enable();
 
    vcpu->stat.generic.blocking = 0;
 
    return waited;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_check_block()

static int kvm_vcpu_check_block ( struct kvm_vcpu *  vcpu )

static

Definition at line 3739 of file kvm_main.c.

{
    int ret = -EINTR;
    int idx = srcu_read_lock(&vcpu->kvm->srcu);
 
    if (kvm_arch_vcpu_runnable(vcpu))
        goto out;
    if (kvm_cpu_has_pending_timer(vcpu))
        goto out;
    if (signal_pending(current))
        goto out;
    if (kvm_check_request(KVM_REQ_UNBLOCK, vcpu))
        goto out;
 
    ret = 0;
out:
    srcu_read_unlock(&vcpu->kvm->srcu, idx);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_destroy()

static void kvm_vcpu_destroy ( struct kvm_vcpu *  vcpu )

static

Definition at line 506 of file kvm_main.c.

{
    kvm_arch_vcpu_destroy(vcpu);
    kvm_dirty_ring_free(&vcpu->dirty_ring);
 
    /*
     * No need for rcu_read_lock as VCPU_RUN is the only place that changes
     * the vcpu->pid pointer, and at destruction time all file descriptors
     * are already gone.
     */
    put_pid(rcu_dereference_protected(vcpu->pid, 1));
 
    free_page((unsigned long)vcpu->run);
    kmem_cache_free(kvm_vcpu_cache, vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_eligible_for_directed_yield()

static bool kvm_vcpu_eligible_for_directed_yield ( struct kvm_vcpu *  vcpu )

static

Definition at line 4011 of file kvm_main.c.

{
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
    bool eligible;
 
    eligible = !vcpu->spin_loop.in_spin_loop ||
            vcpu->spin_loop.dy_eligible;
 
    if (vcpu->spin_loop.in_spin_loop)
        kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
 
    return eligible;
#else
    return true;
#endif
}

Here is the caller graph for this function:

kvm_vcpu_fault()

static vm_fault_t kvm_vcpu_fault ( struct vm_fault *  vmf )

static

Definition at line 4123 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
    struct page *page;
 
    if (vmf->pgoff == 0)
        page = virt_to_page(vcpu->run);
#ifdef CONFIG_X86
    else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
        page = virt_to_page(vcpu->arch.pio_data);
#endif
#ifdef CONFIG_KVM_MMIO
    else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
        page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
#endif
    else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))
        page = kvm_dirty_ring_get_page(
            &vcpu->dirty_ring,
            vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET);
    else
        return kvm_arch_vcpu_fault(vcpu, vmf);
    get_page(page);
    vmf->page = page;
    return 0;
}

Here is the call graph for this function:

kvm_vcpu_gfn_to_hva()

unsigned long kvm_vcpu_gfn_to_hva	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 2748 of file kvm_main.c.

{
    return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
}

Here is the call graph for this function:

kvm_vcpu_gfn_to_hva_prot()

unsigned long kvm_vcpu_gfn_to_hva_prot	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn,
		bool *	writable
	)

Definition at line 2780 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 
    return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_gfn_to_memslot()

struct kvm_memory_slot* kvm_vcpu_gfn_to_memslot	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 2636 of file kvm_main.c.

{
    struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
    u64 gen = slots->generation;
    struct kvm_memory_slot *slot;
 
    /*
     * This also protects against using a memslot from a different address space,
     * since different address spaces have different generation numbers.
     */
    if (unlikely(gen != vcpu->last_used_slot_gen)) {
        vcpu->last_used_slot = NULL;
        vcpu->last_used_slot_gen = gen;
    }
 
    slot = try_get_memslot(vcpu->last_used_slot, gfn);
    if (slot)
        return slot;
 
    /*
     * Fall back to searching all memslots. We purposely use
     * search_memslots() instead of __gfn_to_memslot() to avoid
     * thrashing the VM-wide last_used_slot in kvm_memslots.
     */
    slot = search_memslots(slots, gfn, false);
    if (slot) {
        vcpu->last_used_slot = slot;
        return slot;
    }
 
    return NULL;
}

Here is the caller graph for this function:

kvm_vcpu_gfn_to_pfn()

kvm_pfn_t kvm_vcpu_gfn_to_pfn	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 3097 of file kvm_main.c.

{
    return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}

Here is the call graph for this function:

kvm_vcpu_gfn_to_pfn_atomic()

kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 3085 of file kvm_main.c.

{
    return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}

Here is the call graph for this function:

kvm_vcpu_halt()

void kvm_vcpu_halt

(

struct kvm_vcpu *

vcpu

)

Definition at line 3842 of file kvm_main.c.

{
    unsigned int max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
    bool halt_poll_allowed = !kvm_arch_no_poll(vcpu);
    ktime_t start, cur, poll_end;
    bool waited = false;
    bool do_halt_poll;
    u64 halt_ns;
 
    if (vcpu->halt_poll_ns > max_halt_poll_ns)
        vcpu->halt_poll_ns = max_halt_poll_ns;
 
    do_halt_poll = halt_poll_allowed && vcpu->halt_poll_ns;
 
    start = cur = poll_end = ktime_get();
    if (do_halt_poll) {
        ktime_t stop = ktime_add_ns(start, vcpu->halt_poll_ns);
 
        do {
            if (kvm_vcpu_check_block(vcpu) < 0)
                goto out;
            cpu_relax();
            poll_end = cur = ktime_get();
        } while (kvm_vcpu_can_poll(cur, stop));
    }
 
    waited = kvm_vcpu_block(vcpu);
 
    cur = ktime_get();
    if (waited) {
        vcpu->stat.generic.halt_wait_ns +=
            ktime_to_ns(cur) - ktime_to_ns(poll_end);
        KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.generic.halt_wait_hist,
                ktime_to_ns(cur) - ktime_to_ns(poll_end));
    }
out:
    /* The total time the vCPU was "halted", including polling time. */
    halt_ns = ktime_to_ns(cur) - ktime_to_ns(start);
 
    /*
     * Note, halt-polling is considered successful so long as the vCPU was
     * never actually scheduled out, i.e. even if the wake event arrived
     * after of the halt-polling loop itself, but before the full wait.
     */
    if (do_halt_poll)
        update_halt_poll_stats(vcpu, start, poll_end, !waited);
 
    if (halt_poll_allowed) {
        /* Recompute the max halt poll time in case it changed. */
        max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
 
        if (!vcpu_valid_wakeup(vcpu)) {
            shrink_halt_poll_ns(vcpu);
        } else if (max_halt_poll_ns) {
            if (halt_ns <= vcpu->halt_poll_ns)
                ;
            /* we had a long block, shrink polling */
            else if (vcpu->halt_poll_ns &&
                 halt_ns > max_halt_poll_ns)
                shrink_halt_poll_ns(vcpu);
            /* we had a short halt and our poll time is too small */
            else if (vcpu->halt_poll_ns < max_halt_poll_ns &&
                 halt_ns < max_halt_poll_ns)
                grow_halt_poll_ns(vcpu);
        } else {
            vcpu->halt_poll_ns = 0;
        }
    }
 
    trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu));
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_init()

static void kvm_vcpu_init ( struct kvm_vcpu *  vcpu, struct kvm *  kvm, unsigned  id  )

static

Definition at line 482 of file kvm_main.c.

{
    mutex_init(&vcpu->mutex);
    vcpu->cpu = -1;
    vcpu->kvm = kvm;
    vcpu->vcpu_id = id;
    vcpu->pid = NULL;
#ifndef __KVM_HAVE_ARCH_WQP
    rcuwait_init(&vcpu->wait);
#endif
    kvm_async_pf_vcpu_init(vcpu);
 
    kvm_vcpu_set_in_spin_loop(vcpu, false);
    kvm_vcpu_set_dy_eligible(vcpu, false);
    vcpu->preempted = false;
    vcpu->ready = false;
    preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
    vcpu->last_used_slot = NULL;
 
    /* Fill the stats id string for the vcpu */
    snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d",
         task_pid_nr(current), id);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_ioctl()

static long kvm_vcpu_ioctl ( struct file *  file, unsigned int  ioctl, unsigned long  arg  )

static

Definition at line 4401 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = filp->private_data;
    void __user *argp = (void __user *)arg;
    int r;
    struct kvm_fpu *fpu = NULL;
    struct kvm_sregs *kvm_sregs = NULL;
 
    if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
        return -EIO;
 
    if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
        return -EINVAL;
 
    /*
     * Some architectures have vcpu ioctls that are asynchronous to vcpu
     * execution; mutex_lock() would break them.
     */
    r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
    if (r != -ENOIOCTLCMD)
        return r;
 
    if (mutex_lock_killable(&vcpu->mutex))
        return -EINTR;
    switch (ioctl) {
    case KVM_RUN: {
        struct pid *oldpid;
        r = -EINVAL;
        if (arg)
            goto out;
        oldpid = rcu_access_pointer(vcpu->pid);
        if (unlikely(oldpid != task_pid(current))) {
            /* The thread running this VCPU changed. */
            struct pid *newpid;
 
            r = kvm_arch_vcpu_run_pid_change(vcpu);
            if (r)
                break;
 
            newpid = get_task_pid(current, PIDTYPE_PID);
            rcu_assign_pointer(vcpu->pid, newpid);
            if (oldpid)
                synchronize_rcu();
            put_pid(oldpid);
        }
        r = kvm_arch_vcpu_ioctl_run(vcpu);
        trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
        break;
    }
    case KVM_GET_REGS: {
        struct kvm_regs *kvm_regs;
 
        r = -ENOMEM;
        kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
        if (!kvm_regs)
            goto out;
        r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
        if (r)
            goto out_free1;
        r = -EFAULT;
        if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
            goto out_free1;
        r = 0;
out_free1:
        kfree(kvm_regs);
        break;
    }
    case KVM_SET_REGS: {
        struct kvm_regs *kvm_regs;
 
        kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
        if (IS_ERR(kvm_regs)) {
            r = PTR_ERR(kvm_regs);
            goto out;
        }
        r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
        kfree(kvm_regs);
        break;
    }
    case KVM_GET_SREGS: {
        kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
                    GFP_KERNEL_ACCOUNT);
        r = -ENOMEM;
        if (!kvm_sregs)
            goto out;
        r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
        if (r)
            goto out;
        r = -EFAULT;
        if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
            goto out;
        r = 0;
        break;
    }
    case KVM_SET_SREGS: {
        kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
        if (IS_ERR(kvm_sregs)) {
            r = PTR_ERR(kvm_sregs);
            kvm_sregs = NULL;
            goto out;
        }
        r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
        break;
    }
    case KVM_GET_MP_STATE: {
        struct kvm_mp_state mp_state;
 
        r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
        if (r)
            goto out;
        r = -EFAULT;
        if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
            goto out;
        r = 0;
        break;
    }
    case KVM_SET_MP_STATE: {
        struct kvm_mp_state mp_state;
 
        r = -EFAULT;
        if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
            goto out;
        r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
        break;
    }
    case KVM_TRANSLATE: {
        struct kvm_translation tr;
 
        r = -EFAULT;
        if (copy_from_user(&tr, argp, sizeof(tr)))
            goto out;
        r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
        if (r)
            goto out;
        r = -EFAULT;
        if (copy_to_user(argp, &tr, sizeof(tr)))
            goto out;
        r = 0;
        break;
    }
    case KVM_SET_GUEST_DEBUG: {
        struct kvm_guest_debug dbg;
 
        r = -EFAULT;
        if (copy_from_user(&dbg, argp, sizeof(dbg)))
            goto out;
        r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
        break;
    }
    case KVM_SET_SIGNAL_MASK: {
        struct kvm_signal_mask __user *sigmask_arg = argp;
        struct kvm_signal_mask kvm_sigmask;
        sigset_t sigset, *p;
 
        p = NULL;
        if (argp) {
            r = -EFAULT;
            if (copy_from_user(&kvm_sigmask, argp,
                       sizeof(kvm_sigmask)))
                goto out;
            r = -EINVAL;
            if (kvm_sigmask.len != sizeof(sigset))
                goto out;
            r = -EFAULT;
            if (copy_from_user(&sigset, sigmask_arg->sigset,
                       sizeof(sigset)))
                goto out;
            p = &sigset;
        }
        r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
        break;
    }
    case KVM_GET_FPU: {
        fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
        r = -ENOMEM;
        if (!fpu)
            goto out;
        r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
        if (r)
            goto out;
        r = -EFAULT;
        if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
            goto out;
        r = 0;
        break;
    }
    case KVM_SET_FPU: {
        fpu = memdup_user(argp, sizeof(*fpu));
        if (IS_ERR(fpu)) {
            r = PTR_ERR(fpu);
            fpu = NULL;
            goto out;
        }
        r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
        break;
    }
    case KVM_GET_STATS_FD: {
        r = kvm_vcpu_ioctl_get_stats_fd(vcpu);
        break;
    }
    default:
        r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
    }
out:
    mutex_unlock(&vcpu->mutex);
    kfree(fpu);
    kfree(kvm_sregs);
    return r;
}

Here is the call graph for this function:

kvm_vcpu_ioctl_get_stats_fd()

static int kvm_vcpu_ioctl_get_stats_fd ( struct kvm_vcpu *  vcpu )

static

Definition at line 4375 of file kvm_main.c.

{
    int fd;
    struct file *file;
    char name[15 + ITOA_MAX_LEN + 1];
 
    snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id);
 
    fd = get_unused_fd_flags(O_CLOEXEC);
    if (fd < 0)
        return fd;
 
    file = anon_inode_getfile(name, &kvm_vcpu_stats_fops, vcpu, O_RDONLY);
    if (IS_ERR(file)) {
        put_unused_fd(fd);
        return PTR_ERR(file);
    }
 
    kvm_get_kvm(vcpu->kvm);
 
    file->f_mode |= FMODE_PREAD;
    fd_install(fd, file);
 
    return fd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_ioctl_set_sigmask()

static int kvm_vcpu_ioctl_set_sigmask ( struct kvm_vcpu *  vcpu, sigset_t *  sigset  )

static

Definition at line 4339 of file kvm_main.c.

{
    if (sigset) {
        sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
        vcpu->sigset_active = 1;
        vcpu->sigset = *sigset;
    } else
        vcpu->sigset_active = 0;
    return 0;
}

Here is the caller graph for this function:

kvm_vcpu_is_visible_gfn()

bool kvm_vcpu_is_visible_gfn	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 2677 of file kvm_main.c.

{
    struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 
    return kvm_is_visible_memslot(memslot);
}

Here is the call graph for this function:

kvm_vcpu_kick()

void kvm_vcpu_kick

(

struct kvm_vcpu *

vcpu

)

Definition at line 3931 of file kvm_main.c.

{
    int me, cpu;
 
    if (kvm_vcpu_wake_up(vcpu))
        return;
 
    me = get_cpu();
    /*
     * The only state change done outside the vcpu mutex is IN_GUEST_MODE
     * to EXITING_GUEST_MODE.  Therefore the moderately expensive "should
     * kick" check does not need atomic operations if kvm_vcpu_kick is used
     * within the vCPU thread itself.
     */
    if (vcpu == __this_cpu_read(kvm_running_vcpu)) {
        if (vcpu->mode == IN_GUEST_MODE)
            WRITE_ONCE(vcpu->mode, EXITING_GUEST_MODE);
        goto out;
    }
 
    /*
     * Note, the vCPU could get migrated to a different pCPU at any point
     * after kvm_arch_vcpu_should_kick(), which could result in sending an
     * IPI to the previous pCPU.  But, that's ok because the purpose of the
     * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the
     * vCPU also requires it to leave IN_GUEST_MODE.
     */
    if (kvm_arch_vcpu_should_kick(vcpu)) {
        cpu = READ_ONCE(vcpu->cpu);
        if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
            smp_send_reschedule(cpu);
    }
out:
    put_cpu();
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_map()

int kvm_vcpu_map	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn,
		struct kvm_host_map *	map
	)

Definition at line 3152 of file kvm_main.c.

{
    kvm_pfn_t pfn;
    void *hva = NULL;
    struct page *page = KVM_UNMAPPED_PAGE;
 
    if (!map)
        return -EINVAL;
 
    pfn = gfn_to_pfn(vcpu->kvm, gfn);
    if (is_error_noslot_pfn(pfn))
        return -EINVAL;
 
    if (pfn_valid(pfn)) {
        page = pfn_to_page(pfn);
        hva = kmap(page);
#ifdef CONFIG_HAS_IOMEM
    } else {
        hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
#endif
    }
 
    if (!hva)
        return -EFAULT;
 
    map->page = page;
    map->hva = hva;
    map->pfn = pfn;
    map->gfn = gfn;
 
    return 0;
}

Here is the call graph for this function:

kvm_vcpu_mark_page_dirty()

void kvm_vcpu_mark_page_dirty	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 3669 of file kvm_main.c.

{
    struct kvm_memory_slot *memslot;
 
    memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
    mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_max_halt_poll_ns()

static unsigned int kvm_vcpu_max_halt_poll_ns ( struct kvm_vcpu *  vcpu )

static

Definition at line 3818 of file kvm_main.c.

{
    struct kvm *kvm = vcpu->kvm;
 
    if (kvm->override_halt_poll_ns) {
        /*
         * Ensure kvm->max_halt_poll_ns is not read before
         * kvm->override_halt_poll_ns.
         *
         * Pairs with the smp_wmb() when enabling KVM_CAP_HALT_POLL.
         */
        smp_rmb();
        return READ_ONCE(kvm->max_halt_poll_ns);
    }
 
    return READ_ONCE(halt_poll_ns);
}

Here is the caller graph for this function:

kvm_vcpu_mmap()

static int kvm_vcpu_mmap ( struct file *  file, struct vm_area_struct *  vma  )

static

Definition at line 4153 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = file->private_data;
    unsigned long pages = vma_pages(vma);
 
    if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||
         kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&
        ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED)))
        return -EINVAL;
 
    vma->vm_ops = &kvm_vcpu_vm_ops;
    return 0;
}

Here is the call graph for this function:

kvm_vcpu_on_spin()

void kvm_vcpu_on_spin	(	struct kvm_vcpu *	me,
		bool	yield_to_kernel_mode
	)

Definition at line 4056 of file kvm_main.c.

{
    struct kvm *kvm = me->kvm;
    struct kvm_vcpu *vcpu;
    int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
    unsigned long i;
    int yielded = 0;
    int try = 3;
    int pass;
 
    kvm_vcpu_set_in_spin_loop(me, true);
    /*
     * We boost the priority of a VCPU that is runnable but not
     * currently running, because it got preempted by something
     * else and called schedule in __vcpu_run.  Hopefully that
     * VCPU is holding the lock that we need and will release it.
     * We approximate round-robin by starting at the last boosted VCPU.
     */
    for (pass = 0; pass < 2 && !yielded && try; pass++) {
        kvm_for_each_vcpu(i, vcpu, kvm) {
            if (!pass && i <= last_boosted_vcpu) {
                i = last_boosted_vcpu;
                continue;
            } else if (pass && i > last_boosted_vcpu)
                break;
            if (!READ_ONCE(vcpu->ready))
                continue;
            if (vcpu == me)
                continue;
            if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu))
                continue;
            if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
                !kvm_arch_dy_has_pending_interrupt(vcpu) &&
                !kvm_arch_vcpu_in_kernel(vcpu))
                continue;
            if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
                continue;
 
            yielded = kvm_vcpu_yield_to(vcpu);
            if (yielded > 0) {
                kvm->last_boosted_vcpu = i;
                break;
            } else if (yielded < 0) {
                try--;
                if (!try)
                    break;
            }
        }
    }
    kvm_vcpu_set_in_spin_loop(me, false);
 
    /* Ensure vcpu is not eligible during next spinloop */
    kvm_vcpu_set_dy_eligible(me, false);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_read_guest()

int kvm_vcpu_read_guest	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		void *	data,
		unsigned long	len
	)

Definition at line 3366 of file kvm_main.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    int seg;
    int offset = offset_in_page(gpa);
    int ret;
 
    while ((seg = next_segment(len, offset)) != 0) {
        ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
        if (ret < 0)
            return ret;
        offset = 0;
        len -= seg;
        data += seg;
        ++gfn;
    }
    return 0;
}

Here is the call graph for this function:

kvm_vcpu_read_guest_atomic()

int kvm_vcpu_read_guest_atomic	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		void *	data,
		unsigned long	len
	)

Definition at line 3403 of file kvm_main.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
    int offset = offset_in_page(gpa);
 
    return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}

Here is the call graph for this function:

kvm_vcpu_read_guest_page()

int kvm_vcpu_read_guest_page	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn,
		void *	data,
		int	offset,
		int	len
	)

Definition at line 3337 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 
    return __kvm_read_guest_page(slot, gfn, data, offset, len);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_release()

static int kvm_vcpu_release ( struct inode *  inode, struct file *  filp  )

static

Definition at line 4167 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = filp->private_data;
 
    kvm_put_kvm(vcpu->kvm);
    return 0;
}

kvm_vcpu_stats_read()

static ssize_t kvm_vcpu_stats_read ( struct file *  file, char __user *  user_buffer, size_t  size, loff_t *  offset  )

static

Definition at line 4350 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = file->private_data;
 
    return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header,
            &kvm_vcpu_stats_desc[0], &vcpu->stat,
            sizeof(vcpu->stat), user_buffer, size, offset);
}

Here is the call graph for this function:

kvm_vcpu_stats_release()

static int kvm_vcpu_stats_release ( struct inode *  inode, struct file *  file  )

static

Definition at line 4360 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = file->private_data;
 
    kvm_put_kvm(vcpu->kvm);
    return 0;
}

kvm_vcpu_unmap()

void kvm_vcpu_unmap	(	struct kvm_vcpu *	vcpu,
		struct kvm_host_map *	map,
		bool	dirty
	)

Definition at line 3186 of file kvm_main.c.

{
    if (!map)
        return;
 
    if (!map->hva)
        return;
 
    if (map->page != KVM_UNMAPPED_PAGE)
        kunmap(map->page);
#ifdef CONFIG_HAS_IOMEM
    else
        memunmap(map->hva);
#endif
 
    if (dirty)
        kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
 
    kvm_release_pfn(map->pfn, dirty);
 
    map->hva = NULL;
    map->page = NULL;
}

Here is the call graph for this function:

kvm_vcpu_wake_up()

bool kvm_vcpu_wake_up

(

struct kvm_vcpu *

vcpu

)

Definition at line 3915 of file kvm_main.c.

{
    if (__kvm_vcpu_wake_up(vcpu)) {
        WRITE_ONCE(vcpu->ready, true);
        ++vcpu->stat.generic.halt_wakeup;
        return true;
    }
 
    return false;
}

Here is the caller graph for this function:

kvm_vcpu_write_guest()

int kvm_vcpu_write_guest	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		const void *	data,
		unsigned long	len
	)

Definition at line 3470 of file kvm_main.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    int seg;
    int offset = offset_in_page(gpa);
    int ret;
 
    while ((seg = next_segment(len, offset)) != 0) {
        ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
        if (ret < 0)
            return ret;
        offset = 0;
        len -= seg;
        data += seg;
        ++gfn;
    }
    return 0;
}

Here is the call graph for this function:

kvm_vcpu_write_guest_page()

int kvm_vcpu_write_guest_page	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn,
		const void *	data,
		int	offset,
		int	len
	)

Definition at line 3440 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 
    return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vcpu_yield_to()

int kvm_vcpu_yield_to

(

struct kvm_vcpu *

target

)

Definition at line 3969 of file kvm_main.c.

{
    struct pid *pid;
    struct task_struct *task = NULL;
    int ret = 0;
 
    rcu_read_lock();
    pid = rcu_dereference(target->pid);
    if (pid)
        task = get_pid_task(pid, PIDTYPE_PID);
    rcu_read_unlock();
    if (!task)
        return ret;
    ret = yield_to(task, 1);
    put_task_struct(task);
 
    return ret;
}

Here is the caller graph for this function:

kvm_vm_create_worker_thread()

int kvm_vm_create_worker_thread	(	struct kvm *	kvm,
		kvm_vm_thread_fn_t	thread_fn,
		uintptr_t	data,
		const char *	name,
		struct task_struct **	thread_ptr
	)

Definition at line 6593 of file kvm_main.c.

{
    struct kvm_vm_worker_thread_context init_context = {};
    struct task_struct *thread;
 
    *thread_ptr = NULL;
    init_context.kvm = kvm;
    init_context.parent = current;
    init_context.thread_fn = thread_fn;
    init_context.data = data;
    init_completion(&init_context.init_done);
 
    thread = kthread_run(kvm_vm_worker_thread, &init_context,
                 "%s-%d", name, task_pid_nr(current));
    if (IS_ERR(thread))
        return PTR_ERR(thread);
 
    /* kthread_run is never supposed to return NULL */
    WARN_ON(thread == NULL);
 
    wait_for_completion(&init_context.init_done);
 
    if (!init_context.err)
        *thread_ptr = thread;
 
    return init_context.err;
}

Here is the call graph for this function:

kvm_vm_ioctl()

static long kvm_vm_ioctl ( struct file *  filp, unsigned int  ioctl, unsigned long  arg  )

static

Definition at line 5098 of file kvm_main.c.

{
    struct kvm *kvm = filp->private_data;
    void __user *argp = (void __user *)arg;
    int r;
 
    if (kvm->mm != current->mm || kvm->vm_dead)
        return -EIO;
    switch (ioctl) {
    case KVM_CREATE_VCPU:
        r = kvm_vm_ioctl_create_vcpu(kvm, arg);
        break;
    case KVM_ENABLE_CAP: {
        struct kvm_enable_cap cap;
 
        r = -EFAULT;
        if (copy_from_user(&cap, argp, sizeof(cap)))
            goto out;
        r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
        break;
    }
    case KVM_SET_USER_MEMORY_REGION2:
    case KVM_SET_USER_MEMORY_REGION: {
        struct kvm_userspace_memory_region2 mem;
        unsigned long size;
 
        if (ioctl == KVM_SET_USER_MEMORY_REGION) {
            /*
             * Fields beyond struct kvm_userspace_memory_region shouldn't be
             * accessed, but avoid leaking kernel memory in case of a bug.
             */
            memset(&mem, 0, sizeof(mem));
            size = sizeof(struct kvm_userspace_memory_region);
        } else {
            size = sizeof(struct kvm_userspace_memory_region2);
        }
 
        /* Ensure the common parts of the two structs are identical. */
        SANITY_CHECK_MEM_REGION_FIELD(slot);
        SANITY_CHECK_MEM_REGION_FIELD(flags);
        SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr);
        SANITY_CHECK_MEM_REGION_FIELD(memory_size);
        SANITY_CHECK_MEM_REGION_FIELD(userspace_addr);
 
        r = -EFAULT;
        if (copy_from_user(&mem, argp, size))
            goto out;
 
        r = -EINVAL;
        if (ioctl == KVM_SET_USER_MEMORY_REGION &&
            (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS))
            goto out;
 
        r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
        break;
    }
    case KVM_GET_DIRTY_LOG: {
        struct kvm_dirty_log log;
 
        r = -EFAULT;
        if (copy_from_user(&log, argp, sizeof(log)))
            goto out;
        r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
        break;
    }
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
    case KVM_CLEAR_DIRTY_LOG: {
        struct kvm_clear_dirty_log log;
 
        r = -EFAULT;
        if (copy_from_user(&log, argp, sizeof(log)))
            goto out;
        r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
        break;
    }
#endif
#ifdef CONFIG_KVM_MMIO
    case KVM_REGISTER_COALESCED_MMIO: {
        struct kvm_coalesced_mmio_zone zone;
 
        r = -EFAULT;
        if (copy_from_user(&zone, argp, sizeof(zone)))
            goto out;
        r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
        break;
    }
    case KVM_UNREGISTER_COALESCED_MMIO: {
        struct kvm_coalesced_mmio_zone zone;
 
        r = -EFAULT;
        if (copy_from_user(&zone, argp, sizeof(zone)))
            goto out;
        r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
        break;
    }
#endif
    case KVM_IRQFD: {
        struct kvm_irqfd data;
 
        r = -EFAULT;
        if (copy_from_user(&data, argp, sizeof(data)))
            goto out;
        r = kvm_irqfd(kvm, &data);
        break;
    }
    case KVM_IOEVENTFD: {
        struct kvm_ioeventfd data;
 
        r = -EFAULT;
        if (copy_from_user(&data, argp, sizeof(data)))
            goto out;
        r = kvm_ioeventfd(kvm, &data);
        break;
    }
#ifdef CONFIG_HAVE_KVM_MSI
    case KVM_SIGNAL_MSI: {
        struct kvm_msi msi;
 
        r = -EFAULT;
        if (copy_from_user(&msi, argp, sizeof(msi)))
            goto out;
        r = kvm_send_userspace_msi(kvm, &msi);
        break;
    }
#endif
#ifdef __KVM_HAVE_IRQ_LINE
    case KVM_IRQ_LINE_STATUS:
    case KVM_IRQ_LINE: {
        struct kvm_irq_level irq_event;
 
        r = -EFAULT;
        if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
            goto out;
 
        r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
                    ioctl == KVM_IRQ_LINE_STATUS);
        if (r)
            goto out;
 
        r = -EFAULT;
        if (ioctl == KVM_IRQ_LINE_STATUS) {
            if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
                goto out;
        }
 
        r = 0;
        break;
    }
#endif
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
    case KVM_SET_GSI_ROUTING: {
        struct kvm_irq_routing routing;
        struct kvm_irq_routing __user *urouting;
        struct kvm_irq_routing_entry *entries = NULL;
 
        r = -EFAULT;
        if (copy_from_user(&routing, argp, sizeof(routing)))
            goto out;
        r = -EINVAL;
        if (!kvm_arch_can_set_irq_routing(kvm))
            goto out;
        if (routing.nr > KVM_MAX_IRQ_ROUTES)
            goto out;
        if (routing.flags)
            goto out;
        if (routing.nr) {
            urouting = argp;
            entries = vmemdup_array_user(urouting->entries,
                             routing.nr, sizeof(*entries));
            if (IS_ERR(entries)) {
                r = PTR_ERR(entries);
                goto out;
            }
        }
        r = kvm_set_irq_routing(kvm, entries, routing.nr,
                    routing.flags);
        kvfree(entries);
        break;
    }
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
    case KVM_SET_MEMORY_ATTRIBUTES: {
        struct kvm_memory_attributes attrs;
 
        r = -EFAULT;
        if (copy_from_user(&attrs, argp, sizeof(attrs)))
            goto out;
 
        r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
        break;
    }
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
    case KVM_CREATE_DEVICE: {
        struct kvm_create_device cd;
 
        r = -EFAULT;
        if (copy_from_user(&cd, argp, sizeof(cd)))
            goto out;
 
        r = kvm_ioctl_create_device(kvm, &cd);
        if (r)
            goto out;
 
        r = -EFAULT;
        if (copy_to_user(argp, &cd, sizeof(cd)))
            goto out;
 
        r = 0;
        break;
    }
    case KVM_CHECK_EXTENSION:
        r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
        break;
    case KVM_RESET_DIRTY_RINGS:
        r = kvm_vm_ioctl_reset_dirty_pages(kvm);
        break;
    case KVM_GET_STATS_FD:
        r = kvm_vm_ioctl_get_stats_fd(kvm);
        break;
#ifdef CONFIG_KVM_PRIVATE_MEM
    case KVM_CREATE_GUEST_MEMFD: {
        struct kvm_create_guest_memfd guest_memfd;
 
        r = -EFAULT;
        if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd)))
            goto out;
 
        r = kvm_gmem_create(kvm, &guest_memfd);
        break;
    }
#endif
    default:
        r = kvm_arch_vm_ioctl(filp, ioctl, arg);
    }
out:
    return r;
}

kvm_vm_ioctl_check_extension_generic()

static int kvm_vm_ioctl_check_extension_generic ( struct kvm *  kvm, long  arg  )

static

Definition at line 4818 of file kvm_main.c.

{
    switch (arg) {
    case KVM_CAP_USER_MEMORY:
    case KVM_CAP_USER_MEMORY2:
    case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
    case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
    case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
    case KVM_CAP_SIGNAL_MSI:
#endif
#ifdef CONFIG_HAVE_KVM_IRQCHIP
    case KVM_CAP_IRQFD:
#endif
    case KVM_CAP_IOEVENTFD_ANY_LENGTH:
    case KVM_CAP_CHECK_EXTENSION_VM:
    case KVM_CAP_ENABLE_CAP_VM:
    case KVM_CAP_HALT_POLL:
        return 1;
#ifdef CONFIG_KVM_MMIO
    case KVM_CAP_COALESCED_MMIO:
        return KVM_COALESCED_MMIO_PAGE_OFFSET;
    case KVM_CAP_COALESCED_PIO:
        return 1;
#endif
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
    case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
        return KVM_DIRTY_LOG_MANUAL_CAPS;
#endif
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
    case KVM_CAP_IRQ_ROUTING:
        return KVM_MAX_IRQ_ROUTES;
#endif
#if KVM_MAX_NR_ADDRESS_SPACES > 1
    case KVM_CAP_MULTI_ADDRESS_SPACE:
        if (kvm)
            return kvm_arch_nr_memslot_as_ids(kvm);
        return KVM_MAX_NR_ADDRESS_SPACES;
#endif
    case KVM_CAP_NR_MEMSLOTS:
        return KVM_USER_MEM_SLOTS;
    case KVM_CAP_DIRTY_LOG_RING:
#ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO
        return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
#else
        return 0;
#endif
    case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
#ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL
        return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
#else
        return 0;
#endif
#ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
    case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
#endif
    case KVM_CAP_BINARY_STATS_FD:
    case KVM_CAP_SYSTEM_EVENT_DATA:
    case KVM_CAP_DEVICE_CTRL:
        return 1;
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
    case KVM_CAP_MEMORY_ATTRIBUTES:
        return kvm_supported_mem_attributes(kvm);
#endif
#ifdef CONFIG_KVM_PRIVATE_MEM
    case KVM_CAP_GUEST_MEMFD:
        return !kvm || kvm_arch_has_private_mem(kvm);
#endif
    default:
        break;
    }
    return kvm_vm_ioctl_check_extension(kvm, arg);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vm_ioctl_create_vcpu()

static int kvm_vm_ioctl_create_vcpu ( struct kvm *  kvm, u32  id  )

static

Definition at line 4228 of file kvm_main.c.

{
    int r;
    struct kvm_vcpu *vcpu;
    struct page *page;
 
    if (id >= KVM_MAX_VCPU_IDS)
        return -EINVAL;
 
    mutex_lock(&kvm->lock);
    if (kvm->created_vcpus >= kvm->max_vcpus) {
        mutex_unlock(&kvm->lock);
        return -EINVAL;
    }
 
    r = kvm_arch_vcpu_precreate(kvm, id);
    if (r) {
        mutex_unlock(&kvm->lock);
        return r;
    }
 
    kvm->created_vcpus++;
    mutex_unlock(&kvm->lock);
 
    vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
    if (!vcpu) {
        r = -ENOMEM;
        goto vcpu_decrement;
    }
 
    BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
    page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
    if (!page) {
        r = -ENOMEM;
        goto vcpu_free;
    }
    vcpu->run = page_address(page);
 
    kvm_vcpu_init(vcpu, kvm, id);
 
    r = kvm_arch_vcpu_create(vcpu);
    if (r)
        goto vcpu_free_run_page;
 
    if (kvm->dirty_ring_size) {
        r = kvm_dirty_ring_alloc(&vcpu->dirty_ring,
                     id, kvm->dirty_ring_size);
        if (r)
            goto arch_vcpu_destroy;
    }
 
    mutex_lock(&kvm->lock);
 
#ifdef CONFIG_LOCKDEP
    /* Ensure that lockdep knows vcpu->mutex is taken *inside* kvm->lock */
    mutex_lock(&vcpu->mutex);
    mutex_unlock(&vcpu->mutex);
#endif
 
    if (kvm_get_vcpu_by_id(kvm, id)) {
        r = -EEXIST;
        goto unlock_vcpu_destroy;
    }
 
    vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
    r = xa_reserve(&kvm->vcpu_array, vcpu->vcpu_idx, GFP_KERNEL_ACCOUNT);
    if (r)
        goto unlock_vcpu_destroy;
 
    /* Now it's all set up, let userspace reach it */
    kvm_get_kvm(kvm);
    r = create_vcpu_fd(vcpu);
    if (r < 0)
        goto kvm_put_xa_release;
 
    if (KVM_BUG_ON(xa_store(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, 0), kvm)) {
        r = -EINVAL;
        goto kvm_put_xa_release;
    }
 
    /*
     * Pairs with smp_rmb() in kvm_get_vcpu.  Store the vcpu
     * pointer before kvm->online_vcpu's incremented value.
     */
    smp_wmb();
    atomic_inc(&kvm->online_vcpus);
 
    mutex_unlock(&kvm->lock);
    kvm_arch_vcpu_postcreate(vcpu);
    kvm_create_vcpu_debugfs(vcpu);
    return r;
 
kvm_put_xa_release:
    kvm_put_kvm_no_destroy(kvm);
    xa_release(&kvm->vcpu_array, vcpu->vcpu_idx);
unlock_vcpu_destroy:
    mutex_unlock(&kvm->lock);
    kvm_dirty_ring_free(&vcpu->dirty_ring);
arch_vcpu_destroy:
    kvm_arch_vcpu_destroy(vcpu);
vcpu_free_run_page:
    free_page((unsigned long)vcpu->run);
vcpu_free:
    kmem_cache_free(kvm_vcpu_cache, vcpu);
vcpu_decrement:
    mutex_lock(&kvm->lock);
    kvm->created_vcpus--;
    mutex_unlock(&kvm->lock);
    return r;
}

Here is the call graph for this function:

kvm_vm_ioctl_enable_cap_generic()

static int kvm_vm_ioctl_enable_cap_generic ( struct kvm *  kvm, struct kvm_enable_cap *  cap  )

static

Definition at line 4973 of file kvm_main.c.

{
    switch (cap->cap) {
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
    case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: {
        u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE;
 
        if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE)
            allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS;
 
        if (cap->flags || (cap->args[0] & ~allowed_options))
            return -EINVAL;
        kvm->manual_dirty_log_protect = cap->args[0];
        return 0;
    }
#endif
    case KVM_CAP_HALT_POLL: {
        if (cap->flags || cap->args[0] != (unsigned int)cap->args[0])
            return -EINVAL;
 
        kvm->max_halt_poll_ns = cap->args[0];
 
        /*
         * Ensure kvm->override_halt_poll_ns does not become visible
         * before kvm->max_halt_poll_ns.
         *
         * Pairs with the smp_rmb() in kvm_vcpu_max_halt_poll_ns().
         */
        smp_wmb();
        kvm->override_halt_poll_ns = true;
 
        return 0;
    }
    case KVM_CAP_DIRTY_LOG_RING:
    case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
        if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))
            return -EINVAL;
 
        return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
    case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
        int r = -EINVAL;
 
        if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
            !kvm->dirty_ring_size || cap->flags)
            return r;
 
        mutex_lock(&kvm->slots_lock);
 
        /*
         * For simplicity, allow enabling ring+bitmap if and only if
         * there are no memslots, e.g. to ensure all memslots allocate
         * a bitmap after the capability is enabled.
         */
        if (kvm_are_all_memslots_empty(kvm)) {
            kvm->dirty_ring_with_bitmap = true;
            r = 0;
        }
 
        mutex_unlock(&kvm->slots_lock);
 
        return r;
    }
    default:
        return kvm_vm_ioctl_enable_cap(kvm, cap);
    }
}

Here is the call graph for this function:

kvm_vm_ioctl_enable_dirty_log_ring()

static int kvm_vm_ioctl_enable_dirty_log_ring ( struct kvm *  kvm, u32  size  )

static

Definition at line 4892 of file kvm_main.c.

{
    int r;
 
    if (!KVM_DIRTY_LOG_PAGE_OFFSET)
        return -EINVAL;
 
    /* the size should be power of 2 */
    if (!size || (size & (size - 1)))
        return -EINVAL;
 
    /* Should be bigger to keep the reserved entries, or a page */
    if (size < kvm_dirty_ring_get_rsvd_entries() *
        sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE)
        return -EINVAL;
 
    if (size > KVM_DIRTY_RING_MAX_ENTRIES *
        sizeof(struct kvm_dirty_gfn))
        return -E2BIG;
 
    /* We only allow it to set once */
    if (kvm->dirty_ring_size)
        return -EINVAL;
 
    mutex_lock(&kvm->lock);
 
    if (kvm->created_vcpus) {
        /* We don't allow to change this value after vcpu created */
        r = -EINVAL;
    } else {
        kvm->dirty_ring_size = size;
        r = 0;
    }
 
    mutex_unlock(&kvm->lock);
    return r;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vm_ioctl_get_stats_fd()

static int kvm_vm_ioctl_get_stats_fd ( struct kvm *  kvm )

static

Definition at line 5066 of file kvm_main.c.

{
    int fd;
    struct file *file;
 
    fd = get_unused_fd_flags(O_CLOEXEC);
    if (fd < 0)
        return fd;
 
    file = anon_inode_getfile("kvm-vm-stats",
            &kvm_vm_stats_fops, kvm, O_RDONLY);
    if (IS_ERR(file)) {
        put_unused_fd(fd);
        return PTR_ERR(file);
    }
 
    kvm_get_kvm(kvm);
 
    file->f_mode |= FMODE_PREAD;
    fd_install(fd, file);
 
    return fd;
}

Here is the call graph for this function:

kvm_vm_ioctl_reset_dirty_pages()

static int kvm_vm_ioctl_reset_dirty_pages ( struct kvm *  kvm )

static

Definition at line 4930 of file kvm_main.c.

{
    unsigned long i;
    struct kvm_vcpu *vcpu;
    int cleared = 0;
 
    if (!kvm->dirty_ring_size)
        return -EINVAL;
 
    mutex_lock(&kvm->slots_lock);
 
    kvm_for_each_vcpu(i, vcpu, kvm)
        cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring);
 
    mutex_unlock(&kvm->slots_lock);
 
    if (cleared)
        kvm_flush_remote_tlbs(kvm);
 
    return cleared;
}

Here is the call graph for this function:

kvm_vm_ioctl_set_memory_region()

static int kvm_vm_ioctl_set_memory_region ( struct kvm *  kvm, struct kvm_userspace_memory_region2 *  mem  )

static

Definition at line 2156 of file kvm_main.c.

{
    if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
        return -EINVAL;
 
    return kvm_set_memory_region(kvm, mem);
}

Here is the call graph for this function:

kvm_vm_release()

static int kvm_vm_release ( struct inode *  inode, struct file *  filp  )

static

Definition at line 1439 of file kvm_main.c.

{
    struct kvm *kvm = filp->private_data;
 
    kvm_irqfd_release(kvm);
 
    kvm_put_kvm(kvm);
    return 0;
}

Here is the call graph for this function:

kvm_vm_stats_read()

static ssize_t kvm_vm_stats_read ( struct file *  file, char __user *  user_buffer, size_t  size, loff_t *  offset  )

static

Definition at line 5041 of file kvm_main.c.

{
    struct kvm *kvm = file->private_data;
 
    return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,
                &kvm_vm_stats_desc[0], &kvm->stat,
                sizeof(kvm->stat), user_buffer, size, offset);
}

Here is the call graph for this function:

kvm_vm_stats_release()

static int kvm_vm_stats_release ( struct inode *  inode, struct file *  file  )

static

Definition at line 5051 of file kvm_main.c.

{
    struct kvm *kvm = file->private_data;
 
    kvm_put_kvm(kvm);
    return 0;
}

kvm_vm_worker_thread()

static int kvm_vm_worker_thread ( void *  context )

static

Definition at line 6529 of file kvm_main.c.

{
    /*
     * The init_context is allocated on the stack of the parent thread, so
     * we have to locally copy anything that is needed beyond initialization
     */
    struct kvm_vm_worker_thread_context *init_context = context;
    struct task_struct *parent;
    struct kvm *kvm = init_context->kvm;
    kvm_vm_thread_fn_t thread_fn = init_context->thread_fn;
    uintptr_t data = init_context->data;
    int err;
 
    err = kthread_park(current);
    /* kthread_park(current) is never supposed to return an error */
    WARN_ON(err != 0);
    if (err)
        goto init_complete;
 
    err = cgroup_attach_task_all(init_context->parent, current);
    if (err) {
        kvm_err("%s: cgroup_attach_task_all failed with err %d\n",
            __func__, err);
        goto init_complete;
    }
 
    set_user_nice(current, task_nice(init_context->parent));
 
init_complete:
    init_context->err = err;
    complete(&init_context->init_done);
    init_context = NULL;
 
    if (err)
        goto out;
 
    /* Wait to be woken up by the spawner before proceeding. */
    kthread_parkme();
 
    if (!kthread_should_stop())
        err = thread_fn(kvm, data);
 
out:
    /*
     * Move kthread back to its original cgroup to prevent it lingering in
     * the cgroup of the VM process, after the latter finishes its
     * execution.
     *
     * kthread_stop() waits on the 'exited' completion condition which is
     * set in exit_mm(), via mm_release(), in do_exit(). However, the
     * kthread is removed from the cgroup in the cgroup_exit() which is
     * called after the exit_mm(). This causes the kthread_stop() to return
     * before the kthread actually quits the cgroup.
     */
    rcu_read_lock();
    parent = rcu_dereference(current->real_parent);
    get_task_struct(parent);
    rcu_read_unlock();
    cgroup_attach_task_all(parent, current);
    put_task_struct(parent);
 
    return err;
}

Here is the caller graph for this function:

kvm_write_guest()

int kvm_write_guest	(	struct kvm *	kvm,
		gpa_t	gpa,
		const void *	data,
		unsigned long	len
	)

Definition at line 3449 of file kvm_main.c.

{
    gfn_t gfn = gpa >> PAGE_SHIFT;
    int seg;
    int offset = offset_in_page(gpa);
    int ret;
 
    while ((seg = next_segment(len, offset)) != 0) {
        ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
        if (ret < 0)
            return ret;
        offset = 0;
        len -= seg;
        data += seg;
        ++gfn;
    }
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_write_guest_cached()

int kvm_write_guest_cached	(	struct kvm *	kvm,
		struct gfn_to_hva_cache *	ghc,
		void *	data,
		unsigned long	len
	)

Definition at line 3571 of file kvm_main.c.

{
    return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
}

Here is the call graph for this function:

kvm_write_guest_offset_cached()

int kvm_write_guest_offset_cached	(	struct kvm *	kvm,
		struct gfn_to_hva_cache *	ghc,
		void *	data,
		unsigned int	offset,
		unsigned long	len
	)

Definition at line 3540 of file kvm_main.c.

{
    struct kvm_memslots *slots = kvm_memslots(kvm);
    int r;
    gpa_t gpa = ghc->gpa + offset;
 
    if (WARN_ON_ONCE(len + offset > ghc->len))
        return -EINVAL;
 
    if (slots->generation != ghc->generation) {
        if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
            return -EFAULT;
    }
 
    if (kvm_is_error_hva(ghc->hva))
        return -EFAULT;
 
    if (unlikely(!ghc->memslot))
        return kvm_write_guest(kvm, gpa, data, len);
 
    r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
    if (r)
        return -EFAULT;
    mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_write_guest_page()

int kvm_write_guest_page	(	struct kvm *	kvm,
		gfn_t	gfn,
		const void *	data,
		int	offset,
		int	len
	)

Definition at line 3431 of file kvm_main.c.

{
    struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
 
    return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);
}

Here is the call graph for this function:

Here is the caller graph for this function:

LIST_HEAD()

LIST_HEAD

(

vm_list

)

mark_page_dirty()

void mark_page_dirty	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 3660 of file kvm_main.c.

{
    struct kvm_memory_slot *memslot;
 
    memslot = gfn_to_memslot(kvm, gfn);
    mark_page_dirty_in_slot(kvm, memslot, gfn);
}

Here is the call graph for this function:

mark_page_dirty_in_slot()

void mark_page_dirty_in_slot	(	struct kvm *	kvm,
		const struct kvm_memory_slot *	memslot,
		gfn_t	gfn
	)

Definition at line 3635 of file kvm_main.c.

{
    struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 
#ifdef CONFIG_HAVE_KVM_DIRTY_RING
    if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
        return;
 
    WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
#endif
 
    if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
        unsigned long rel_gfn = gfn - memslot->base_gfn;
        u32 slot = (memslot->as_id << 16) | memslot->id;
 
        if (kvm->dirty_ring_size && vcpu)
            kvm_dirty_ring_push(vcpu, slot, rel_gfn);
        else if (memslot->dirty_bitmap)
            set_bit_le(rel_gfn, memslot->dirty_bitmap);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

memslot_is_readonly()

static bool memslot_is_readonly ( const struct kvm_memory_slot *  slot )

static

Definition at line 2709 of file kvm_main.c.

{
    return slot->flags & KVM_MEM_READONLY;
}

Here is the caller graph for this function:

MODULE_AUTHOR()

MODULE_AUTHOR

(

"Qumranet"

)

MODULE_LICENSE()

MODULE_LICENSE

(

"GPL"

)

module_param() [1/4]

module_param	(	halt_poll_ns	,
		uint	,
		0644
	)

module_param() [2/4]

module_param	(	halt_poll_ns_grow	,
		uint	,
		0644
	)

module_param() [3/4]

module_param	(	halt_poll_ns_grow_start	,
		uint	,
		0644
	)

module_param() [4/4]

module_param	(	halt_poll_ns_shrink	,
		uint	,
		0644
	)

next_segment()

static int next_segment ( unsigned long  len, int  offset  )

static

Definition at line 3305 of file kvm_main.c.

{
    if (len > PAGE_SIZE - offset)
        return PAGE_SIZE - offset;
    else
        return len;
}

Here is the caller graph for this function:

preempt_notifier_to_vcpu()

static struct kvm_vcpu* preempt_notifier_to_vcpu ( struct preempt_notifier *  pn )

inlinestatic

Definition at line 6299 of file kvm_main.c.

{
    return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

Here is the caller graph for this function:

shrink_halt_poll_ns()

static void shrink_halt_poll_ns ( struct kvm_vcpu *  vcpu )

static

Definition at line 3720 of file kvm_main.c.

{
    unsigned int old, val, shrink, grow_start;
 
    old = val = vcpu->halt_poll_ns;
    shrink = READ_ONCE(halt_poll_ns_shrink);
    grow_start = READ_ONCE(halt_poll_ns_grow_start);
    if (shrink == 0)
        val = 0;
    else
        val /= shrink;
 
    if (val < grow_start)
        val = 0;
 
    vcpu->halt_poll_ns = val;
    trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
}

Here is the caller graph for this function:

update_halt_poll_stats()

static void update_halt_poll_stats ( struct kvm_vcpu *  vcpu, ktime_t  start, ktime_t  end, bool  success  )

inlinestatic

Definition at line 3796 of file kvm_main.c.

{
    struct kvm_vcpu_stat_generic *stats = &vcpu->stat.generic;
    u64 poll_ns = ktime_to_ns(ktime_sub(end, start));
 
    ++vcpu->stat.generic.halt_attempted_poll;
 
    if (success) {
        ++vcpu->stat.generic.halt_successful_poll;
 
        if (!vcpu_valid_wakeup(vcpu))
            ++vcpu->stat.generic.halt_poll_invalid;
 
        stats->halt_poll_success_ns += poll_ns;
        KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_success_hist, poll_ns);
    } else {
        stats->halt_poll_fail_ns += poll_ns;
        KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_fail_hist, poll_ns);
    }
}

Here is the caller graph for this function:

vcpu_dy_runnable()

static bool vcpu_dy_runnable ( struct kvm_vcpu *  vcpu )

static

Definition at line 4038 of file kvm_main.c.

{
    if (kvm_arch_dy_runnable(vcpu))
        return true;
 
#ifdef CONFIG_KVM_ASYNC_PF
    if (!list_empty_careful(&vcpu->async_pf.done))
        return true;
#endif
 
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_load()

void vcpu_load

(

struct kvm_vcpu *

vcpu

)

Definition at line 208 of file kvm_main.c.

{
    int cpu = get_cpu();
 
    __this_cpu_write(kvm_running_vcpu, vcpu);
    preempt_notifier_register(&vcpu->preempt_notifier);
    kvm_arch_vcpu_load(vcpu, cpu);
    put_cpu();
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_put()

void vcpu_put

(

struct kvm_vcpu *

vcpu

)

Definition at line 219 of file kvm_main.c.

{
    preempt_disable();
    kvm_arch_vcpu_put(vcpu);
    preempt_notifier_unregister(&vcpu->preempt_notifier);
    __this_cpu_write(kvm_running_vcpu, NULL);
    preempt_enable();
}

Here is the call graph for this function:

Here is the caller graph for this function:

vcpu_stat_clear()

static int vcpu_stat_clear ( void *  _offset, u64  val  )

static

Definition at line 6196 of file kvm_main.c.

{
    unsigned offset = (long)_offset;
    struct kvm *kvm;
 
    if (val)
        return -EINVAL;
 
    mutex_lock(&kvm_lock);
    list_for_each_entry(kvm, &vm_list, vm_list) {
        kvm_clear_stat_per_vcpu(kvm, offset);
    }
    mutex_unlock(&kvm_lock);
 
    return 0;
}

Here is the call graph for this function:

vcpu_stat_get()

static int vcpu_stat_get ( void *  _offset, u64 *  val  )

static

Definition at line 6180 of file kvm_main.c.

{
    unsigned offset = (long)_offset;
    struct kvm *kvm;
    u64 tmp_val;
 
    *val = 0;
    mutex_lock(&kvm_lock);
    list_for_each_entry(kvm, &vm_list, vm_list) {
        kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
        *val += tmp_val;
    }
    mutex_unlock(&kvm_lock);
    return 0;
}

Here is the call graph for this function:

vm_stat_clear()

static int vm_stat_clear ( void *  _offset, u64  val  )

static

Definition at line 6160 of file kvm_main.c.

{
    unsigned offset = (long)_offset;
    struct kvm *kvm;
 
    if (val)
        return -EINVAL;
 
    mutex_lock(&kvm_lock);
    list_for_each_entry(kvm, &vm_list, vm_list) {
        kvm_clear_stat_per_vm(kvm, offset);
    }
    mutex_unlock(&kvm_lock);
 
    return 0;
}

Here is the call graph for this function:

vm_stat_get()

static int vm_stat_get ( void *  _offset, u64 *  val  )

static

Definition at line 6144 of file kvm_main.c.

{
    unsigned offset = (long)_offset;
    struct kvm *kvm;
    u64 tmp_val;
 
    *val = 0;
    mutex_lock(&kvm_lock);
    list_for_each_entry(kvm, &vm_list, vm_list) {
        kvm_get_stat_per_vm(kvm, offset, &tmp_val);
        *val += tmp_val;
    }
    mutex_unlock(&kvm_lock);
    return 0;
}

Here is the call graph for this function:

vma_is_valid()

static bool vma_is_valid ( struct vm_area_struct *  vma, bool  write_fault  )

static

Definition at line 2876 of file kvm_main.c.

{
    if (unlikely(!(vma->vm_flags & VM_READ)))
        return false;
 
    if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
        return false;
 
    return true;
}

Here is the caller graph for this function:

Variable Documentation

halt_poll_ns

unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT

Definition at line 80 of file kvm_main.c.

halt_poll_ns_grow

unsigned int halt_poll_ns_grow = 2

Definition at line 85 of file kvm_main.c.

halt_poll_ns_grow_start

unsigned int halt_poll_ns_grow_start = 10000

Definition at line 90 of file kvm_main.c.

halt_poll_ns_shrink

unsigned int halt_poll_ns_shrink

Definition at line 95 of file kvm_main.c.

kvm_active_vms

unsigned long long kvm_active_vms

static

Definition at line 151 of file kvm_main.c.

kvm_chardev_ops

struct file_operations kvm_chardev_ops

static

Initial value:

= {
    .unlocked_ioctl = kvm_dev_ioctl,
    .llseek     = noop_llseek,
    KVM_COMPAT(kvm_dev_ioctl),
}

Definition at line 5473 of file kvm_main.c.

kvm_createvm_count

unsigned long long kvm_createvm_count

static

Definition at line 150 of file kvm_main.c.

kvm_debugfs_dir

struct dentry* kvm_debugfs_dir

Definition at line 113 of file kvm_main.c.

kvm_dev

struct miscdevice kvm_dev

static

Initial value:

= {
    KVM_MINOR,
    "kvm",
    &kvm_chardev_ops,
}

Definition at line 5473 of file kvm_main.c.

kvm_device_fops

struct file_operations kvm_device_fops

static

Initial value:

= {
    .unlocked_ioctl = kvm_device_ioctl,
    .release = kvm_device_release,
    KVM_COMPAT(kvm_device_ioctl),
    .mmap = kvm_device_mmap,
}

Definition at line 4704 of file kvm_main.c.

kvm_device_ops_table

const struct kvm_device_ops* kvm_device_ops_table[KVM_DEV_TYPE_MAX]

static

Initial value:

= {
 
 
 
 
}

Definition at line 4735 of file kvm_main.c.

kvm_preempt_ops

__read_mostly struct preempt_ops kvm_preempt_ops

static

Definition at line 110 of file kvm_main.c.

kvm_vcpu_cache

struct kmem_cache* kvm_vcpu_cache

static

Definition at line 108 of file kvm_main.c.

kvm_vcpu_fops

struct file_operations kvm_vcpu_fops

static

Initial value:

= {
    .release        = kvm_vcpu_release,
    .unlocked_ioctl = kvm_vcpu_ioctl,
    .mmap           = kvm_vcpu_mmap,
    .llseek     = noop_llseek,
    KVM_COMPAT(kvm_vcpu_compat_ioctl),
}

Definition at line 4167 of file kvm_main.c.

kvm_vcpu_stats_fops

const struct file_operations kvm_vcpu_stats_fops

static

Initial value:

= {
    .owner = THIS_MODULE,
    .read = kvm_vcpu_stats_read,
    .release = kvm_vcpu_stats_release,
    .llseek = noop_llseek,
}

Definition at line 4360 of file kvm_main.c.

kvm_vcpu_vm_ops

const struct vm_operations_struct kvm_vcpu_vm_ops

static

Initial value:

= {
    .fault = kvm_vcpu_fault,
}

Definition at line 4123 of file kvm_main.c.

kvm_vm_fops

struct file_operations kvm_vm_fops

static

Initial value:

= {
    .release        = kvm_vm_release,
    .unlocked_ioctl = kvm_vm_ioctl,
    .llseek     = noop_llseek,
    KVM_COMPAT(kvm_vm_compat_ioctl),
}

Definition at line 5098 of file kvm_main.c.

kvm_vm_stats_fops

const struct file_operations kvm_vm_stats_fops

static

Initial value:

= {
    .owner = THIS_MODULE,
    .read = kvm_vm_stats_read,
    .release = kvm_vm_stats_release,
    .llseek = noop_llseek,
}

Definition at line 5051 of file kvm_main.c.

stat_fops_per_vm

static const struct file_operations stat_fops_per_vm

static

Initial value:

= {
    .owner = THIS_MODULE,
    .open = kvm_stat_data_open,
    .release = kvm_debugfs_release,
    .read = simple_attr_read,
    .write = simple_attr_write,
    .llseek = no_llseek,
}

Definition at line 114 of file kvm_main.c.