dirty_ring.c File Reference

#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/vmalloc.h>
#include <linux/kvm_dirty_ring.h>
#include <trace/events/kvm.h>
#include "kvm_mm.h"

Include dependency graph for dirty_ring.c:

Go to the source code of this file.

Functions
int __weak	kvm_cpu_dirty_log_size (void)
u32	kvm_dirty_ring_get_rsvd_entries (void)
bool	kvm_use_dirty_bitmap (struct kvm *kvm)
bool	kvm_arch_allow_write_without_running_vcpu (struct kvm *kvm)
static u32	kvm_dirty_ring_used (struct kvm_dirty_ring *ring)
static bool	kvm_dirty_ring_soft_full (struct kvm_dirty_ring *ring)
static bool	kvm_dirty_ring_full (struct kvm_dirty_ring *ring)
static void	kvm_reset_dirty_gfn (struct kvm *kvm, u32 slot, u64 offset, u64 mask)
int	kvm_dirty_ring_alloc (struct kvm_dirty_ring *ring, int index, u32 size)
static void	kvm_dirty_gfn_set_invalid (struct kvm_dirty_gfn *gfn)
static void	kvm_dirty_gfn_set_dirtied (struct kvm_dirty_gfn *gfn)
static bool	kvm_dirty_gfn_harvested (struct kvm_dirty_gfn *gfn)
int	kvm_dirty_ring_reset (struct kvm *kvm, struct kvm_dirty_ring *ring)
void	kvm_dirty_ring_push (struct kvm_vcpu *vcpu, u32 slot, u64 offset)
bool	kvm_dirty_ring_check_request (struct kvm_vcpu *vcpu)
struct page *	kvm_dirty_ring_get_page (struct kvm_dirty_ring *ring, u32 offset)
void	kvm_dirty_ring_free (struct kvm_dirty_ring *ring)

Function Documentation

kvm_arch_allow_write_without_running_vcpu()

bool kvm_arch_allow_write_without_running_vcpu

(

struct kvm *

kvm

)

Definition at line 32 of file dirty_ring.c.

{
    return false;
}

kvm_cpu_dirty_log_size()

int __weak kvm_cpu_dirty_log_size

(

void

)

Definition at line 14 of file dirty_ring.c.

{
    return 0;
}

Here is the caller graph for this function:

kvm_dirty_gfn_harvested()

static bool kvm_dirty_gfn_harvested ( struct kvm_dirty_gfn *  gfn )

inlinestatic

Definition at line 99 of file dirty_ring.c.

{
    return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET;
}

Here is the caller graph for this function:

kvm_dirty_gfn_set_dirtied()

static void kvm_dirty_gfn_set_dirtied ( struct kvm_dirty_gfn *  gfn )

inlinestatic

Definition at line 94 of file dirty_ring.c.

{
    gfn->flags = KVM_DIRTY_GFN_F_DIRTY;
}

Here is the caller graph for this function:

kvm_dirty_gfn_set_invalid()

static void kvm_dirty_gfn_set_invalid ( struct kvm_dirty_gfn *  gfn )

inlinestatic

Definition at line 89 of file dirty_ring.c.

{
    smp_store_release(&gfn->flags, 0);
}

Here is the caller graph for this function:

kvm_dirty_ring_alloc()

int kvm_dirty_ring_alloc	(	struct kvm_dirty_ring *	ring,
		int	index,
		u32	size
	)

Definition at line 74 of file dirty_ring.c.

{
    ring->dirty_gfns = vzalloc(size);
    if (!ring->dirty_gfns)
        return -ENOMEM;
 
    ring->size = size / sizeof(struct kvm_dirty_gfn);
    ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();
    ring->dirty_index = 0;
    ring->reset_index = 0;
    ring->index = index;
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_check_request()

bool kvm_dirty_ring_check_request

(

struct kvm_vcpu *

vcpu

)

Definition at line 194 of file dirty_ring.c.

{
    /*
     * The VCPU isn't runnable when the dirty ring becomes soft full.
     * The KVM_REQ_DIRTY_RING_SOFT_FULL event is always set to prevent
     * the VCPU from running until the dirty pages are harvested and
     * the dirty ring is reset by userspace.
     */
    if (kvm_check_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu) &&
        kvm_dirty_ring_soft_full(&vcpu->dirty_ring)) {
        kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
        vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
        trace_kvm_dirty_ring_exit(vcpu);
        return true;
    }
 
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_free()

void kvm_dirty_ring_free

(

struct kvm_dirty_ring *

ring

)

Definition at line 218 of file dirty_ring.c.

{
    vfree(ring->dirty_gfns);
    ring->dirty_gfns = NULL;
}

Here is the caller graph for this function:

kvm_dirty_ring_full()

static bool kvm_dirty_ring_full ( struct kvm_dirty_ring *  ring )

static

Definition at line 48 of file dirty_ring.c.

{
    return kvm_dirty_ring_used(ring) >= ring->size;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_get_page()

struct page* kvm_dirty_ring_get_page	(	struct kvm_dirty_ring *	ring,
		u32	offset
	)

Definition at line 213 of file dirty_ring.c.

{
    return vmalloc_to_page((void *)ring->dirty_gfns + offset * PAGE_SIZE);
}

Here is the caller graph for this function:

kvm_dirty_ring_get_rsvd_entries()

u32 kvm_dirty_ring_get_rsvd_entries

(

void

)

Definition at line 19 of file dirty_ring.c.

{
    return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_push()

void kvm_dirty_ring_push	(	struct kvm_vcpu *	vcpu,
		u32	slot,
		u64	offset
	)

Definition at line 169 of file dirty_ring.c.

{
    struct kvm_dirty_ring *ring = &vcpu->dirty_ring;
    struct kvm_dirty_gfn *entry;
 
    /* It should never get full */
    WARN_ON_ONCE(kvm_dirty_ring_full(ring));
 
    entry = &ring->dirty_gfns[ring->dirty_index & (ring->size - 1)];
 
    entry->slot = slot;
    entry->offset = offset;
    /*
     * Make sure the data is filled in before we publish this to
     * the userspace program.  There's no paired kernel-side reader.
     */
    smp_wmb();
    kvm_dirty_gfn_set_dirtied(entry);
    ring->dirty_index++;
    trace_kvm_dirty_ring_push(ring, slot, offset);
 
    if (kvm_dirty_ring_soft_full(ring))
        kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_reset()

int kvm_dirty_ring_reset	(	struct kvm *	kvm,
		struct kvm_dirty_ring *	ring
	)

Definition at line 104 of file dirty_ring.c.

{
    u32 cur_slot, next_slot;
    u64 cur_offset, next_offset;
    unsigned long mask;
    int count = 0;
    struct kvm_dirty_gfn *entry;
    bool first_round = true;
 
    /* This is only needed to make compilers happy */
    cur_slot = cur_offset = mask = 0;
 
    while (true) {
        entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)];
 
        if (!kvm_dirty_gfn_harvested(entry))
            break;
 
        next_slot = READ_ONCE(entry->slot);
        next_offset = READ_ONCE(entry->offset);
 
        /* Update the flags to reflect that this GFN is reset */
        kvm_dirty_gfn_set_invalid(entry);
 
        ring->reset_index++;
        count++;
        /*
         * Try to coalesce the reset operations when the guest is
         * scanning pages in the same slot.
         */
        if (!first_round && next_slot == cur_slot) {
            s64 delta = next_offset - cur_offset;
 
            if (delta >= 0 && delta < BITS_PER_LONG) {
                mask |= 1ull << delta;
                continue;
            }
 
            /* Backwards visit, careful about overflows!  */
            if (delta > -BITS_PER_LONG && delta < 0 &&
                (mask << -delta >> -delta) == mask) {
                cur_offset = next_offset;
                mask = (mask << -delta) | 1;
                continue;
            }
        }
        kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
        cur_slot = next_slot;
        cur_offset = next_offset;
        mask = 1;
        first_round = false;
    }
 
    kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
 
    /*
     * The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared
     * by the VCPU thread next time when it enters the guest.
     */
 
    trace_kvm_dirty_ring_reset(ring);
 
    return count;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_soft_full()

static bool kvm_dirty_ring_soft_full ( struct kvm_dirty_ring *  ring )

static

Definition at line 43 of file dirty_ring.c.

{
    return kvm_dirty_ring_used(ring) >= ring->soft_limit;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_dirty_ring_used()

static u32 kvm_dirty_ring_used ( struct kvm_dirty_ring *  ring )

static

Definition at line 38 of file dirty_ring.c.

{
    return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
}

Here is the caller graph for this function:

kvm_reset_dirty_gfn()

static void kvm_reset_dirty_gfn ( struct kvm *  kvm, u32  slot, u64  offset, u64  mask  )

static

Definition at line 53 of file dirty_ring.c.

{
    struct kvm_memory_slot *memslot;
    int as_id, id;
 
    as_id = slot >> 16;
    id = (u16)slot;
 
    if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
        return;
 
    memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
 
    if (!memslot || (offset + __fls(mask)) >= memslot->npages)
        return;
 
    KVM_MMU_LOCK(kvm);
    kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
    KVM_MMU_UNLOCK(kvm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_use_dirty_bitmap()

bool kvm_use_dirty_bitmap

(

struct kvm *

kvm

)

Definition at line 24 of file dirty_ring.c.

{
    lockdep_assert_held(&kvm->slots_lock);
 
    return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap;
}

Here is the caller graph for this function: