vgic.c File Reference

#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/list_sort.h>
#include <linux/nospec.h>
#include <asm/kvm_hyp.h>
#include "vgic.h"
#include "trace.h"

Include dependency graph for vgic.c:

Go to the source code of this file.

Macros
#define	CREATE_TRACE_POINTS

Functions
static struct vgic_irq *	vgic_get_lpi (struct kvm *kvm, u32 intid)
struct vgic_irq *	vgic_get_irq (struct kvm *kvm, struct kvm_vcpu *vcpu, u32 intid)
static void	vgic_irq_release (struct kref *ref)
void	__vgic_put_lpi_locked (struct kvm *kvm, struct vgic_irq *irq)
void	vgic_put_irq (struct kvm *kvm, struct vgic_irq *irq)
void	vgic_flush_pending_lpis (struct kvm_vcpu *vcpu)
void	vgic_irq_set_phys_pending (struct vgic_irq *irq, bool pending)
bool	vgic_get_phys_line_level (struct vgic_irq *irq)
void	vgic_irq_set_phys_active (struct vgic_irq *irq, bool active)
static struct kvm_vcpu *	vgic_target_oracle (struct vgic_irq *irq)
static int	vgic_irq_cmp (void *priv, const struct list_head *a, const struct list_head *b)
static void	vgic_sort_ap_list (struct kvm_vcpu *vcpu)
static bool	vgic_validate_injection (struct vgic_irq *irq, bool level, void *owner)
bool	vgic_queue_irq_unlock (struct kvm *kvm, struct vgic_irq *irq, unsigned long flags)
int	kvm_vgic_inject_irq (struct kvm *kvm, struct kvm_vcpu *vcpu, unsigned int intid, bool level, void *owner)
static int	kvm_vgic_map_irq (struct kvm_vcpu *vcpu, struct vgic_irq *irq, unsigned int host_irq, struct irq_ops *ops)
static void	kvm_vgic_unmap_irq (struct vgic_irq *irq)
int	kvm_vgic_map_phys_irq (struct kvm_vcpu *vcpu, unsigned int host_irq, u32 vintid, struct irq_ops *ops)
void	kvm_vgic_reset_mapped_irq (struct kvm_vcpu *vcpu, u32 vintid)
int	kvm_vgic_unmap_phys_irq (struct kvm_vcpu *vcpu, unsigned int vintid)
int	kvm_vgic_get_map (struct kvm_vcpu *vcpu, unsigned int vintid)
int	kvm_vgic_set_owner (struct kvm_vcpu *vcpu, unsigned int intid, void *owner)
static void	vgic_prune_ap_list (struct kvm_vcpu *vcpu)
static void	vgic_fold_lr_state (struct kvm_vcpu *vcpu)
static void	vgic_populate_lr (struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr)
static void	vgic_clear_lr (struct kvm_vcpu *vcpu, int lr)
static void	vgic_set_underflow (struct kvm_vcpu *vcpu)
static int	compute_ap_list_depth (struct kvm_vcpu *vcpu, bool *multi_sgi)
static void	vgic_flush_lr_state (struct kvm_vcpu *vcpu)
static bool	can_access_vgic_from_kernel (void)
static void	vgic_save_state (struct kvm_vcpu *vcpu)
void	kvm_vgic_sync_hwstate (struct kvm_vcpu *vcpu)
static void	vgic_restore_state (struct kvm_vcpu *vcpu)
void	kvm_vgic_flush_hwstate (struct kvm_vcpu *vcpu)
void	kvm_vgic_load (struct kvm_vcpu *vcpu)
void	kvm_vgic_put (struct kvm_vcpu *vcpu)
void	kvm_vgic_vmcr_sync (struct kvm_vcpu *vcpu)
int	kvm_vgic_vcpu_pending_irq (struct kvm_vcpu *vcpu)
void	vgic_kick_vcpus (struct kvm *kvm)
bool	kvm_vgic_map_is_active (struct kvm_vcpu *vcpu, unsigned int vintid)
void	vgic_irq_handle_resampling (struct vgic_irq *irq, bool lr_deactivated, bool lr_pending)

Variables
struct vgic_global kvm_vgic_global_state	__ro_after_init

Macro Definition Documentation

CREATE_TRACE_POINTS

#define CREATE_TRACE_POINTS

Definition at line 17 of file vgic.c.

Function Documentation

__vgic_put_lpi_locked()

void __vgic_put_lpi_locked	(	struct kvm *	kvm,
		struct vgic_irq *	irq
	)

Definition at line 126 of file vgic.c.

{
    struct vgic_dist *dist = &kvm->arch.vgic;
 
    if (!kref_put(&irq->refcount, vgic_irq_release))
        return;
 
    list_del(&irq->lpi_list);
    dist->lpi_list_count--;
 
    kfree(irq);
}

Here is the call graph for this function:

Here is the caller graph for this function:

can_access_vgic_from_kernel()

static bool can_access_vgic_from_kernel ( void  )

inlinestatic

Definition at line 856 of file vgic.c.

{
    /*
     * GICv2 can always be accessed from the kernel because it is
     * memory-mapped, and VHE systems can access GICv3 EL2 system
     * registers.
     */
    return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe();
}

Here is the caller graph for this function:

compute_ap_list_depth()

static int compute_ap_list_depth ( struct kvm_vcpu *  vcpu, bool *  multi_sgi  )

static

Definition at line 771 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
    struct vgic_irq *irq;
    int count = 0;
 
    *multi_sgi = false;
 
    lockdep_assert_held(&vgic_cpu->ap_list_lock);
 
    list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
        int w;
 
        raw_spin_lock(&irq->irq_lock);
        /* GICv2 SGIs can count for more than one... */
        w = vgic_irq_get_lr_count(irq);
        raw_spin_unlock(&irq->irq_lock);
 
        count += w;
        *multi_sgi |= (w > 1);
    }
    return count;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_flush_hwstate()

void kvm_vgic_flush_hwstate

(

struct kvm_vcpu *

vcpu

)

Definition at line 905 of file vgic.c.

{
    /*
     * If there are no virtual interrupts active or pending for this
     * VCPU, then there is no work to do and we can bail out without
     * taking any lock.  There is a potential race with someone injecting
     * interrupts to the VCPU, but it is a benign race as the VCPU will
     * either observe the new interrupt before or after doing this check,
     * and introducing additional synchronization mechanism doesn't change
     * this.
     *
     * Note that we still need to go through the whole thing if anything
     * can be directly injected (GICv4).
     */
    if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
        !vgic_supports_direct_msis(vcpu->kvm))
        return;
 
    DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
    if (!list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) {
        raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
        vgic_flush_lr_state(vcpu);
        raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
    }
 
    if (can_access_vgic_from_kernel())
        vgic_restore_state(vcpu);
 
    if (vgic_supports_direct_msis(vcpu->kvm))
        vgic_v4_commit(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_get_map()

int kvm_vgic_get_map	(	struct kvm_vcpu *	vcpu,
		unsigned int	vintid
	)

Definition at line 576 of file vgic.c.

{
    struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
    unsigned long flags;
    int ret = -1;
 
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    if (irq->hw)
        ret = irq->hwintid;
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
    vgic_put_irq(vcpu->kvm, irq);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_inject_irq()

int kvm_vgic_inject_irq	(	struct kvm *	kvm,
		struct kvm_vcpu *	vcpu,
		unsigned int	intid,
		bool	level,
		void *	owner
	)

kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic @kvm: The VM structure pointer @vcpu: The CPU for PPIs or NULL for global interrupts @intid: The INTID to inject a new state to. @level: Edge-triggered: true: to trigger the interrupt false: to ignore the call Level-sensitive true: raise the input signal false: lower the input signal @owner: The opaque pointer to the owner of the IRQ being raised to verify that the caller is allowed to inject this IRQ. Userspace injections will have owner == NULL.

The VGIC is not concerned with devices being active-LOW or active-HIGH for level-sensitive interrupts. You can think of the level parameter as 1 being HIGH and 0 being LOW and all devices being active-HIGH.

Definition at line 439 of file vgic.c.

{
    struct vgic_irq *irq;
    unsigned long flags;
    int ret;
 
    ret = vgic_lazy_init(kvm);
    if (ret)
        return ret;
 
    if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS)
        return -EINVAL;
 
    trace_vgic_update_irq_pending(vcpu ? vcpu->vcpu_idx : 0, intid, level);
 
    irq = vgic_get_irq(kvm, vcpu, intid);
    if (!irq)
        return -EINVAL;
 
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
    if (!vgic_validate_injection(irq, level, owner)) {
        /* Nothing to see here, move along... */
        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
        vgic_put_irq(kvm, irq);
        return 0;
    }
 
    if (irq->config == VGIC_CONFIG_LEVEL)
        irq->line_level = level;
    else
        irq->pending_latch = true;
 
    vgic_queue_irq_unlock(kvm, irq, flags);
    vgic_put_irq(kvm, irq);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_load()

void kvm_vgic_load

(

struct kvm_vcpu *

vcpu

)

Definition at line 938 of file vgic.c.

{
    if (unlikely(!vgic_initialized(vcpu->kvm)))
        return;
 
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_load(vcpu);
    else
        vgic_v3_load(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_map_irq()

static int kvm_vgic_map_irq ( struct kvm_vcpu *  vcpu, struct vgic_irq *  irq, unsigned int  host_irq, struct irq_ops *  ops  )

static

Definition at line 480 of file vgic.c.

{
    struct irq_desc *desc;
    struct irq_data *data;
 
    /*
     * Find the physical IRQ number corresponding to @host_irq
     */
    desc = irq_to_desc(host_irq);
    if (!desc) {
        kvm_err("%s: no interrupt descriptor\n", __func__);
        return -EINVAL;
    }
    data = irq_desc_get_irq_data(desc);
    while (data->parent_data)
        data = data->parent_data;
 
    irq->hw = true;
    irq->host_irq = host_irq;
    irq->hwintid = data->hwirq;
    irq->ops = ops;
    return 0;
}

Here is the caller graph for this function:

kvm_vgic_map_is_active()

bool kvm_vgic_map_is_active	(	struct kvm_vcpu *	vcpu,
		unsigned int	vintid
	)

Definition at line 1022 of file vgic.c.

{
    struct vgic_irq *irq;
    bool map_is_active;
    unsigned long flags;
 
    if (!vgic_initialized(vcpu->kvm))
        return false;
 
    irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    map_is_active = irq->hw && irq->active;
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
    vgic_put_irq(vcpu->kvm, irq);
 
    return map_is_active;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_map_phys_irq()

int kvm_vgic_map_phys_irq	(	struct kvm_vcpu *	vcpu,
		unsigned int	host_irq,
		u32	vintid,
		struct irq_ops *	ops
	)

Definition at line 514 of file vgic.c.

{
    struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
    unsigned long flags;
    int ret;
 
    BUG_ON(!irq);
 
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
    vgic_put_irq(vcpu->kvm, irq);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_put()

void kvm_vgic_put

(

struct kvm_vcpu *

vcpu

)

Definition at line 949 of file vgic.c.

{
    if (unlikely(!vgic_initialized(vcpu->kvm)))
        return;
 
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_put(vcpu);
    else
        vgic_v3_put(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_reset_mapped_irq()

void kvm_vgic_reset_mapped_irq	(	struct kvm_vcpu *	vcpu,
		u32	vintid
	)

kvm_vgic_reset_mapped_irq - Reset a mapped IRQ @vcpu: The VCPU pointer @vintid: The INTID of the interrupt

Reset the active and pending states of a mapped interrupt. Kernel subsystems injecting mapped interrupts should reset their interrupt lines when we are doing a reset of the VM.

Definition at line 540 of file vgic.c.

{
    struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
    unsigned long flags;
 
    if (!irq->hw)
        goto out;
 
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    irq->active = false;
    irq->pending_latch = false;
    irq->line_level = false;
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
out:
    vgic_put_irq(vcpu->kvm, irq);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_set_owner()

int kvm_vgic_set_owner	(	struct kvm_vcpu *	vcpu,
		unsigned int	intid,
		void *	owner
	)

kvm_vgic_set_owner - Set the owner of an interrupt for a VM

@vcpu: Pointer to the VCPU (used for PPIs) @intid: The virtual INTID identifying the interrupt (PPI or SPI) @owner: Opaque pointer to the owner

Returns 0 if intid is not already used by another in-kernel device and the owner is set, otherwise returns an error code.

Definition at line 601 of file vgic.c.

{
    struct vgic_irq *irq;
    unsigned long flags;
    int ret = 0;
 
    if (!vgic_initialized(vcpu->kvm))
        return -EAGAIN;
 
    /* SGIs and LPIs cannot be wired up to any device */
    if (!irq_is_ppi(intid) && !vgic_valid_spi(vcpu->kvm, intid))
        return -EINVAL;
 
    irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    if (irq->owner && irq->owner != owner)
        ret = -EEXIST;
    else
        irq->owner = owner;
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_sync_hwstate()

void kvm_vgic_sync_hwstate

(

struct kvm_vcpu *

vcpu

)

Definition at line 875 of file vgic.c.

{
    int used_lrs;
 
    /* An empty ap_list_head implies used_lrs == 0 */
    if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
        return;
 
    if (can_access_vgic_from_kernel())
        vgic_save_state(vcpu);
 
    if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
        used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;
    else
        used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
 
    if (used_lrs)
        vgic_fold_lr_state(vcpu);
    vgic_prune_ap_list(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_unmap_irq()

static void kvm_vgic_unmap_irq ( struct vgic_irq *  irq )

inlinestatic

Definition at line 507 of file vgic.c.

{
    irq->hw = false;
    irq->hwintid = 0;
    irq->ops = NULL;
}

Here is the caller graph for this function:

kvm_vgic_unmap_phys_irq()

int kvm_vgic_unmap_phys_irq	(	struct kvm_vcpu *	vcpu,
		unsigned int	vintid
	)

Definition at line 557 of file vgic.c.

{
    struct vgic_irq *irq;
    unsigned long flags;
 
    if (!vgic_initialized(vcpu->kvm))
        return -EAGAIN;
 
    irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
    BUG_ON(!irq);
 
    raw_spin_lock_irqsave(&irq->irq_lock, flags);
    kvm_vgic_unmap_irq(irq);
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
    vgic_put_irq(vcpu->kvm, irq);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_vcpu_pending_irq()

int kvm_vgic_vcpu_pending_irq

(

struct kvm_vcpu *

vcpu

)

Definition at line 971 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
    struct vgic_irq *irq;
    bool pending = false;
    unsigned long flags;
    struct vgic_vmcr vmcr;
 
    if (!vcpu->kvm->arch.vgic.enabled)
        return false;
 
    if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last)
        return true;
 
    vgic_get_vmcr(vcpu, &vmcr);
 
    raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
 
    list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
        raw_spin_lock(&irq->irq_lock);
        pending = irq_is_pending(irq) && irq->enabled &&
              !irq->active &&
              irq->priority < vmcr.pmr;
        raw_spin_unlock(&irq->irq_lock);
 
        if (pending)
            break;
    }
 
    raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
 
    return pending;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_vgic_vmcr_sync()

void kvm_vgic_vmcr_sync

(

struct kvm_vcpu *

vcpu

)

Definition at line 960 of file vgic.c.

{
    if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
        return;
 
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_vmcr_sync(vcpu);
    else
        vgic_v3_vmcr_sync(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_clear_lr()

static void vgic_clear_lr ( struct kvm_vcpu *  vcpu, int  lr  )

inlinestatic

Definition at line 754 of file vgic.c.

{
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_clear_lr(vcpu, lr);
    else
        vgic_v3_clear_lr(vcpu, lr);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_flush_lr_state()

static void vgic_flush_lr_state ( struct kvm_vcpu *  vcpu )

static

Definition at line 797 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
    struct vgic_irq *irq;
    int count;
    bool multi_sgi;
    u8 prio = 0xff;
    int i = 0;
 
    lockdep_assert_held(&vgic_cpu->ap_list_lock);
 
    count = compute_ap_list_depth(vcpu, &multi_sgi);
    if (count > kvm_vgic_global_state.nr_lr || multi_sgi)
        vgic_sort_ap_list(vcpu);
 
    count = 0;
 
    list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
        raw_spin_lock(&irq->irq_lock);
 
        /*
         * If we have multi-SGIs in the pipeline, we need to
         * guarantee that they are all seen before any IRQ of
         * lower priority. In that case, we need to filter out
         * these interrupts by exiting early. This is easy as
         * the AP list has been sorted already.
         */
        if (multi_sgi && irq->priority > prio) {
            _raw_spin_unlock(&irq->irq_lock);
            break;
        }
 
        if (likely(vgic_target_oracle(irq) == vcpu)) {
            vgic_populate_lr(vcpu, irq, count++);
 
            if (irq->source)
                prio = irq->priority;
        }
 
        raw_spin_unlock(&irq->irq_lock);
 
        if (count == kvm_vgic_global_state.nr_lr) {
            if (!list_is_last(&irq->ap_list,
                      &vgic_cpu->ap_list_head))
                vgic_set_underflow(vcpu);
            break;
        }
    }
 
    /* Nuke remaining LRs */
    for (i = count ; i < kvm_vgic_global_state.nr_lr; i++)
        vgic_clear_lr(vcpu, i);
 
    if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
        vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count;
    else
        vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_flush_pending_lpis()

void vgic_flush_pending_lpis

(

struct kvm_vcpu *

vcpu

)

Definition at line 152 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
    struct vgic_irq *irq, *tmp;
    unsigned long flags;
 
    raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
 
    list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
        if (irq->intid >= VGIC_MIN_LPI) {
            raw_spin_lock(&irq->irq_lock);
            list_del(&irq->ap_list);
            irq->vcpu = NULL;
            raw_spin_unlock(&irq->irq_lock);
            vgic_put_irq(vcpu->kvm, irq);
        }
    }
 
    raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_fold_lr_state()

static void vgic_fold_lr_state ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 734 of file vgic.c.

{
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_fold_lr_state(vcpu);
    else
        vgic_v3_fold_lr_state(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_get_irq()

struct vgic_irq* vgic_get_irq	(	struct kvm *	kvm,
		struct kvm_vcpu *	vcpu,
		u32	intid
	)

Definition at line 92 of file vgic.c.

{
    /* SGIs and PPIs */
    if (intid <= VGIC_MAX_PRIVATE) {
        intid = array_index_nospec(intid, VGIC_MAX_PRIVATE + 1);
        return &vcpu->arch.vgic_cpu.private_irqs[intid];
    }
 
    /* SPIs */
    if (intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
        intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS);
        return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
    }
 
    /* LPIs */
    if (intid >= VGIC_MIN_LPI)
        return vgic_get_lpi(kvm, intid);
 
    return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_get_lpi()

static struct vgic_irq* vgic_get_lpi ( struct kvm *  kvm, u32  intid  )

static

Definition at line 60 of file vgic.c.

{
    struct vgic_dist *dist = &kvm->arch.vgic;
    struct vgic_irq *irq = NULL;
    unsigned long flags;
 
    raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 
    list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
        if (irq->intid != intid)
            continue;
 
        /*
         * This increases the refcount, the caller is expected to
         * call vgic_put_irq() later once it's finished with the IRQ.
         */
        vgic_get_irq_kref(irq);
        goto out_unlock;
    }
    irq = NULL;
 
out_unlock:
    raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 
    return irq;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_get_phys_line_level()

bool vgic_get_phys_line_level

(

struct vgic_irq *

irq

)

Definition at line 180 of file vgic.c.

{
    bool line_level;
 
    BUG_ON(!irq->hw);
 
    if (irq->ops && irq->ops->get_input_level)
        return irq->ops->get_input_level(irq->intid);
 
    WARN_ON(irq_get_irqchip_state(irq->host_irq,
                      IRQCHIP_STATE_PENDING,
                      &line_level));
    return line_level;
}

Here is the caller graph for this function:

vgic_irq_cmp()

static int vgic_irq_cmp ( void *  priv, const struct list_head *  a, const struct list_head *  b  )

static

Definition at line 259 of file vgic.c.

{
    struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
    struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);
    bool penda, pendb;
    int ret;
 
    /*
     * list_sort may call this function with the same element when
     * the list is fairly long.
     */
    if (unlikely(irqa == irqb))
        return 0;
 
    raw_spin_lock(&irqa->irq_lock);
    raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
 
    if (irqa->active || irqb->active) {
        ret = (int)irqb->active - (int)irqa->active;
        goto out;
    }
 
    penda = irqa->enabled && irq_is_pending(irqa);
    pendb = irqb->enabled && irq_is_pending(irqb);
 
    if (!penda || !pendb) {
        ret = (int)pendb - (int)penda;
        goto out;
    }
 
    /* Both pending and enabled, sort by priority */
    ret = irqa->priority - irqb->priority;
out:
    raw_spin_unlock(&irqb->irq_lock);
    raw_spin_unlock(&irqa->irq_lock);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_irq_handle_resampling()

void vgic_irq_handle_resampling	(	struct vgic_irq *	irq,
		bool	lr_deactivated,
		bool	lr_pending
	)

Definition at line 1060 of file vgic.c.

{
    if (vgic_irq_is_mapped_level(irq)) {
        bool resample = false;
 
        if (unlikely(vgic_irq_needs_resampling(irq))) {
            resample = !(irq->active || irq->pending_latch);
        } else if (lr_pending || (lr_deactivated && irq->line_level)) {
            irq->line_level = vgic_get_phys_line_level(irq);
            resample = !irq->line_level;
        }
 
        if (resample)
            vgic_irq_set_phys_active(irq, false);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_irq_release()

static void vgic_irq_release ( struct kref *  ref )

static

Definition at line 119 of file vgic.c.

{
}

Here is the caller graph for this function:

vgic_irq_set_phys_active()

void vgic_irq_set_phys_active	(	struct vgic_irq *	irq,
		bool	active
	)

Definition at line 196 of file vgic.c.

{
 
    BUG_ON(!irq->hw);
    WARN_ON(irq_set_irqchip_state(irq->host_irq,
                      IRQCHIP_STATE_ACTIVE,
                      active));
}

Here is the caller graph for this function:

vgic_irq_set_phys_pending()

void vgic_irq_set_phys_pending	(	struct vgic_irq *	irq,
		bool	pending
	)

Definition at line 173 of file vgic.c.

{
    WARN_ON(irq_set_irqchip_state(irq->host_irq,
                      IRQCHIP_STATE_PENDING,
                      pending));
}

Here is the caller graph for this function:

vgic_kick_vcpus()

void vgic_kick_vcpus

(

struct kvm *

kvm

)

Definition at line 1005 of file vgic.c.

{
    struct kvm_vcpu *vcpu;
    unsigned long c;
 
    /*
     * We've injected an interrupt, time to find out who deserves
     * a good kick...
     */
    kvm_for_each_vcpu(c, vcpu, kvm) {
        if (kvm_vgic_vcpu_pending_irq(vcpu)) {
            kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
            kvm_vcpu_kick(vcpu);
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_populate_lr()

static void vgic_populate_lr ( struct kvm_vcpu *  vcpu, struct vgic_irq *  irq, int  lr  )

inlinestatic

Definition at line 743 of file vgic.c.

{
    lockdep_assert_held(&irq->irq_lock);
 
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_populate_lr(vcpu, irq, lr);
    else
        vgic_v3_populate_lr(vcpu, irq, lr);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_prune_ap_list()

static void vgic_prune_ap_list ( struct kvm_vcpu *  vcpu )

static

vgic_prune_ap_list - Remove non-relevant interrupts from the list

@vcpu: The VCPU pointer

Go over the list of "interesting" interrupts, and prune those that we won't have to consider in the near future.

Definition at line 633 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
    struct vgic_irq *irq, *tmp;
 
    DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
retry:
    raw_spin_lock(&vgic_cpu->ap_list_lock);
 
    list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
        struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
        bool target_vcpu_needs_kick = false;
 
        raw_spin_lock(&irq->irq_lock);
 
        BUG_ON(vcpu != irq->vcpu);
 
        target_vcpu = vgic_target_oracle(irq);
 
        if (!target_vcpu) {
            /*
             * We don't need to process this interrupt any
             * further, move it off the list.
             */
            list_del(&irq->ap_list);
            irq->vcpu = NULL;
            raw_spin_unlock(&irq->irq_lock);
 
            /*
             * This vgic_put_irq call matches the
             * vgic_get_irq_kref in vgic_queue_irq_unlock,
             * where we added the LPI to the ap_list. As
             * we remove the irq from the list, we drop
             * also drop the refcount.
             */
            vgic_put_irq(vcpu->kvm, irq);
            continue;
        }
 
        if (target_vcpu == vcpu) {
            /* We're on the right CPU */
            raw_spin_unlock(&irq->irq_lock);
            continue;
        }
 
        /* This interrupt looks like it has to be migrated. */
 
        raw_spin_unlock(&irq->irq_lock);
        raw_spin_unlock(&vgic_cpu->ap_list_lock);
 
        /*
         * Ensure locking order by always locking the smallest
         * ID first.
         */
        if (vcpu->vcpu_id < target_vcpu->vcpu_id) {
            vcpuA = vcpu;
            vcpuB = target_vcpu;
        } else {
            vcpuA = target_vcpu;
            vcpuB = vcpu;
        }
 
        raw_spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
        raw_spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
                      SINGLE_DEPTH_NESTING);
        raw_spin_lock(&irq->irq_lock);
 
        /*
         * If the affinity has been preserved, move the
         * interrupt around. Otherwise, it means things have
         * changed while the interrupt was unlocked, and we
         * need to replay this.
         *
         * In all cases, we cannot trust the list not to have
         * changed, so we restart from the beginning.
         */
        if (target_vcpu == vgic_target_oracle(irq)) {
            struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu;
 
            list_del(&irq->ap_list);
            irq->vcpu = target_vcpu;
            list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);
            target_vcpu_needs_kick = true;
        }
 
        raw_spin_unlock(&irq->irq_lock);
        raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
        raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
 
        if (target_vcpu_needs_kick) {
            kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu);
            kvm_vcpu_kick(target_vcpu);
        }
 
        goto retry;
    }
 
    raw_spin_unlock(&vgic_cpu->ap_list_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_put_irq()

void vgic_put_irq	(	struct kvm *	kvm,
		struct vgic_irq *	irq
	)

Definition at line 139 of file vgic.c.

{
    struct vgic_dist *dist = &kvm->arch.vgic;
    unsigned long flags;
 
    if (irq->intid < VGIC_MIN_LPI)
        return;
 
    raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
    __vgic_put_lpi_locked(kvm, irq);
    raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_queue_irq_unlock()

bool vgic_queue_irq_unlock	(	struct kvm *	kvm,
		struct vgic_irq *	irq,
		unsigned long	flags
	)

Definition at line 336 of file vgic.c.

{
    struct kvm_vcpu *vcpu;
 
    lockdep_assert_held(&irq->irq_lock);
 
retry:
    vcpu = vgic_target_oracle(irq);
    if (irq->vcpu || !vcpu) {
        /*
         * If this IRQ is already on a VCPU's ap_list, then it
         * cannot be moved or modified and there is no more work for
         * us to do.
         *
         * Otherwise, if the irq is not pending and enabled, it does
         * not need to be inserted into an ap_list and there is also
         * no more work for us to do.
         */
        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
        /*
         * We have to kick the VCPU here, because we could be
         * queueing an edge-triggered interrupt for which we
         * get no EOI maintenance interrupt. In that case,
         * while the IRQ is already on the VCPU's AP list, the
         * VCPU could have EOI'ed the original interrupt and
         * won't see this one until it exits for some other
         * reason.
         */
        if (vcpu) {
            kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
            kvm_vcpu_kick(vcpu);
        }
        return false;
    }
 
    /*
     * We must unlock the irq lock to take the ap_list_lock where
     * we are going to insert this new pending interrupt.
     */
    raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
    /* someone can do stuff here, which we re-check below */
 
    raw_spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
    raw_spin_lock(&irq->irq_lock);
 
    /*
     * Did something change behind our backs?
     *
     * There are two cases:
     * 1) The irq lost its pending state or was disabled behind our
     *    backs and/or it was queued to another VCPU's ap_list.
     * 2) Someone changed the affinity on this irq behind our
     *    backs and we are now holding the wrong ap_list_lock.
     *
     * In both cases, drop the locks and retry.
     */
 
    if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {
        raw_spin_unlock(&irq->irq_lock);
        raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock,
                       flags);
 
        raw_spin_lock_irqsave(&irq->irq_lock, flags);
        goto retry;
    }
 
    /*
     * Grab a reference to the irq to reflect the fact that it is
     * now in the ap_list.
     */
    vgic_get_irq_kref(irq);
    list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
    irq->vcpu = vcpu;
 
    raw_spin_unlock(&irq->irq_lock);
    raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
 
    kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
    kvm_vcpu_kick(vcpu);
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_restore_state()

static void vgic_restore_state ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 896 of file vgic.c.

{
    if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
        vgic_v2_restore_state(vcpu);
    else
        __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_save_state()

static void vgic_save_state ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 866 of file vgic.c.

{
    if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
        vgic_v2_save_state(vcpu);
    else
        __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_set_underflow()

static void vgic_set_underflow ( struct kvm_vcpu *  vcpu )

inlinestatic

Definition at line 762 of file vgic.c.

{
    if (kvm_vgic_global_state.type == VGIC_V2)
        vgic_v2_set_underflow(vcpu);
    else
        vgic_v3_set_underflow(vcpu);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_sort_ap_list()

static void vgic_sort_ap_list ( struct kvm_vcpu *  vcpu )

static

Definition at line 299 of file vgic.c.

{
    struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 
    lockdep_assert_held(&vgic_cpu->ap_list_lock);
 
    list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_target_oracle()

static struct kvm_vcpu* vgic_target_oracle ( struct vgic_irq *  irq )

static

kvm_vgic_target_oracle - compute the target vcpu for an irq

@irq: The irq to route. Must be already locked.

Based on the current state of the interrupt (enabled, pending, active, vcpu and target_vcpu), compute the next vcpu this should be given to. Return NULL if this shouldn't be injected at all.

Requires the IRQ lock to be held.

Definition at line 216 of file vgic.c.

{
    lockdep_assert_held(&irq->irq_lock);
 
    /* If the interrupt is active, it must stay on the current vcpu */
    if (irq->active)
        return irq->vcpu ? : irq->target_vcpu;
 
    /*
     * If the IRQ is not active but enabled and pending, we should direct
     * it to its configured target VCPU.
     * If the distributor is disabled, pending interrupts shouldn't be
     * forwarded.
     */
    if (irq->enabled && irq_is_pending(irq)) {
        if (unlikely(irq->target_vcpu &&
                 !irq->target_vcpu->kvm->arch.vgic.enabled))
            return NULL;
 
        return irq->target_vcpu;
    }
 
    /* If neither active nor pending and enabled, then this IRQ should not
     * be queued to any VCPU.
     */
    return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

vgic_validate_injection()

static bool vgic_validate_injection ( struct vgic_irq *  irq, bool  level, void *  owner  )

static

Definition at line 313 of file vgic.c.

{
    if (irq->owner != owner)
        return false;
 
    switch (irq->config) {
    case VGIC_CONFIG_LEVEL:
        return irq->line_level != level;
    case VGIC_CONFIG_EDGE:
        return level;
    }
 
    return false;
}

Here is the caller graph for this function:

Variable Documentation

__ro_after_init

struct vgic_global kvm_vgic_global_state __ro_after_init

Initial value:

= {
    .gicv3_cpuif = STATIC_KEY_FALSE_INIT,
}

Definition at line 1 of file vgic.c.