posted_intr.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	pi_desc

Macros
#define	POSTED_INTR_ON   0
#define	POSTED_INTR_SN   1
#define	PID_TABLE_ENTRY_VALID   1

Functions
struct pi_desc	__aligned (64)
static bool	pi_test_and_set_on (struct pi_desc *pi_desc)
static bool	pi_test_and_clear_on (struct pi_desc *pi_desc)
static bool	pi_test_and_clear_sn (struct pi_desc *pi_desc)
static bool	pi_test_and_set_pir (int vector, struct pi_desc *pi_desc)
static bool	pi_is_pir_empty (struct pi_desc *pi_desc)
static void	pi_set_sn (struct pi_desc *pi_desc)
static void	pi_set_on (struct pi_desc *pi_desc)
static void	pi_clear_on (struct pi_desc *pi_desc)
static void	pi_clear_sn (struct pi_desc *pi_desc)
static bool	pi_test_on (struct pi_desc *pi_desc)
static bool	pi_test_sn (struct pi_desc *pi_desc)
void	vmx_vcpu_pi_load (struct kvm_vcpu *vcpu, int cpu)
void	vmx_vcpu_pi_put (struct kvm_vcpu *vcpu)
void	pi_wakeup_handler (void)
void __init	pi_init_cpu (int cpu)
bool	pi_has_pending_interrupt (struct kvm_vcpu *vcpu)
int	vmx_pi_update_irte (struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, bool set)
void	vmx_pi_start_assignment (struct kvm *kvm)

Variables
u32	pir [8]
union {
struct {
u16   on: 1
u16   sn: 1
u16   rsvd_1: 14
u8   nv
u8   rsvd_2
u32   ndst
}
u64   control
};
u32	rsvd [6]

Macro Definition Documentation

PID_TABLE_ENTRY_VALID

#define PID_TABLE_ENTRY_VALID   1

Definition at line 8 of file posted_intr.h.

POSTED_INTR_ON

#define POSTED_INTR_ON   0

Definition at line 5 of file posted_intr.h.

POSTED_INTR_SN

#define POSTED_INTR_SN   1

Definition at line 6 of file posted_intr.h.

Function Documentation

__aligned()

struct pi_desc __aligned

(

64

)

pi_clear_on()

static void pi_clear_on ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 73 of file posted_intr.h.

{
    clear_bit(POSTED_INTR_ON,
        (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_clear_sn()

static void pi_clear_sn ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 79 of file posted_intr.h.

{
    clear_bit(POSTED_INTR_SN,
        (unsigned long *)&pi_desc->control);
}

pi_has_pending_interrupt()

bool pi_has_pending_interrupt

(

struct kvm_vcpu *

vcpu

)

Definition at line 240 of file posted_intr.c.

{
    struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 
    return pi_test_on(pi_desc) ||
        (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc));
}

Here is the call graph for this function:

pi_init_cpu()

void __init pi_init_cpu

(

int

cpu

)

Definition at line 234 of file posted_intr.c.

{
    INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
    raw_spin_lock_init(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu));
}

Here is the caller graph for this function:

pi_is_pir_empty()

static bool pi_is_pir_empty ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 56 of file posted_intr.h.

{
    return bitmap_empty((unsigned long *)pi_desc->pir, NR_VECTORS);
}

Here is the caller graph for this function:

pi_set_on()

static void pi_set_on ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 67 of file posted_intr.h.

{
    set_bit(POSTED_INTR_ON,
        (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_set_sn()

static void pi_set_sn ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 61 of file posted_intr.h.

{
    set_bit(POSTED_INTR_SN,
        (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_test_and_clear_on()

static bool pi_test_and_clear_on ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 39 of file posted_intr.h.

{
    return test_and_clear_bit(POSTED_INTR_ON,
            (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_test_and_clear_sn()

static bool pi_test_and_clear_sn ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 45 of file posted_intr.h.

{
    return test_and_clear_bit(POSTED_INTR_SN,
            (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_test_and_set_on()

static bool pi_test_and_set_on ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 33 of file posted_intr.h.

{
    return test_and_set_bit(POSTED_INTR_ON,
            (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_test_and_set_pir()

static bool pi_test_and_set_pir ( int  vector, struct pi_desc *  pi_desc  )

inlinestatic

Definition at line 51 of file posted_intr.h.

{
    return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
}

Here is the caller graph for this function:

pi_test_on()

static bool pi_test_on ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 85 of file posted_intr.h.

{
    return test_bit(POSTED_INTR_ON,
            (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_test_sn()

static bool pi_test_sn ( struct pi_desc *  pi_desc )

inlinestatic

Definition at line 91 of file posted_intr.h.

{
    return test_bit(POSTED_INTR_SN,
            (unsigned long *)&pi_desc->control);
}

Here is the caller graph for this function:

pi_wakeup_handler()

void pi_wakeup_handler

(

void

)

Definition at line 218 of file posted_intr.c.

{
    int cpu = smp_processor_id();
    struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
    raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
    struct vcpu_vmx *vmx;
 
    raw_spin_lock(spinlock);
    list_for_each_entry(vmx, wakeup_list, pi_wakeup_list) {
 
        if (pi_test_on(&vmx->pi_desc))
            kvm_vcpu_wake_up(&vmx->vcpu);
    }
    raw_spin_unlock(spinlock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vmx_pi_start_assignment()

void vmx_pi_start_assignment

(

struct kvm *

kvm

)

Definition at line 255 of file posted_intr.c.

{
    if (!irq_remapping_cap(IRQ_POSTING_CAP))
        return;
 
    kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
}

Here is the call graph for this function:

vmx_pi_update_irte()

int vmx_pi_update_irte	(	struct kvm *	kvm,
		unsigned int	host_irq,
		uint32_t	guest_irq,
		bool	set
	)

Definition at line 272 of file posted_intr.c.

{
    struct kvm_kernel_irq_routing_entry *e;
    struct kvm_irq_routing_table *irq_rt;
    struct kvm_lapic_irq irq;
    struct kvm_vcpu *vcpu;
    struct vcpu_data vcpu_info;
    int idx, ret = 0;
 
    if (!vmx_can_use_vtd_pi(kvm))
        return 0;
 
    idx = srcu_read_lock(&kvm->irq_srcu);
    irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
    if (guest_irq >= irq_rt->nr_rt_entries ||
        hlist_empty(&irq_rt->map[guest_irq])) {
        pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
                 guest_irq, irq_rt->nr_rt_entries);
        goto out;
    }
 
    hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
        if (e->type != KVM_IRQ_ROUTING_MSI)
            continue;
        /*
         * VT-d PI cannot support posting multicast/broadcast
         * interrupts to a vCPU, we still use interrupt remapping
         * for these kind of interrupts.
         *
         * For lowest-priority interrupts, we only support
         * those with single CPU as the destination, e.g. user
         * configures the interrupts via /proc/irq or uses
         * irqbalance to make the interrupts single-CPU.
         *
         * We will support full lowest-priority interrupt later.
         *
         * In addition, we can only inject generic interrupts using
         * the PI mechanism, refuse to route others through it.
         */
 
        kvm_set_msi_irq(kvm, e, &irq);
        if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
            !kvm_irq_is_postable(&irq)) {
            /*
             * Make sure the IRTE is in remapped mode if
             * we don't handle it in posted mode.
             */
            ret = irq_set_vcpu_affinity(host_irq, NULL);
            if (ret < 0) {
                printk(KERN_INFO
                   "failed to back to remapped mode, irq: %u\n",
                   host_irq);
                goto out;
            }
 
            continue;
        }
 
        vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
        vcpu_info.vector = irq.vector;
 
        trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
                vcpu_info.vector, vcpu_info.pi_desc_addr, set);
 
        if (set)
            ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
        else
            ret = irq_set_vcpu_affinity(host_irq, NULL);
 
        if (ret < 0) {
            printk(KERN_INFO "%s: failed to update PI IRTE\n",
                    __func__);
            goto out;
        }
    }
 
    ret = 0;
out:
    srcu_read_unlock(&kvm->irq_srcu, idx);
    return ret;
}

Here is the call graph for this function:

vmx_vcpu_pi_load()

void vmx_vcpu_pi_load	(	struct kvm_vcpu *	vcpu,
		int	cpu
	)

Definition at line 53 of file posted_intr.c.

{
    struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
    struct vcpu_vmx *vmx = to_vmx(vcpu);
    struct pi_desc old, new;
    unsigned long flags;
    unsigned int dest;
 
    /*
     * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and
     * PI.SN up-to-date even if there is no assigned device or if APICv is
     * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.
     */
    if (!enable_apicv || !lapic_in_kernel(vcpu))
        return;
 
    /*
     * If the vCPU wasn't on the wakeup list and wasn't migrated, then the
     * full update can be skipped as neither the vector nor the destination
     * needs to be changed.
     */
    if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {
        /*
         * Clear SN if it was set due to being preempted.  Again, do
         * this even if there is no assigned device for simplicity.
         */
        if (pi_test_and_clear_sn(pi_desc))
            goto after_clear_sn;
        return;
    }
 
    local_irq_save(flags);
 
    /*
     * If the vCPU was waiting for wakeup, remove the vCPU from the wakeup
     * list of the _previous_ pCPU, which will not be the same as the
     * current pCPU if the task was migrated.
     */
    if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {
        raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
        list_del(&vmx->pi_wakeup_list);
        raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
    }
 
    dest = cpu_physical_id(cpu);
    if (!x2apic_mode)
        dest = (dest << 8) & 0xFF00;
 
    old.control = READ_ONCE(pi_desc->control);
    do {
        new.control = old.control;
 
        /*
         * Clear SN (as above) and refresh the destination APIC ID to
         * handle task migration (@cpu != vcpu->cpu).
         */
        new.ndst = dest;
        new.sn = 0;
 
        /*
         * Restore the notification vector; in the blocking case, the
         * descriptor was modified on "put" to use the wakeup vector.
         */
        new.nv = POSTED_INTR_VECTOR;
    } while (pi_try_set_control(pi_desc, &old.control, new.control));
 
    local_irq_restore(flags);
 
after_clear_sn:
 
    /*
     * Clear SN before reading the bitmap.  The VT-d firmware
     * writes the bitmap and reads SN atomically (5.2.3 in the
     * spec), so it doesn't really have a memory barrier that
     * pairs with this, but we cannot do that and we need one.
     */
    smp_mb__after_atomic();
 
    if (!pi_is_pir_empty(pi_desc))
        pi_set_on(pi_desc);
}

Here is the call graph for this function:

Here is the caller graph for this function:

vmx_vcpu_pi_put()

void vmx_vcpu_pi_put

(

struct kvm_vcpu *

vcpu

)

Definition at line 196 of file posted_intr.c.

{
    struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 
    if (!vmx_needs_pi_wakeup(vcpu))
        return;
 
    if (kvm_vcpu_is_blocking(vcpu) && !vmx_interrupt_blocked(vcpu))
        pi_enable_wakeup_handler(vcpu);
 
    /*
     * Set SN when the vCPU is preempted.  Note, the vCPU can both be seen
     * as blocking and preempted, e.g. if it's preempted between setting
     * its wait state and manually scheduling out.
     */
    if (vcpu->preempted)
        pi_set_sn(pi_desc);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

union { ... }

control

u64 control

Definition at line 16 of file posted_intr.h.

ndst

u32 ndst

Definition at line 14 of file posted_intr.h.

nv

u8 nv

Definition at line 10 of file posted_intr.h.

on

u16 on

Definition at line 4 of file posted_intr.h.

pir

u32 pir[8]

Definition at line 0 of file posted_intr.h.

rsvd

u32 rsvd[6]

Definition at line 18 of file posted_intr.h.

rsvd_1

u16 rsvd_1

Definition at line 8 of file posted_intr.h.

rsvd_2

u8 rsvd_2

Definition at line 12 of file posted_intr.h.

sn

u16 sn

Definition at line 6 of file posted_intr.h.