spinlock.h

Go to the documentation of this file.

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * A stand-alone ticket spinlock implementation for use by the non-VHE
 * KVM hypervisor code running at EL2.
 *
 * Copyright (C) 2020 Google LLC
 * Author: Will Deacon <will@kernel.org>
 *
 * Heavily based on the implementation removed by c11090474d70 which was:
 * Copyright (C) 2012 ARM Ltd.
 */
 
#ifndef __ARM64_KVM_NVHE_SPINLOCK_H__
#define __ARM64_KVM_NVHE_SPINLOCK_H__
 
#include <asm/alternative.h>
#include <asm/lse.h>
#include <asm/rwonce.h>
 
typedef union hyp_spinlock {
    u32 __val;
    struct {
#ifdef __AARCH64EB__
        u16 next, owner;
#else
        u16 owner, next;
#endif
    };
} hyp_spinlock_t;
 
#define __HYP_SPIN_LOCK_INITIALIZER \
    { .__val = 0 }
 
#define __HYP_SPIN_LOCK_UNLOCKED \
    ((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER)
 
#define DEFINE_HYP_SPINLOCK(x)  hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED
 
#define hyp_spin_lock_init(l)                       \
do {                                    \
    *(l) = __HYP_SPIN_LOCK_UNLOCKED;                \
} while (0)
 
static inline void hyp_spin_lock(hyp_spinlock_t *lock)
{
    u32 tmp;
    hyp_spinlock_t lockval, newval;
 
    asm volatile(
    /* Atomically increment the next ticket. */
    ARM64_LSE_ATOMIC_INSN(
    /* LL/SC */
"   prfm    pstl1strm, %3\n"
"1: ldaxr   %w0, %3\n"
"   add %w1, %w0, #(1 << 16)\n"
"   stxr    %w2, %w1, %3\n"
"   cbnz    %w2, 1b\n",
    /* LSE atomics */
"   mov %w2, #(1 << 16)\n"
"   ldadda  %w2, %w0, %3\n"
    __nops(3))
 
    /* Did we get the lock? */
"   eor %w1, %w0, %w0, ror #16\n"
"   cbz %w1, 3f\n"
    /*
     * No: spin on the owner. Send a local event to avoid missing an
     * unlock before the exclusive load.
     */
"   sevl\n"
"2: wfe\n"
"   ldaxrh  %w2, %4\n"
"   eor %w1, %w2, %w0, lsr #16\n"
"   cbnz    %w1, 2b\n"
    /* We got the lock. Critical section starts here. */
"3:"
    : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
    : "Q" (lock->owner)
    : "memory");
}
 
static inline void hyp_spin_unlock(hyp_spinlock_t *lock)
{
    u64 tmp;
 
    asm volatile(
    ARM64_LSE_ATOMIC_INSN(
    /* LL/SC */
    "   ldrh    %w1, %0\n"
    "   add %w1, %w1, #1\n"
    "   stlrh   %w1, %0",
    /* LSE atomics */
    "   mov %w1, #1\n"
    "   staddlh %w1, %0\n"
    __nops(1))
    : "=Q" (lock->owner), "=&r" (tmp)
    :
    : "memory");
}
 
static inline bool hyp_spin_is_locked(hyp_spinlock_t *lock)
{
    hyp_spinlock_t lockval = READ_ONCE(*lock);
 
    return lockval.owner != lockval.next;
}
 
#ifdef CONFIG_NVHE_EL2_DEBUG
static inline void hyp_assert_lock_held(hyp_spinlock_t *lock)
{
    /*
     * The __pkvm_init() path accesses protected data-structures without
     * holding locks as the other CPUs are guaranteed to not enter EL2
     * concurrently at this point in time. The point by which EL2 is
     * initialized on all CPUs is reflected in the pkvm static key, so
     * wait until it is set before checking the lock state.
     */
    if (static_branch_likely(&kvm_protected_mode_initialized))
        BUG_ON(!hyp_spin_is_locked(lock));
}
#else
static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { }
#endif
 
#endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */