pkvm.c File Reference

#include <linux/init.h>
#include <linux/kmemleak.h>
#include <linux/kvm_host.h>
#include <linux/memblock.h>
#include <linux/mutex.h>
#include <linux/sort.h>
#include <asm/kvm_pkvm.h>
#include "hyp_constants.h"

Include dependency graph for pkvm.c:

Go to the source code of this file.

Functions
	DEFINE_STATIC_KEY_FALSE (kvm_protected_mode_initialized)
static int	cmp_hyp_memblock (const void *p1, const void *p2)
static void __init	sort_memblock_regions (void)
static int __init	register_memblock_regions (void)
void __init	kvm_hyp_reserve (void)
static void	__pkvm_destroy_hyp_vm (struct kvm *host_kvm)
static int	__pkvm_create_hyp_vm (struct kvm *host_kvm)
int	pkvm_create_hyp_vm (struct kvm *host_kvm)
void	pkvm_destroy_hyp_vm (struct kvm *host_kvm)
int	pkvm_init_host_vm (struct kvm *host_kvm)
static void __init	_kvm_host_prot_finalize (void *arg)
static int __init	pkvm_drop_host_privileges (void)
static int __init	finalize_pkvm (void)
	device_initcall_sync (finalize_pkvm)

Variables
static struct memblock_region *	hyp_memory = kvm_nvhe_sym(hyp_memory)
static unsigned int *	hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr)
phys_addr_t	hyp_mem_base
phys_addr_t	hyp_mem_size

Function Documentation

__pkvm_create_hyp_vm()

static int __pkvm_create_hyp_vm ( struct kvm *  host_kvm )

static

Definition at line 125 of file pkvm.c.

{
    size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz;
    struct kvm_vcpu *host_vcpu;
    pkvm_handle_t handle;
    void *pgd, *hyp_vm;
    unsigned long idx;
    int ret;
 
    if (host_kvm->created_vcpus < 1)
        return -EINVAL;
 
    pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.mmu.vtcr);
 
    /*
     * The PGD pages will be reclaimed using a hyp_memcache which implies
     * page granularity. So, use alloc_pages_exact() to get individual
     * refcounts.
     */
    pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
    if (!pgd)
        return -ENOMEM;
 
    /* Allocate memory to donate to hyp for vm and vcpu pointers. */
    hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
                    size_mul(sizeof(void *),
                         host_kvm->created_vcpus)));
    hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
    if (!hyp_vm) {
        ret = -ENOMEM;
        goto free_pgd;
    }
 
    /* Donate the VM memory to hyp and let hyp initialize it. */
    ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd);
    if (ret < 0)
        goto free_vm;
 
    handle = ret;
 
    host_kvm->arch.pkvm.handle = handle;
 
    /* Donate memory for the vcpus at hyp and initialize it. */
    hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
    kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
        void *hyp_vcpu;
 
        /* Indexing of the vcpus to be sequential starting at 0. */
        if (WARN_ON(host_vcpu->vcpu_idx != idx)) {
            ret = -EINVAL;
            goto destroy_vm;
        }
 
        hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
        if (!hyp_vcpu) {
            ret = -ENOMEM;
            goto destroy_vm;
        }
 
        ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu,
                    hyp_vcpu);
        if (ret) {
            free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
            goto destroy_vm;
        }
    }
 
    return 0;
 
destroy_vm:
    __pkvm_destroy_hyp_vm(host_kvm);
    return ret;
free_vm:
    free_pages_exact(hyp_vm, hyp_vm_sz);
free_pgd:
    free_pages_exact(pgd, pgd_sz);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_destroy_hyp_vm()

static void __pkvm_destroy_hyp_vm ( struct kvm *  host_kvm )

static

Definition at line 104 of file pkvm.c.

{
    if (host_kvm->arch.pkvm.handle) {
        WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm,
                      host_kvm->arch.pkvm.handle));
    }
 
    host_kvm->arch.pkvm.handle = 0;
    free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
}

Here is the call graph for this function:

Here is the caller graph for this function:

_kvm_host_prot_finalize()

static void __init _kvm_host_prot_finalize ( void *  arg )

static

Definition at line 229 of file pkvm.c.

{
    int *err = arg;
 
    if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)))
        WRITE_ONCE(*err, -EINVAL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

cmp_hyp_memblock()

static int cmp_hyp_memblock ( const void *  p1, const void *  p2  )

static

Definition at line 26 of file pkvm.c.

{
    const struct memblock_region *r1 = p1;
    const struct memblock_region *r2 = p2;
 
    return r1->base < r2->base ? -1 : (r1->base > r2->base);
}

Here is the caller graph for this function:

DEFINE_STATIC_KEY_FALSE()

DEFINE_STATIC_KEY_FALSE

(

kvm_protected_mode_initialized

)

device_initcall_sync()

device_initcall_sync

(

finalize_pkvm

)

finalize_pkvm()

static int __init finalize_pkvm ( void  )

static

Definition at line 250 of file pkvm.c.

{
    int ret;
 
    if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised())
        return 0;
 
    /*
     * Exclude HYP sections from kmemleak so that they don't get peeked
     * at, which would end badly once inaccessible.
     */
    kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
    kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size);
 
    ret = pkvm_drop_host_privileges();
    if (ret)
        pr_err("Failed to finalize Hyp protection: %d\n", ret);
 
    return ret;
}

Here is the call graph for this function:

kvm_hyp_reserve()

void __init kvm_hyp_reserve

(

void

)

Definition at line 59 of file pkvm.c.

{
    u64 hyp_mem_pages = 0;
    int ret;
 
    if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
        return;
 
    if (kvm_get_mode() != KVM_MODE_PROTECTED)
        return;
 
    ret = register_memblock_regions();
    if (ret) {
        *hyp_memblock_nr_ptr = 0;
        kvm_err("Failed to register hyp memblocks: %d\n", ret);
        return;
    }
 
    hyp_mem_pages += hyp_s1_pgtable_pages();
    hyp_mem_pages += host_s2_pgtable_pages();
    hyp_mem_pages += hyp_vm_table_pages();
    hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
    hyp_mem_pages += hyp_ffa_proxy_pages();
 
    /*
     * Try to allocate a PMD-aligned region to reduce TLB pressure once
     * this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
     */
    hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
    hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE),
                       PMD_SIZE);
    if (!hyp_mem_base)
        hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE);
    else
        hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
 
    if (!hyp_mem_base) {
        kvm_err("Failed to reserve hyp memory\n");
        return;
    }
 
    kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
         hyp_mem_base);
}

Here is the call graph for this function:

pkvm_create_hyp_vm()

int pkvm_create_hyp_vm

(

struct kvm *

host_kvm

)

Definition at line 204 of file pkvm.c.

{
    int ret = 0;
 
    mutex_lock(&host_kvm->arch.config_lock);
    if (!host_kvm->arch.pkvm.handle)
        ret = __pkvm_create_hyp_vm(host_kvm);
    mutex_unlock(&host_kvm->arch.config_lock);
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

pkvm_destroy_hyp_vm()

void pkvm_destroy_hyp_vm

(

struct kvm *

host_kvm

)

Definition at line 216 of file pkvm.c.

{
    mutex_lock(&host_kvm->arch.config_lock);
    __pkvm_destroy_hyp_vm(host_kvm);
    mutex_unlock(&host_kvm->arch.config_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

pkvm_drop_host_privileges()

static int __init pkvm_drop_host_privileges ( void  )

static

Definition at line 237 of file pkvm.c.

{
    int ret = 0;
 
    /*
     * Flip the static key upfront as that may no longer be possible
     * once the host stage 2 is installed.
     */
    static_branch_enable(&kvm_protected_mode_initialized);
    on_each_cpu(_kvm_host_prot_finalize, &ret, 1);
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

pkvm_init_host_vm()

int pkvm_init_host_vm

(

struct kvm *

host_kvm

)

Definition at line 223 of file pkvm.c.

{
    mutex_init(&host_kvm->lock);
    return 0;
}

Here is the caller graph for this function:

register_memblock_regions()

static int __init register_memblock_regions ( void  )

static

Definition at line 43 of file pkvm.c.

{
    struct memblock_region *reg;
 
    for_each_mem_region(reg) {
        if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
            return -ENOMEM;
 
        hyp_memory[*hyp_memblock_nr_ptr] = *reg;
        (*hyp_memblock_nr_ptr)++;
    }
    sort_memblock_regions();
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

sort_memblock_regions()

static void __init sort_memblock_regions ( void  )

static

Definition at line 34 of file pkvm.c.

{
    sort(hyp_memory,
         *hyp_memblock_nr_ptr,
         sizeof(struct memblock_region),
         cmp_hyp_memblock,
         NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

hyp_mem_base

phys_addr_t hyp_mem_base

Definition at line 23 of file pkvm.c.

hyp_mem_size

phys_addr_t hyp_mem_size

Definition at line 24 of file pkvm.c.

hyp_memblock_nr_ptr

unsigned int* hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr)

static

Definition at line 21 of file pkvm.c.

hyp_memory

struct memblock_region* hyp_memory = kvm_nvhe_sym(hyp_memory)

static

Definition at line 20 of file pkvm.c.