mem_protect.h File Reference

#include <linux/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_pgtable.h>
#include <asm/virt.h>
#include <nvhe/pkvm.h>
#include <nvhe/spinlock.h>

Include dependency graph for mem_protect.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	host_mmu

Macros
#define	PKVM_PAGE_STATE_PROT_MASK   (KVM_PGTABLE_PROT_SW0 | KVM_PGTABLE_PROT_SW1)

Enumerations
enum	pkvm_page_state {   PKVM_PAGE_OWNED = 0ULL , PKVM_PAGE_SHARED_OWNED = KVM_PGTABLE_PROT_SW0 , PKVM_PAGE_SHARED_BORROWED = KVM_PGTABLE_PROT_SW1 , __PKVM_PAGE_RESERVED ,   PKVM_NOPAGE }
enum	pkvm_component_id { PKVM_ID_HOST , PKVM_ID_HYP , PKVM_ID_FFA }

Functions
static enum kvm_pgtable_prot	pkvm_mkstate (enum kvm_pgtable_prot prot, enum pkvm_page_state state)
static enum pkvm_page_state	pkvm_getstate (enum kvm_pgtable_prot prot)
int	__pkvm_prot_finalize (void)
int	__pkvm_host_share_hyp (u64 pfn)
int	__pkvm_host_unshare_hyp (u64 pfn)
int	__pkvm_host_donate_hyp (u64 pfn, u64 nr_pages)
int	__pkvm_hyp_donate_host (u64 pfn, u64 nr_pages)
int	__pkvm_host_share_ffa (u64 pfn, u64 nr_pages)
int	__pkvm_host_unshare_ffa (u64 pfn, u64 nr_pages)
bool	addr_is_memory (phys_addr_t phys)
int	host_stage2_idmap_locked (phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot)
int	host_stage2_set_owner_locked (phys_addr_t addr, u64 size, u8 owner_id)
int	kvm_host_prepare_stage2 (void *pgt_pool_base)
int	kvm_guest_prepare_stage2 (struct pkvm_hyp_vm *vm, void *pgd)
void	handle_host_mem_abort (struct kvm_cpu_context *host_ctxt)
int	hyp_pin_shared_mem (void *from, void *to)
void	hyp_unpin_shared_mem (void *from, void *to)
void	reclaim_guest_pages (struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc)
int	refill_memcache (struct kvm_hyp_memcache *mc, unsigned long min_pages, struct kvm_hyp_memcache *host_mc)
static __always_inline void	__load_host_stage2 (void)

Variables
struct host_mmu	host_mmu
unsigned long	hyp_nr_cpus

Macro Definition Documentation

PKVM_PAGE_STATE_PROT_MASK

#define PKVM_PAGE_STATE_PROT_MASK   (KVM_PGTABLE_PROT_SW0 | KVM_PGTABLE_PROT_SW1)

Definition at line 36 of file mem_protect.h.

Enumeration Type Documentation

pkvm_component_id

enum pkvm_component_id

Enumerator
PKVM_ID_HOST
PKVM_ID_HYP
PKVM_ID_FFA

Definition at line 57 of file mem_protect.h.

                       {
    PKVM_ID_HOST,
    PKVM_ID_HYP,
    PKVM_ID_FFA,
};

pkvm_page_state

enum pkvm_page_state

Enumerator
PKVM_PAGE_OWNED
PKVM_PAGE_SHARED_OWNED
PKVM_PAGE_SHARED_BORROWED
__PKVM_PAGE_RESERVED
PKVM_NOPAGE

Definition at line 25 of file mem_protect.h.

                     {
    PKVM_PAGE_OWNED         = 0ULL,
    PKVM_PAGE_SHARED_OWNED      = KVM_PGTABLE_PROT_SW0,
    PKVM_PAGE_SHARED_BORROWED   = KVM_PGTABLE_PROT_SW1,
    __PKVM_PAGE_RESERVED        = KVM_PGTABLE_PROT_SW0 |
                      KVM_PGTABLE_PROT_SW1,
 
    /* Meta-states which aren't encoded directly in the PTE's SW bits */
    PKVM_NOPAGE,
};

Function Documentation

__load_host_stage2()

static __always_inline void __load_host_stage2 ( void  )

static

Definition at line 86 of file mem_protect.h.

{
    if (static_branch_likely(&kvm_protected_mode_initialized))
        __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
    else
        write_sysreg(0, vttbr_el2);
}

Here is the caller graph for this function:

__pkvm_host_donate_hyp()

int __pkvm_host_donate_hyp	(	u64	pfn,
		u64	nr_pages
	)

Definition at line 1152 of file mem_protect.c.

{
    int ret;
    u64 host_addr = hyp_pfn_to_phys(pfn);
    u64 hyp_addr = (u64)__hyp_va(host_addr);
    struct pkvm_mem_donation donation = {
        .tx = {
            .nr_pages   = nr_pages,
            .initiator  = {
                .id = PKVM_ID_HOST,
                .addr   = host_addr,
                .host   = {
                    .completer_addr = hyp_addr,
                },
            },
            .completer  = {
                .id = PKVM_ID_HYP,
            },
        },
    };
 
    host_lock_component();
    hyp_lock_component();
 
    ret = do_donate(&donation);
 
    hyp_unlock_component();
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_host_share_ffa()

int __pkvm_host_share_ffa	(	u64	pfn,
		u64	nr_pages
	)

Definition at line 1261 of file mem_protect.c.

{
    int ret;
    struct pkvm_mem_share share = {
        .tx = {
            .nr_pages   = nr_pages,
            .initiator  = {
                .id = PKVM_ID_HOST,
                .addr   = hyp_pfn_to_phys(pfn),
            },
            .completer  = {
                .id = PKVM_ID_FFA,
            },
        },
    };
 
    host_lock_component();
    ret = do_share(&share);
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_host_share_hyp()

int __pkvm_host_share_hyp

(

u64

pfn

)

Definition at line 1086 of file mem_protect.c.

{
    int ret;
    u64 host_addr = hyp_pfn_to_phys(pfn);
    u64 hyp_addr = (u64)__hyp_va(host_addr);
    struct pkvm_mem_share share = {
        .tx = {
            .nr_pages   = 1,
            .initiator  = {
                .id = PKVM_ID_HOST,
                .addr   = host_addr,
                .host   = {
                    .completer_addr = hyp_addr,
                },
            },
            .completer  = {
                .id = PKVM_ID_HYP,
            },
        },
        .completer_prot = PAGE_HYP,
    };
 
    host_lock_component();
    hyp_lock_component();
 
    ret = do_share(&share);
 
    hyp_unlock_component();
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_host_unshare_ffa()

int __pkvm_host_unshare_ffa	(	u64	pfn,
		u64	nr_pages
	)

Definition at line 1284 of file mem_protect.c.

{
    int ret;
    struct pkvm_mem_share share = {
        .tx = {
            .nr_pages   = nr_pages,
            .initiator  = {
                .id = PKVM_ID_HOST,
                .addr   = hyp_pfn_to_phys(pfn),
            },
            .completer  = {
                .id = PKVM_ID_FFA,
            },
        },
    };
 
    host_lock_component();
    ret = do_unshare(&share);
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_host_unshare_hyp()

int __pkvm_host_unshare_hyp

(

u64

pfn

)

Definition at line 1119 of file mem_protect.c.

{
    int ret;
    u64 host_addr = hyp_pfn_to_phys(pfn);
    u64 hyp_addr = (u64)__hyp_va(host_addr);
    struct pkvm_mem_share share = {
        .tx = {
            .nr_pages   = 1,
            .initiator  = {
                .id = PKVM_ID_HOST,
                .addr   = host_addr,
                .host   = {
                    .completer_addr = hyp_addr,
                },
            },
            .completer  = {
                .id = PKVM_ID_HYP,
            },
        },
        .completer_prot = PAGE_HYP,
    };
 
    host_lock_component();
    hyp_lock_component();
 
    ret = do_unshare(&share);
 
    hyp_unlock_component();
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_hyp_donate_host()

int __pkvm_hyp_donate_host	(	u64	pfn,
		u64	nr_pages
	)

Definition at line 1184 of file mem_protect.c.

{
    int ret;
    u64 host_addr = hyp_pfn_to_phys(pfn);
    u64 hyp_addr = (u64)__hyp_va(host_addr);
    struct pkvm_mem_donation donation = {
        .tx = {
            .nr_pages   = nr_pages,
            .initiator  = {
                .id = PKVM_ID_HYP,
                .addr   = hyp_addr,
                .hyp    = {
                    .completer_addr = host_addr,
                },
            },
            .completer  = {
                .id = PKVM_ID_HOST,
            },
        },
    };
 
    host_lock_component();
    hyp_lock_component();
 
    ret = do_donate(&donation);
 
    hyp_unlock_component();
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__pkvm_prot_finalize()

int __pkvm_prot_finalize

(

void

)

Definition at line 289 of file mem_protect.c.

{
    struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
    struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
 
    if (params->hcr_el2 & HCR_VM)
        return -EPERM;
 
    params->vttbr = kvm_get_vttbr(mmu);
    params->vtcr = mmu->vtcr;
    params->hcr_el2 |= HCR_VM;
 
    /*
     * The CMO below not only cleans the updated params to the
     * PoC, but also provides the DSB that ensures ongoing
     * page-table walks that have started before we trapped to EL2
     * have completed.
     */
    kvm_flush_dcache_to_poc(params, sizeof(*params));
 
    write_sysreg(params->hcr_el2, hcr_el2);
    __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
 
    /*
     * Make sure to have an ISB before the TLB maintenance below but only
     * when __load_stage2() doesn't include one already.
     */
    asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));
 
    /* Invalidate stale HCR bits that may be cached in TLBs */
    __tlbi(vmalls12e1);
    dsb(nsh);
    isb();
 
    return 0;
}

Here is the caller graph for this function:

addr_is_memory()

bool addr_is_memory

(

phys_addr_t

phys

)

Definition at line 378 of file mem_protect.c.

{
    struct kvm_mem_range range;
 
    return !!find_mem_range(phys, &range);
}

Here is the call graph for this function:

Here is the caller graph for this function:

handle_host_mem_abort()

void handle_host_mem_abort

(

struct kvm_cpu_context *

host_ctxt

)

Definition at line 529 of file mem_protect.c.

{
    struct kvm_vcpu_fault_info fault;
    u64 esr, addr;
    int ret = 0;
 
    esr = read_sysreg_el2(SYS_ESR);
    BUG_ON(!__get_fault_info(esr, &fault));
 
    addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
    ret = host_stage2_idmap(addr);
    BUG_ON(ret && ret != -EAGAIN);
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_stage2_idmap_locked()

int host_stage2_idmap_locked	(	phys_addr_t	addr,
		u64	size,
		enum kvm_pgtable_prot	prot
	)

Definition at line 474 of file mem_protect.c.

{
    return host_stage2_try(__host_stage2_idmap, addr, addr + size, prot);
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_stage2_set_owner_locked()

int host_stage2_set_owner_locked	(	phys_addr_t	addr,
		u64	size,
		u8	owner_id
	)

Definition at line 480 of file mem_protect.c.

{
    return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
                   addr, size, &host_s2_pool, owner_id);
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_pin_shared_mem()

int hyp_pin_shared_mem	(	void *	from,
		void *	to
	)

Definition at line 1216 of file mem_protect.c.

{
    u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
    u64 end = PAGE_ALIGN((u64)to);
    u64 size = end - start;
    int ret;
 
    host_lock_component();
    hyp_lock_component();
 
    ret = __host_check_page_state_range(__hyp_pa(start), size,
                        PKVM_PAGE_SHARED_OWNED);
    if (ret)
        goto unlock;
 
    ret = __hyp_check_page_state_range(start, size,
                       PKVM_PAGE_SHARED_BORROWED);
    if (ret)
        goto unlock;
 
    for (cur = start; cur < end; cur += PAGE_SIZE)
        hyp_page_ref_inc(hyp_virt_to_page(cur));
 
unlock:
    hyp_unlock_component();
    host_unlock_component();
 
    return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_unpin_shared_mem()

void hyp_unpin_shared_mem	(	void *	from,
		void *	to
	)

Definition at line 1246 of file mem_protect.c.

{
    u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
    u64 end = PAGE_ALIGN((u64)to);
 
    host_lock_component();
    hyp_lock_component();
 
    for (cur = start; cur < end; cur += PAGE_SIZE)
        hyp_page_ref_dec(hyp_virt_to_page(cur));
 
    hyp_unlock_component();
    host_unlock_component();
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_guest_prepare_stage2()

int kvm_guest_prepare_stage2	(	struct pkvm_hyp_vm *	vm,
		void *	pgd
	)

Definition at line 232 of file mem_protect.c.

{
    struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
    unsigned long nr_pages;
    int ret;
 
    nr_pages = kvm_pgtable_stage2_pgd_size(mmu->vtcr) >> PAGE_SHIFT;
    ret = hyp_pool_init(&vm->pool, hyp_virt_to_pfn(pgd), nr_pages, 0);
    if (ret)
        return ret;
 
    hyp_spin_lock_init(&vm->lock);
    vm->mm_ops = (struct kvm_pgtable_mm_ops) {
        .zalloc_pages_exact = guest_s2_zalloc_pages_exact,
        .free_pages_exact   = guest_s2_free_pages_exact,
        .zalloc_page        = guest_s2_zalloc_page,
        .phys_to_virt       = hyp_phys_to_virt,
        .virt_to_phys       = hyp_virt_to_phys,
        .page_count     = hyp_page_count,
        .get_page       = guest_s2_get_page,
        .put_page       = guest_s2_put_page,
        .dcache_clean_inval_poc = clean_dcache_guest_page,
        .icache_inval_pou   = invalidate_icache_guest_page,
    };
 
    guest_lock_component(vm);
    ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0,
                    guest_stage2_force_pte_cb);
    guest_unlock_component(vm);
    if (ret)
        return ret;
 
    vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

kvm_host_prepare_stage2()

int kvm_host_prepare_stage2

(

void *

pgt_pool_base

)

Definition at line 138 of file mem_protect.c.

{
    struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
    int ret;
 
    prepare_host_vtcr();
    hyp_spin_lock_init(&host_mmu.lock);
    mmu->arch = &host_mmu.arch;
 
    ret = prepare_s2_pool(pgt_pool_base);
    if (ret)
        return ret;
 
    ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,
                    &host_mmu.mm_ops, KVM_HOST_S2_FLAGS,
                    host_stage2_force_pte_cb);
    if (ret)
        return ret;
 
    mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);
    mmu->pgt = &host_mmu.pgt;
    atomic64_set(&mmu->vmid.id, 0);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

pkvm_getstate()

static enum pkvm_page_state pkvm_getstate ( enum kvm_pgtable_prot  prot )

inlinestatic

Definition at line 1 of file mem_protect.h.

{
    return prot & PKVM_PAGE_STATE_PROT_MASK;
}

Here is the caller graph for this function:

pkvm_mkstate()

static enum kvm_pgtable_prot pkvm_mkstate ( enum kvm_pgtable_prot  prot, enum pkvm_page_state  state  )

inlinestatic

Definition at line 1 of file mem_protect.h.

{
    return (prot & ~PKVM_PAGE_STATE_PROT_MASK) | state;
}

Here is the caller graph for this function:

reclaim_guest_pages()

void reclaim_guest_pages	(	struct pkvm_hyp_vm *	vm,
		struct kvm_hyp_memcache *	mc
	)

Definition at line 269 of file mem_protect.c.

{
    void *addr;
 
    /* Dump all pgtable pages in the hyp_pool */
    guest_lock_component(vm);
    kvm_pgtable_stage2_destroy(&vm->pgt);
    vm->kvm.arch.mmu.pgd_phys = 0ULL;
    guest_unlock_component(vm);
 
    /* Drain the hyp_pool into the memcache */
    addr = hyp_alloc_pages(&vm->pool, 0);
    while (addr) {
        memset(hyp_virt_to_page(addr), 0, sizeof(struct hyp_page));
        push_hyp_memcache(mc, addr, hyp_virt_to_phys);
        WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1));
        addr = hyp_alloc_pages(&vm->pool, 0);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

refill_memcache()

int refill_memcache	(	struct kvm_hyp_memcache *	mc,
		unsigned long	min_pages,
		struct kvm_hyp_memcache *	host_mc
	)

Definition at line 412 of file mm.c.

{
    struct kvm_hyp_memcache tmp = *host_mc;
    int ret;
 
    ret =  __topup_hyp_memcache(mc, min_pages, admit_host_page,
                    hyp_virt_to_phys, &tmp);
    *host_mc = tmp;
 
    return ret;
}

Here is the call graph for this function:

Variable Documentation

host_mmu

struct host_mmu host_mmu

extern

Definition at line 1 of file mem_protect.c.

hyp_nr_cpus

unsigned long hyp_nr_cpus

extern

Definition at line 23 of file setup.c.