page_alloc.c File Reference

#include <asm/kvm_hyp.h>
#include <nvhe/gfp.h>

Include dependency graph for page_alloc.c:

Go to the source code of this file.

Functions
static struct hyp_page *	__find_buddy_nocheck (struct hyp_pool *pool, struct hyp_page *p, unsigned short order)
static struct hyp_page *	__find_buddy_avail (struct hyp_pool *pool, struct hyp_page *p, unsigned short order)
static void	page_remove_from_list (struct hyp_page *p)
static void	page_add_to_list (struct hyp_page *p, struct list_head *head)
static struct hyp_page *	node_to_page (struct list_head *node)
static void	__hyp_attach_page (struct hyp_pool *pool, struct hyp_page *p)
static struct hyp_page *	__hyp_extract_page (struct hyp_pool *pool, struct hyp_page *p, unsigned short order)
static void	__hyp_put_page (struct hyp_pool *pool, struct hyp_page *p)
void	hyp_put_page (struct hyp_pool *pool, void *addr)
void	hyp_get_page (struct hyp_pool *pool, void *addr)
void	hyp_split_page (struct hyp_page *p)
void *	hyp_alloc_pages (struct hyp_pool *pool, unsigned short order)
int	hyp_pool_init (struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, unsigned int reserved_pages)

Variables
u64	__hyp_vmemmap

Function Documentation

__find_buddy_avail()

static struct hyp_page* __find_buddy_avail ( struct hyp_pool *  pool, struct hyp_page *  p, unsigned short  order  )

static

Definition at line 52 of file page_alloc.c.

{
    struct hyp_page *buddy = __find_buddy_nocheck(pool, p, order);
 
    if (!buddy || buddy->order != order || buddy->refcount)
        return NULL;
 
    return buddy;
 
}

Here is the call graph for this function:

Here is the caller graph for this function:

__find_buddy_nocheck()

static struct hyp_page* __find_buddy_nocheck ( struct hyp_pool *  pool, struct hyp_page *  p, unsigned short  order  )

static

Definition at line 33 of file page_alloc.c.

{
    phys_addr_t addr = hyp_page_to_phys(p);
 
    addr ^= (PAGE_SIZE << order);
 
    /*
     * Don't return a page outside the pool range -- it belongs to
     * something else and may not be mapped in hyp_vmemmap.
     */
    if (addr < pool->range_start || addr >= pool->range_end)
        return NULL;
 
    return hyp_phys_to_page(addr);
}

Here is the caller graph for this function:

__hyp_attach_page()

static void __hyp_attach_page ( struct hyp_pool *  pool, struct hyp_page *  p  )

static

Definition at line 93 of file page_alloc.c.

{
    phys_addr_t phys = hyp_page_to_phys(p);
    unsigned short order = p->order;
    struct hyp_page *buddy;
 
    memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
 
    /* Skip coalescing for 'external' pages being freed into the pool. */
    if (phys < pool->range_start || phys >= pool->range_end)
        goto insert;
 
    /*
     * Only the first struct hyp_page of a high-order page (otherwise known
     * as the 'head') should have p->order set. The non-head pages should
     * have p->order = HYP_NO_ORDER. Here @p may no longer be the head
     * after coalescing, so make sure to mark it HYP_NO_ORDER proactively.
     */
    p->order = HYP_NO_ORDER;
    for (; (order + 1) <= pool->max_order; order++) {
        buddy = __find_buddy_avail(pool, p, order);
        if (!buddy)
            break;
 
        /* Take the buddy out of its list, and coalesce with @p */
        page_remove_from_list(buddy);
        buddy->order = HYP_NO_ORDER;
        p = min(p, buddy);
    }
 
insert:
    /* Mark the new head, and insert it */
    p->order = order;
    page_add_to_list(p, &pool->free_area[order]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

__hyp_extract_page()

static struct hyp_page* __hyp_extract_page ( struct hyp_pool *  pool, struct hyp_page *  p, unsigned short  order  )

static

Definition at line 130 of file page_alloc.c.

{
    struct hyp_page *buddy;
 
    page_remove_from_list(p);
    while (p->order > order) {
        /*
         * The buddy of order n - 1 currently has HYP_NO_ORDER as it
         * is covered by a higher-level page (whose head is @p). Use
         * __find_buddy_nocheck() to find it and inject it in the
         * free_list[n - 1], effectively splitting @p in half.
         */
        p->order--;
        buddy = __find_buddy_nocheck(pool, p, p->order);
        buddy->order = p->order;
        page_add_to_list(buddy, &pool->free_area[buddy->order]);
    }
 
    return p;
}

Here is the call graph for this function:

Here is the caller graph for this function:

__hyp_put_page()

static void __hyp_put_page ( struct hyp_pool *  pool, struct hyp_page *  p  )

static

Definition at line 153 of file page_alloc.c.

{
    if (hyp_page_ref_dec_and_test(p))
        __hyp_attach_page(pool, p);
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_alloc_pages()

void* hyp_alloc_pages	(	struct hyp_pool *	pool,
		unsigned short	order
	)

Definition at line 198 of file page_alloc.c.

{
    unsigned short i = order;
    struct hyp_page *p;
 
    hyp_spin_lock(&pool->lock);
 
    /* Look for a high-enough-order page */
    while (i <= pool->max_order && list_empty(&pool->free_area[i]))
        i++;
    if (i > pool->max_order) {
        hyp_spin_unlock(&pool->lock);
        return NULL;
    }
 
    /* Extract it from the tree at the right order */
    p = node_to_page(pool->free_area[i].next);
    p = __hyp_extract_page(pool, p, order);
 
    hyp_set_page_refcounted(p);
    hyp_spin_unlock(&pool->lock);
 
    return hyp_page_to_virt(p);
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_get_page()

void hyp_get_page	(	struct hyp_pool *	pool,
		void *	addr
	)

Definition at line 175 of file page_alloc.c.

{
    struct hyp_page *p = hyp_virt_to_page(addr);
 
    hyp_spin_lock(&pool->lock);
    hyp_page_ref_inc(p);
    hyp_spin_unlock(&pool->lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_pool_init()

int hyp_pool_init	(	struct hyp_pool *	pool,
		u64	pfn,
		unsigned int	nr_pages,
		unsigned int	reserved_pages
	)

Definition at line 223 of file page_alloc.c.

{
    phys_addr_t phys = hyp_pfn_to_phys(pfn);
    struct hyp_page *p;
    int i;
 
    hyp_spin_lock_init(&pool->lock);
    pool->max_order = min(MAX_PAGE_ORDER,
                  get_order(nr_pages << PAGE_SHIFT));
    for (i = 0; i <= pool->max_order; i++)
        INIT_LIST_HEAD(&pool->free_area[i]);
    pool->range_start = phys;
    pool->range_end = phys + (nr_pages << PAGE_SHIFT);
 
    /* Init the vmemmap portion */
    p = hyp_phys_to_page(phys);
    for (i = 0; i < nr_pages; i++)
        hyp_set_page_refcounted(&p[i]);
 
    /* Attach the unused pages to the buddy tree */
    for (i = reserved_pages; i < nr_pages; i++)
        __hyp_put_page(pool, &p[i]);
 
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_put_page()

void hyp_put_page	(	struct hyp_pool *	pool,
		void *	addr
	)

Definition at line 166 of file page_alloc.c.

{
    struct hyp_page *p = hyp_virt_to_page(addr);
 
    hyp_spin_lock(&pool->lock);
    __hyp_put_page(pool, p);
    hyp_spin_unlock(&pool->lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

hyp_split_page()

void hyp_split_page

(

struct hyp_page *

p

)

Definition at line 184 of file page_alloc.c.

{
    unsigned short order = p->order;
    unsigned int i;
 
    p->order = 0;
    for (i = 1; i < (1 << order); i++) {
        struct hyp_page *tail = p + i;
 
        tail->order = 0;
        hyp_set_page_refcounted(tail);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

node_to_page()

static struct hyp_page* node_to_page ( struct list_head *  node )

inlinestatic

Definition at line 88 of file page_alloc.c.

{
    return hyp_virt_to_page(node);
}

Here is the caller graph for this function:

page_add_to_list()

static void page_add_to_list ( struct hyp_page *  p, struct list_head *  head  )

inlinestatic

Definition at line 80 of file page_alloc.c.

{
    struct list_head *node = hyp_page_to_virt(p);
 
    INIT_LIST_HEAD(node);
    list_add_tail(node, head);
}

Here is the caller graph for this function:

page_remove_from_list()

static void page_remove_from_list ( struct hyp_page *  p )

inlinestatic

Definition at line 72 of file page_alloc.c.

{
    struct list_head *node = hyp_page_to_virt(p);
 
    __list_del_entry(node);
    memset(node, 0, sizeof(*node));
}

Here is the caller graph for this function:

Variable Documentation

__hyp_vmemmap

u64 __hyp_vmemmap

Definition at line 10 of file page_alloc.c.