2 * Jailhouse, a Linux-based partitioning hypervisor
4 * Copyright (c) Siemens AG, 2013
7 * Jan Kiszka <jan.kiszka@siemens.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
13 #include <jailhouse/paging.h>
14 #include <jailhouse/printk.h>
15 #include <jailhouse/string.h>
16 #include <jailhouse/control.h>
17 #include <asm/bitops.h>
19 #define BITS_PER_PAGE (PAGE_SIZE * 8)
21 #define INVALID_PAGE_NR (~0UL)
23 #define PAGE_SCRUB_ON_FREE 0x1
25 extern u8 __page_pool[];
27 unsigned long page_offset;
29 struct page_pool mem_pool;
30 struct page_pool remap_pool = {
31 .base_address = (void *)REMAP_BASE,
32 .pages = BITS_PER_PAGE * NUM_REMAP_BITMAP_PAGES,
35 struct paging_structures hv_paging_structs;
37 unsigned long page_map_get_phys_invalid(pt_entry_t pte, unsigned long virt)
39 return INVALID_PHYS_ADDR;
42 static unsigned long find_next_free_page(struct page_pool *pool,
45 unsigned long bmp_pos, bmp_val, page_nr;
46 unsigned long start_mask = 0;
48 if (start >= pool->pages)
49 return INVALID_PAGE_NR;
52 * If we don't start on the beginning of a bitmap word, create a mask
53 * to mark the pages before the start page as (virtually) used.
55 if (start % BITS_PER_LONG > 0)
56 start_mask = ~0UL >> (BITS_PER_LONG - (start % BITS_PER_LONG));
58 for (bmp_pos = start / BITS_PER_LONG;
59 bmp_pos < pool->pages / BITS_PER_LONG; bmp_pos++) {
60 bmp_val = pool->used_bitmap[bmp_pos] | start_mask;
62 if (bmp_val != ~0UL) {
63 page_nr = ffzl(bmp_val) + bmp_pos * BITS_PER_LONG;
64 if (page_nr >= pool->pages)
70 return INVALID_PAGE_NR;
73 void *page_alloc(struct page_pool *pool, unsigned int num)
75 unsigned long start, last, next;
76 unsigned int allocated;
78 start = find_next_free_page(pool, 0);
79 if (start == INVALID_PAGE_NR)
83 for (allocated = 1, last = start; allocated < num;
84 allocated++, last = next) {
85 next = find_next_free_page(pool, last + 1);
86 if (next == INVALID_PAGE_NR)
88 if (next != last + 1) {
94 for (allocated = 0; allocated < num; allocated++)
95 set_bit(start + allocated, pool->used_bitmap);
97 pool->used_pages += num;
99 return pool->base_address + start * PAGE_SIZE;
102 void page_free(struct page_pool *pool, void *page, unsigned int num)
104 unsigned long page_nr;
110 if (pool->flags & PAGE_SCRUB_ON_FREE)
111 memset(page, 0, PAGE_SIZE);
112 page_nr = (page - pool->base_address) / PAGE_SIZE;
113 clear_bit(page_nr, pool->used_bitmap);
119 unsigned long page_map_virt2phys(const struct paging_structures *pg_structs,
120 unsigned long virt, unsigned long flags)
122 const struct paging *paging = pg_structs->root_paging;
123 page_table_t pt = pg_structs->root_table;
128 pte = paging->get_entry(pt, virt);
129 if (!paging->entry_valid(pte, flags))
130 return INVALID_PHYS_ADDR;
131 phys = paging->get_phys(pte, virt);
132 if (phys != INVALID_PHYS_ADDR)
134 pt = page_map_phys2hvirt(paging->get_next_pt(pte));
139 static void flush_pt_entry(pt_entry_t pte, enum page_map_coherent coherent)
141 if (coherent == PAGE_MAP_COHERENT)
142 flush_cache(pte, sizeof(*pte));
145 static int split_hugepage(const struct paging *paging, pt_entry_t pte,
146 unsigned long virt, enum page_map_coherent coherent)
148 unsigned long phys = paging->get_phys(pte, virt);
149 struct paging_structures sub_structs;
150 unsigned long page_mask, flags;
152 if (phys == INVALID_PHYS_ADDR)
155 page_mask = ~(paging->page_size - 1);
159 flags = paging->get_flags(pte);
161 sub_structs.root_paging = paging + 1;
162 sub_structs.root_table = page_alloc(&mem_pool, 1);
163 if (!sub_structs.root_table)
165 paging->set_next_pt(pte, page_map_hvirt2phys(sub_structs.root_table));
166 flush_pt_entry(pte, coherent);
168 return page_map_create(&sub_structs, phys, paging->page_size, virt,
172 int page_map_create(const struct paging_structures *pg_structs,
173 unsigned long phys, unsigned long size, unsigned long virt,
174 unsigned long flags, enum page_map_coherent coherent)
178 size = PAGE_ALIGN(size);
181 const struct paging *paging = pg_structs->root_paging;
182 page_table_t pt = pg_structs->root_table;
187 pte = paging->get_entry(pt, virt);
188 if (paging->page_size > 0 &&
189 paging->page_size <= size &&
190 ((phys | virt) & (paging->page_size - 1)) == 0) {
192 * We might be overwriting a more fine-grained
193 * mapping, so release it first. This cannot
194 * fail as we are working along hugepage
197 if (paging->page_size > PAGE_SIZE)
198 page_map_destroy(pg_structs, virt,
201 paging->set_terminal(pte, phys, flags);
202 flush_pt_entry(pte, coherent);
205 if (paging->entry_valid(pte, PAGE_PRESENT_FLAGS)) {
206 err = split_hugepage(paging, pte, virt,
210 pt = page_map_phys2hvirt(
211 paging->get_next_pt(pte));
213 pt = page_alloc(&mem_pool, 1);
216 paging->set_next_pt(pte,
217 page_map_hvirt2phys(pt));
218 flush_pt_entry(pte, coherent);
222 if (pg_structs == &hv_paging_structs)
223 arch_tlb_flush_page(virt);
225 phys += paging->page_size;
226 virt += paging->page_size;
227 size -= paging->page_size;
232 int page_map_destroy(const struct paging_structures *pg_structs,
233 unsigned long virt, unsigned long size,
234 enum page_map_coherent coherent)
236 size = PAGE_ALIGN(size);
239 const struct paging *paging = pg_structs->root_paging;
240 page_table_t pt[MAX_PAGE_DIR_LEVELS];
241 unsigned long page_size;
246 /* walk down the page table, saving intermediate tables */
247 pt[0] = pg_structs->root_table;
249 pte = paging->get_entry(pt[n], virt);
250 if (!paging->entry_valid(pte, PAGE_PRESENT_FLAGS))
252 if (paging->get_phys(pte, virt) != INVALID_PHYS_ADDR) {
253 if (paging->page_size > size) {
254 err = split_hugepage(paging, pte, virt,
261 pt[++n] = page_map_phys2hvirt(
262 paging->get_next_pt(pte));
265 /* advance by page size of current level paging */
266 page_size = paging->page_size ? paging->page_size : PAGE_SIZE;
268 /* walk up again, clearing entries, releasing empty tables */
270 paging->clear_entry(pte);
271 flush_pt_entry(pte, coherent);
272 if (n == 0 || !paging->page_table_empty(pt[n]))
274 page_free(&mem_pool, pt[n], 1);
276 pte = paging->get_entry(pt[--n], virt);
278 if (pg_structs == &hv_paging_structs)
279 arch_tlb_flush_page(virt);
281 if (page_size > size)
290 page_map_gvirt2gphys(const struct guest_paging_structures *pg_structs,
291 unsigned long gvirt, unsigned long tmp_page,
294 unsigned long page_table_gphys = pg_structs->root_table_gphys;
295 const struct paging *paging = pg_structs->root_paging;
296 unsigned long gphys, phys;
301 /* map guest page table */
302 phys = arch_page_map_gphys2phys(this_cpu_data(),
304 PAGE_READONLY_FLAGS);
305 if (phys == INVALID_PHYS_ADDR)
306 return INVALID_PHYS_ADDR;
307 err = page_map_create(&hv_paging_structs, phys,
310 PAGE_MAP_NON_COHERENT);
312 return INVALID_PHYS_ADDR;
314 /* evaluate page table entry */
315 pte = paging->get_entry((page_table_t)tmp_page, gvirt);
316 if (!paging->entry_valid(pte, flags))
317 return INVALID_PHYS_ADDR;
318 gphys = paging->get_phys(pte, gvirt);
319 if (gphys != INVALID_PHYS_ADDR)
321 page_table_gphys = paging->get_next_pt(pte);
327 page_map_get_guest_pages(const struct guest_paging_structures *pg_structs,
328 unsigned long gaddr, unsigned int num,
331 unsigned long page_base = TEMPORARY_MAPPING_BASE +
332 this_cpu_id() * PAGE_SIZE * NUM_TEMPORARY_PAGES;
333 unsigned long phys, gphys, page_virt = page_base;
336 if (num > NUM_TEMPORARY_PAGES)
340 gphys = page_map_gvirt2gphys(pg_structs, gaddr,
345 phys = arch_page_map_gphys2phys(this_cpu_data(), gphys, flags);
346 if (phys == INVALID_PHYS_ADDR)
349 err = page_map_create(&hv_paging_structs, phys, PAGE_SIZE,
350 page_virt, flags, PAGE_MAP_NON_COHERENT);
354 page_virt += PAGE_SIZE;
356 return (void *)page_base;
359 int paging_init(void)
361 unsigned long n, per_cpu_pages, config_pages, bitmap_pages;
364 per_cpu_pages = hypervisor_header.possible_cpus *
365 sizeof(struct per_cpu) / PAGE_SIZE;
367 system_config = (struct jailhouse_system *)
368 (__page_pool + per_cpu_pages * PAGE_SIZE);
369 config_pages = PAGES(jailhouse_system_config_size(system_config));
371 page_offset = JAILHOUSE_BASE -
372 system_config->hypervisor_memory.phys_start;
374 mem_pool.pages = (system_config->hypervisor_memory.size -
375 (__page_pool - (u8 *)&hypervisor_header)) / PAGE_SIZE;
376 bitmap_pages = (mem_pool.pages + BITS_PER_PAGE - 1) / BITS_PER_PAGE;
378 if (mem_pool.pages <= per_cpu_pages + config_pages + bitmap_pages)
381 mem_pool.base_address = __page_pool;
382 mem_pool.used_bitmap =
383 (unsigned long *)(__page_pool + per_cpu_pages * PAGE_SIZE +
384 config_pages * PAGE_SIZE);
385 mem_pool.used_pages = per_cpu_pages + config_pages + bitmap_pages;
386 for (n = 0; n < mem_pool.used_pages; n++)
387 set_bit(n, mem_pool.used_bitmap);
388 mem_pool.flags = PAGE_SCRUB_ON_FREE;
390 remap_pool.used_bitmap = page_alloc(&mem_pool, NUM_REMAP_BITMAP_PAGES);
391 remap_pool.used_pages =
392 hypervisor_header.possible_cpus * NUM_TEMPORARY_PAGES;
393 for (n = 0; n < remap_pool.used_pages; n++)
394 set_bit(n, remap_pool.used_bitmap);
398 hv_paging_structs.root_paging = hv_paging;
399 hv_paging_structs.root_table = page_alloc(&mem_pool, 1);
400 if (!hv_paging_structs.root_table)
403 /* Replicate hypervisor mapping of Linux */
404 err = page_map_create(&hv_paging_structs,
405 page_map_hvirt2phys(&hypervisor_header),
406 system_config->hypervisor_memory.size,
407 (unsigned long)&hypervisor_header,
408 PAGE_DEFAULT_FLAGS, PAGE_MAP_NON_COHERENT);
412 /* Make sure any remappings to the temporary regions can be performed
413 * without allocations of page table pages. */
414 err = page_map_create(&hv_paging_structs, 0,
415 remap_pool.used_pages * PAGE_SIZE,
416 TEMPORARY_MAPPING_BASE, PAGE_NONPRESENT_FLAGS,
417 PAGE_MAP_NON_COHERENT);
424 printk("FATAL: page pool much too small\n");
428 void page_map_dump_stats(const char *when)
430 printk("Page pool usage %s: mem %d/%d, remap %d/%d\n", when,
431 mem_pool.used_pages, mem_pool.pages,
432 remap_pool.used_pages, remap_pool.pages);