hypervisor/paging.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2013
   5  *
   6  * Authors:
   7  *  Jan Kiszka <jan.kiszka@siemens.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  */
  12
  13 #include <jailhouse/paging.h>
  14 #include <jailhouse/printk.h>
  15 #include <jailhouse/string.h>
  16 #include <jailhouse/control.h>
  17 #include <asm/bitops.h>
  18
  19 #define BITS_PER_PAGE           (PAGE_SIZE * 8)
  20
  21 #define INVALID_PAGE_NR         (~0UL)
  22
  23 #define PAGE_SCRUB_ON_FREE      0x1
  24
  25 extern u8 __page_pool[];
  26
  27 unsigned long page_offset;
  28
  29 struct page_pool mem_pool;
  30 struct page_pool remap_pool = {
  31         .base_address = (void *)REMAP_BASE,
  32         .pages = BITS_PER_PAGE * NUM_REMAP_BITMAP_PAGES,
  33 };
  34
  35 struct paging_structures hv_paging_structs;
  36
  37 unsigned long page_map_get_phys_invalid(pt_entry_t pte, unsigned long virt)
  38 {
  39         return INVALID_PHYS_ADDR;
  40 }
  41
  42 static unsigned long find_next_free_page(struct page_pool *pool,
  43                                          unsigned long start)
  44 {
  45         unsigned long start_mask =
  46                 ~0UL >> (BITS_PER_LONG - (start % BITS_PER_LONG));
  47         unsigned long bmp_pos, bmp_val, page_nr;
  48
  49         if (start >= pool->pages)
  50                 return INVALID_PAGE_NR;
  51
  52         for (bmp_pos = start / BITS_PER_LONG;
  53              bmp_pos < pool->pages / BITS_PER_LONG; bmp_pos++) {
  54                 bmp_val = pool->used_bitmap[bmp_pos] | start_mask;
  55                 start_mask = 0;
  56                 if (bmp_val != ~0UL) {
  57                         page_nr = ffzl(bmp_val) + bmp_pos * BITS_PER_LONG;
  58                         if (page_nr >= pool->pages)
  59                                 break;
  60                         return page_nr;
  61                 }
  62         }
  63
  64         return INVALID_PAGE_NR;
  65 }
  66
  67 void *page_alloc(struct page_pool *pool, unsigned int num)
  68 {
  69         unsigned long start, last, next;
  70         unsigned int allocated;
  71
  72         start = find_next_free_page(pool, 0);
  73         if (start == INVALID_PAGE_NR)
  74                 return NULL;
  75
  76 restart:
  77         for (allocated = 1, last = start; allocated < num;
  78              allocated++, last = next) {
  79                 next = find_next_free_page(pool, last + 1);
  80                 if (next == INVALID_PAGE_NR)
  81                         return NULL;
  82                 if (next != last + 1) {
  83                         start = next;
  84                         goto restart;
  85                 }
  86         }
  87
  88         for (allocated = 0; allocated < num; allocated++)
  89                 set_bit(start + allocated, pool->used_bitmap);
  90
  91         pool->used_pages += num;
  92
  93         return pool->base_address + start * PAGE_SIZE;
  94 }
  95
  96 void page_free(struct page_pool *pool, void *page, unsigned int num)
  97 {
  98         unsigned long page_nr;
  99
 100         if (!page)
 101                 return;
 102
 103         while (num-- > 0) {
 104                 if (pool->flags & PAGE_SCRUB_ON_FREE)
 105                         memset(page, 0, PAGE_SIZE);
 106                 page_nr = (page - pool->base_address) / PAGE_SIZE;
 107                 clear_bit(page_nr, pool->used_bitmap);
 108                 pool->used_pages--;
 109                 page += PAGE_SIZE;
 110         }
 111 }
 112
 113 unsigned long page_map_virt2phys(const struct paging_structures *pg_structs,
 114                                  unsigned long virt)
 115 {
 116         const struct paging *paging = pg_structs->root_paging;
 117         page_table_t pt = pg_structs->root_table;
 118         unsigned long phys;
 119         pt_entry_t pte;
 120
 121         while (1) {
 122                 pte = paging->get_entry(pt, virt);
 123                 if (!paging->entry_valid(pte))
 124                         return INVALID_PHYS_ADDR;
 125                 phys = paging->get_phys(pte, virt);
 126                 if (phys != INVALID_PHYS_ADDR)
 127                         return phys;
 128                 pt = page_map_phys2hvirt(paging->get_next_pt(pte));
 129                 paging++;
 130         }
 131 }
 132
 133 static void flush_pt_entry(pt_entry_t pte, enum page_map_coherent coherent)
 134 {
 135         if (coherent == PAGE_MAP_COHERENT)
 136                 flush_cache(pte, sizeof(*pte));
 137 }
 138
 139 static int split_hugepage(const struct paging *paging, pt_entry_t pte,
 140                           unsigned long virt, enum page_map_coherent coherent)
 141 {
 142         unsigned long phys = paging->get_phys(pte, virt);
 143         struct paging_structures sub_structs;
 144         unsigned long page_mask, flags;
 145
 146         if (phys == INVALID_PHYS_ADDR)
 147                 return 0;
 148
 149         page_mask = ~(paging->page_size - 1);
 150         phys &= page_mask;
 151         virt &= page_mask;
 152
 153         flags = paging->get_flags(pte);
 154
 155         sub_structs.root_paging = paging + 1;
 156         sub_structs.root_table = page_alloc(&mem_pool, 1);
 157         if (!sub_structs.root_table)
 158                 return -ENOMEM;
 159         paging->set_next_pt(pte, page_map_hvirt2phys(sub_structs.root_table));
 160         flush_pt_entry(pte, coherent);
 161
 162         return page_map_create(&sub_structs, phys, paging->page_size, virt,
 163                                flags, coherent);
 164 }
 165
 166 int page_map_create(const struct paging_structures *pg_structs,
 167                     unsigned long phys, unsigned long size, unsigned long virt,
 168                     unsigned long flags, enum page_map_coherent coherent)
 169 {
 170         phys &= PAGE_MASK;
 171         virt &= PAGE_MASK;
 172         size = PAGE_ALIGN(size);
 173
 174         while (size > 0) {
 175                 const struct paging *paging = pg_structs->root_paging;
 176                 page_table_t pt = pg_structs->root_table;
 177                 pt_entry_t pte;
 178                 int err;
 179
 180                 while (1) {
 181                         pte = paging->get_entry(pt, virt);
 182                         if (paging->page_size > 0 &&
 183                             paging->page_size <= size &&
 184                             ((phys | virt) & (paging->page_size - 1)) == 0) {
 185                                 /*
 186                                  * We might be overwriting a more fine-grained
 187                                  * mapping, so release it first. This cannot
 188                                  * fail as we are working along hugepage
 189                                  * boundaries.
 190                                  */
 191                                 if (paging->page_size > PAGE_SIZE)
 192                                         page_map_destroy(pg_structs, virt,
 193                                                          paging->page_size,
 194                                                          coherent);
 195                                 paging->set_terminal(pte, phys, flags);
 196                                 flush_pt_entry(pte, coherent);
 197                                 break;
 198                         }
 199                         if (paging->entry_valid(pte)) {
 200                                 err = split_hugepage(paging, pte, virt,
 201                                                      coherent);
 202                                 if (err)
 203                                         return err;
 204                                 pt = page_map_phys2hvirt(
 205                                                 paging->get_next_pt(pte));
 206                         } else {
 207                                 pt = page_alloc(&mem_pool, 1);
 208                                 if (!pt)
 209                                         return -ENOMEM;
 210                                 paging->set_next_pt(pte,
 211                                                     page_map_hvirt2phys(pt));
 212                                 flush_pt_entry(pte, coherent);
 213                         }
 214                         paging++;
 215                 }
 216                 arch_tlb_flush_page(virt);
 217
 218                 phys += paging->page_size;
 219                 virt += paging->page_size;
 220                 size -= paging->page_size;
 221         }
 222         return 0;
 223 }
 224
 225 int page_map_destroy(const struct paging_structures *pg_structs,
 226                      unsigned long virt, unsigned long size,
 227                      enum page_map_coherent coherent)
 228 {
 229         size = PAGE_ALIGN(size);
 230
 231         while (size > 0) {
 232                 const struct paging *paging = pg_structs->root_paging;
 233                 page_table_t pt[MAX_PAGE_DIR_LEVELS];
 234                 unsigned long page_size;
 235                 pt_entry_t pte;
 236                 int n = 0;
 237                 int err;
 238
 239                 /* walk down the page table, saving intermediate tables */
 240                 pt[0] = pg_structs->root_table;
 241                 while (1) {
 242                         pte = paging->get_entry(pt[n], virt);
 243                         if (!paging->entry_valid(pte))
 244                                 break;
 245                         if (paging->get_phys(pte, virt) != INVALID_PHYS_ADDR) {
 246                                 if (paging->page_size > size) {
 247                                         err = split_hugepage(paging, pte, virt,
 248                                                              coherent);
 249                                         if (err)
 250                                                 return err;
 251                                 } else
 252                                         break;
 253                         }
 254                         pt[++n] = page_map_phys2hvirt(
 255                                         paging->get_next_pt(pte));
 256                         paging++;
 257                 }
 258                 /* advance by page size of current level paging */
 259                 page_size = paging->page_size ? paging->page_size : PAGE_SIZE;
 260
 261                 /* walk up again, clearing entries, releasing empty tables */
 262                 while (1) {
 263                         paging->clear_entry(pte);
 264                         flush_pt_entry(pte, coherent);
 265                         if (n == 0 || !paging->page_table_empty(pt[n]))
 266                                 break;
 267                         page_free(&mem_pool, pt[n], 1);
 268                         paging--;
 269                         pte = paging->get_entry(pt[--n], virt);
 270                 }
 271                 arch_tlb_flush_page(virt);
 272
 273                 if (page_size > size)
 274                         break;
 275                 virt += page_size;
 276                 size -= page_size;
 277         }
 278         return 0;
 279 }
 280
 281 void *page_map_get_guest_page(struct per_cpu *cpu_data,
 282                               const struct guest_paging_structures *pg_structs,
 283                               unsigned long virt, unsigned long flags)
 284 {
 285         unsigned long page_table_gphys = pg_structs->root_table_gphys;
 286         const struct paging *paging = pg_structs->root_paging;
 287         unsigned long page_virt, phys, gphys;
 288         pt_entry_t pte;
 289         int err;
 290
 291         page_virt = TEMPORARY_MAPPING_BASE +
 292                 cpu_data->cpu_id * PAGE_SIZE * NUM_TEMPORARY_PAGES;
 293
 294         while (1) {
 295                 /* map guest page table */
 296                 phys = arch_page_map_gphys2phys(cpu_data, page_table_gphys);
 297                 if (phys == INVALID_PHYS_ADDR)
 298                         return NULL;
 299                 err = page_map_create(&hv_paging_structs, phys,
 300                                       PAGE_SIZE, page_virt,
 301                                       PAGE_READONLY_FLAGS,
 302                                       PAGE_MAP_NON_COHERENT);
 303                 if (err)
 304                         return NULL;
 305
 306                 /* evaluate page table entry */
 307                 pte = paging->get_entry((page_table_t)page_virt, virt);
 308                 if (!paging->entry_valid(pte))
 309                         return NULL;
 310                 gphys = paging->get_phys(pte, virt);
 311                 if (gphys != INVALID_PHYS_ADDR)
 312                         break;
 313                 page_table_gphys = paging->get_next_pt(pte);
 314                 paging++;
 315         }
 316
 317         phys = arch_page_map_gphys2phys(cpu_data, gphys);
 318         if (phys == INVALID_PHYS_ADDR)
 319                 return NULL;
 320         /* map guest page */
 321         err = page_map_create(&hv_paging_structs, phys, PAGE_SIZE, page_virt,
 322                               flags, PAGE_MAP_NON_COHERENT);
 323         if (err)
 324                 return NULL;
 325
 326         return (void *)page_virt;
 327 }
 328
 329 int paging_init(void)
 330 {
 331         unsigned long per_cpu_pages, config_pages, bitmap_pages;
 332         unsigned long n;
 333         int err;
 334
 335         per_cpu_pages = hypervisor_header.possible_cpus *
 336                 sizeof(struct per_cpu) / PAGE_SIZE;
 337
 338         system_config = (struct jailhouse_system *)
 339                 (__page_pool + per_cpu_pages * PAGE_SIZE);
 340         config_pages = (jailhouse_system_config_size(system_config) +
 341                         PAGE_SIZE - 1) / PAGE_SIZE;
 342
 343         page_offset = JAILHOUSE_BASE -
 344                 system_config->hypervisor_memory.phys_start;
 345
 346         mem_pool.pages = (system_config->hypervisor_memory.size -
 347                 (__page_pool - (u8 *)&hypervisor_header)) / PAGE_SIZE;
 348         bitmap_pages = (mem_pool.pages + BITS_PER_PAGE - 1) / BITS_PER_PAGE;
 349
 350         if (mem_pool.pages <= per_cpu_pages + config_pages + bitmap_pages)
 351                 goto error_nomem;
 352
 353         mem_pool.base_address = __page_pool;
 354         mem_pool.used_bitmap =
 355                 (unsigned long *)(__page_pool + per_cpu_pages * PAGE_SIZE +
 356                                   config_pages * PAGE_SIZE);
 357         mem_pool.used_pages = per_cpu_pages + config_pages + bitmap_pages;
 358         for (n = 0; n < mem_pool.used_pages; n++)
 359                 set_bit(n, mem_pool.used_bitmap);
 360         mem_pool.flags = PAGE_SCRUB_ON_FREE;
 361
 362         remap_pool.used_bitmap = page_alloc(&mem_pool, NUM_REMAP_BITMAP_PAGES);
 363         remap_pool.used_pages =
 364                 hypervisor_header.possible_cpus * NUM_TEMPORARY_PAGES;
 365         for (n = 0; n < remap_pool.used_pages; n++)
 366                 set_bit(n, remap_pool.used_bitmap);
 367
 368         arch_paging_init();
 369
 370         hv_paging_structs.root_paging = hv_paging;
 371         hv_paging_structs.root_table = page_alloc(&mem_pool, 1);
 372         if (!hv_paging_structs.root_table)
 373                 goto error_nomem;
 374
 375         /* Replicate hypervisor mapping of Linux */
 376         err = page_map_create(&hv_paging_structs,
 377                               page_map_hvirt2phys(&hypervisor_header),
 378                               system_config->hypervisor_memory.size,
 379                               (unsigned long)&hypervisor_header,
 380                               PAGE_DEFAULT_FLAGS, PAGE_MAP_NON_COHERENT);
 381         if (err)
 382                 goto error_nomem;
 383
 384         /* Make sure any remappings to the temporary regions can be performed
 385          * without allocations of page table pages. */
 386         err = page_map_create(&hv_paging_structs, 0,
 387                               remap_pool.used_pages * PAGE_SIZE,
 388                               TEMPORARY_MAPPING_BASE, PAGE_NONPRESENT_FLAGS,
 389                               PAGE_MAP_NON_COHERENT);
 390         if (err)
 391                 goto error_nomem;
 392
 393         return 0;
 394
 395 error_nomem:
 396         printk("FATAL: page pool much too small\n");
 397         return -ENOMEM;
 398 }
 399
 400 void page_map_dump_stats(const char *when)
 401 {
 402         printk("Page pool usage %s: mem %d/%d, remap %d/%d\n", when,
 403                mem_pool.used_pages, mem_pool.pages,
 404                remap_pool.used_pages, remap_pool.pages);
 405 }