hypervisor/paging.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2013
   5  *
   6  * Authors:
   7  *  Jan Kiszka <jan.kiszka@siemens.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  */
  12
  13 #include <jailhouse/paging.h>
  14 #include <jailhouse/printk.h>
  15 #include <jailhouse/string.h>
  16 #include <jailhouse/control.h>
  17 #include <asm/bitops.h>
  18
  19 #define BITS_PER_PAGE           (PAGE_SIZE * 8)
  20
  21 #define INVALID_PAGE_NR         (~0UL)
  22
  23 #define PAGE_SCRUB_ON_FREE      0x1
  24
  25 extern u8 __page_pool[];
  26
  27 unsigned long page_offset;
  28
  29 struct page_pool mem_pool;
  30 struct page_pool remap_pool = {
  31         .base_address = (void *)REMAP_BASE,
  32         .pages = BITS_PER_PAGE * NUM_REMAP_BITMAP_PAGES,
  33 };
  34
  35 struct paging_structures hv_paging_structs;
  36
  37 unsigned long page_map_get_phys_invalid(pt_entry_t pte, unsigned long virt)
  38 {
  39         return INVALID_PHYS_ADDR;
  40 }
  41
  42 static unsigned long find_next_free_page(struct page_pool *pool,
  43                                          unsigned long start)
  44 {
  45         unsigned long bmp_pos, bmp_val, page_nr;
  46         unsigned long start_mask = 0;
  47
  48         if (start >= pool->pages)
  49                 return INVALID_PAGE_NR;
  50
  51         /*
  52          * If we don't start on the beginning of a bitmap word, create a mask
  53          * to mark the pages before the start page as (virtually) used.
  54          */
  55         if (start % BITS_PER_LONG > 0)
  56                 start_mask = ~0UL >> (BITS_PER_LONG - (start % BITS_PER_LONG));
  57
  58         for (bmp_pos = start / BITS_PER_LONG;
  59              bmp_pos < pool->pages / BITS_PER_LONG; bmp_pos++) {
  60                 bmp_val = pool->used_bitmap[bmp_pos] | start_mask;
  61                 start_mask = 0;
  62                 if (bmp_val != ~0UL) {
  63                         page_nr = ffzl(bmp_val) + bmp_pos * BITS_PER_LONG;
  64                         if (page_nr >= pool->pages)
  65                                 break;
  66                         return page_nr;
  67                 }
  68         }
  69
  70         return INVALID_PAGE_NR;
  71 }
  72
  73 void *page_alloc(struct page_pool *pool, unsigned int num)
  74 {
  75         unsigned long start, last, next;
  76         unsigned int allocated;
  77
  78         start = find_next_free_page(pool, 0);
  79         if (start == INVALID_PAGE_NR)
  80                 return NULL;
  81
  82 restart:
  83         for (allocated = 1, last = start; allocated < num;
  84              allocated++, last = next) {
  85                 next = find_next_free_page(pool, last + 1);
  86                 if (next == INVALID_PAGE_NR)
  87                         return NULL;
  88                 if (next != last + 1) {
  89                         start = next;
  90                         goto restart;
  91                 }
  92         }
  93
  94         for (allocated = 0; allocated < num; allocated++)
  95                 set_bit(start + allocated, pool->used_bitmap);
  96
  97         pool->used_pages += num;
  98
  99         return pool->base_address + start * PAGE_SIZE;
 100 }
 101
 102 void page_free(struct page_pool *pool, void *page, unsigned int num)
 103 {
 104         unsigned long page_nr;
 105
 106         if (!page)
 107                 return;
 108
 109         while (num-- > 0) {
 110                 if (pool->flags & PAGE_SCRUB_ON_FREE)
 111                         memset(page, 0, PAGE_SIZE);
 112                 page_nr = (page - pool->base_address) / PAGE_SIZE;
 113                 clear_bit(page_nr, pool->used_bitmap);
 114                 pool->used_pages--;
 115                 page += PAGE_SIZE;
 116         }
 117 }
 118
 119 unsigned long page_map_virt2phys(const struct paging_structures *pg_structs,
 120                                  unsigned long virt)
 121 {
 122         const struct paging *paging = pg_structs->root_paging;
 123         page_table_t pt = pg_structs->root_table;
 124         unsigned long phys;
 125         pt_entry_t pte;
 126
 127         while (1) {
 128                 pte = paging->get_entry(pt, virt);
 129                 if (!paging->entry_valid(pte))
 130                         return INVALID_PHYS_ADDR;
 131                 phys = paging->get_phys(pte, virt);
 132                 if (phys != INVALID_PHYS_ADDR)
 133                         return phys;
 134                 pt = page_map_phys2hvirt(paging->get_next_pt(pte));
 135                 paging++;
 136         }
 137 }
 138
 139 static void flush_pt_entry(pt_entry_t pte, enum page_map_coherent coherent)
 140 {
 141         if (coherent == PAGE_MAP_COHERENT)
 142                 flush_cache(pte, sizeof(*pte));
 143 }
 144
 145 static int split_hugepage(const struct paging *paging, pt_entry_t pte,
 146                           unsigned long virt, enum page_map_coherent coherent)
 147 {
 148         unsigned long phys = paging->get_phys(pte, virt);
 149         struct paging_structures sub_structs;
 150         unsigned long page_mask, flags;
 151
 152         if (phys == INVALID_PHYS_ADDR)
 153                 return 0;
 154
 155         page_mask = ~(paging->page_size - 1);
 156         phys &= page_mask;
 157         virt &= page_mask;
 158
 159         flags = paging->get_flags(pte);
 160
 161         sub_structs.root_paging = paging + 1;
 162         sub_structs.root_table = page_alloc(&mem_pool, 1);
 163         if (!sub_structs.root_table)
 164                 return -ENOMEM;
 165         paging->set_next_pt(pte, page_map_hvirt2phys(sub_structs.root_table));
 166         flush_pt_entry(pte, coherent);
 167
 168         return page_map_create(&sub_structs, phys, paging->page_size, virt,
 169                                flags, coherent);
 170 }
 171
 172 int page_map_create(const struct paging_structures *pg_structs,
 173                     unsigned long phys, unsigned long size, unsigned long virt,
 174                     unsigned long flags, enum page_map_coherent coherent)
 175 {
 176         phys &= PAGE_MASK;
 177         virt &= PAGE_MASK;
 178         size = PAGE_ALIGN(size);
 179
 180         while (size > 0) {
 181                 const struct paging *paging = pg_structs->root_paging;
 182                 page_table_t pt = pg_structs->root_table;
 183                 pt_entry_t pte;
 184                 int err;
 185
 186                 while (1) {
 187                         pte = paging->get_entry(pt, virt);
 188                         if (paging->page_size > 0 &&
 189                             paging->page_size <= size &&
 190                             ((phys | virt) & (paging->page_size - 1)) == 0) {
 191                                 /*
 192                                  * We might be overwriting a more fine-grained
 193                                  * mapping, so release it first. This cannot
 194                                  * fail as we are working along hugepage
 195                                  * boundaries.
 196                                  */
 197                                 if (paging->page_size > PAGE_SIZE)
 198                                         page_map_destroy(pg_structs, virt,
 199                                                          paging->page_size,
 200                                                          coherent);
 201                                 paging->set_terminal(pte, phys, flags);
 202                                 flush_pt_entry(pte, coherent);
 203                                 break;
 204                         }
 205                         if (paging->entry_valid(pte)) {
 206                                 err = split_hugepage(paging, pte, virt,
 207                                                      coherent);
 208                                 if (err)
 209                                         return err;
 210                                 pt = page_map_phys2hvirt(
 211                                                 paging->get_next_pt(pte));
 212                         } else {
 213                                 pt = page_alloc(&mem_pool, 1);
 214                                 if (!pt)
 215                                         return -ENOMEM;
 216                                 paging->set_next_pt(pte,
 217                                                     page_map_hvirt2phys(pt));
 218                                 flush_pt_entry(pte, coherent);
 219                         }
 220                         paging++;
 221                 }
 222                 if (pg_structs == &hv_paging_structs)
 223                         arch_tlb_flush_page(virt);
 224
 225                 phys += paging->page_size;
 226                 virt += paging->page_size;
 227                 size -= paging->page_size;
 228         }
 229         return 0;
 230 }
 231
 232 int page_map_destroy(const struct paging_structures *pg_structs,
 233                      unsigned long virt, unsigned long size,
 234                      enum page_map_coherent coherent)
 235 {
 236         size = PAGE_ALIGN(size);
 237
 238         while (size > 0) {
 239                 const struct paging *paging = pg_structs->root_paging;
 240                 page_table_t pt[MAX_PAGE_DIR_LEVELS];
 241                 unsigned long page_size;
 242                 pt_entry_t pte;
 243                 int n = 0;
 244                 int err;
 245
 246                 /* walk down the page table, saving intermediate tables */
 247                 pt[0] = pg_structs->root_table;
 248                 while (1) {
 249                         pte = paging->get_entry(pt[n], virt);
 250                         if (!paging->entry_valid(pte))
 251                                 break;
 252                         if (paging->get_phys(pte, virt) != INVALID_PHYS_ADDR) {
 253                                 if (paging->page_size > size) {
 254                                         err = split_hugepage(paging, pte, virt,
 255                                                              coherent);
 256                                         if (err)
 257                                                 return err;
 258                                 } else
 259                                         break;
 260                         }
 261                         pt[++n] = page_map_phys2hvirt(
 262                                         paging->get_next_pt(pte));
 263                         paging++;
 264                 }
 265                 /* advance by page size of current level paging */
 266                 page_size = paging->page_size ? paging->page_size : PAGE_SIZE;
 267
 268                 /* walk up again, clearing entries, releasing empty tables */
 269                 while (1) {
 270                         paging->clear_entry(pte);
 271                         flush_pt_entry(pte, coherent);
 272                         if (n == 0 || !paging->page_table_empty(pt[n]))
 273                                 break;
 274                         page_free(&mem_pool, pt[n], 1);
 275                         paging--;
 276                         pte = paging->get_entry(pt[--n], virt);
 277                 }
 278                 if (pg_structs == &hv_paging_structs)
 279                         arch_tlb_flush_page(virt);
 280
 281                 if (page_size > size)
 282                         break;
 283                 virt += page_size;
 284                 size -= page_size;
 285         }
 286         return 0;
 287 }
 288
 289 static unsigned long
 290 page_map_gvirt2gphys(struct per_cpu *cpu_data,
 291                      const struct guest_paging_structures *pg_structs,
 292                      unsigned long gvirt, unsigned long tmp_page)
 293 {
 294         unsigned long page_table_gphys = pg_structs->root_table_gphys;
 295         const struct paging *paging = pg_structs->root_paging;
 296         unsigned long gphys, phys;
 297         pt_entry_t pte;
 298         int err;
 299
 300         while (1) {
 301                 /* map guest page table */
 302                 phys = arch_page_map_gphys2phys(cpu_data, page_table_gphys);
 303                 if (phys == INVALID_PHYS_ADDR)
 304                         return INVALID_PHYS_ADDR;
 305                 err = page_map_create(&hv_paging_structs, phys,
 306                                       PAGE_SIZE, tmp_page,
 307                                       PAGE_READONLY_FLAGS,
 308                                       PAGE_MAP_NON_COHERENT);
 309                 if (err)
 310                         return INVALID_PHYS_ADDR;
 311
 312                 /* evaluate page table entry */
 313                 pte = paging->get_entry((page_table_t)tmp_page, gvirt);
 314                 if (!paging->entry_valid(pte))
 315                         return INVALID_PHYS_ADDR;
 316                 gphys = paging->get_phys(pte, gvirt);
 317                 if (gphys != INVALID_PHYS_ADDR)
 318                         return gphys;
 319                 page_table_gphys = paging->get_next_pt(pte);
 320                 paging++;
 321         }
 322 }
 323
 324 void *
 325 page_map_get_guest_pages(struct per_cpu *cpu_data,
 326                          const struct guest_paging_structures *pg_structs,
 327                          unsigned long gaddr, unsigned int num,
 328                          unsigned long flags)
 329 {
 330         unsigned long page_base = TEMPORARY_MAPPING_BASE +
 331                 cpu_data->cpu_id * PAGE_SIZE * NUM_TEMPORARY_PAGES;
 332         unsigned long phys, gphys, page_virt = page_base;
 333         int err;
 334
 335         if (num > NUM_TEMPORARY_PAGES)
 336                 return NULL;
 337         while (num-- > 0) {
 338                 if (pg_structs)
 339                         gphys = page_map_gvirt2gphys(cpu_data, pg_structs,
 340                                                      gaddr, page_virt);
 341                 else
 342                         gphys = gaddr;
 343
 344                 phys = arch_page_map_gphys2phys(cpu_data, gphys);
 345                 if (phys == INVALID_PHYS_ADDR)
 346                         return NULL;
 347                 /* map guest page */
 348                 err = page_map_create(&hv_paging_structs, phys, PAGE_SIZE,
 349                                       page_virt, flags, PAGE_MAP_NON_COHERENT);
 350                 if (err)
 351                         return NULL;
 352                 gaddr += PAGE_SIZE;
 353                 page_virt += PAGE_SIZE;
 354         }
 355         return (void *)page_base;
 356 }
 357
 358 int paging_init(void)
 359 {
 360         unsigned long per_cpu_pages, config_pages, bitmap_pages;
 361         unsigned long n;
 362         int err;
 363
 364         per_cpu_pages = hypervisor_header.possible_cpus *
 365                 sizeof(struct per_cpu) / PAGE_SIZE;
 366
 367         system_config = (struct jailhouse_system *)
 368                 (__page_pool + per_cpu_pages * PAGE_SIZE);
 369         config_pages = (jailhouse_system_config_size(system_config) +
 370                         PAGE_SIZE - 1) / PAGE_SIZE;
 371
 372         page_offset = JAILHOUSE_BASE -
 373                 system_config->hypervisor_memory.phys_start;
 374
 375         mem_pool.pages = (system_config->hypervisor_memory.size -
 376                 (__page_pool - (u8 *)&hypervisor_header)) / PAGE_SIZE;
 377         bitmap_pages = (mem_pool.pages + BITS_PER_PAGE - 1) / BITS_PER_PAGE;
 378
 379         if (mem_pool.pages <= per_cpu_pages + config_pages + bitmap_pages)
 380                 goto error_nomem;
 381
 382         mem_pool.base_address = __page_pool;
 383         mem_pool.used_bitmap =
 384                 (unsigned long *)(__page_pool + per_cpu_pages * PAGE_SIZE +
 385                                   config_pages * PAGE_SIZE);
 386         mem_pool.used_pages = per_cpu_pages + config_pages + bitmap_pages;
 387         for (n = 0; n < mem_pool.used_pages; n++)
 388                 set_bit(n, mem_pool.used_bitmap);
 389         mem_pool.flags = PAGE_SCRUB_ON_FREE;
 390
 391         remap_pool.used_bitmap = page_alloc(&mem_pool, NUM_REMAP_BITMAP_PAGES);
 392         remap_pool.used_pages =
 393                 hypervisor_header.possible_cpus * NUM_TEMPORARY_PAGES;
 394         for (n = 0; n < remap_pool.used_pages; n++)
 395                 set_bit(n, remap_pool.used_bitmap);
 396
 397         arch_paging_init();
 398
 399         hv_paging_structs.root_paging = hv_paging;
 400         hv_paging_structs.root_table = page_alloc(&mem_pool, 1);
 401         if (!hv_paging_structs.root_table)
 402                 goto error_nomem;
 403
 404         /* Replicate hypervisor mapping of Linux */
 405         err = page_map_create(&hv_paging_structs,
 406                               page_map_hvirt2phys(&hypervisor_header),
 407                               system_config->hypervisor_memory.size,
 408                               (unsigned long)&hypervisor_header,
 409                               PAGE_DEFAULT_FLAGS, PAGE_MAP_NON_COHERENT);
 410         if (err)
 411                 goto error_nomem;
 412
 413         /* Make sure any remappings to the temporary regions can be performed
 414          * without allocations of page table pages. */
 415         err = page_map_create(&hv_paging_structs, 0,
 416                               remap_pool.used_pages * PAGE_SIZE,
 417                               TEMPORARY_MAPPING_BASE, PAGE_NONPRESENT_FLAGS,
 418                               PAGE_MAP_NON_COHERENT);
 419         if (err)
 420                 goto error_nomem;
 421
 422         return 0;
 423
 424 error_nomem:
 425         printk("FATAL: page pool much too small\n");
 426         return -ENOMEM;
 427 }
 428
 429 void page_map_dump_stats(const char *when)
 430 {
 431         printk("Page pool usage %s: mem %d/%d, remap %d/%d\n", when,
 432                mem_pool.used_pages, mem_pool.pages,
 433                remap_pool.used_pages, remap_pool.pages);
 434 }