driver/main.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2013-2015
   5  * Copyright (c) Valentine Sinitsyn, 2014
   6  *
   7  * Authors:
   8  *  Jan Kiszka <jan.kiszka@siemens.com>
   9  *  Valentine Sinitsyn <valentine.sinitsyn@gmail.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/cpu.h>
  18 #include <linux/device.h>
  19 #include <linux/fs.h>
  20 #include <linux/miscdevice.h>
  21 #include <linux/firmware.h>
  22 #include <linux/mm.h>
  23 #include <linux/slab.h>
  24 #include <linux/smp.h>
  25 #include <linux/uaccess.h>
  26 #include <linux/reboot.h>
  27 #include <linux/vmalloc.h>
  28 #include <linux/io.h>
  29 #include <asm/smp.h>
  30 #include <asm/cacheflush.h>
  31 #include <asm/tlbflush.h>
  32
  33 #include "cell.h"
  34 #include "jailhouse.h"
  35 #include "pci.h"
  36
  37 #include <jailhouse/header.h>
  38 #include <jailhouse/hypercall.h>
  39 #include <generated/version.h>
  40
  41 #ifdef CONFIG_X86_32
  42 #error 64-bit kernel required!
  43 #endif
  44
  45 #if JAILHOUSE_CELL_ID_NAMELEN != JAILHOUSE_CELL_NAME_MAXLEN
  46 # warning JAILHOUSE_CELL_ID_NAMELEN and JAILHOUSE_CELL_NAME_MAXLEN out of sync!
  47 #endif
  48
  49 /* For compatibility with older kernel versions */
  50 #include <linux/version.h>
  51
  52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0)
  53 #define DEVICE_ATTR_RO(_name) \
  54         struct device_attribute dev_attr_##_name = __ATTR_RO(_name)
  55 #endif /* < 3.11 */
  56
  57 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0)
  58 static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr,
  59                               char *buf)
  60 {
  61         struct kobj_attribute *kattr;
  62         ssize_t ret = -EIO;
  63
  64         kattr = container_of(attr, struct kobj_attribute, attr);
  65         if (kattr->show)
  66                 ret = kattr->show(kobj, kattr, buf);
  67         return ret;
  68 }
  69
  70 static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr,
  71                                const char *buf, size_t count)
  72 {
  73         struct kobj_attribute *kattr;
  74         ssize_t ret = -EIO;
  75
  76         kattr = container_of(attr, struct kobj_attribute, attr);
  77         if (kattr->store)
  78                 ret = kattr->store(kobj, kattr, buf, count);
  79         return ret;
  80 }
  81
  82 static const struct sysfs_ops cell_sysfs_ops = {
  83         .show   = kobj_attr_show,
  84         .store  = kobj_attr_store,
  85 };
  86 #define kobj_sysfs_ops cell_sysfs_ops
  87 #endif /* < 3.14 */
  88 /* End of compatibility section - remove as version become obsolete */
  89
  90 #ifdef CONFIG_X86
  91 #define JAILHOUSE_AMD_FW_NAME   "jailhouse-amd.bin"
  92 #define JAILHOUSE_INTEL_FW_NAME "jailhouse-intel.bin"
  93 #else
  94 #define JAILHOUSE_FW_NAME       "jailhouse.bin"
  95 #endif
  96
  97 MODULE_DESCRIPTION("Management driver for Jailhouse partitioning hypervisor");
  98 MODULE_LICENSE("GPL");
  99 #ifdef CONFIG_X86
 100 MODULE_FIRMWARE(JAILHOUSE_AMD_FW_NAME);
 101 MODULE_FIRMWARE(JAILHOUSE_INTEL_FW_NAME);
 102 #else
 103 MODULE_FIRMWARE(JAILHOUSE_FW_NAME);
 104 #endif
 105 MODULE_VERSION(JAILHOUSE_VERSION);
 106
 107 static struct device *jailhouse_dev;
 108 static DEFINE_MUTEX(lock);
 109 static bool enabled;
 110 static void *hypervisor_mem;
 111 static unsigned long hv_core_and_percpu_size;
 112 static cpumask_t offlined_cpus;
 113 static atomic_t call_done;
 114 static int error_code;
 115 static LIST_HEAD(cells);
 116 static struct cell *root_cell;
 117 static struct kobject *cells_dir;
 118
 119 #ifdef CONFIG_X86
 120 bool jailhouse_use_vmcall;
 121
 122 static void init_hypercall(void)
 123 {
 124         jailhouse_use_vmcall = boot_cpu_has(X86_FEATURE_VMX);
 125 }
 126 #else /* !CONFIG_X86 */
 127 static void init_hypercall(void)
 128 {
 129 }
 130 #endif
 131
 132 struct jailhouse_cpu_stats_attr {
 133         struct kobj_attribute kattr;
 134         unsigned int code;
 135 };
 136
 137 static ssize_t stats_show(struct kobject *kobj, struct kobj_attribute *attr,
 138                           char *buffer)
 139 {
 140         struct jailhouse_cpu_stats_attr *stats_attr =
 141                 container_of(attr, struct jailhouse_cpu_stats_attr, kattr);
 142         unsigned int code = JAILHOUSE_CPU_INFO_STAT_BASE + stats_attr->code;
 143         struct cell *cell = container_of(kobj, struct cell, kobj);
 144         unsigned long sum = 0;
 145         unsigned int cpu;
 146         int value;
 147
 148         for_each_cpu(cpu, &cell->cpus_assigned) {
 149                 value = jailhouse_call_arg2(JAILHOUSE_HC_CPU_GET_INFO, cpu,
 150                                             code);
 151                 if (value > 0)
 152                         sum += value;
 153         }
 154
 155         return sprintf(buffer, "%lu\n", sum);
 156 }
 157
 158 #define JAILHOUSE_CPU_STATS_ATTR(_name, _code) \
 159         static struct jailhouse_cpu_stats_attr _name##_attr = { \
 160                 .kattr = __ATTR(_name, S_IRUGO, stats_show, NULL), \
 161                 .code = _code, \
 162         }
 163
 164 JAILHOUSE_CPU_STATS_ATTR(vmexits_total, JAILHOUSE_CPU_STAT_VMEXITS_TOTAL);
 165 JAILHOUSE_CPU_STATS_ATTR(vmexits_mmio, JAILHOUSE_CPU_STAT_VMEXITS_MMIO);
 166 JAILHOUSE_CPU_STATS_ATTR(vmexits_management,
 167                          JAILHOUSE_CPU_STAT_VMEXITS_MANAGEMENT);
 168 JAILHOUSE_CPU_STATS_ATTR(vmexits_hypercall,
 169                          JAILHOUSE_CPU_STAT_VMEXITS_HYPERCALL);
 170 #ifdef CONFIG_X86
 171 JAILHOUSE_CPU_STATS_ATTR(vmexits_pio, JAILHOUSE_CPU_STAT_VMEXITS_PIO);
 172 JAILHOUSE_CPU_STATS_ATTR(vmexits_xapic, JAILHOUSE_CPU_STAT_VMEXITS_XAPIC);
 173 JAILHOUSE_CPU_STATS_ATTR(vmexits_cr, JAILHOUSE_CPU_STAT_VMEXITS_CR);
 174 JAILHOUSE_CPU_STATS_ATTR(vmexits_msr, JAILHOUSE_CPU_STAT_VMEXITS_MSR);
 175 JAILHOUSE_CPU_STATS_ATTR(vmexits_cpuid, JAILHOUSE_CPU_STAT_VMEXITS_CPUID);
 176 JAILHOUSE_CPU_STATS_ATTR(vmexits_xsetbv, JAILHOUSE_CPU_STAT_VMEXITS_XSETBV);
 177 #elif defined(CONFIG_ARM)
 178 JAILHOUSE_CPU_STATS_ATTR(vmexits_maintenance, JAILHOUSE_CPU_STAT_VMEXITS_MAINTENANCE);
 179 JAILHOUSE_CPU_STATS_ATTR(vmexits_virt_irq, JAILHOUSE_CPU_STAT_VMEXITS_VIRQ);
 180 JAILHOUSE_CPU_STATS_ATTR(vmexits_virt_sgi, JAILHOUSE_CPU_STAT_VMEXITS_VSGI);
 181 #endif
 182
 183 static struct attribute *no_attrs[] = {
 184         &vmexits_total_attr.kattr.attr,
 185         &vmexits_mmio_attr.kattr.attr,
 186         &vmexits_management_attr.kattr.attr,
 187         &vmexits_hypercall_attr.kattr.attr,
 188 #ifdef CONFIG_X86
 189         &vmexits_pio_attr.kattr.attr,
 190         &vmexits_xapic_attr.kattr.attr,
 191         &vmexits_cr_attr.kattr.attr,
 192         &vmexits_msr_attr.kattr.attr,
 193         &vmexits_cpuid_attr.kattr.attr,
 194         &vmexits_xsetbv_attr.kattr.attr,
 195 #elif defined(CONFIG_ARM)
 196         &vmexits_maintenance_attr.kattr.attr,
 197         &vmexits_virt_irq_attr.kattr.attr,
 198         &vmexits_virt_sgi_attr.kattr.attr,
 199 #endif
 200         NULL
 201 };
 202
 203 static struct attribute_group stats_attr_group = {
 204         .attrs = no_attrs,
 205         .name = "statistics"
 206 };
 207
 208 static ssize_t id_show(struct kobject *kobj, struct kobj_attribute *attr,
 209                        char *buffer)
 210 {
 211         struct cell *cell = container_of(kobj, struct cell, kobj);
 212
 213         return sprintf(buffer, "%u\n", cell->id);
 214 }
 215
 216 static ssize_t state_show(struct kobject *kobj, struct kobj_attribute *attr,
 217                           char *buffer)
 218 {
 219         struct cell *cell = container_of(kobj, struct cell, kobj);
 220
 221         switch (jailhouse_call_arg1(JAILHOUSE_HC_CELL_GET_STATE, cell->id)) {
 222         case JAILHOUSE_CELL_RUNNING:
 223                 return sprintf(buffer, "running\n");
 224         case JAILHOUSE_CELL_RUNNING_LOCKED:
 225                 return sprintf(buffer, "running/locked\n");
 226         case JAILHOUSE_CELL_SHUT_DOWN:
 227                 return sprintf(buffer, "shut down\n");
 228         case JAILHOUSE_CELL_FAILED:
 229                 return sprintf(buffer, "failed\n");
 230         default:
 231                 return sprintf(buffer, "invalid\n");
 232         }
 233 }
 234
 235 static ssize_t cpus_assigned_show(struct kobject *kobj,
 236                                   struct kobj_attribute *attr, char *buf)
 237 {
 238         struct cell *cell = container_of(kobj, struct cell, kobj);
 239         int written;
 240
 241 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 242         written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
 243                             cpumask_pr_args(&cell->cpus_assigned));
 244 #else
 245         written = cpumask_scnprintf(buf, PAGE_SIZE, &cell->cpus_assigned);
 246         written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
 247 #endif
 248         return written;
 249 }
 250
 251 static ssize_t cpus_failed_show(struct kobject *kobj,
 252                                 struct kobj_attribute *attr, char *buf)
 253 {
 254         struct cell *cell = container_of(kobj, struct cell, kobj);
 255         cpumask_var_t cpus_failed;
 256         unsigned int cpu;
 257         int written;
 258
 259         if (!zalloc_cpumask_var(&cpus_failed, GFP_KERNEL))
 260                 return -ENOMEM;
 261
 262         for_each_cpu(cpu, &cell->cpus_assigned)
 263                 if (jailhouse_call_arg2(JAILHOUSE_HC_CPU_GET_INFO, cpu,
 264                                         JAILHOUSE_CPU_INFO_STATE) ==
 265                     JAILHOUSE_CPU_FAILED)
 266                         cpu_set(cpu, *cpus_failed);
 267
 268 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 269         written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
 270                             cpumask_pr_args(cpus_failed));
 271 #else
 272         written = cpumask_scnprintf(buf, PAGE_SIZE, cpus_failed);
 273         written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
 274 #endif
 275
 276         free_cpumask_var(cpus_failed);
 277
 278         return written;
 279 }
 280
 281 static struct kobj_attribute cell_id_attr = __ATTR_RO(id);
 282 static struct kobj_attribute cell_state_attr = __ATTR_RO(state);
 283 static struct kobj_attribute cell_cpus_assigned_attr =
 284         __ATTR_RO(cpus_assigned);
 285 static struct kobj_attribute cell_cpus_failed_attr = __ATTR_RO(cpus_failed);
 286
 287 static struct attribute *cell_attrs[] = {
 288         &cell_id_attr.attr,
 289         &cell_state_attr.attr,
 290         &cell_cpus_assigned_attr.attr,
 291         &cell_cpus_failed_attr.attr,
 292         NULL,
 293 };
 294
 295 static void cell_kobj_release(struct kobject *kobj)
 296 {
 297         struct cell *cell = container_of(kobj, struct cell, kobj);
 298
 299         jailhouse_pci_cell_cleanup(cell);
 300         vfree(cell->memory_regions);
 301         kfree(cell);
 302 }
 303
 304 static struct kobj_type cell_type = {
 305         .release = cell_kobj_release,
 306         .sysfs_ops = &kobj_sysfs_ops,
 307         .default_attrs = cell_attrs,
 308 };
 309
 310 static struct cell *create_cell(const struct jailhouse_cell_desc *cell_desc)
 311 {
 312         struct cell *cell;
 313         int err;
 314
 315         cell = kzalloc(sizeof(*cell), GFP_KERNEL);
 316         if (!cell)
 317                 return ERR_PTR(-ENOMEM);
 318
 319         INIT_LIST_HEAD(&cell->entry);
 320
 321         bitmap_copy(cpumask_bits(&cell->cpus_assigned),
 322                     jailhouse_cell_cpu_set(cell_desc),
 323                     min(nr_cpumask_bits, (int)cell_desc->cpu_set_size * 8));
 324
 325         cell->num_memory_regions = cell_desc->num_memory_regions;
 326         cell->memory_regions = vmalloc(sizeof(struct jailhouse_memory) *
 327                                        cell->num_memory_regions);
 328         if (!cell->memory_regions) {
 329                 kfree(cell);
 330                 return ERR_PTR(-ENOMEM);
 331         }
 332
 333         memcpy(cell->memory_regions, jailhouse_cell_mem_regions(cell_desc),
 334                sizeof(struct jailhouse_memory) * cell->num_memory_regions);
 335
 336         err = jailhouse_pci_cell_setup(cell, cell_desc);
 337         if (err) {
 338                 vfree(cell->memory_regions);
 339                 kfree(cell);
 340                 return ERR_PTR(err);
 341         }
 342
 343         err = kobject_init_and_add(&cell->kobj, &cell_type, cells_dir, "%s",
 344                                    cell_desc->name);
 345         if (err) {
 346                 cell_kobj_release(&cell->kobj);
 347                 return ERR_PTR(err);
 348         }
 349
 350         err = sysfs_create_group(&cell->kobj, &stats_attr_group);
 351         if (err) {
 352                 kobject_put(&cell->kobj);
 353                 return ERR_PTR(err);
 354         }
 355
 356         return cell;
 357 }
 358
 359 static void register_cell(struct cell *cell)
 360 {
 361         list_add_tail(&cell->entry, &cells);
 362         kobject_uevent(&cell->kobj, KOBJ_ADD);
 363 }
 364
 365 static struct cell *find_cell(struct jailhouse_cell_id *cell_id)
 366 {
 367         struct cell *cell;
 368
 369         list_for_each_entry(cell, &cells, entry)
 370                 if (cell_id->id == cell->id ||
 371                     (cell_id->id == JAILHOUSE_CELL_ID_UNUSED &&
 372                      strcmp(kobject_name(&cell->kobj), cell_id->name) == 0))
 373                         return cell;
 374         return NULL;
 375 }
 376
 377 static void delete_cell(struct cell *cell)
 378 {
 379         list_del(&cell->entry);
 380         sysfs_remove_group(&cell->kobj, &stats_attr_group);
 381         kobject_put(&cell->kobj);
 382 }
 383
 384 static long get_max_cpus(u32 cpu_set_size,
 385                          const struct jailhouse_system __user *system_config)
 386 {
 387         u8 __user *cpu_set =
 388                 (u8 __user *)jailhouse_cell_cpu_set(&system_config->root_cell);
 389         unsigned int pos = cpu_set_size;
 390         long max_cpu_id;
 391         u8 bitmap;
 392
 393         while (pos-- > 0) {
 394                 if (get_user(bitmap, cpu_set + pos))
 395                         return -EFAULT;
 396                 max_cpu_id = fls(bitmap);
 397                 if (max_cpu_id > 0)
 398                         return pos * 8 + max_cpu_id;
 399         }
 400         return -EINVAL;
 401 }
 402
 403 static void *jailhouse_ioremap(phys_addr_t phys, unsigned long virt,
 404                                unsigned long size)
 405 {
 406         struct vm_struct *vma;
 407
 408         size = PAGE_ALIGN(size);
 409         if (virt)
 410                 vma = __get_vm_area(size, VM_IOREMAP, virt,
 411                                     virt + size + PAGE_SIZE);
 412         else
 413                 vma = __get_vm_area(size, VM_IOREMAP, VMALLOC_START,
 414                                     VMALLOC_END);
 415         if (!vma)
 416                 return NULL;
 417         vma->phys_addr = phys;
 418
 419         if (ioremap_page_range((unsigned long)vma->addr,
 420                                (unsigned long)vma->addr + size, phys,
 421                                PAGE_KERNEL_EXEC)) {
 422                 vunmap(vma->addr);
 423                 return NULL;
 424         }
 425
 426         return vma->addr;
 427 }
 428
 429 static void enter_hypervisor(void *info)
 430 {
 431         struct jailhouse_header *header = info;
 432         unsigned int cpu = smp_processor_id();
 433         int err;
 434
 435         if (cpu < header->max_cpus)
 436                 /* either returns 0 or the same error code across all CPUs */
 437                 err = header->entry(cpu);
 438         else
 439                 err = -EINVAL;
 440
 441         if (err)
 442                 error_code = err;
 443
 444 #if defined(CONFIG_X86) && LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 445         /* on Intel, VMXE is now on - update the shadow */
 446         cr4_init_shadow();
 447 #endif
 448
 449         atomic_inc(&call_done);
 450 }
 451
 452 static inline const char * jailhouse_fw_name(void)
 453 {
 454 #ifdef CONFIG_X86
 455         if (boot_cpu_has(X86_FEATURE_SVM))
 456                 return JAILHOUSE_AMD_FW_NAME;
 457         if (boot_cpu_has(X86_FEATURE_VMX))
 458                 return JAILHOUSE_INTEL_FW_NAME;
 459         return NULL;
 460 #else
 461         return JAILHOUSE_FW_NAME;
 462 #endif
 463 }
 464
 465 static int jailhouse_enable(struct jailhouse_system __user *arg)
 466 {
 467         const struct firmware *hypervisor;
 468         struct jailhouse_system config_header;
 469         struct jailhouse_system *config;
 470         struct jailhouse_memory *hv_mem = &config_header.hypervisor_memory;
 471         struct jailhouse_header *header;
 472         void __iomem *uart = NULL;
 473         unsigned long config_size;
 474         const char *fw_name;
 475         long max_cpus;
 476         int err;
 477
 478         fw_name = jailhouse_fw_name();
 479         if (!fw_name) {
 480                 pr_err("jailhouse: Missing or unsupported HVM technology\n");
 481                 return -ENODEV;
 482         }
 483
 484         if (copy_from_user(&config_header, arg, sizeof(config_header)))
 485                 return -EFAULT;
 486         config_header.root_cell.name[JAILHOUSE_CELL_NAME_MAXLEN] = 0;
 487
 488         max_cpus = get_max_cpus(config_header.root_cell.cpu_set_size, arg);
 489         if (max_cpus < 0)
 490                 return max_cpus;
 491         if (max_cpus > UINT_MAX)
 492                 return -EINVAL;
 493
 494         if (mutex_lock_interruptible(&lock) != 0)
 495                 return -EINTR;
 496
 497         err = -EBUSY;
 498         if (enabled || !try_module_get(THIS_MODULE))
 499                 goto error_unlock;
 500
 501         err = request_firmware(&hypervisor, fw_name, jailhouse_dev);
 502         if (err) {
 503                 pr_err("jailhouse: Missing hypervisor image %s\n", fw_name);
 504                 goto error_put_module;
 505         }
 506
 507         header = (struct jailhouse_header *)hypervisor->data;
 508
 509         err = -EINVAL;
 510         if (memcmp(header->signature, JAILHOUSE_SIGNATURE,
 511                    sizeof(header->signature)) != 0)
 512                 goto error_release_fw;
 513
 514         hv_core_and_percpu_size = PAGE_ALIGN(header->core_size) +
 515                 max_cpus * header->percpu_size;
 516         config_size = jailhouse_system_config_size(&config_header);
 517         if (hv_mem->size <= hv_core_and_percpu_size + config_size)
 518                 goto error_release_fw;
 519
 520         hypervisor_mem = jailhouse_ioremap(hv_mem->phys_start, JAILHOUSE_BASE,
 521                                            hv_mem->size);
 522         if (!hypervisor_mem) {
 523                 pr_err("jailhouse: Unable to map RAM reserved for hypervisor "
 524                        "at %08lx\n", (unsigned long)hv_mem->phys_start);
 525                 goto error_release_fw;
 526         }
 527
 528         memcpy(hypervisor_mem, hypervisor->data, hypervisor->size);
 529         memset(hypervisor_mem + hypervisor->size, 0,
 530                hv_mem->size - hypervisor->size);
 531
 532         header = (struct jailhouse_header *)hypervisor_mem;
 533         header->max_cpus = max_cpus;
 534
 535         config = (struct jailhouse_system *)
 536                 (hypervisor_mem + hv_core_and_percpu_size);
 537         if (copy_from_user(config, arg, config_size)) {
 538                 err = -EFAULT;
 539                 goto error_unmap;
 540         }
 541
 542         if (config->debug_uart.flags & JAILHOUSE_MEM_IO) {
 543                 uart = ioremap(config->debug_uart.phys_start,
 544                                config->debug_uart.size);
 545                 if (!uart) {
 546                         err = -EINVAL;
 547                         pr_err("jailhouse: Unable to map hypervisor UART at "
 548                                "%08lx\n",
 549                                (unsigned long)config->debug_uart.phys_start);
 550                         goto error_unmap;
 551                 }
 552                 header->debug_uart_base = (void *)uart;
 553         }
 554
 555         root_cell = create_cell(&config->root_cell);
 556         if (IS_ERR(root_cell)) {
 557                 err = PTR_ERR(root_cell);
 558                 goto error_unmap;
 559         }
 560
 561         cpumask_and(&root_cell->cpus_assigned, &root_cell->cpus_assigned,
 562                     cpu_online_mask);
 563
 564         error_code = 0;
 565
 566         preempt_disable();
 567
 568         header->online_cpus = num_online_cpus();
 569
 570         atomic_set(&call_done, 0);
 571         on_each_cpu(enter_hypervisor, header, 0);
 572         while (atomic_read(&call_done) != num_online_cpus())
 573                 cpu_relax();
 574
 575         preempt_enable();
 576
 577         if (error_code) {
 578                 err = error_code;
 579                 goto error_free_cell;
 580         }
 581
 582         jailhouse_pci_do_all_devices(root_cell, JAILHOUSE_PCI_TYPE_IVSHMEM,
 583                                      JAILHOUSE_PCI_ACTION_ADD);
 584
 585         if (uart)
 586                 iounmap(uart);
 587
 588         release_firmware(hypervisor);
 589
 590         enabled = true;
 591         root_cell->id = 0;
 592         register_cell(root_cell);
 593
 594         mutex_unlock(&lock);
 595
 596         pr_info("The Jailhouse is opening.\n");
 597
 598         return 0;
 599
 600 error_free_cell:
 601         delete_cell(root_cell);
 602
 603 error_unmap:
 604         vunmap(hypervisor_mem);
 605         if (uart)
 606                 iounmap(uart);
 607
 608 error_release_fw:
 609         release_firmware(hypervisor);
 610
 611 error_put_module:
 612         module_put(THIS_MODULE);
 613
 614 error_unlock:
 615         mutex_unlock(&lock);
 616         return err;
 617 }
 618
 619 static void leave_hypervisor(void *info)
 620 {
 621         unsigned long size;
 622         void *page;
 623         int err;
 624
 625         /* Touch each hypervisor page we may need during the switch so that
 626          * the active mm definitely contains all mappings. At least x86 does
 627          * not support taking any faults while switching worlds. */
 628         for (page = hypervisor_mem, size = hv_core_and_percpu_size; size > 0;
 629              size -= PAGE_SIZE, page += PAGE_SIZE)
 630                 readl(page);
 631
 632         /* either returns 0 or the same error code across all CPUs */
 633         err = jailhouse_call(JAILHOUSE_HC_DISABLE);
 634         if (err)
 635                 error_code = err;
 636
 637 #if defined(CONFIG_X86) && LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 638         /* on Intel, VMXE is now off - update the shadow */
 639         cr4_init_shadow();
 640 #endif
 641
 642         atomic_inc(&call_done);
 643 }
 644
 645 static int jailhouse_disable(void)
 646 {
 647         struct cell *cell, *tmp;
 648         unsigned int cpu;
 649         int err;
 650
 651         if (mutex_lock_interruptible(&lock) != 0)
 652                 return -EINTR;
 653
 654         if (!enabled) {
 655                 err = -EINVAL;
 656                 goto unlock_out;
 657         }
 658
 659         error_code = 0;
 660
 661         preempt_disable();
 662
 663         atomic_set(&call_done, 0);
 664         on_each_cpu(leave_hypervisor, NULL, 0);
 665         while (atomic_read(&call_done) != num_online_cpus())
 666                 cpu_relax();
 667
 668         preempt_enable();
 669
 670         err = error_code;
 671         if (err)
 672                 goto unlock_out;
 673
 674         vunmap(hypervisor_mem);
 675
 676         for_each_cpu(cpu, &offlined_cpus) {
 677                 if (cpu_up(cpu) != 0)
 678                         pr_err("Jailhouse: failed to bring CPU %d back "
 679                                "online\n", cpu);
 680                 cpu_clear(cpu, offlined_cpus);
 681         }
 682
 683         jailhouse_pci_do_all_devices(root_cell, JAILHOUSE_PCI_TYPE_IVSHMEM,
 684                                      JAILHOUSE_PCI_ACTION_DEL);
 685
 686         list_for_each_entry_safe(cell, tmp, &cells, entry)
 687                 delete_cell(cell);
 688         enabled = false;
 689         module_put(THIS_MODULE);
 690
 691         pr_info("The Jailhouse was closed.\n");
 692
 693 unlock_out:
 694         mutex_unlock(&lock);
 695
 696         return err;
 697 }
 698
 699 static int jailhouse_cell_create(struct jailhouse_cell_create __user *arg)
 700 {
 701         struct jailhouse_cell_create cell_params;
 702         struct jailhouse_cell_desc *config;
 703         struct jailhouse_cell_id cell_id;
 704         struct cell *cell;
 705         unsigned int cpu;
 706         int id, err = 0;
 707
 708         if (copy_from_user(&cell_params, arg, sizeof(cell_params)))
 709                 return -EFAULT;
 710
 711         config = kmalloc(cell_params.config_size, GFP_KERNEL | GFP_DMA);
 712         if (!config)
 713                 return -ENOMEM;
 714
 715         if (copy_from_user(config,
 716                            (void *)(unsigned long)cell_params.config_address,
 717                            cell_params.config_size)) {
 718                 err = -EFAULT;
 719                 goto kfree_config_out;
 720         }
 721         config->name[JAILHOUSE_CELL_NAME_MAXLEN] = 0;
 722
 723         if (mutex_lock_interruptible(&lock) != 0) {
 724                 err = -EINTR;
 725                 goto kfree_config_out;
 726         }
 727
 728         if (!enabled) {
 729                 err = -EINVAL;
 730                 goto unlock_out;
 731         }
 732
 733         cell_id.id = JAILHOUSE_CELL_ID_UNUSED;
 734         memcpy(cell_id.name, config->name, sizeof(cell_id.name));
 735         if (find_cell(&cell_id) != NULL) {
 736                 err = -EEXIST;
 737                 goto unlock_out;
 738         }
 739
 740         cell = create_cell(config);
 741         if (IS_ERR(cell)) {
 742                 err = PTR_ERR(cell);
 743                 goto unlock_out;
 744         }
 745
 746         if (!cpumask_subset(&cell->cpus_assigned, &root_cell->cpus_assigned)) {
 747                 err = -EBUSY;
 748                 goto error_cell_delete;
 749         }
 750
 751         for_each_cpu(cpu, &cell->cpus_assigned) {
 752                 if (cpu_online(cpu)) {
 753                         err = cpu_down(cpu);
 754                         if (err)
 755                                 goto error_cpu_online;
 756                         cpu_set(cpu, offlined_cpus);
 757                 }
 758                 cpu_clear(cpu, root_cell->cpus_assigned);
 759         }
 760
 761         id = jailhouse_call_arg1(JAILHOUSE_HC_CELL_CREATE, __pa(config));
 762         if (id < 0) {
 763                 err = id;
 764                 goto error_cpu_online;
 765         }
 766
 767         cell->id = id;
 768         register_cell(cell);
 769
 770         pr_info("Created Jailhouse cell \"%s\"\n", config->name);
 771
 772 unlock_out:
 773         mutex_unlock(&lock);
 774
 775 kfree_config_out:
 776         kfree(config);
 777
 778         return err;
 779
 780 error_cpu_online:
 781         for_each_cpu(cpu, &cell->cpus_assigned) {
 782                 if (!cpu_online(cpu) && cpu_up(cpu) == 0)
 783                         cpu_clear(cpu, offlined_cpus);
 784                 cpu_set(cpu, root_cell->cpus_assigned);
 785         }
 786
 787 error_cell_delete:
 788         delete_cell(cell);
 789         goto unlock_out;
 790 }
 791
 792 static int cell_management_prologue(struct jailhouse_cell_id *cell_id,
 793                                     struct cell **cell_ptr)
 794 {
 795         cell_id->name[JAILHOUSE_CELL_ID_NAMELEN] = 0;
 796
 797         if (mutex_lock_interruptible(&lock) != 0)
 798                 return -EINTR;
 799
 800         if (!enabled) {
 801                 mutex_unlock(&lock);
 802                 return -EINVAL;
 803         }
 804
 805         *cell_ptr = find_cell(cell_id);
 806         if (*cell_ptr == NULL) {
 807                 mutex_unlock(&lock);
 808                 return -ENOENT;
 809         }
 810         return 0;
 811 }
 812
 813 #define MEM_REQ_FLAGS   (JAILHOUSE_MEM_WRITE | JAILHOUSE_MEM_LOADABLE)
 814
 815 static int load_image(struct cell *cell,
 816                       struct jailhouse_preload_image __user *uimage)
 817 {
 818         struct jailhouse_preload_image image;
 819         const struct jailhouse_memory *mem;
 820         unsigned int regions;
 821         u64 image_offset;
 822         void *image_mem;
 823         int err = 0;
 824
 825         if (copy_from_user(&image, uimage, sizeof(image)))
 826                 return -EFAULT;
 827
 828         mem = cell->memory_regions;
 829         for (regions = cell->num_memory_regions; regions > 0; regions--) {
 830                 image_offset = image.target_address - mem->virt_start;
 831                 if (image.target_address >= mem->virt_start &&
 832                     image_offset < mem->size) {
 833                         if (image.size > mem->size - image_offset ||
 834                             (mem->flags & MEM_REQ_FLAGS) != MEM_REQ_FLAGS)
 835                                 return -EINVAL;
 836                         break;
 837                 }
 838                 mem++;
 839         }
 840         if (regions == 0)
 841                 return -EINVAL;
 842
 843         image_mem = jailhouse_ioremap(mem->phys_start + image_offset, 0,
 844                                       image.size);
 845         if (!image_mem) {
 846                 pr_err("jailhouse: Unable to map cell RAM at %08llx "
 847                        "for image loading\n",
 848                        (unsigned long long)(mem->phys_start + image_offset));
 849                 return -EBUSY;
 850         }
 851
 852         if (copy_from_user(image_mem,
 853                            (void *)(unsigned long)image.source_address,
 854                            image.size))
 855                 err = -EFAULT;
 856
 857         vunmap(image_mem);
 858
 859         return err;
 860 }
 861
 862 static int jailhouse_cell_load(struct jailhouse_cell_load __user *arg)
 863 {
 864         struct jailhouse_preload_image __user *image = arg->image;
 865         struct jailhouse_cell_load cell_load;
 866         struct cell *cell;
 867         unsigned int n;
 868         int err;
 869
 870         if (copy_from_user(&cell_load, arg, sizeof(cell_load)))
 871                 return -EFAULT;
 872
 873         err = cell_management_prologue(&cell_load.cell_id, &cell);
 874         if (err)
 875                 return err;
 876
 877         err = jailhouse_call_arg1(JAILHOUSE_HC_CELL_SET_LOADABLE, cell->id);
 878         if (err)
 879                 goto unlock_out;
 880
 881         for (n = cell_load.num_preload_images; n > 0; n--, image++) {
 882                 err = load_image(cell, image);
 883                 if (err)
 884                         break;
 885         }
 886
 887 unlock_out:
 888         mutex_unlock(&lock);
 889
 890         return err;
 891 }
 892
 893 static int jailhouse_cell_start(const char __user *arg)
 894 {
 895         struct jailhouse_cell_id cell_id;
 896         struct cell *cell;
 897         int err;
 898
 899         if (copy_from_user(&cell_id, arg, sizeof(cell_id)))
 900                 return -EFAULT;
 901
 902         err = cell_management_prologue(&cell_id, &cell);
 903         if (err)
 904                 return err;
 905
 906         err = jailhouse_call_arg1(JAILHOUSE_HC_CELL_START, cell->id);
 907
 908         mutex_unlock(&lock);
 909
 910         return err;
 911 }
 912
 913 static int jailhouse_cell_destroy(const char __user *arg)
 914 {
 915         struct jailhouse_cell_id cell_id;
 916         struct cell *cell;
 917         unsigned int cpu;
 918         int err;
 919
 920         if (copy_from_user(&cell_id, arg, sizeof(cell_id)))
 921                 return -EFAULT;
 922
 923         err = cell_management_prologue(&cell_id, &cell);
 924         if (err)
 925                 return err;
 926
 927         err = jailhouse_call_arg1(JAILHOUSE_HC_CELL_DESTROY, cell->id);
 928         if (err)
 929                 goto unlock_out;
 930
 931         for_each_cpu(cpu, &cell->cpus_assigned) {
 932                 if (cpu_isset(cpu, offlined_cpus)) {
 933                         if (cpu_up(cpu) != 0)
 934                                 pr_err("Jailhouse: failed to bring CPU %d "
 935                                        "back online\n", cpu);
 936                         cpu_clear(cpu, offlined_cpus);
 937                 }
 938                 cpu_set(cpu, root_cell->cpus_assigned);
 939         }
 940
 941         pr_info("Destroyed Jailhouse cell \"%s\"\n",
 942                 kobject_name(&cell->kobj));
 943
 944         delete_cell(cell);
 945
 946 unlock_out:
 947         mutex_unlock(&lock);
 948
 949         return err;
 950 }
 951
 952 static long jailhouse_ioctl(struct file *file, unsigned int ioctl,
 953                             unsigned long arg)
 954 {
 955         long err;
 956
 957         switch (ioctl) {
 958         case JAILHOUSE_ENABLE:
 959                 err = jailhouse_enable(
 960                         (struct jailhouse_system __user *)arg);
 961                 break;
 962         case JAILHOUSE_DISABLE:
 963                 err = jailhouse_disable();
 964                 break;
 965         case JAILHOUSE_CELL_CREATE:
 966                 err = jailhouse_cell_create(
 967                         (struct jailhouse_cell_create __user *)arg);
 968                 break;
 969         case JAILHOUSE_CELL_LOAD:
 970                 err = jailhouse_cell_load(
 971                         (struct jailhouse_cell_load __user *)arg);
 972                 break;
 973         case JAILHOUSE_CELL_START:
 974                 err = jailhouse_cell_start((const char __user *)arg);
 975                 break;
 976         case JAILHOUSE_CELL_DESTROY:
 977                 err = jailhouse_cell_destroy((const char __user *)arg);
 978                 break;
 979         default:
 980                 err = -EINVAL;
 981                 break;
 982         }
 983
 984         return err;
 985 }
 986
 987 static const struct file_operations jailhouse_fops = {
 988         .owner = THIS_MODULE,
 989         .unlocked_ioctl = jailhouse_ioctl,
 990         .compat_ioctl = jailhouse_ioctl,
 991         .llseek = noop_llseek,
 992 };
 993
 994 static struct miscdevice jailhouse_misc_dev = {
 995         .minor = MISC_DYNAMIC_MINOR,
 996         .name = "jailhouse",
 997         .fops = &jailhouse_fops,
 998 };
 999
1000 static int jailhouse_shutdown_notify(struct notifier_block *unused1,
1001                                      unsigned long unused2, void *unused3)
1002 {
1003         int err;
1004
1005         err = jailhouse_disable();
1006         if (err && err != -EINVAL)
1007                 pr_emerg("jailhouse: ordered shutdown failed!\n");
1008
1009         return NOTIFY_DONE;
1010 }
1011
1012 static struct notifier_block jailhouse_shutdown_nb = {
1013         .notifier_call = jailhouse_shutdown_notify,
1014 };
1015
1016 static ssize_t enabled_show(struct device *dev, struct device_attribute *attr,
1017                             char *buffer)
1018 {
1019         return sprintf(buffer, "%d\n", enabled);
1020 }
1021
1022 static ssize_t info_show(struct device *dev, char *buffer, unsigned int type)
1023 {
1024         ssize_t result;
1025         long val = 0;
1026
1027         if (mutex_lock_interruptible(&lock) != 0)
1028                 return -EINTR;
1029
1030         if (enabled)
1031                 val = jailhouse_call_arg1(JAILHOUSE_HC_HYPERVISOR_GET_INFO,
1032                                           type);
1033         if (val >= 0)
1034                 result = sprintf(buffer, "%ld\n", val);
1035         else
1036                 result = val;
1037
1038         mutex_unlock(&lock);
1039         return result;
1040 }
1041
1042 static ssize_t mem_pool_size_show(struct device *dev,
1043                                   struct device_attribute *attr, char *buffer)
1044 {
1045         return info_show(dev, buffer, JAILHOUSE_INFO_MEM_POOL_SIZE);
1046 }
1047
1048 static ssize_t mem_pool_used_show(struct device *dev,
1049                                   struct device_attribute *attr, char *buffer)
1050 {
1051         return info_show(dev, buffer, JAILHOUSE_INFO_MEM_POOL_USED);
1052 }
1053
1054 static ssize_t remap_pool_size_show(struct device *dev,
1055                                     struct device_attribute *attr,
1056                                     char *buffer)
1057 {
1058         return info_show(dev, buffer, JAILHOUSE_INFO_REMAP_POOL_SIZE);
1059 }
1060
1061 static ssize_t remap_pool_used_show(struct device *dev,
1062                                     struct device_attribute *attr,
1063                                     char *buffer)
1064 {
1065         return info_show(dev, buffer, JAILHOUSE_INFO_REMAP_POOL_USED);
1066 }
1067
1068 static DEVICE_ATTR_RO(enabled);
1069 static DEVICE_ATTR_RO(mem_pool_size);
1070 static DEVICE_ATTR_RO(mem_pool_used);
1071 static DEVICE_ATTR_RO(remap_pool_size);
1072 static DEVICE_ATTR_RO(remap_pool_used);
1073
1074 static struct attribute *jailhouse_sysfs_entries[] = {
1075         &dev_attr_enabled.attr,
1076         &dev_attr_mem_pool_size.attr,
1077         &dev_attr_mem_pool_used.attr,
1078         &dev_attr_remap_pool_size.attr,
1079         &dev_attr_remap_pool_used.attr,
1080         NULL
1081 };
1082
1083 static struct attribute_group jailhouse_attribute_group = {
1084         .name = NULL,
1085         .attrs = jailhouse_sysfs_entries,
1086 };
1087
1088 static int __init jailhouse_init(void)
1089 {
1090         int err;
1091
1092         jailhouse_dev = root_device_register("jailhouse");
1093         if (IS_ERR(jailhouse_dev))
1094                 return PTR_ERR(jailhouse_dev);
1095
1096         err = sysfs_create_group(&jailhouse_dev->kobj,
1097                                  &jailhouse_attribute_group);
1098         if (err)
1099                 goto unreg_dev;
1100
1101         cells_dir = kobject_create_and_add("cells", &jailhouse_dev->kobj);
1102         if (!cells_dir) {
1103                 err = -ENOMEM;
1104                 goto remove_attrs;
1105         }
1106
1107         err = misc_register(&jailhouse_misc_dev);
1108         if (err)
1109                 goto remove_cells_dir;
1110
1111         register_reboot_notifier(&jailhouse_shutdown_nb);
1112
1113         init_hypercall();
1114
1115         return 0;
1116
1117 remove_cells_dir:
1118         kobject_put(cells_dir);
1119
1120 remove_attrs:
1121         sysfs_remove_group(&jailhouse_dev->kobj, &jailhouse_attribute_group);
1122
1123 unreg_dev:
1124         root_device_unregister(jailhouse_dev);
1125         return err;
1126 }
1127
1128 static void __exit jailhouse_exit(void)
1129 {
1130         unregister_reboot_notifier(&jailhouse_shutdown_nb);
1131         misc_deregister(&jailhouse_misc_dev);
1132         kobject_put(cells_dir);
1133         sysfs_remove_group(&jailhouse_dev->kobj, &jailhouse_attribute_group);
1134         root_device_unregister(jailhouse_dev);
1135 }
1136
1137 module_init(jailhouse_init);
1138 module_exit(jailhouse_exit);