driver/main.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2013-2015
   5  * Copyright (c) Valentine Sinitsyn, 2014
   6  *
   7  * Authors:
   8  *  Jan Kiszka <jan.kiszka@siemens.com>
   9  *  Valentine Sinitsyn <valentine.sinitsyn@gmail.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/device.h>
  18 #include <linux/fs.h>
  19 #include <linux/miscdevice.h>
  20 #include <linux/firmware.h>
  21 #include <linux/mm.h>
  22 #include <linux/smp.h>
  23 #include <linux/uaccess.h>
  24 #include <linux/reboot.h>
  25 #include <linux/vmalloc.h>
  26 #include <linux/io.h>
  27 #include <asm/smp.h>
  28 #include <asm/cacheflush.h>
  29 #include <asm/tlbflush.h>
  30
  31 #include "cell.h"
  32 #include "jailhouse.h"
  33 #include "main.h"
  34 #include "pci.h"
  35 #include "sysfs.h"
  36
  37 #include <jailhouse/header.h>
  38 #include <jailhouse/hypercall.h>
  39 #include <generated/version.h>
  40
  41 #ifdef CONFIG_X86_32
  42 #error 64-bit kernel required!
  43 #endif
  44
  45 #if JAILHOUSE_CELL_ID_NAMELEN != JAILHOUSE_CELL_NAME_MAXLEN
  46 # warning JAILHOUSE_CELL_ID_NAMELEN and JAILHOUSE_CELL_NAME_MAXLEN out of sync!
  47 #endif
  48
  49 /* For compatibility with older kernel versions */
  50 #include <linux/version.h>
  51
  52 #ifdef CONFIG_X86
  53 #define JAILHOUSE_AMD_FW_NAME   "jailhouse-amd.bin"
  54 #define JAILHOUSE_INTEL_FW_NAME "jailhouse-intel.bin"
  55 #else
  56 #define JAILHOUSE_FW_NAME       "jailhouse.bin"
  57 #endif
  58
  59 MODULE_DESCRIPTION("Management driver for Jailhouse partitioning hypervisor");
  60 MODULE_LICENSE("GPL");
  61 #ifdef CONFIG_X86
  62 MODULE_FIRMWARE(JAILHOUSE_AMD_FW_NAME);
  63 MODULE_FIRMWARE(JAILHOUSE_INTEL_FW_NAME);
  64 #else
  65 MODULE_FIRMWARE(JAILHOUSE_FW_NAME);
  66 #endif
  67 MODULE_VERSION(JAILHOUSE_VERSION);
  68
  69 DEFINE_MUTEX(jailhouse_lock);
  70 bool jailhouse_enabled;
  71
  72 static struct device *jailhouse_dev;
  73 static void *hypervisor_mem;
  74 static unsigned long hv_core_and_percpu_size;
  75 static atomic_t call_done;
  76 static int error_code;
  77
  78 #ifdef CONFIG_X86
  79 bool jailhouse_use_vmcall;
  80
  81 static void init_hypercall(void)
  82 {
  83         jailhouse_use_vmcall = boot_cpu_has(X86_FEATURE_VMX);
  84 }
  85 #else /* !CONFIG_X86 */
  86 static void init_hypercall(void)
  87 {
  88 }
  89 #endif
  90
  91 static long get_max_cpus(u32 cpu_set_size,
  92                          const struct jailhouse_system __user *system_config)
  93 {
  94         u8 __user *cpu_set =
  95                 (u8 __user *)jailhouse_cell_cpu_set(
  96                                 (const struct jailhouse_cell_desc * __force)
  97                                 &system_config->root_cell);
  98         unsigned int pos = cpu_set_size;
  99         long max_cpu_id;
 100         u8 bitmap;
 101
 102         while (pos-- > 0) {
 103                 if (get_user(bitmap, cpu_set + pos))
 104                         return -EFAULT;
 105                 max_cpu_id = fls(bitmap);
 106                 if (max_cpu_id > 0)
 107                         return pos * 8 + max_cpu_id;
 108         }
 109         return -EINVAL;
 110 }
 111
 112 void *jailhouse_ioremap(phys_addr_t phys, unsigned long virt,
 113                         unsigned long size)
 114 {
 115         struct vm_struct *vma;
 116
 117         size = PAGE_ALIGN(size);
 118         if (virt)
 119                 vma = __get_vm_area(size, VM_IOREMAP, virt,
 120                                     virt + size + PAGE_SIZE);
 121         else
 122                 vma = __get_vm_area(size, VM_IOREMAP, VMALLOC_START,
 123                                     VMALLOC_END);
 124         if (!vma)
 125                 return NULL;
 126         vma->phys_addr = phys;
 127
 128         if (ioremap_page_range((unsigned long)vma->addr,
 129                                (unsigned long)vma->addr + size, phys,
 130                                PAGE_KERNEL_EXEC)) {
 131                 vunmap(vma->addr);
 132                 return NULL;
 133         }
 134
 135         return vma->addr;
 136 }
 137
 138 static void enter_hypervisor(void *info)
 139 {
 140         struct jailhouse_header *header = info;
 141         unsigned int cpu = smp_processor_id();
 142         int (*entry)(unsigned int);
 143         int err;
 144
 145         entry = header->entry + (unsigned long) hypervisor_mem;
 146
 147         if (cpu < header->max_cpus)
 148                 /* either returns 0 or the same error code across all CPUs */
 149                 err = entry(cpu);
 150         else
 151                 err = -EINVAL;
 152
 153         if (err)
 154                 error_code = err;
 155
 156 #if defined(CONFIG_X86) && LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 157         /* on Intel, VMXE is now on - update the shadow */
 158         cr4_init_shadow();
 159 #endif
 160
 161         atomic_inc(&call_done);
 162 }
 163
 164 static inline const char * jailhouse_fw_name(void)
 165 {
 166 #ifdef CONFIG_X86
 167         if (boot_cpu_has(X86_FEATURE_SVM))
 168                 return JAILHOUSE_AMD_FW_NAME;
 169         if (boot_cpu_has(X86_FEATURE_VMX))
 170                 return JAILHOUSE_INTEL_FW_NAME;
 171         return NULL;
 172 #else
 173         return JAILHOUSE_FW_NAME;
 174 #endif
 175 }
 176
 177 static int jailhouse_cmd_enable(struct jailhouse_system __user *arg)
 178 {
 179         const struct firmware *hypervisor;
 180         struct jailhouse_system config_header;
 181         struct jailhouse_system *config;
 182         struct jailhouse_memory *hv_mem = &config_header.hypervisor_memory;
 183         struct jailhouse_header *header;
 184         unsigned long remap_addr = 0;
 185         void __iomem *console = NULL;
 186         unsigned long config_size;
 187         const char *fw_name;
 188         long max_cpus;
 189         int err;
 190
 191         fw_name = jailhouse_fw_name();
 192         if (!fw_name) {
 193                 pr_err("jailhouse: Missing or unsupported HVM technology\n");
 194                 return -ENODEV;
 195         }
 196
 197         if (copy_from_user(&config_header, arg, sizeof(config_header)))
 198                 return -EFAULT;
 199
 200         if (memcmp(config_header.signature, JAILHOUSE_SYSTEM_SIGNATURE,
 201                    sizeof(config_header.signature)) != 0) {
 202                 pr_err("jailhouse: Not a system configuration\n");
 203                 return -EINVAL;
 204         }
 205
 206         config_header.root_cell.name[JAILHOUSE_CELL_NAME_MAXLEN] = 0;
 207
 208         max_cpus = get_max_cpus(config_header.root_cell.cpu_set_size, arg);
 209         if (max_cpus < 0)
 210                 return max_cpus;
 211         if (max_cpus > UINT_MAX)
 212                 return -EINVAL;
 213
 214         if (mutex_lock_interruptible(&jailhouse_lock) != 0)
 215                 return -EINTR;
 216
 217         err = -EBUSY;
 218         if (jailhouse_enabled || !try_module_get(THIS_MODULE))
 219                 goto error_unlock;
 220
 221         err = request_firmware(&hypervisor, fw_name, jailhouse_dev);
 222         if (err) {
 223                 pr_err("jailhouse: Missing hypervisor image %s\n", fw_name);
 224                 goto error_put_module;
 225         }
 226
 227         header = (struct jailhouse_header *)hypervisor->data;
 228
 229         err = -EINVAL;
 230         if (memcmp(header->signature, JAILHOUSE_SIGNATURE,
 231                    sizeof(header->signature)) != 0 ||
 232             hypervisor->size >= hv_mem->size)
 233                 goto error_release_fw;
 234
 235         hv_core_and_percpu_size = header->core_size +
 236                 max_cpus * header->percpu_size;
 237         config_size = jailhouse_system_config_size(&config_header);
 238         if (hv_core_and_percpu_size >= hv_mem->size ||
 239             config_size >= hv_mem->size - hv_core_and_percpu_size)
 240                 goto error_release_fw;
 241
 242 #ifdef JAILHOUSE_BORROW_ROOT_PT
 243         remap_addr = JAILHOUSE_BASE;
 244 #endif
 245         hypervisor_mem = jailhouse_ioremap(hv_mem->phys_start, remap_addr,
 246                                            hv_mem->size);
 247         if (!hypervisor_mem) {
 248                 pr_err("jailhouse: Unable to map RAM reserved for hypervisor "
 249                        "at %08lx\n", (unsigned long)hv_mem->phys_start);
 250                 goto error_release_fw;
 251         }
 252
 253         memcpy(hypervisor_mem, hypervisor->data, hypervisor->size);
 254         memset(hypervisor_mem + hypervisor->size, 0,
 255                hv_mem->size - hypervisor->size);
 256
 257         header = (struct jailhouse_header *)hypervisor_mem;
 258         header->max_cpus = max_cpus;
 259
 260         /*
 261          * ARMv8 requires to clean D-cache and invalidate I-cache for memory
 262          * containing new instructions. On x86 this is a NOP. On ARMv7 the
 263          * firmware does its own cache maintenance, so it is an
 264          * extraneous (but harmless) flush.
 265          */
 266         flush_icache_range((unsigned long)hypervisor_mem,
 267                            (unsigned long)(hypervisor_mem + header->core_size));
 268
 269         config = (struct jailhouse_system *)
 270                 (hypervisor_mem + hv_core_and_percpu_size);
 271         if (copy_from_user(config, arg, config_size)) {
 272                 err = -EFAULT;
 273                 goto error_unmap;
 274         }
 275
 276         if (config->debug_console.flags & JAILHOUSE_MEM_IO) {
 277 #ifdef JAILHOUSE_BORROW_ROOT_PT
 278                 console = ioremap(config->debug_console.phys_start,
 279                                   config->debug_console.size);
 280                 if (!console) {
 281                         err = -EINVAL;
 282                         pr_err("jailhouse: Unable to map hypervisor debug "
 283                                "console at %08lx\n",
 284                                (unsigned long)config->debug_console.phys_start);
 285                         goto error_unmap;
 286                 }
 287                 /* The hypervisor has no notion of address spaces, so we need
 288                  * to enforce conversion. */
 289                 header->debug_console_base = (void * __force)console;
 290 #else
 291                 header->debug_console_base =
 292                         (void * __force) config->debug_console.phys_start;
 293 #endif
 294         }
 295
 296         err = jailhouse_cell_prepare_root(&config->root_cell);
 297         if (err)
 298                 goto error_unmap;
 299
 300         error_code = 0;
 301
 302         preempt_disable();
 303
 304         header->online_cpus = num_online_cpus();
 305
 306         atomic_set(&call_done, 0);
 307         on_each_cpu(enter_hypervisor, header, 0);
 308         while (atomic_read(&call_done) != num_online_cpus())
 309                 cpu_relax();
 310
 311         preempt_enable();
 312
 313         if (error_code) {
 314                 err = error_code;
 315                 goto error_free_cell;
 316         }
 317
 318         if (console)
 319                 iounmap(console);
 320
 321         release_firmware(hypervisor);
 322
 323         jailhouse_cell_register_root();
 324
 325         jailhouse_enabled = true;
 326
 327         mutex_unlock(&jailhouse_lock);
 328
 329         pr_info("The Jailhouse is opening.\n");
 330
 331         return 0;
 332
 333 error_free_cell:
 334         jailhouse_cell_delete_root();
 335
 336 error_unmap:
 337         vunmap(hypervisor_mem);
 338         if (console)
 339                 iounmap(console);
 340
 341 error_release_fw:
 342         release_firmware(hypervisor);
 343
 344 error_put_module:
 345         module_put(THIS_MODULE);
 346
 347 error_unlock:
 348         mutex_unlock(&jailhouse_lock);
 349         return err;
 350 }
 351
 352 static void leave_hypervisor(void *info)
 353 {
 354         void *page;
 355         int err;
 356
 357         /* Touch each hypervisor page we may need during the switch so that
 358          * the active mm definitely contains all mappings. At least x86 does
 359          * not support taking any faults while switching worlds. */
 360         for (page = hypervisor_mem;
 361              page < hypervisor_mem + hv_core_and_percpu_size;
 362              page += PAGE_SIZE)
 363                 readl((void __iomem *)page);
 364
 365         /* either returns 0 or the same error code across all CPUs */
 366         err = jailhouse_call(JAILHOUSE_HC_DISABLE);
 367         if (err)
 368                 error_code = err;
 369
 370 #if defined(CONFIG_X86) && LINUX_VERSION_CODE >= KERNEL_VERSION(4,0,0)
 371         /* on Intel, VMXE is now off - update the shadow */
 372         cr4_init_shadow();
 373 #endif
 374
 375         atomic_inc(&call_done);
 376 }
 377
 378 static int jailhouse_cmd_disable(void)
 379 {
 380         int err;
 381
 382         if (mutex_lock_interruptible(&jailhouse_lock) != 0)
 383                 return -EINTR;
 384
 385         if (!jailhouse_enabled) {
 386                 err = -EINVAL;
 387                 goto unlock_out;
 388         }
 389
 390         error_code = 0;
 391
 392         preempt_disable();
 393
 394         if (num_online_cpus() != cpumask_weight(&root_cell->cpus_assigned)) {
 395                 /*
 396                  * Not all assigned CPUs are currently online. If we disable
 397                  * now, we will loose the offlined ones.
 398                  */
 399
 400                 preempt_enable();
 401
 402                 err = -EBUSY;
 403                 goto unlock_out;
 404         }
 405
 406         atomic_set(&call_done, 0);
 407         on_each_cpu(leave_hypervisor, NULL, 0);
 408         while (atomic_read(&call_done) != num_online_cpus())
 409                 cpu_relax();
 410
 411         preempt_enable();
 412
 413         err = error_code;
 414         if (err)
 415                 goto unlock_out;
 416
 417         vunmap(hypervisor_mem);
 418
 419         jailhouse_cell_delete_all();
 420         jailhouse_enabled = false;
 421         module_put(THIS_MODULE);
 422
 423         pr_info("The Jailhouse was closed.\n");
 424
 425 unlock_out:
 426         mutex_unlock(&jailhouse_lock);
 427
 428         return err;
 429 }
 430
 431 static long jailhouse_ioctl(struct file *file, unsigned int ioctl,
 432                             unsigned long arg)
 433 {
 434         long err;
 435
 436         switch (ioctl) {
 437         case JAILHOUSE_ENABLE:
 438                 err = jailhouse_cmd_enable(
 439                         (struct jailhouse_system __user *)arg);
 440                 break;
 441         case JAILHOUSE_DISABLE:
 442                 err = jailhouse_cmd_disable();
 443                 break;
 444         case JAILHOUSE_CELL_CREATE:
 445                 err = jailhouse_cmd_cell_create(
 446                         (struct jailhouse_cell_create __user *)arg);
 447                 break;
 448         case JAILHOUSE_CELL_LOAD:
 449                 err = jailhouse_cmd_cell_load(
 450                         (struct jailhouse_cell_load __user *)arg);
 451                 break;
 452         case JAILHOUSE_CELL_START:
 453                 err = jailhouse_cmd_cell_start((const char __user *)arg);
 454                 break;
 455         case JAILHOUSE_CELL_DESTROY:
 456                 err = jailhouse_cmd_cell_destroy((const char __user *)arg);
 457                 break;
 458         default:
 459                 err = -EINVAL;
 460                 break;
 461         }
 462
 463         return err;
 464 }
 465
 466 static const struct file_operations jailhouse_fops = {
 467         .owner = THIS_MODULE,
 468         .unlocked_ioctl = jailhouse_ioctl,
 469         .compat_ioctl = jailhouse_ioctl,
 470         .llseek = noop_llseek,
 471 };
 472
 473 static struct miscdevice jailhouse_misc_dev = {
 474         .minor = MISC_DYNAMIC_MINOR,
 475         .name = "jailhouse",
 476         .fops = &jailhouse_fops,
 477 };
 478
 479 static int jailhouse_shutdown_notify(struct notifier_block *unused1,
 480                                      unsigned long unused2, void *unused3)
 481 {
 482         int err;
 483
 484         err = jailhouse_cmd_disable();
 485         if (err && err != -EINVAL)
 486                 pr_emerg("jailhouse: ordered shutdown failed!\n");
 487
 488         return NOTIFY_DONE;
 489 }
 490
 491 static struct notifier_block jailhouse_shutdown_nb = {
 492         .notifier_call = jailhouse_shutdown_notify,
 493 };
 494
 495 static int __init jailhouse_init(void)
 496 {
 497         int err;
 498
 499         jailhouse_dev = root_device_register("jailhouse");
 500         if (IS_ERR(jailhouse_dev))
 501                 return PTR_ERR(jailhouse_dev);
 502
 503         err = jailhouse_sysfs_init(jailhouse_dev);
 504         if (err)
 505                 goto unreg_dev;
 506
 507         err = misc_register(&jailhouse_misc_dev);
 508         if (err)
 509                 goto exit_sysfs;
 510
 511         err = jailhouse_pci_register();
 512         if (err)
 513                 goto exit_misc;
 514
 515         register_reboot_notifier(&jailhouse_shutdown_nb);
 516
 517         init_hypercall();
 518
 519         return 0;
 520 exit_misc:
 521         misc_deregister(&jailhouse_misc_dev);
 522
 523 exit_sysfs:
 524         jailhouse_sysfs_exit(jailhouse_dev);
 525
 526 unreg_dev:
 527         root_device_unregister(jailhouse_dev);
 528         return err;
 529 }
 530
 531 static void __exit jailhouse_exit(void)
 532 {
 533         unregister_reboot_notifier(&jailhouse_shutdown_nb);
 534         misc_deregister(&jailhouse_misc_dev);
 535         jailhouse_sysfs_exit(jailhouse_dev);
 536         jailhouse_pci_unregister();
 537         root_device_unregister(jailhouse_dev);
 538 }
 539
 540 module_init(jailhouse_init);
 541 module_exit(jailhouse_exit);