hypervisor/pci.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2014
   5  *
   6  * Authors:
   7  *  Ivan Kolchin <ivan.kolchin@siemens.com>
   8  *  Jan Kiszka <jan.kiszka@siemens.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include <jailhouse/control.h>
  15 #include <jailhouse/mmio.h>
  16 #include <jailhouse/pci.h>
  17 #include <jailhouse/printk.h>
  18 #include <jailhouse/utils.h>
  19
  20 #define PCI_CONFIG_HEADER_SIZE          0x40
  21
  22 #define PCI_CAP_MSI                     0x05
  23 #define PCI_CAP_MSIX                    0x11
  24
  25 #define MSIX_VECTOR_CTRL_DWORD          3
  26
  27 #define for_each_configured_pci_device(dev, cell)                       \
  28         for ((dev) = (cell)->pci_devices;                               \
  29              (dev) - (cell)->pci_devices < (cell)->config->num_pci_devices; \
  30              (dev)++)
  31
  32 #define for_each_pci_cap(cap, dev, counter)                             \
  33         for ((cap) = jailhouse_cell_pci_caps((dev)->cell->config) +     \
  34                 (dev)->info->caps_start, (counter) = 0;                 \
  35              (counter) < (dev)->info->num_caps;                         \
  36              (cap)++, (counter)++)
  37
  38 /* entry for PCI config space whitelist (granting access) */
  39 struct pci_cfg_access {
  40         u32 reg_num; /* Register number (4-byte aligned) */
  41         u32 mask; /* Bit set: access allowed */
  42 };
  43
  44 /* --- Whilelists for writing to PCI config space registers --- */
  45 /* Type 1: Endpoints */
  46 static const struct pci_cfg_access endpoint_write_access[] = {
  47         { 0x04, 0xffffffff }, /* Command, Status */
  48         { 0x0c, 0xff00ffff }, /* BIST, Latency Timer, Cacheline */
  49         { 0x3c, 0x000000ff }, /* Int Line */
  50 };
  51 /* Type 2: Bridges */
  52 static const struct pci_cfg_access bridge_write_access[] = {
  53         { 0x04, 0xffffffff }, /* Command, Status */
  54         { 0x0c, 0xff00ffff }, /* BIST, Latency Timer, Cacheline */
  55         { 0x3c, 0xffff00ff }, /* Int Line, Bridge Control */
  56 };
  57
  58 static void *pci_space;
  59 static u64 mmcfg_start, mmcfg_end;
  60 static u8 end_bus;
  61
  62 static void *pci_get_device_mmcfg_base(u16 bdf)
  63 {
  64         return pci_space + ((unsigned long)bdf << 12);
  65 }
  66
  67 /**
  68  * Read from PCI config space.
  69  * @param bdf           16-bit bus/device/function ID of target.
  70  * @param address       Config space access address.
  71  * @param size          Access size (1, 2 or 4 bytes).
  72  *
  73  * @return Read value.
  74  *
  75  * @see pci_write_config
  76  */
  77 u32 pci_read_config(u16 bdf, u16 address, unsigned int size)
  78 {
  79         void *mmcfg_addr = pci_get_device_mmcfg_base(bdf) + address;
  80
  81         if (!pci_space || PCI_BUS(bdf) > end_bus)
  82                 return arch_pci_read_config(bdf, address, size);
  83
  84         if (size == 1)
  85                 return mmio_read8(mmcfg_addr);
  86         else if (size == 2)
  87                 return mmio_read16(mmcfg_addr);
  88         else
  89                 return mmio_read32(mmcfg_addr);
  90 }
  91
  92 /**
  93  * Write to PCI config space.
  94  * @param bdf           16-bit bus/device/function ID of target.
  95  * @param address       Config space access address.
  96  * @param value         Value to be written.
  97  * @param size          Access size (1, 2 or 4 bytes).
  98  *
  99  * @see pci_read_config
 100  */
 101 void pci_write_config(u16 bdf, u16 address, u32 value, unsigned int size)
 102 {
 103         void *mmcfg_addr = pci_get_device_mmcfg_base(bdf) + address;
 104
 105         if (!pci_space || PCI_BUS(bdf) > end_bus)
 106                 return arch_pci_write_config(bdf, address, value, size);
 107
 108         if (size == 1)
 109                 mmio_write8(mmcfg_addr, value);
 110         else if (size == 2)
 111                 mmio_write16(mmcfg_addr, value);
 112         else
 113                 mmio_write32(mmcfg_addr, value);
 114 }
 115
 116 /**
 117  * Look up device owned by a cell.
 118  * @param[in] cell      Owning cell.
 119  * @param bdf           16-bit bus/device/function ID.
 120  *
 121  * @return Pointer to owned PCI device or NULL.
 122  */
 123 struct pci_device *pci_get_assigned_device(const struct cell *cell, u16 bdf)
 124 {
 125         const struct jailhouse_pci_device *dev_info =
 126                 jailhouse_cell_pci_devices(cell->config);
 127         u32 n;
 128
 129         /* We iterate over the static device information to increase cache
 130          * locality. */
 131         for (n = 0; n < cell->config->num_pci_devices; n++)
 132                 if (dev_info[n].bdf == bdf)
 133                         return cell->pci_devices[n].cell ?
 134                                 &cell->pci_devices[n] : NULL;
 135
 136         return NULL;
 137 }
 138
 139 /**
 140  * Look up capability at given config space address.
 141  * @param device        The device to be accessed.
 142  * @param address       Config space access address.
 143  *
 144  * @return Corresponding capability structure or NULL if none found.
 145  *
 146  * @private
 147  */
 148 static const struct jailhouse_pci_capability *
 149 pci_find_capability(struct pci_device *device, u16 address)
 150 {
 151         const struct jailhouse_pci_capability *cap =
 152                 jailhouse_cell_pci_caps(device->cell->config) +
 153                 device->info->caps_start;
 154         u32 n;
 155
 156         for (n = 0; n < device->info->num_caps; n++, cap++)
 157                 if (cap->start <= address && cap->start + cap->len > address)
 158                         return cap;
 159
 160         return NULL;
 161 }
 162
 163 /**
 164  * Moderate config space read access.
 165  * @param device        The device to be accessed. If NULL, access will be
 166  *                      emulated, returning a value of -1.
 167  * @param address       Config space address.
 168  * @param size          Access size (1, 2 or 4 bytes).
 169  * @param value         Pointer to buffer to receive the emulated value if
 170  *                      PCI_ACCESS_DONE is returned.
 171  *
 172  * @return PCI_ACCESS_PERFORM or PCI_ACCESS_DONE.
 173  *
 174  * @see pci_cfg_write_moderate
 175  */
 176 enum pci_access pci_cfg_read_moderate(struct pci_device *device, u16 address,
 177                                       unsigned int size, u32 *value)
 178 {
 179         const struct jailhouse_pci_capability *cap;
 180         unsigned int cap_offs;
 181
 182         if (!device) {
 183                 *value = -1;
 184                 return PCI_ACCESS_DONE;
 185         }
 186
 187         if (address < PCI_CONFIG_HEADER_SIZE)
 188                 return PCI_ACCESS_PERFORM;
 189
 190         cap = pci_find_capability(device, address);
 191         if (!cap)
 192                 return PCI_ACCESS_PERFORM;
 193
 194         cap_offs = address - cap->start;
 195         if (cap->id == PCI_CAP_MSI && cap_offs >= 4 &&
 196             (cap_offs < 10 || (device->info->msi_64bits && cap_offs < 14))) {
 197                 *value = device->msi_registers.raw[cap_offs / 4] >>
 198                         ((cap_offs % 4) * 8);
 199                 return PCI_ACCESS_DONE;
 200         }
 201
 202         return PCI_ACCESS_PERFORM;
 203 }
 204
 205 static int pci_update_msix(struct pci_device *device,
 206                            const struct jailhouse_pci_capability *cap)
 207 {
 208         unsigned int n;
 209         int result;
 210
 211         for (n = 0; n < device->info->num_msix_vectors; n++) {
 212                 result = arch_pci_update_msix_vector(device, n);
 213                 if (result < 0)
 214                         return result;
 215         }
 216         return 0;
 217 }
 218
 219 /**
 220  * Moderate config space write access.
 221  * @param device        The device to be accessed. If NULL, access will be
 222  *                      rejected.
 223  * @param address       Config space address.
 224  * @param size          Access size (1, 2 or 4 bytes).
 225  * @param value         Value to be written.
 226  *
 227  * @return PCI_ACCESS_REJECT, PCI_ACCESS_PERFORM or PCI_ACCESS_DONE.
 228  *
 229  * @see pci_cfg_read_moderate
 230  */
 231 enum pci_access pci_cfg_write_moderate(struct pci_device *device, u16 address,
 232                                        unsigned int size, u32 value)
 233 {
 234         const struct jailhouse_pci_capability *cap;
 235         /* initialize list to work around wrong compiler warning */
 236         const struct pci_cfg_access *list = NULL;
 237         unsigned int bias_shift = (address % 4) * 8;
 238         u32 mask = BYTE_MASK(size) << bias_shift;
 239         unsigned int n, cap_offs, len = 0;
 240
 241         if (!device)
 242                 return PCI_ACCESS_REJECT;
 243
 244         if (address < PCI_CONFIG_HEADER_SIZE) {
 245                 if (device->info->type == JAILHOUSE_PCI_TYPE_DEVICE) {
 246                         list = endpoint_write_access;
 247                         len = ARRAY_SIZE(endpoint_write_access);
 248                 } else if (device->info->type == JAILHOUSE_PCI_TYPE_BRIDGE) {
 249                         list = bridge_write_access;
 250                         len = ARRAY_SIZE(bridge_write_access);
 251                 }
 252
 253                 for (n = 0; n < len; n++) {
 254                         if (list[n].reg_num == (address & 0xffc) &&
 255                             (list[n].mask & mask) == mask)
 256                                 return PCI_ACCESS_PERFORM;
 257                 }
 258
 259                 return PCI_ACCESS_REJECT;
 260         }
 261
 262         cap = pci_find_capability(device, address);
 263         if (!cap || !(cap->flags & JAILHOUSE_PCICAPS_WRITE))
 264                 return PCI_ACCESS_REJECT;
 265
 266         value <<= bias_shift;
 267
 268         cap_offs = address - cap->start;
 269         if (cap->id == PCI_CAP_MSI &&
 270             (cap_offs < 10 || (device->info->msi_64bits && cap_offs < 14))) {
 271                 device->msi_registers.raw[cap_offs / 4] &= ~mask;
 272                 device->msi_registers.raw[cap_offs / 4] |= value;
 273
 274                 if (arch_pci_update_msi(device, cap) < 0)
 275                         return PCI_ACCESS_REJECT;
 276
 277                 /*
 278                  * Address and data words are emulated, the control word is
 279                  * written as-is.
 280                  */
 281                 if (cap_offs >= 4)
 282                         return PCI_ACCESS_DONE;
 283         } else if (cap->id == PCI_CAP_MSIX && cap_offs < 4) {
 284                 device->msix_registers.raw &= ~mask;
 285                 device->msix_registers.raw |= value;
 286
 287                 if (pci_update_msix(device, cap) < 0)
 288                         return PCI_ACCESS_REJECT;
 289         }
 290
 291         return PCI_ACCESS_PERFORM;
 292 }
 293
 294 /**
 295  * Initialization of PCI subsystem.
 296  *
 297  * @return 0 on success, negative error code otherwise.
 298  */
 299 int pci_init(void)
 300 {
 301         unsigned int mmcfg_size;
 302         int err;
 303
 304         err = pci_cell_init(&root_cell);
 305         if (err)
 306                 return err;
 307
 308         mmcfg_start = system_config->platform_info.x86.mmconfig_base;
 309         if (mmcfg_start == 0)
 310                 return 0;
 311
 312         end_bus = system_config->platform_info.x86.mmconfig_end_bus;
 313         mmcfg_size = (end_bus + 1) * 256 * 4096;
 314         mmcfg_end = mmcfg_start + mmcfg_size - 4;
 315
 316         pci_space = page_alloc(&remap_pool, mmcfg_size / PAGE_SIZE);
 317         if (!pci_space)
 318                 return -ENOMEM;
 319
 320         return page_map_create(&hv_paging_structs, mmcfg_start, mmcfg_size,
 321                                (unsigned long)pci_space,
 322                                PAGE_DEFAULT_FLAGS | PAGE_FLAG_UNCACHED,
 323                                PAGE_MAP_NON_COHERENT);
 324 }
 325
 326 static int pci_msix_access_handler(const struct cell *cell, bool is_write,
 327                                    u64 addr, u32 *value)
 328 {
 329         unsigned int dword = (addr % sizeof(union pci_msix_vector)) >> 2;
 330         struct pci_device *device = cell->msix_device_list;
 331         unsigned int index;
 332         u64 offs;
 333
 334         while (device) {
 335                 if (addr >= device->info->msix_address &&
 336                     addr < device->info->msix_address +
 337                            device->info->msix_region_size)
 338                         goto found;
 339                 device = device->next_msix_device;
 340         }
 341         return 0;
 342
 343 found:
 344         /* access must be DWORD-aligned */
 345         if (addr & 0x3)
 346                 goto invalid_access;
 347
 348         offs = addr - device->info->msix_address;
 349         index = offs / sizeof(union pci_msix_vector);
 350
 351         if (is_write) {
 352                 /*
 353                  * The PBA may share a page with the MSI-X table. Writing to
 354                  * PBA entries is undefined. We declare it as invalid.
 355                  */
 356                 if (index >= device->info->num_msix_vectors)
 357                         goto invalid_access;
 358                 if (dword == MSIX_VECTOR_CTRL_DWORD) {
 359                         mmio_write32(&device->msix_table[index].field.ctrl,
 360                                      *value);
 361                 } else {
 362                         device->msix_vectors[index].raw[dword] = *value;
 363                         if (arch_pci_update_msix_vector(device, index) < 0)
 364                                 goto invalid_access;
 365                 }
 366         } else {
 367                 if (index >= device->info->num_msix_vectors ||
 368                     dword == MSIX_VECTOR_CTRL_DWORD)
 369                         *value =
 370                             mmio_read32(((void *)device->msix_table) + offs);
 371                 else
 372                         *value = device->msix_vectors[index].raw[dword];
 373         }
 374         return 1;
 375
 376 invalid_access:
 377         panic_printk("FATAL: Invalid PCI MSIX BAR write, device "
 378                      "%02x:%02x.%x\n", PCI_BDF_PARAMS(device->info->bdf));
 379         return -1;
 380 }
 381
 382 /**
 383  * Handler for MMIO-accesses to PCI config space.
 384  * @param cell          Request issuing cell.
 385  * @param is_write      True if write access.
 386  * @param addr          Address accessed.
 387  * @param value         Pointer to value for reading/writing.
 388  *
 389  * @return 1 if handled successfully, 0 if unhandled, -1 on access error.
 390  */
 391 int pci_mmio_access_handler(const struct cell *cell, bool is_write,
 392                             u64 addr, u32 *value)
 393 {
 394         u32 mmcfg_offset, reg_addr;
 395         struct pci_device *device;
 396         enum pci_access access;
 397
 398         if (!pci_space || addr < mmcfg_start || addr > mmcfg_end)
 399                 return pci_msix_access_handler(cell, is_write, addr, value);
 400
 401         mmcfg_offset = addr - mmcfg_start;
 402         reg_addr = mmcfg_offset & 0xfff;
 403         /* access must be DWORD-aligned */
 404         if (reg_addr & 0x3)
 405                 goto invalid_access;
 406
 407         device = pci_get_assigned_device(cell, mmcfg_offset >> 12);
 408
 409         if (is_write) {
 410                 access = pci_cfg_write_moderate(device, reg_addr, 4, *value);
 411                 if (access == PCI_ACCESS_REJECT)
 412                         goto invalid_access;
 413                 if (access == PCI_ACCESS_PERFORM)
 414                         mmio_write32(pci_space + mmcfg_offset, *value);
 415         } else {
 416                 access = pci_cfg_read_moderate(device, reg_addr, 4, value);
 417                 if (access == PCI_ACCESS_PERFORM)
 418                         *value = mmio_read32(pci_space + mmcfg_offset);
 419         }
 420
 421         return 1;
 422
 423 invalid_access:
 424         panic_printk("FATAL: Invalid PCI MMCONFIG write, device %02x:%02x.%x, "
 425                      "reg: %\n", PCI_BDF_PARAMS(mmcfg_offset >> 12), reg_addr);
 426         return -1;
 427
 428 }
 429
 430 /**
 431  * Retrieve number of enabled MSI vector of a device.
 432  * @param device        The device to be examined.
 433  *
 434  * @return number of vectors.
 435  */
 436 unsigned int pci_enabled_msi_vectors(struct pci_device *device)
 437 {
 438         return device->msi_registers.msg32.enable ?
 439                 1 << device->msi_registers.msg32.mme : 0;
 440 }
 441
 442 static void pci_save_msi(struct pci_device *device,
 443                          const struct jailhouse_pci_capability *cap)
 444 {
 445         u16 bdf = device->info->bdf;
 446         unsigned int n;
 447
 448         for (n = 0; n < (device->info->msi_64bits ? 4 : 3); n++)
 449                 device->msi_registers.raw[n] =
 450                         pci_read_config(bdf, cap->start + n * 4, 4);
 451 }
 452
 453 static void pci_restore_msi(struct pci_device *device,
 454                             const struct jailhouse_pci_capability *cap)
 455 {
 456         unsigned int n;
 457
 458         for (n = 1; n < (device->info->msi_64bits ? 4 : 3); n++)
 459                 pci_write_config(device->info->bdf, cap->start + n * 4,
 460                                  device->msi_registers.raw[n], 4);
 461 }
 462
 463 static void pci_suppress_msix(struct pci_device *device,
 464                               const struct jailhouse_pci_capability *cap,
 465                               bool suppressed)
 466 {
 467         union pci_msix_registers regs = device->msix_registers;
 468
 469         if (suppressed)
 470                 regs.field.fmask = 1;
 471         pci_write_config(device->info->bdf, cap->start, regs.raw, 4);
 472 }
 473
 474 static void pci_save_msix(struct pci_device *device,
 475                           const struct jailhouse_pci_capability *cap)
 476 {
 477         unsigned int n, r;
 478
 479         device->msix_registers.raw =
 480                 pci_read_config(device->info->bdf, cap->start, 4);
 481
 482         for (n = 0; n < device->info->num_msix_vectors; n++)
 483                 for (r = 0; r < 3; r++)
 484                         device->msix_vectors[n].raw[r] =
 485                                 mmio_read32(&device->msix_table[n].raw[r]);
 486 }
 487
 488 static void pci_restore_msix(struct pci_device *device,
 489                              const struct jailhouse_pci_capability *cap)
 490 {
 491         unsigned int n, r;
 492
 493         for (n = 0; n < device->info->num_msix_vectors; n++)
 494                 for (r = 0; r < 3; r++)
 495                         mmio_write32(&device->msix_table[n].raw[r],
 496                                      device->msix_vectors[n].raw[r]);
 497         pci_suppress_msix(device, cap, false);
 498 }
 499
 500 /**
 501  * Prepare the handover of PCI devices to Jailhouse or back to Linux.
 502  */
 503 void pci_prepare_handover(void)
 504 {
 505         const struct jailhouse_pci_capability *cap;
 506         struct pci_device *device;
 507         unsigned int n;
 508
 509         if (!root_cell.pci_devices)
 510                 return;
 511
 512         for_each_configured_pci_device(device, &root_cell) {
 513                 if (device->cell)
 514                         for_each_pci_cap(cap, device, n)
 515                                 if (cap->id == PCI_CAP_MSI)
 516                                         arch_pci_suppress_msi(device, cap);
 517                                 else if (cap->id == PCI_CAP_MSIX)
 518                                         pci_suppress_msix(device, cap, true);
 519         }
 520 }
 521
 522 static int pci_add_device(struct cell *cell, struct pci_device *device)
 523 {
 524         unsigned int size = device->info->msix_region_size;
 525         int err;
 526
 527         printk("Adding PCI device %02x:%02x.%x to cell \"%s\"\n",
 528                PCI_BDF_PARAMS(device->info->bdf), cell->config->name);
 529
 530         err = arch_pci_add_device(cell, device);
 531
 532         if (!err && device->info->msix_address) {
 533                 device->msix_table = page_alloc(&remap_pool, size / PAGE_SIZE);
 534                 if (!device->msix_table) {
 535                         err = -ENOMEM;
 536                         goto error_remove_dev;
 537                 }
 538
 539                 err = page_map_create(&hv_paging_structs,
 540                                       device->info->msix_address, size,
 541                                       (unsigned long)device->msix_table,
 542                                       PAGE_DEFAULT_FLAGS | PAGE_FLAG_UNCACHED,
 543                                       PAGE_MAP_NON_COHERENT);
 544                 if (err)
 545                         goto error_page_free;
 546
 547                 device->next_msix_device = cell->msix_device_list;
 548                 cell->msix_device_list = device;
 549         }
 550         return 0;
 551
 552 error_page_free:
 553         page_free(&remap_pool, device->msix_table, size / PAGE_SIZE);
 554 error_remove_dev:
 555         arch_pci_remove_device(device);
 556         return err;
 557 }
 558
 559 static void pci_remove_device(struct pci_device *device)
 560 {
 561         unsigned int size = device->info->msix_region_size;
 562         struct pci_device *prev_msix_device;
 563
 564         printk("Removing PCI device %02x:%02x.%x from cell \"%s\"\n",
 565                PCI_BDF_PARAMS(device->info->bdf), device->cell->config->name);
 566         arch_pci_remove_device(device);
 567         pci_write_config(device->info->bdf, PCI_CFG_COMMAND,
 568                          PCI_CMD_INTX_OFF, 2);
 569
 570         if (!device->msix_table)
 571                 return;
 572
 573         /* cannot fail, destruction of same size as construction */
 574         page_map_destroy(&hv_paging_structs, (unsigned long)device->msix_table,
 575                          size, PAGE_MAP_NON_COHERENT);
 576         page_free(&remap_pool, device->msix_table, size / PAGE_SIZE);
 577
 578         prev_msix_device = device->cell->msix_device_list;
 579         if (prev_msix_device == device) {
 580                 device->cell->msix_device_list = NULL;
 581         } else {
 582                 while (prev_msix_device->next_msix_device != device)
 583                         prev_msix_device = prev_msix_device->next_msix_device;
 584                 prev_msix_device->next_msix_device = NULL;
 585         }
 586 }
 587
 588 /**
 589  * Perform PCI-specific initialization for a new cell.
 590  * @param cell  Cell to be initialized.
 591  *
 592  * @return 0 on success, negative error code otherwise.
 593  *
 594  * @see pci_cell_exit
 595  */
 596 int pci_cell_init(struct cell *cell)
 597 {
 598         unsigned int devlist_pages = PAGES(cell->config->num_pci_devices *
 599                                            sizeof(struct pci_device));
 600         const struct jailhouse_pci_device *dev_infos =
 601                 jailhouse_cell_pci_devices(cell->config);
 602         const struct jailhouse_pci_capability *cap;
 603         struct pci_device *device, *root_device;
 604         unsigned int ndev, ncap;
 605         int err;
 606
 607         cell->pci_devices = page_alloc(&mem_pool, devlist_pages);
 608         if (!cell->pci_devices)
 609                 return -ENOMEM;
 610
 611         /*
 612          * We order device states in the same way as the static information
 613          * so that we can use the index of the latter to find the former. For
 614          * the other way around and for obtaining the owner cell, we use more
 615          * handy pointers. The cell pointer also encodes active ownership.
 616          */
 617         for (ndev = 0; ndev < cell->config->num_pci_devices; ndev++) {
 618                 if (dev_infos[ndev].num_msix_vectors > PCI_MAX_MSIX_VECTORS) {
 619                         pci_cell_exit(cell);
 620                         return -ERANGE;
 621                 }
 622
 623                 device = &cell->pci_devices[ndev];
 624                 device->info = &dev_infos[ndev];
 625
 626                 root_device = pci_get_assigned_device(&root_cell,
 627                                                       dev_infos[ndev].bdf);
 628                 if (root_device) {
 629                         pci_remove_device(root_device);
 630                         root_device->cell = NULL;
 631                 }
 632
 633                 err = pci_add_device(cell, device);
 634                 if (err) {
 635                         pci_cell_exit(cell);
 636                         return err;
 637                 }
 638
 639                 device->cell = cell;
 640
 641                 for_each_pci_cap(cap, device, ncap)
 642                         if (cap->id == PCI_CAP_MSI)
 643                                 pci_save_msi(device, cap);
 644                         else if (cap->id == PCI_CAP_MSIX)
 645                                 pci_save_msix(device, cap);
 646         }
 647
 648         if (cell == &root_cell)
 649                 pci_prepare_handover();
 650
 651         return 0;
 652 }
 653
 654 static void pci_return_device_to_root_cell(struct pci_device *device)
 655 {
 656         struct pci_device *root_device;
 657
 658         for_each_configured_pci_device(root_device, &root_cell)
 659                 if (root_device->info->domain == device->info->domain &&
 660                     root_device->info->bdf == device->info->bdf) {
 661                         if (pci_add_device(&root_cell, root_device) < 0)
 662                                 printk("WARNING: Failed to re-assign PCI "
 663                                        "device to root cell\n");
 664                         else
 665                                 root_device->cell = &root_cell;
 666                         break;
 667                 }
 668 }
 669
 670 /**
 671  * Perform PCI-specific cleanup for a cell under destruction.
 672  * @param cell  Cell to be destructed.
 673  *
 674  * @see pci_cell_init
 675  */
 676 void pci_cell_exit(struct cell *cell)
 677 {
 678         unsigned int devlist_pages = PAGES(cell->config->num_pci_devices *
 679                                            sizeof(struct pci_device));
 680         struct pci_device *device;
 681
 682         /*
 683          * Do not destroy the root cell. We will shut down the complete
 684          * hypervisor instead.
 685          */
 686         if (cell == &root_cell)
 687                 return;
 688
 689         for_each_configured_pci_device(device, cell)
 690                 if (device->cell) {
 691                         pci_remove_device(device);
 692                         pci_return_device_to_root_cell(device);
 693                 }
 694
 695         page_free(&mem_pool, cell->pci_devices, devlist_pages);
 696 }
 697
 698 /**
 699  * Apply PCI-specific configuration changes.
 700  * @param cell_added_removed    Cell that was added or removed to/from the
 701  *                              system or NULL.
 702  */
 703 void pci_config_commit(struct cell *cell_added_removed)
 704 {
 705         const struct jailhouse_pci_capability *cap;
 706         struct pci_device *device;
 707         unsigned int n;
 708         int err = 0;
 709
 710         if (!cell_added_removed)
 711                 return;
 712
 713         for_each_configured_pci_device(device, &root_cell)
 714                 if (device->cell)
 715                         for_each_pci_cap(cap, device, n) {
 716                                 if (cap->id == PCI_CAP_MSI) {
 717                                         err = arch_pci_update_msi(device, cap);
 718                                 } else if (cap->id == PCI_CAP_MSIX) {
 719                                         err = pci_update_msix(device, cap);
 720                                         pci_suppress_msix(device, cap, false);
 721                                 }
 722                                 if (err)
 723                                         goto error;
 724                         }
 725         return;
 726
 727 error:
 728         panic_printk("FATAL: Unsupported MSI/MSI-X state, device %02x:%02x.%x,"
 729                      " cap %d\n", PCI_BDF_PARAMS(device->info->bdf), cap->id);
 730         panic_stop();
 731 }
 732
 733 /**
 734  * Shut down the PCI layer during hypervisor deactivation.
 735  */
 736 void pci_shutdown(void)
 737 {
 738         const struct jailhouse_pci_capability *cap;
 739         struct pci_device *device;
 740         unsigned int n;
 741
 742         if (!root_cell.pci_devices)
 743                 return;
 744
 745         for_each_configured_pci_device(device, &root_cell)
 746                 if (device->cell)
 747                         for_each_pci_cap(cap, device, n)
 748                                 if (cap->id == PCI_CAP_MSI)
 749                                         pci_restore_msi(device, cap);
 750                                 else if (cap->id == PCI_CAP_MSIX)
 751                                         pci_restore_msix(device, cap);
 752 }