hypervisor/pci.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2014, 2015
   5  *
   6  * Authors:
   7  *  Ivan Kolchin <ivan.kolchin@siemens.com>
   8  *  Jan Kiszka <jan.kiszka@siemens.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include <jailhouse/control.h>
  15 #include <jailhouse/mmio.h>
  16 #include <jailhouse/pci.h>
  17 #include <jailhouse/printk.h>
  18 #include <jailhouse/utils.h>
  19
  20 #define MSIX_VECTOR_CTRL_DWORD          3
  21
  22 #define for_each_configured_pci_device(dev, cell)                       \
  23         for ((dev) = (cell)->pci_devices;                               \
  24              (dev) - (cell)->pci_devices < (cell)->config->num_pci_devices; \
  25              (dev)++)
  26
  27 #define for_each_pci_cap(cap, dev, counter)                             \
  28         for ((cap) = jailhouse_cell_pci_caps((dev)->cell->config) +     \
  29                 (dev)->info->caps_start, (counter) = 0;                 \
  30              (counter) < (dev)->info->num_caps;                         \
  31              (cap)++, (counter)++)
  32
  33 /* entry for PCI config space access control */
  34 struct pci_cfg_control {
  35         enum {
  36                 PCI_CONFIG_DENY,
  37                 PCI_CONFIG_ALLOW,
  38                 PCI_CONFIG_RDONLY,
  39         } type;   /* Access type */
  40         u32 mask; /* Bit set: access type applies; bit cleared: deny access */
  41 };
  42
  43 /* --- Access control for writing to PCI config space registers --- */
  44 /* Type 1: Endpoints */
  45 static const struct pci_cfg_control endpoint_write[PCI_CONFIG_HEADER_SIZE] = {
  46         [0x04/4] = {PCI_CONFIG_ALLOW,  0xffffffff}, /* Command, Status */
  47         [0x0c/4] = {PCI_CONFIG_ALLOW,  0xff00ffff}, /* BIST, Lat., Cacheline */
  48         [0x30/4] = {PCI_CONFIG_RDONLY, 0xffffffff}, /* ROM BAR */
  49         [0x3c/4] = {PCI_CONFIG_ALLOW,  0x000000ff}, /* Int Line */
  50 };
  51
  52 /* Type 2: Bridges
  53  * Note: Ignore limit/base reprogramming attempts because the root cell will
  54  *       perform them on bus rescans. */
  55 static const struct pci_cfg_control bridge_write[PCI_CONFIG_HEADER_SIZE] = {
  56         [0x04/4] = {PCI_CONFIG_ALLOW,  0xffffffff}, /* Command, Status */
  57         [0x0c/4] = {PCI_CONFIG_ALLOW,  0xff00ffff}, /* BIST, Lat., Cacheline */
  58         [0x1c/4] = {PCI_CONFIG_RDONLY, 0x0000ffff}, /* I/O Limit & Base */
  59         [0x20/4 ...      /* Memory Limit/Base, Prefetch Memory Limit/Base, */
  60          0x30/4] = {PCI_CONFIG_RDONLY, 0xffffffff}, /* I/O Limit & Base */
  61         [0x3c/4] = {PCI_CONFIG_ALLOW,  0xffff00ff}, /* Int Line, Bridge Ctrl */
  62 };
  63
  64 static void *pci_space;
  65 static u64 mmcfg_start, mmcfg_end;
  66 static u8 end_bus;
  67
  68 static void *pci_get_device_mmcfg_base(u16 bdf)
  69 {
  70         return pci_space + ((unsigned long)bdf << 12);
  71 }
  72
  73 /**
  74  * Read from PCI config space.
  75  * @param bdf           16-bit bus/device/function ID of target.
  76  * @param address       Config space access address.
  77  * @param size          Access size (1, 2 or 4 bytes).
  78  *
  79  * @return Read value.
  80  *
  81  * @see pci_write_config
  82  */
  83 u32 pci_read_config(u16 bdf, u16 address, unsigned int size)
  84 {
  85         void *mmcfg_addr = pci_get_device_mmcfg_base(bdf) + address;
  86
  87         if (!pci_space || PCI_BUS(bdf) > end_bus)
  88                 return arch_pci_read_config(bdf, address, size);
  89
  90         if (size == 1)
  91                 return mmio_read8(mmcfg_addr);
  92         else if (size == 2)
  93                 return mmio_read16(mmcfg_addr);
  94         else
  95                 return mmio_read32(mmcfg_addr);
  96 }
  97
  98 /**
  99  * Write to PCI config space.
 100  * @param bdf           16-bit bus/device/function ID of target.
 101  * @param address       Config space access address.
 102  * @param value         Value to be written.
 103  * @param size          Access size (1, 2 or 4 bytes).
 104  *
 105  * @see pci_read_config
 106  */
 107 void pci_write_config(u16 bdf, u16 address, u32 value, unsigned int size)
 108 {
 109         void *mmcfg_addr = pci_get_device_mmcfg_base(bdf) + address;
 110
 111         if (!pci_space || PCI_BUS(bdf) > end_bus)
 112                 return arch_pci_write_config(bdf, address, value, size);
 113
 114         if (size == 1)
 115                 mmio_write8(mmcfg_addr, value);
 116         else if (size == 2)
 117                 mmio_write16(mmcfg_addr, value);
 118         else
 119                 mmio_write32(mmcfg_addr, value);
 120 }
 121
 122 /**
 123  * Look up device owned by a cell.
 124  * @param[in] cell      Owning cell.
 125  * @param bdf           16-bit bus/device/function ID.
 126  *
 127  * @return Pointer to owned PCI device or NULL.
 128  */
 129 struct pci_device *pci_get_assigned_device(const struct cell *cell, u16 bdf)
 130 {
 131         const struct jailhouse_pci_device *dev_info =
 132                 jailhouse_cell_pci_devices(cell->config);
 133         u32 n;
 134
 135         /* We iterate over the static device information to increase cache
 136          * locality. */
 137         for (n = 0; n < cell->config->num_pci_devices; n++)
 138                 if (dev_info[n].bdf == bdf)
 139                         return cell->pci_devices[n].cell ?
 140                                 &cell->pci_devices[n] : NULL;
 141
 142         return NULL;
 143 }
 144
 145 /**
 146  * Look up capability at given config space address.
 147  * @param device        The device to be accessed.
 148  * @param address       Config space access address.
 149  *
 150  * @return Corresponding capability structure or NULL if none found.
 151  *
 152  * @private
 153  */
 154 static const struct jailhouse_pci_capability *
 155 pci_find_capability(struct pci_device *device, u16 address)
 156 {
 157         const struct jailhouse_pci_capability *cap =
 158                 jailhouse_cell_pci_caps(device->cell->config) +
 159                 device->info->caps_start;
 160         u32 n;
 161
 162         for (n = 0; n < device->info->num_caps; n++, cap++)
 163                 if (cap->start <= address && cap->start + cap->len > address)
 164                         return cap;
 165
 166         return NULL;
 167 }
 168
 169 /**
 170  * Moderate config space read access.
 171  * @param device        The device to be accessed. If NULL, access will be
 172  *                      emulated, returning a value of -1.
 173  * @param address       Config space address.
 174  * @param size          Access size (1, 2 or 4 bytes).
 175  * @param value         Pointer to buffer to receive the emulated value if
 176  *                      PCI_ACCESS_DONE is returned.
 177  *
 178  * @return PCI_ACCESS_PERFORM or PCI_ACCESS_DONE.
 179  *
 180  * @see pci_cfg_write_moderate
 181  */
 182 enum pci_access pci_cfg_read_moderate(struct pci_device *device, u16 address,
 183                                       unsigned int size, u32 *value)
 184 {
 185         const struct jailhouse_pci_capability *cap;
 186         unsigned int bar_no, cap_offs;
 187
 188         if (!device) {
 189                 *value = -1;
 190                 return PCI_ACCESS_DONE;
 191         }
 192
 193         if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM)
 194                 return pci_ivshmem_cfg_read(device, address, size, value);
 195
 196         /* Emulate BARs for physical devices */
 197         if (device->info->type == JAILHOUSE_PCI_TYPE_DEVICE) {
 198                 /* Emulate BAR access, always returning the shadow value. */
 199                 if (address >= PCI_CFG_BAR && address <= PCI_CFG_BAR_END) {
 200                         bar_no = (address - PCI_CFG_BAR) / 4;
 201                         *value = device->bar[bar_no] >> ((address % 4) * 8);
 202                         return PCI_ACCESS_DONE;
 203                 }
 204
 205                 /* We do not expose ROMs. */
 206                 if (address >= PCI_CFG_ROMBAR && address < PCI_CFG_CAPS) {
 207                         *value = 0;
 208                         return PCI_ACCESS_DONE;
 209                 }
 210         }
 211
 212         if (address < PCI_CONFIG_HEADER_SIZE)
 213                 return PCI_ACCESS_PERFORM;
 214
 215         cap = pci_find_capability(device, address);
 216         if (!cap)
 217                 return PCI_ACCESS_PERFORM;
 218
 219         cap_offs = address - cap->start;
 220         if (cap->id == PCI_CAP_MSI && cap_offs >= 4 &&
 221             (cap_offs < 10 || (device->info->msi_64bits && cap_offs < 14))) {
 222                 *value = device->msi_registers.raw[cap_offs / 4] >>
 223                         ((cap_offs % 4) * 8);
 224                 return PCI_ACCESS_DONE;
 225         }
 226
 227         return PCI_ACCESS_PERFORM;
 228 }
 229
 230 static int pci_update_msix(struct pci_device *device,
 231                            const struct jailhouse_pci_capability *cap)
 232 {
 233         unsigned int n;
 234         int result;
 235
 236         for (n = 0; n < device->info->num_msix_vectors; n++) {
 237                 result = arch_pci_update_msix_vector(device, n);
 238                 if (result < 0)
 239                         return result;
 240         }
 241         return 0;
 242 }
 243
 244 /**
 245  * Moderate config space write access.
 246  * @param device        The device to be accessed. If NULL, access will be
 247  *                      rejected.
 248  * @param address       Config space address.
 249  * @param size          Access size (1, 2 or 4 bytes).
 250  * @param value         Value to be written.
 251  *
 252  * @return PCI_ACCESS_REJECT, PCI_ACCESS_PERFORM or PCI_ACCESS_DONE.
 253  *
 254  * @see pci_cfg_read_moderate
 255  */
 256 enum pci_access pci_cfg_write_moderate(struct pci_device *device, u16 address,
 257                                        unsigned int size, u32 value)
 258 {
 259         const struct jailhouse_pci_capability *cap;
 260         /* initialize list to work around wrong compiler warning */
 261         unsigned int bias_shift = (address % 4) * 8;
 262         u32 mask = BYTE_MASK(size) << bias_shift;
 263         struct pci_cfg_control cfg_control;
 264         unsigned int bar_no, cap_offs;
 265
 266         if (!device)
 267                 return PCI_ACCESS_REJECT;
 268
 269         value <<= bias_shift;
 270
 271         if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM)
 272                 return pci_ivshmem_cfg_write(device, address / 4, mask, value);
 273
 274         /* Emulate BARs for physical devices */
 275         if (device->info->type == JAILHOUSE_PCI_TYPE_DEVICE &&
 276             address >= PCI_CFG_BAR && address <= PCI_CFG_BAR_END) {
 277                 bar_no = (address - PCI_CFG_BAR) / 4;
 278                 mask &= device->info->bar_mask[bar_no];
 279                 device->bar[bar_no] &= ~mask;
 280                 device->bar[bar_no] |= value & mask;
 281                 return PCI_ACCESS_DONE;
 282         }
 283
 284         if (address < PCI_CONFIG_HEADER_SIZE) {
 285                 if (device->info->type == JAILHOUSE_PCI_TYPE_BRIDGE)
 286                         cfg_control = bridge_write[address / 4];
 287                 else /* physical device */
 288                         cfg_control = endpoint_write[address / 4];
 289
 290                 if ((cfg_control.mask & mask) != mask)
 291                         return PCI_ACCESS_REJECT;
 292
 293                 switch (cfg_control.type) {
 294                 case PCI_CONFIG_ALLOW:
 295                         return PCI_ACCESS_PERFORM;
 296                 case PCI_CONFIG_RDONLY:
 297                         return PCI_ACCESS_DONE;
 298                 default:
 299                         return PCI_ACCESS_REJECT;
 300                 }
 301         }
 302
 303         cap = pci_find_capability(device, address);
 304         if (!cap || !(cap->flags & JAILHOUSE_PCICAPS_WRITE))
 305                 return PCI_ACCESS_REJECT;
 306
 307         cap_offs = address - cap->start;
 308         if (cap->id == PCI_CAP_MSI &&
 309             (cap_offs < 10 || (device->info->msi_64bits && cap_offs < 14))) {
 310                 device->msi_registers.raw[cap_offs / 4] &= ~mask;
 311                 device->msi_registers.raw[cap_offs / 4] |= value;
 312
 313                 if (arch_pci_update_msi(device, cap) < 0)
 314                         return PCI_ACCESS_REJECT;
 315
 316                 /*
 317                  * Address and data words are emulated, the control word is
 318                  * written as-is.
 319                  */
 320                 if (cap_offs >= 4)
 321                         return PCI_ACCESS_DONE;
 322         } else if (cap->id == PCI_CAP_MSIX && cap_offs < 4) {
 323                 device->msix_registers.raw &= ~mask;
 324                 device->msix_registers.raw |= value;
 325
 326                 if (pci_update_msix(device, cap) < 0)
 327                         return PCI_ACCESS_REJECT;
 328         }
 329
 330         return PCI_ACCESS_PERFORM;
 331 }
 332
 333 /**
 334  * Initialization of PCI subsystem.
 335  *
 336  * @return 0 on success, negative error code otherwise.
 337  */
 338 int pci_init(void)
 339 {
 340         unsigned int mmcfg_size;
 341         int err;
 342
 343         err = pci_cell_init(&root_cell);
 344         if (err)
 345                 return err;
 346
 347         mmcfg_start = system_config->platform_info.x86.mmconfig_base;
 348         if (mmcfg_start == 0)
 349                 return 0;
 350
 351         end_bus = system_config->platform_info.x86.mmconfig_end_bus;
 352         mmcfg_size = (end_bus + 1) * 256 * 4096;
 353         mmcfg_end = mmcfg_start + mmcfg_size - 4;
 354
 355         pci_space = page_alloc(&remap_pool, mmcfg_size / PAGE_SIZE);
 356         if (!pci_space)
 357                 return trace_error(-ENOMEM);
 358
 359         return paging_create(&hv_paging_structs, mmcfg_start, mmcfg_size,
 360                              (unsigned long)pci_space,
 361                              PAGE_DEFAULT_FLAGS | PAGE_FLAG_DEVICE,
 362                              PAGING_NON_COHERENT);
 363 }
 364
 365 static int pci_msix_access_handler(const struct cell *cell, bool is_write,
 366                                    u64 addr, u32 *value)
 367 {
 368         unsigned int dword = (addr % sizeof(union pci_msix_vector)) >> 2;
 369         struct pci_device *device = cell->msix_device_list;
 370         unsigned int index;
 371         u64 offs;
 372
 373         while (device) {
 374                 if (addr >= device->info->msix_address &&
 375                     addr < device->info->msix_address +
 376                            device->info->msix_region_size)
 377                         goto found;
 378                 device = device->next_msix_device;
 379         }
 380         return 0;
 381
 382 found:
 383         /* access must be DWORD-aligned */
 384         if (addr & 0x3)
 385                 goto invalid_access;
 386
 387         offs = addr - device->info->msix_address;
 388         index = offs / sizeof(union pci_msix_vector);
 389
 390         if (is_write) {
 391                 /*
 392                  * The PBA may share a page with the MSI-X table. Writing to
 393                  * PBA entries is undefined. We declare it as invalid.
 394                  */
 395                 if (index >= device->info->num_msix_vectors)
 396                         goto invalid_access;
 397
 398                 device->msix_vectors[index].raw[dword] = *value;
 399                 if (arch_pci_update_msix_vector(device, index) < 0)
 400                         goto invalid_access;
 401
 402                 if (dword == MSIX_VECTOR_CTRL_DWORD)
 403                         mmio_write32(&device->msix_table[index].raw[dword],
 404                                      *value);
 405         } else {
 406                 if (index >= device->info->num_msix_vectors ||
 407                     dword == MSIX_VECTOR_CTRL_DWORD)
 408                         *value =
 409                             mmio_read32(((void *)device->msix_table) + offs);
 410                 else
 411                         *value = device->msix_vectors[index].raw[dword];
 412         }
 413         return 1;
 414
 415 invalid_access:
 416         panic_printk("FATAL: Invalid PCI MSI-X table/PBA access, device "
 417                      "%02x:%02x.%x\n", PCI_BDF_PARAMS(device->info->bdf));
 418         return -1;
 419 }
 420
 421 /**
 422  * Handler for MMIO-accesses to PCI config space.
 423  * @param cell          Request issuing cell.
 424  * @param is_write      True if write access.
 425  * @param addr          Address accessed.
 426  * @param value         Pointer to value for reading/writing.
 427  *
 428  * @return 1 if handled successfully, 0 if unhandled, -1 on access error.
 429  */
 430 int pci_mmio_access_handler(const struct cell *cell, bool is_write,
 431                             u64 addr, u32 *value)
 432 {
 433         u32 mmcfg_offset, reg_addr;
 434         struct pci_device *device;
 435         enum pci_access access;
 436         int ret;
 437
 438         if (!pci_space || addr < mmcfg_start || addr > mmcfg_end) {
 439                 ret = pci_msix_access_handler(cell, is_write, addr, value);
 440                 if (ret == 0)
 441                         ret = ivshmem_mmio_access_handler(cell, is_write, addr,
 442                                                           value);
 443                 return ret;
 444         }
 445
 446         mmcfg_offset = addr - mmcfg_start;
 447         reg_addr = mmcfg_offset & 0xfff;
 448         /* access must be DWORD-aligned */
 449         if (reg_addr & 0x3)
 450                 goto invalid_access;
 451
 452         device = pci_get_assigned_device(cell, mmcfg_offset >> 12);
 453
 454         if (is_write) {
 455                 access = pci_cfg_write_moderate(device, reg_addr, 4, *value);
 456                 if (access == PCI_ACCESS_REJECT)
 457                         goto invalid_access;
 458                 if (access == PCI_ACCESS_PERFORM)
 459                         mmio_write32(pci_space + mmcfg_offset, *value);
 460         } else {
 461                 access = pci_cfg_read_moderate(device, reg_addr, 4, value);
 462                 if (access == PCI_ACCESS_PERFORM)
 463                         *value = mmio_read32(pci_space + mmcfg_offset);
 464         }
 465
 466         return 1;
 467
 468 invalid_access:
 469         panic_printk("FATAL: Invalid PCI MMCONFIG write, device %02x:%02x.%x, "
 470                      "reg: %\n", PCI_BDF_PARAMS(mmcfg_offset >> 12), reg_addr);
 471         return -1;
 472
 473 }
 474
 475 /**
 476  * Retrieve number of enabled MSI vector of a device.
 477  * @param device        The device to be examined.
 478  *
 479  * @return number of vectors.
 480  */
 481 unsigned int pci_enabled_msi_vectors(struct pci_device *device)
 482 {
 483         return device->msi_registers.msg32.enable ?
 484                 1 << device->msi_registers.msg32.mme : 0;
 485 }
 486
 487 static void pci_save_msi(struct pci_device *device,
 488                          const struct jailhouse_pci_capability *cap)
 489 {
 490         u16 bdf = device->info->bdf;
 491         unsigned int n;
 492
 493         for (n = 0; n < (device->info->msi_64bits ? 4 : 3); n++)
 494                 device->msi_registers.raw[n] =
 495                         pci_read_config(bdf, cap->start + n * 4, 4);
 496 }
 497
 498 static void pci_restore_msi(struct pci_device *device,
 499                             const struct jailhouse_pci_capability *cap)
 500 {
 501         unsigned int n;
 502
 503         for (n = 1; n < (device->info->msi_64bits ? 4 : 3); n++)
 504                 pci_write_config(device->info->bdf, cap->start + n * 4,
 505                                  device->msi_registers.raw[n], 4);
 506 }
 507
 508 static void pci_suppress_msix(struct pci_device *device,
 509                               const struct jailhouse_pci_capability *cap,
 510                               bool suppressed)
 511 {
 512         union pci_msix_registers regs = device->msix_registers;
 513
 514         if (suppressed)
 515                 regs.fmask = 1;
 516         pci_write_config(device->info->bdf, cap->start, regs.raw, 4);
 517 }
 518
 519 static void pci_save_msix(struct pci_device *device,
 520                           const struct jailhouse_pci_capability *cap)
 521 {
 522         unsigned int n, r;
 523
 524         device->msix_registers.raw =
 525                 pci_read_config(device->info->bdf, cap->start, 4);
 526
 527         for (n = 0; n < device->info->num_msix_vectors; n++)
 528                 for (r = 0; r < 4; r++)
 529                         device->msix_vectors[n].raw[r] =
 530                                 mmio_read32(&device->msix_table[n].raw[r]);
 531 }
 532
 533 static void pci_restore_msix(struct pci_device *device,
 534                              const struct jailhouse_pci_capability *cap)
 535 {
 536         unsigned int n, r;
 537
 538         for (n = 0; n < device->info->num_msix_vectors; n++)
 539                 /* only restore address/data, control is write-through */
 540                 for (r = 0; r < 3; r++)
 541                         mmio_write32(&device->msix_table[n].raw[r],
 542                                      device->msix_vectors[n].raw[r]);
 543         pci_suppress_msix(device, cap, false);
 544 }
 545
 546 /**
 547  * Prepare the handover of PCI devices to Jailhouse or back to Linux.
 548  */
 549 void pci_prepare_handover(void)
 550 {
 551         const struct jailhouse_pci_capability *cap;
 552         struct pci_device *device;
 553         unsigned int n;
 554
 555         if (!root_cell.pci_devices)
 556                 return;
 557
 558         for_each_configured_pci_device(device, &root_cell) {
 559                 if (device->cell)
 560                         for_each_pci_cap(cap, device, n)
 561                                 if (cap->id == PCI_CAP_MSI)
 562                                         arch_pci_suppress_msi(device, cap);
 563                                 else if (cap->id == PCI_CAP_MSIX)
 564                                         pci_suppress_msix(device, cap, true);
 565         }
 566 }
 567
 568 static int pci_add_virtual_device(struct cell *cell, struct pci_device *device)
 569 {
 570         device->cell = cell;
 571         device->next_virtual_device = cell->virtual_device_list;
 572         cell->virtual_device_list = device;
 573         return 0;
 574 }
 575
 576 static int pci_add_physical_device(struct cell *cell, struct pci_device *device)
 577 {
 578         unsigned int n, size = device->info->msix_region_size;
 579         int err;
 580
 581         printk("Adding PCI device %02x:%02x.%x to cell \"%s\"\n",
 582                PCI_BDF_PARAMS(device->info->bdf), cell->config->name);
 583
 584         for (n = 0; n < PCI_NUM_BARS; n ++)
 585                 device->bar[n] = pci_read_config(device->info->bdf,
 586                                                  PCI_CFG_BAR + n * 4, 4);
 587
 588         err = arch_pci_add_physical_device(cell, device);
 589
 590         if (!err && device->info->msix_address) {
 591                 device->msix_table = page_alloc(&remap_pool, size / PAGE_SIZE);
 592                 if (!device->msix_table) {
 593                         err = trace_error(-ENOMEM);
 594                         goto error_remove_dev;
 595                 }
 596
 597                 err = paging_create(&hv_paging_structs,
 598                                     device->info->msix_address, size,
 599                                     (unsigned long)device->msix_table,
 600                                     PAGE_DEFAULT_FLAGS | PAGE_FLAG_DEVICE,
 601                                     PAGING_NON_COHERENT);
 602                 if (err)
 603                         goto error_page_free;
 604
 605                 device->next_msix_device = cell->msix_device_list;
 606                 cell->msix_device_list = device;
 607         }
 608         return err;
 609
 610 error_page_free:
 611         page_free(&remap_pool, device->msix_table, size / PAGE_SIZE);
 612 error_remove_dev:
 613         arch_pci_remove_physical_device(device);
 614         return err;
 615 }
 616
 617 static void pci_remove_virtual_device(struct pci_device *device)
 618 {
 619         struct pci_device *prev = device->cell->virtual_device_list;
 620
 621         if (prev == device) {
 622                 device->cell->virtual_device_list = device->next_virtual_device;
 623         } else {
 624                 while (prev->next_virtual_device != device)
 625                         prev = prev->next_virtual_device;
 626                 prev->next_virtual_device = device->next_virtual_device;
 627         }
 628 }
 629
 630 static void pci_remove_physical_device(struct pci_device *device)
 631 {
 632         unsigned int size = device->info->msix_region_size;
 633         struct pci_device *prev_msix_device;
 634
 635         printk("Removing PCI device %02x:%02x.%x from cell \"%s\"\n",
 636                PCI_BDF_PARAMS(device->info->bdf), device->cell->config->name);
 637         arch_pci_remove_physical_device(device);
 638         pci_write_config(device->info->bdf, PCI_CFG_COMMAND,
 639                          PCI_CMD_INTX_OFF, 2);
 640
 641         if (!device->msix_table)
 642                 return;
 643
 644         /* cannot fail, destruction of same size as construction */
 645         paging_destroy(&hv_paging_structs, (unsigned long)device->msix_table,
 646                        size, PAGING_NON_COHERENT);
 647         page_free(&remap_pool, device->msix_table, size / PAGE_SIZE);
 648
 649         prev_msix_device = device->cell->msix_device_list;
 650         if (prev_msix_device == device) {
 651                 device->cell->msix_device_list = device->next_msix_device;
 652         } else {
 653                 while (prev_msix_device->next_msix_device != device)
 654                         prev_msix_device = prev_msix_device->next_msix_device;
 655                 prev_msix_device->next_msix_device = device->next_msix_device;
 656         }
 657 }
 658
 659 /**
 660  * Perform PCI-specific initialization for a new cell.
 661  * @param cell  Cell to be initialized.
 662  *
 663  * @return 0 on success, negative error code otherwise.
 664  *
 665  * @see pci_cell_exit
 666  */
 667 int pci_cell_init(struct cell *cell)
 668 {
 669         unsigned int devlist_pages = PAGES(cell->config->num_pci_devices *
 670                                            sizeof(struct pci_device));
 671         const struct jailhouse_pci_device *dev_infos =
 672                 jailhouse_cell_pci_devices(cell->config);
 673         const struct jailhouse_pci_capability *cap;
 674         struct pci_device *device, *root_device;
 675         unsigned int ndev, ncap;
 676         int err;
 677
 678         cell->pci_devices = page_alloc(&mem_pool, devlist_pages);
 679         if (!cell->pci_devices)
 680                 return -ENOMEM;
 681
 682         /*
 683          * We order device states in the same way as the static information
 684          * so that we can use the index of the latter to find the former. For
 685          * the other way around and for obtaining the owner cell, we use more
 686          * handy pointers. The cell pointer also encodes active ownership.
 687          */
 688         for (ndev = 0; ndev < cell->config->num_pci_devices; ndev++) {
 689                 if (dev_infos[ndev].num_msix_vectors > PCI_MAX_MSIX_VECTORS) {
 690                         err = trace_error(-ERANGE);
 691                         goto error;
 692                 }
 693
 694                 device = &cell->pci_devices[ndev];
 695                 device->info = &dev_infos[ndev];
 696
 697                 if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM) {
 698                         err = pci_ivshmem_init(cell, device);
 699                         if (err)
 700                                 goto error;
 701                         err = pci_add_virtual_device(cell, device);
 702                         if (err)
 703                                 goto error;
 704                         continue;
 705                 }
 706
 707                 root_device = pci_get_assigned_device(&root_cell,
 708                                                       dev_infos[ndev].bdf);
 709                 if (root_device) {
 710                         pci_remove_physical_device(root_device);
 711                         root_device->cell = NULL;
 712                 }
 713
 714                 err = pci_add_physical_device(cell, device);
 715                 if (err)
 716                         goto error;
 717
 718                 device->cell = cell;
 719
 720                 for_each_pci_cap(cap, device, ncap)
 721                         if (cap->id == PCI_CAP_MSI)
 722                                 pci_save_msi(device, cap);
 723                         else if (cap->id == PCI_CAP_MSIX)
 724                                 pci_save_msix(device, cap);
 725         }
 726
 727         if (cell == &root_cell)
 728                 pci_prepare_handover();
 729
 730         return 0;
 731 error:
 732         pci_cell_exit(cell);
 733         return err;
 734 }
 735
 736 static void pci_return_device_to_root_cell(struct pci_device *device)
 737 {
 738         struct pci_device *root_device;
 739
 740         for_each_configured_pci_device(root_device, &root_cell)
 741                 if (root_device->info->domain == device->info->domain &&
 742                     root_device->info->bdf == device->info->bdf) {
 743                         if (pci_add_physical_device(&root_cell,
 744                                                     root_device) < 0)
 745                                 printk("WARNING: Failed to re-assign PCI "
 746                                        "device to root cell\n");
 747                         else
 748                                 root_device->cell = &root_cell;
 749                         break;
 750                 }
 751 }
 752
 753 /**
 754  * Perform PCI-specific cleanup for a cell under destruction.
 755  * @param cell  Cell to be destructed.
 756  *
 757  * @see pci_cell_init
 758  */
 759 void pci_cell_exit(struct cell *cell)
 760 {
 761         unsigned int devlist_pages = PAGES(cell->config->num_pci_devices *
 762                                            sizeof(struct pci_device));
 763         struct pci_device *device;
 764
 765         /*
 766          * Do not destroy the root cell. We will shut down the complete
 767          * hypervisor instead.
 768          */
 769         if (cell == &root_cell)
 770                 return;
 771
 772         for_each_configured_pci_device(device, cell)
 773                 if (device->cell) {
 774                         if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM) {
 775                                 pci_ivshmem_exit(device);
 776                                 pci_remove_virtual_device(device);
 777                         } else {
 778                                 pci_remove_physical_device(device);
 779                                 pci_return_device_to_root_cell(device);
 780                         }
 781                 }
 782
 783         page_free(&mem_pool, cell->pci_devices, devlist_pages);
 784 }
 785
 786 /**
 787  * Apply PCI-specific configuration changes.
 788  * @param cell_added_removed    Cell that was added or removed to/from the
 789  *                              system or NULL.
 790  *
 791  * @see arch_config_commit
 792  */
 793 void pci_config_commit(struct cell *cell_added_removed)
 794 {
 795         const struct jailhouse_pci_capability *cap;
 796         struct pci_device *device;
 797         unsigned int n;
 798         int err = 0;
 799
 800         if (!cell_added_removed)
 801                 return;
 802
 803         for_each_configured_pci_device(device, &root_cell)
 804                 if (device->cell) {
 805                         for_each_pci_cap(cap, device, n) {
 806                                 if (cap->id == PCI_CAP_MSI) {
 807                                         err = arch_pci_update_msi(device, cap);
 808                                 } else if (cap->id == PCI_CAP_MSIX) {
 809                                         err = pci_update_msix(device, cap);
 810                                         pci_suppress_msix(device, cap, false);
 811                                 }
 812                                 if (err)
 813                                         goto error;
 814                         }
 815                         if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM) {
 816                                 err = pci_ivshmem_update_msix(device);
 817                                 if (err) {
 818                                         cap = NULL;
 819                                         goto error;
 820                                 }
 821                         }
 822                 }
 823         return;
 824
 825 error:
 826         panic_printk("FATAL: Unsupported MSI/MSI-X state, device %02x:%02x.%x",
 827                      PCI_BDF_PARAMS(device->info->bdf));
 828         if (cap)
 829                 panic_printk(", cap %d\n", cap->id);
 830         else
 831                 panic_printk("\n");
 832         panic_stop();
 833 }
 834
 835 /**
 836  * Shut down the PCI layer during hypervisor deactivation.
 837  */
 838 void pci_shutdown(void)
 839 {
 840         const struct jailhouse_pci_capability *cap;
 841         struct pci_device *device;
 842         unsigned int n;
 843
 844         if (!root_cell.pci_devices)
 845                 return;
 846
 847         for_each_configured_pci_device(device, &root_cell) {
 848                 if (!device->cell)
 849                         continue;
 850
 851                 for_each_pci_cap(cap, device, n)
 852                         if (cap->id == PCI_CAP_MSI)
 853                                 pci_restore_msi(device, cap);
 854                         else if (cap->id == PCI_CAP_MSIX)
 855                                 pci_restore_msix(device, cap);
 856
 857                 if (device->cell != &root_cell)
 858                         pci_write_config(device->info->bdf, PCI_CFG_COMMAND,
 859                                          PCI_CMD_INTX_OFF, 2);
 860         }
 861 }