hypervisor/pci_ivshmem.c

   1 /*
   2  * Jailhouse, a Linux-based partitioning hypervisor
   3  *
   4  * Copyright (c) Siemens AG, 2014, 2015
   5  *
   6  * Author:
   7  *  Henning Schild <henning.schild@siemens.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  */
  12
  13 /** @addtogroup PCI-IVSHMEM
  14  * Inter Cell communication using a virtual PCI device. The device provides
  15  * shared memory and interrupts based on MSI-X.
  16  *
  17  * The implementation in Jailhouse provides a shared memory device between
  18  * exactly 2 cells. The link between the two PCI devices is established by
  19  * choosing the same BDF, memory location, and memory size.
  20  */
  21
  22 #include <jailhouse/control.h>
  23 #include <jailhouse/pci.h>
  24 #include <jailhouse/printk.h>
  25 #include <jailhouse/string.h>
  26 #include <jailhouse/utils.h>
  27 #include <jailhouse/processor.h>
  28 #include <asm/apic.h>
  29
  30 #define VIRTIO_VENDOR_ID        0x1af4
  31 #define IVSHMEM_DEVICE_ID       0x1110
  32
  33 /* in jailhouse we can not allow dynamic remapping of the actual shared memory
  34  * the location and the size are stored here. A memory-BAR size of 0 will tell
  35  * device drivers that they are dealing with a special ivshmem device */
  36 #define IVSHMEM_CFG_SHMEM_PTR   0x40
  37 #define IVSHMEM_CFG_SHMEM_SZ    0x48
  38
  39 #define IVSHMEM_MSIX_VECTORS    1
  40 #define IVSHMEM_CFG_MSIX_CAP    0x50
  41
  42 #define IVSHMEM_REG_IVPOS       8
  43 #define IVSHMEM_REG_DBELL       12
  44
  45 #define IVSHMEM_CFG_SIZE        (IVSHMEM_CFG_MSIX_CAP + 12)
  46
  47 #define IVSHMEM_BAR0_SIZE       256
  48 #define IVSHMEM_BAR4_SIZE       ((0x18 * IVSHMEM_MSIX_VECTORS + 0xf) & ~0xf)
  49
  50 struct pci_ivshmem_endpoint {
  51         u32 cspace[IVSHMEM_CFG_SIZE / sizeof(u32)];
  52         u32 ivpos;
  53         struct pci_device *device;
  54         struct pci_ivshmem_endpoint *remote;
  55         struct apic_irq_message irq_msg;
  56 };
  57
  58 struct pci_ivshmem_data {
  59         struct pci_ivshmem_endpoint eps[2];
  60         struct pci_ivshmem_data *next;
  61 };
  62
  63 static struct pci_ivshmem_data *ivshmem_list;
  64
  65 static const u32 default_cspace[IVSHMEM_CFG_SIZE / sizeof(u32)] = {
  66         [0x00/4] = (IVSHMEM_DEVICE_ID << 16) | VIRTIO_VENDOR_ID,
  67         [0x04/4] = (PCI_STS_CAPS << 16),
  68         [0x08/4] = PCI_DEV_CLASS_MEM << 24,
  69         [0x2c/4] = (IVSHMEM_DEVICE_ID << 16) | VIRTIO_VENDOR_ID,
  70         [0x34/4] = IVSHMEM_CFG_MSIX_CAP,
  71         /* MSI-X capability */
  72         [IVSHMEM_CFG_MSIX_CAP/4] = (0xC000 + IVSHMEM_MSIX_VECTORS - 1) << 16
  73                                    | (0x00 << 8) | PCI_CAP_MSIX,
  74         [(IVSHMEM_CFG_MSIX_CAP + 0x4)/4] = PCI_CFG_BAR/8 + 2,
  75         [(IVSHMEM_CFG_MSIX_CAP + 0x8)/4] = 0x10 * IVSHMEM_MSIX_VECTORS |
  76                                            (PCI_CFG_BAR/8 + 2),
  77 };
  78
  79 static bool ivshmem_is_msix_masked(struct pci_ivshmem_endpoint *ive)
  80 {
  81         union pci_msix_registers c;
  82
  83         /* global mask */
  84         c.raw = ive->cspace[IVSHMEM_CFG_MSIX_CAP/4];
  85         if (!c.enable || c.fmask)
  86                 return true;
  87
  88         /* local mask */
  89         if (ive->device->msix_vectors[0].masked)
  90                 return true;
  91
  92         /* PCI Bus Master */
  93         if (!(ive->cspace[PCI_CFG_COMMAND/4] & PCI_CMD_MASTER))
  94                 return true;
  95
  96         return false;
  97 }
  98
  99 static int ivshmem_update_msix(struct pci_ivshmem_endpoint *ive)
 100 {
 101         union x86_msi_vector msi = {
 102                 .raw.address = ive->device->msix_vectors[0].address,
 103                 .raw.data = ive->device->msix_vectors[0].data,
 104         };
 105         struct apic_irq_message irq_msg;
 106
 107         /* before doing anything mark the cached irq_msg as invalid,
 108          * on success it will be valid on return. */
 109         ive->irq_msg.valid = 0;
 110         memory_barrier();
 111
 112         if (ivshmem_is_msix_masked(ive))
 113                 return 0;
 114
 115         irq_msg = pci_translate_msi_vector(ive->device, 0, 0, msi);
 116         if (!irq_msg.valid)
 117                 return 0;
 118
 119         if (!apic_filter_irq_dest(ive->device->cell, &irq_msg)) {
 120                 panic_printk("FATAL: ivshmem MSI-X target outside of "
 121                              "cell \"%s\" device %02x:%02x.%x\n",
 122                              ive->device->cell->config->name,
 123                              PCI_BDF_PARAMS(ive->device->info->bdf));
 124                 return -EPERM;
 125         }
 126         /* now copy the whole struct into our cache and mark the cache
 127          * valid at the end */
 128         irq_msg.valid = 0;
 129         ive->irq_msg = irq_msg;
 130         memory_barrier();
 131         ive->irq_msg.valid = 1;
 132
 133         return 0;
 134 }
 135
 136 /**
 137  * update the command register
 138  * note that we only accept writes to two flags
 139  */
 140 static int ivshmem_write_command(struct pci_ivshmem_endpoint *ive, u16 val)
 141 {
 142         u16 *cmd = (u16 *)&ive->cspace[PCI_CFG_COMMAND/4];
 143         int err;
 144
 145         if ((val & PCI_CMD_MASTER) != (*cmd & PCI_CMD_MASTER)) {
 146                 *cmd = (*cmd & ~PCI_CMD_MASTER) | (val & PCI_CMD_MASTER);
 147                 err = ivshmem_update_msix(ive);
 148                 if (err)
 149                         return err;
 150         }
 151
 152         *cmd = (*cmd & ~PCI_CMD_MEM) | (val & PCI_CMD_MEM);
 153         return 0;
 154 }
 155
 156 static int ivshmem_msix_mmio(struct pci_ivshmem_endpoint *ive, bool is_write,
 157                              u32 offset, u32 *value)
 158 {
 159         u32 *msix_table = (u32 *)ive->device->msix_vectors;
 160
 161         if (offset % 4)
 162                 goto fail;
 163
 164         /* MSI-X PBA */
 165         if (offset >= 0x10 * IVSHMEM_MSIX_VECTORS) {
 166                 if (is_write) {
 167                         goto fail;
 168                 } else {
 169                         *value = 0;
 170                         return 1;
 171                 }
 172         /* MSI-X Table */
 173         } else {
 174                 if (is_write) {
 175                         msix_table[offset/4] = *value;
 176                         if (ivshmem_update_msix(ive))
 177                                 return -1;
 178                 } else {
 179                         *value = msix_table[offset/4];
 180                 }
 181                 return 1;
 182         }
 183
 184 fail:
 185         panic_printk("FATAL: Invalid PCI MSI-X table/PBA access, device "
 186                      "%02x:%02x.%x\n", PCI_BDF_PARAMS(ive->device->info->bdf));
 187         return -1;
 188 }
 189
 190 static void ivshmem_write_doorbell(struct pci_ivshmem_endpoint *ive)
 191 {
 192         struct pci_ivshmem_endpoint *remote = ive->remote;
 193         struct apic_irq_message irq_msg;
 194
 195         if (!remote)
 196                 return;
 197
 198         /* get a copy of the struct before using it, the read barrier makes
 199          * sure the copy is consistent */
 200         irq_msg = remote->irq_msg;
 201         memory_load_barrier();
 202         if (irq_msg.valid)
 203                 apic_send_irq(irq_msg);
 204 }
 205
 206 static int ivshmem_register_mmio(struct pci_ivshmem_endpoint *ive,
 207                                  bool is_write, u32 offset, u32 *value)
 208 {
 209         /* read-only IVPosition */
 210         if (offset == IVSHMEM_REG_IVPOS && !is_write) {
 211                 *value = ive->ivpos;
 212                 return 1;
 213         }
 214
 215         if (offset == IVSHMEM_REG_DBELL) {
 216                 if (is_write) {
 217                         ivshmem_write_doorbell(ive);
 218                 } else {
 219                         *value = 0;
 220                 }
 221                 return 1;
 222         }
 223         panic_printk("FATAL: Invalid ivshmem register %s, number %02x\n",
 224                      is_write ? "write" : "read", offset);
 225         return -1;
 226 }
 227
 228 static int ivshmem_write_msix_control(struct pci_ivshmem_endpoint *ive, u32 val)
 229 {
 230         union pci_msix_registers *p = (union pci_msix_registers *)&val;
 231         union pci_msix_registers newval = {
 232                 .raw = ive->cspace[IVSHMEM_CFG_MSIX_CAP/4]
 233         };
 234
 235         newval.enable = p->enable;
 236         newval.fmask = p->fmask;
 237         if (ive->cspace[IVSHMEM_CFG_MSIX_CAP/4] != newval.raw) {
 238                 ive->cspace[IVSHMEM_CFG_MSIX_CAP/4] = newval.raw;
 239                 return ivshmem_update_msix(ive);
 240         }
 241         return 0;
 242 }
 243
 244 static struct pci_ivshmem_data **ivshmem_find(struct pci_device *d,
 245                                               int *cellnum)
 246 {
 247         struct pci_ivshmem_data **ivp, *iv;
 248         u16 bdf2;
 249
 250         for (ivp = &ivshmem_list; *ivp; ivp = &((*ivp)->next)) {
 251                 iv = *ivp;
 252                 bdf2 = iv->eps[0].device->info->bdf;
 253                 if (d->info->bdf == bdf2) {
 254                         if (iv->eps[0].device == d) {
 255                                 if (cellnum)
 256                                         *cellnum = 0;
 257                                 return ivp;
 258                         }
 259                         if (iv->eps[1].device == d) {
 260                                 if (cellnum)
 261                                         *cellnum = 1;
 262                                 return ivp;
 263                         }
 264                         if (!cellnum)
 265                                 return ivp;
 266                 }
 267         }
 268
 269         return NULL;
 270 }
 271
 272 static void ivshmem_connect_cell(struct pci_ivshmem_data *iv,
 273                                  struct pci_device *d,
 274                                  const struct jailhouse_memory *mem,
 275                                  int cellnum)
 276 {
 277         struct pci_ivshmem_endpoint *remote = &iv->eps[(cellnum + 1) % 2];
 278         struct pci_ivshmem_endpoint *ive = &iv->eps[cellnum];
 279
 280         d->bar[0] = PCI_BAR_64BIT;
 281         d->bar[4] = PCI_BAR_64BIT;
 282
 283         memcpy(ive->cspace, &default_cspace, sizeof(default_cspace));
 284
 285         ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4] = (u32)mem->virt_start;
 286         ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4 + 1] = (u32)(mem->virt_start >> 32);
 287         ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4] = (u32)mem->size;
 288         ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4 + 1] = (u32)(mem->size >> 32);
 289
 290         ive->device = d;
 291         if (remote->device) {
 292                 ive->remote = remote;
 293                 remote->remote = ive;
 294                 ive->ivpos = (remote->ivpos + 1) % 2;
 295         } else {
 296                 ive->ivpos = cellnum;
 297                 ive->remote = NULL;
 298                 remote->remote = NULL;
 299         }
 300         d->ivshmem_endpoint = ive;
 301 }
 302
 303 static void ivshmem_disconnect_cell(struct pci_ivshmem_data *iv, int cellnum)
 304 {
 305         struct pci_ivshmem_endpoint *remote = &iv->eps[(cellnum + 1) % 2];
 306         struct pci_ivshmem_endpoint *ive = &iv->eps[cellnum];
 307
 308         ive->device->ivshmem_endpoint = NULL;
 309         ive->device = NULL;
 310         ive->remote = NULL;
 311         remote->remote = NULL;
 312 }
 313
 314 /**
 315  * Handler for MMIO-accesses to this virtual PCI devices memory. Both for the
 316  * BAR containing the registers, and the MSI-X BAR.
 317  * @param cell          The cell that issued the access.
 318  * @param is_write      True if write access.
 319  * @param addr          Address accessed.
 320  * @param value         Pointer to value for reading/writing.
 321  *
 322  * @return 1 if handled successfully, 0 if unhandled, -1 on access error.
 323  *
 324  * @see pci_mmio_access_handler
 325  */
 326 int ivshmem_mmio_access_handler(const struct cell *cell, bool is_write,
 327                                 u64 addr, u32 *value)
 328 {
 329         struct pci_ivshmem_endpoint *ive;
 330         struct pci_device *device;
 331         u64 mem_start;
 332
 333         for (device = cell->virtual_device_list; device;
 334              device = device->next_virtual_device) {
 335                 ive = device->ivshmem_endpoint;
 336                 if ((ive->cspace[PCI_CFG_COMMAND/4] & PCI_CMD_MEM) == 0)
 337                         continue;
 338
 339                 /* BAR0: registers */
 340                 mem_start = (*(u64 *)&device->bar[0]) & ~0xfL;
 341                 if (addr >= mem_start &&
 342                     addr <= (mem_start + IVSHMEM_BAR0_SIZE - 4))
 343                         return ivshmem_register_mmio(ive, is_write,
 344                                                      addr - mem_start,
 345                                                      value);
 346
 347                 /* BAR4: MSI-X */
 348                 mem_start = (*(u64 *)&device->bar[4]) & ~0xfL;
 349                 if (addr >= mem_start &&
 350                     addr <= (mem_start + IVSHMEM_BAR4_SIZE - 4))
 351                         return ivshmem_msix_mmio(ive, is_write,
 352                                                  addr - mem_start, value);
 353         }
 354
 355         return 0;
 356 }
 357
 358 /**
 359  * Handler for MMIO-write-accesses to PCI config space of this virtual device.
 360  * @param device        The device that access should be performed on.
 361  * @param row           Config space DWORD row of the access.
 362  * @param mask          Mask selected the DWORD bytes to write.
 363  * @param value         DWORD to write to the config space.
 364  *
 365  * @return PCI_ACCESS_REJECT or PCI_ACCESS_DONE.
 366  *
 367  * @see pci_cfg_write_moderate
 368  */
 369 enum pci_access pci_ivshmem_cfg_write(struct pci_device *dev, unsigned int row,
 370                                       u32 mask, u32 value)
 371 {
 372         struct pci_ivshmem_endpoint *ive = dev->ivshmem_endpoint;
 373
 374         if (row >= ARRAY_SIZE(default_cspace))
 375                 return PCI_ACCESS_REJECT;
 376
 377         value |= ive->cspace[row] & ~mask;
 378
 379         switch (row) {
 380         case PCI_CFG_COMMAND / 4:
 381                 if (ivshmem_write_command(ive, value))
 382                         return PCI_ACCESS_REJECT;
 383                 break;
 384         case IVSHMEM_CFG_MSIX_CAP / 4:
 385                 if (ivshmem_write_msix_control(ive, value))
 386                         return PCI_ACCESS_REJECT;
 387         }
 388         return PCI_ACCESS_DONE;
 389 }
 390
 391 /**
 392  * Handler for MMIO-read-accesses to PCI config space of this virtual device.
 393  * @param dev           The device that access should be performed on.
 394  * @param address       Config space address accessed.
 395  * @param value         Pointer to the return value.
 396  *
 397  * @return PCI_ACCESS_DONE.
 398  *
 399  * @see pci_cfg_read_moderate
 400  */
 401 enum pci_access pci_ivshmem_cfg_read(struct pci_device *dev, u16 address,
 402                                      u32 *value)
 403 {
 404         struct pci_ivshmem_endpoint *ive = dev->ivshmem_endpoint;
 405
 406         if (address < sizeof(default_cspace))
 407                 *value = ive->cspace[address / 4] >> ((address % 4) * 8);
 408         else
 409                 *value = -1;
 410         return PCI_ACCESS_DONE;
 411 }
 412
 413 /**
 414  * Update cached MSI-X state of the given ivshmem device.
 415  * @param dev   The device to be updated.
 416  *
 417  * @return 0 on success, negative error code otherwise.
 418  */
 419 int pci_ivshmem_update_msix(struct pci_device *dev)
 420 {
 421         return ivshmem_update_msix(dev->ivshmem_endpoint);
 422 }
 423
 424 /**
 425  * Register a new ivshmem device.
 426  * @param cell          The cell the device should be attached to.
 427  * @param dev           The device to be registered.
 428  *
 429  * @return 0 on success, negative error code otherwise.
 430  */
 431 int pci_ivshmem_init(struct cell *cell, struct pci_device *dev)
 432 {
 433         const struct jailhouse_memory *mem, *mem0;
 434         struct pci_ivshmem_data **ivp;
 435         struct pci_device *dev0;
 436
 437         if (dev->info->num_msix_vectors != 1)
 438                 return trace_error(-EINVAL);
 439
 440         if (dev->info->shmem_region >= cell->config->num_memory_regions)
 441                 return trace_error(-EINVAL);
 442
 443         mem = jailhouse_cell_mem_regions(cell->config)
 444                 + dev->info->shmem_region;
 445         ivp = ivshmem_find(dev, NULL);
 446         if (ivp) {
 447                 dev0 = (*ivp)->eps[0].device;
 448                 mem0 = jailhouse_cell_mem_regions(dev0->cell->config) +
 449                         dev0->info->shmem_region;
 450
 451                 /* we already have a datastructure, connect second endpoint */
 452                 if ((mem0->phys_start == mem->phys_start) &&
 453                     (mem0->size == mem->size)) {
 454                         if ((*ivp)->eps[1].device)
 455                                 return trace_error(-EBUSY);
 456                         ivshmem_connect_cell(*ivp, dev, mem, 1);
 457                         printk("Virtual PCI connection established "
 458                                 "\"%s\" <--> \"%s\"\n",
 459                                 cell->config->name, dev0->cell->config->name);
 460                         goto connected;
 461                 }
 462         }
 463
 464         /* this is the first endpoint, allocate a new datastructure */
 465         for (ivp = &ivshmem_list; *ivp; ivp = &((*ivp)->next))
 466                 ; /* empty loop */
 467         *ivp = page_alloc(&mem_pool, 1);
 468         if (!(*ivp))
 469                 return -ENOMEM;
 470         ivshmem_connect_cell(*ivp, dev, mem, 0);
 471
 472 connected:
 473         dev->cell = cell;
 474         printk("Adding virtual PCI device %02x:%02x.%x to cell \"%s\"\n",
 475                PCI_BDF_PARAMS(dev->info->bdf), cell->config->name);
 476
 477         return 0;
 478 }
 479
 480 /**
 481  * Unregister a ivshmem device, typically when the corresponding cell exits.
 482  * @param dev           The device to be stopped.
 483  *
 484  */
 485 void pci_ivshmem_exit(struct pci_device *dev)
 486 {
 487         struct pci_ivshmem_data **ivp, *iv;
 488         int cellnum;
 489
 490         ivp = ivshmem_find(dev, &cellnum);
 491         if (!ivp || !(*ivp))
 492                 return;
 493
 494         iv = *ivp;
 495
 496         ivshmem_disconnect_cell(iv, cellnum);
 497
 498         if (cellnum == 0) {
 499                 if (!iv->eps[1].device) {
 500                         *ivp = iv->next;
 501                         page_free(&mem_pool, iv, 1);
 502                         return;
 503                 }
 504                 iv->eps[0] = iv->eps[1];
 505         }
 506 }