usr/src/cmd/bhyve/pci_passthru.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/types.h>
  36 #include <sys/mman.h>
  37 #include <sys/pciio.h>
  38 #include <sys/ioctl.h>
  39 #include <sys/stat.h>
  40
  41 #include <sys/pci.h>
  42
  43 #include <dev/io/iodev.h>
  44 #include <dev/pci/pcireg.h>
  45
  46 #include <machine/iodev.h>
  47
  48 #include <stdio.h>
  49 #include <stdlib.h>
  50 #include <string.h>
  51 #include <err.h>
  52 #include <errno.h>
  53 #include <fcntl.h>
  54 #include <sysexits.h>
  55 #include <unistd.h>
  56
  57 #include <machine/vmm.h>
  58 #include <vmmapi.h>
  59 #include <sys/ppt_dev.h>
  60
  61 #include "config.h"
  62 #include "debug.h"
  63 #include "pci_passthru.h"
  64 #include "mem.h"
  65
  66 #define LEGACY_SUPPORT  1
  67
  68 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
  69 #define MSIX_CAPLEN 12
  70
  71 struct passthru_softc {
  72         struct pci_devinst *psc_pi;
  73         /* ROM is handled like a BAR */
  74         struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1];
  75         struct {
  76                 int             capoff;
  77                 int             msgctrl;
  78                 int             emulated;
  79         } psc_msi;
  80         struct {
  81                 int             capoff;
  82         } psc_msix;
  83         int pptfd;
  84         int msi_limit;
  85         int msix_limit;
  86 };
  87
  88 static int
  89 msi_caplen(int msgctrl)
  90 {
  91         int len;
  92
  93         len = 10;               /* minimum length of msi capability */
  94
  95         if (msgctrl & PCIM_MSICTRL_64BIT)
  96                 len += 4;
  97
  98 #if 0
  99         /*
 100          * Ignore the 'mask' and 'pending' bits in the MSI capability.
 101          * We'll let the guest manipulate them directly.
 102          */
 103         if (msgctrl & PCIM_MSICTRL_VECTOR)
 104                 len += 10;
 105 #endif
 106
 107         return (len);
 108 }
 109
 110 static uint32_t
 111 passthru_read_config(const struct passthru_softc *sc, long reg, int width)
 112 {
 113         struct ppt_cfg_io pi;
 114
 115         pi.pci_off = reg;
 116         pi.pci_width = width;
 117
 118         if (ioctl(sc->pptfd, PPT_CFG_READ, &pi) != 0) {
 119                 return (0);
 120         }
 121         return (pi.pci_data);
 122 }
 123
 124 static void
 125 passthru_write_config(const struct passthru_softc *sc, long reg, int width,
 126     uint32_t data)
 127 {
 128         struct ppt_cfg_io pi;
 129
 130         pi.pci_off = reg;
 131         pi.pci_width = width;
 132         pi.pci_data = data;
 133
 134         (void) ioctl(sc->pptfd, PPT_CFG_WRITE, &pi);
 135 }
 136
 137 uint32_t
 138 read_config(struct pci_devinst *pi, long reg, int width)
 139 {
 140         struct passthru_softc *sc = pi->pi_arg;
 141
 142         return (passthru_read_config(sc, reg, width));
 143 }
 144
 145 void
 146 write_config(struct pci_devinst *pi, long reg, int width, uint32_t data)
 147 {
 148         struct passthru_softc *sc = pi->pi_arg;
 149
 150         passthru_write_config(sc, reg, width, data);
 151 }
 152
 153 static int
 154 passthru_get_bar(struct passthru_softc *sc, int bar, enum pcibar_type *type,
 155     uint64_t *base, uint64_t *size)
 156 {
 157         struct ppt_bar_query pb;
 158
 159         pb.pbq_baridx = bar;
 160
 161         if (ioctl(sc->pptfd, PPT_BAR_QUERY, &pb) != 0) {
 162                 return (-1);
 163         }
 164
 165         switch (pb.pbq_type) {
 166         case PCI_ADDR_IO:
 167                 *type = PCIBAR_IO;
 168                 break;
 169         case PCI_ADDR_MEM32:
 170                 *type = PCIBAR_MEM32;
 171                 break;
 172         case PCI_ADDR_MEM64:
 173                 *type = PCIBAR_MEM64;
 174                 break;
 175         default:
 176                 err(1, "unrecognized BAR type: %u\n", pb.pbq_type);
 177                 break;
 178         }
 179
 180         *base = pb.pbq_base;
 181         *size = pb.pbq_size;
 182         return (0);
 183 }
 184
 185 static int
 186 passthru_dev_open(const char *path, int *pptfdp)
 187 {
 188         int pptfd;
 189
 190         if ((pptfd = open(path, O_RDWR)) < 0) {
 191                 return (errno);
 192         }
 193
 194         /* XXX: verify fd with ioctl? */
 195         *pptfdp = pptfd;
 196         return (0);
 197 }
 198
 199 #ifdef LEGACY_SUPPORT
 200 static int
 201 passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)
 202 {
 203         int capoff;
 204         struct msicap msicap;
 205         u_char *capdata;
 206
 207         pci_populate_msicap(&msicap, msgnum, nextptr);
 208
 209         /*
 210          * XXX
 211          * Copy the msi capability structure in the last 16 bytes of the
 212          * config space. This is wrong because it could shadow something
 213          * useful to the device.
 214          */
 215         capoff = 256 - roundup(sizeof(msicap), 4);
 216         capdata = (u_char *)&msicap;
 217         for (size_t i = 0; i < sizeof(msicap); i++)
 218                 pci_set_cfgdata8(pi, capoff + i, capdata[i]);
 219
 220         return (capoff);
 221 }
 222 #endif  /* LEGACY_SUPPORT */
 223
 224 static void
 225 passthru_intr_limit(struct passthru_softc *sc, struct msixcap *msixcap)
 226 {
 227         struct pci_devinst *pi = sc->psc_pi;
 228         int off;
 229
 230         /* Reduce the number of MSI vectors if higher than OS limit */
 231         if ((off = sc->psc_msi.capoff) != 0 && sc->msi_limit != -1) {
 232                 int msi_limit, mmc;
 233
 234                 msi_limit =
 235                     sc->msi_limit > 16 ? PCIM_MSICTRL_MMC_32 :
 236                     sc->msi_limit > 8 ? PCIM_MSICTRL_MMC_16 :
 237                     sc->msi_limit > 4 ? PCIM_MSICTRL_MMC_8 :
 238                     sc->msi_limit > 2 ? PCIM_MSICTRL_MMC_4 :
 239                     sc->msi_limit > 1 ? PCIM_MSICTRL_MMC_2 :
 240                     PCIM_MSICTRL_MMC_1;
 241                 mmc = sc->psc_msi.msgctrl & PCIM_MSICTRL_MMC_MASK;
 242
 243                 if (mmc > msi_limit) {
 244                         sc->psc_msi.msgctrl &= ~PCIM_MSICTRL_MMC_MASK;
 245                         sc->psc_msi.msgctrl |= msi_limit;
 246                         pci_set_cfgdata16(pi, off + 2, sc->psc_msi.msgctrl);
 247                 }
 248         }
 249
 250         /* Reduce the number of MSI-X vectors if higher than OS limit */
 251         if ((off = sc->psc_msix.capoff) != 0 && sc->msix_limit != -1) {
 252                 if (MSIX_TABLE_COUNT(msixcap->msgctrl) > sc->msix_limit) {
 253                         msixcap->msgctrl &= ~PCIM_MSIXCTRL_TABLE_SIZE;
 254                         msixcap->msgctrl |= sc->msix_limit - 1;
 255                         pci_set_cfgdata16(pi, off + 2, msixcap->msgctrl);
 256                 }
 257         }
 258 }
 259
 260 static int
 261 cfginitmsi(struct passthru_softc *sc)
 262 {
 263         int i, ptr, capptr, cap, sts, caplen, table_size;
 264         uint32_t u32;
 265         struct pci_devinst *pi = sc->psc_pi;
 266         struct msixcap msixcap;
 267         char *msixcap_ptr;
 268
 269         /*
 270          * Parse the capabilities and cache the location of the MSI
 271          * and MSI-X capabilities.
 272          */
 273         sts = passthru_read_config(sc, PCIR_STATUS, 2);
 274         if (sts & PCIM_STATUS_CAPPRESENT) {
 275                 ptr = passthru_read_config(sc, PCIR_CAP_PTR, 1);
 276                 while (ptr != 0 && ptr != 0xff) {
 277                         cap = passthru_read_config(sc, ptr + PCICAP_ID, 1);
 278                         if (cap == PCIY_MSI) {
 279                                 /*
 280                                  * Copy the MSI capability into the config
 281                                  * space of the emulated pci device
 282                                  */
 283                                 sc->psc_msi.capoff = ptr;
 284                                 sc->psc_msi.msgctrl = passthru_read_config(sc,
 285                                     ptr + 2, 2);
 286                                 sc->psc_msi.emulated = 0;
 287                                 caplen = msi_caplen(sc->psc_msi.msgctrl);
 288                                 capptr = ptr;
 289                                 while (caplen > 0) {
 290                                         u32 = passthru_read_config(sc,
 291                                             capptr, 4);
 292                                         pci_set_cfgdata32(pi, capptr, u32);
 293                                         caplen -= 4;
 294                                         capptr += 4;
 295                                 }
 296                         } else if (cap == PCIY_MSIX) {
 297                                 /*
 298                                  * Copy the MSI-X capability
 299                                  */
 300                                 sc->psc_msix.capoff = ptr;
 301                                 caplen = 12;
 302                                 msixcap_ptr = (char *)&msixcap;
 303                                 capptr = ptr;
 304                                 while (caplen > 0) {
 305                                         u32 = passthru_read_config(sc,
 306                                             capptr, 4);
 307                                         memcpy(msixcap_ptr, &u32, 4);
 308                                         pci_set_cfgdata32(pi, capptr, u32);
 309                                         caplen -= 4;
 310                                         capptr += 4;
 311                                         msixcap_ptr += 4;
 312                                 }
 313                         }
 314                         ptr = passthru_read_config(sc, ptr + PCICAP_NEXTPTR, 1);
 315                 }
 316         }
 317
 318         passthru_intr_limit(sc, &msixcap);
 319
 320         if (sc->psc_msix.capoff != 0) {
 321                 pi->pi_msix.pba_bar =
 322                     msixcap.pba_info & PCIM_MSIX_BIR_MASK;
 323                 pi->pi_msix.pba_offset =
 324                     msixcap.pba_info & ~PCIM_MSIX_BIR_MASK;
 325                 pi->pi_msix.table_bar =
 326                     msixcap.table_info & PCIM_MSIX_BIR_MASK;
 327                 pi->pi_msix.table_offset =
 328                     msixcap.table_info & ~PCIM_MSIX_BIR_MASK;
 329                 pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl);
 330                 pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count);
 331
 332                 /* Allocate the emulated MSI-X table array */
 333                 table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
 334                 pi->pi_msix.table = calloc(1, table_size);
 335
 336                 /* Mask all table entries */
 337                 for (i = 0; i < pi->pi_msix.table_count; i++) {
 338                         pi->pi_msix.table[i].vector_control |=
 339                                                 PCIM_MSIX_VCTRL_MASK;
 340                 }
 341         }
 342
 343 #ifdef LEGACY_SUPPORT
 344         /*
 345          * If the passthrough device does not support MSI then craft a
 346          * MSI capability for it. We link the new MSI capability at the
 347          * head of the list of capabilities.
 348          */
 349         if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) {
 350                 int origptr, msiptr;
 351                 origptr = passthru_read_config(sc, PCIR_CAP_PTR, 1);
 352                 msiptr = passthru_add_msicap(pi, 1, origptr);
 353                 sc->psc_msi.capoff = msiptr;
 354                 sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2);
 355                 sc->psc_msi.emulated = 1;
 356                 pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr);
 357         }
 358 #endif
 359
 360         /* Make sure one of the capabilities is present */
 361         if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0)
 362                 return (-1);
 363         else
 364                 return (0);
 365 }
 366
 367 static uint64_t
 368 msix_table_read(struct passthru_softc *sc, uint64_t offset, int size)
 369 {
 370         struct pci_devinst *pi;
 371         struct msix_table_entry *entry;
 372         uint8_t *src8;
 373         uint16_t *src16;
 374         uint32_t *src32;
 375         uint64_t *src64;
 376         uint64_t data;
 377         size_t entry_offset;
 378         uint32_t table_offset;
 379         int index, table_count;
 380
 381         pi = sc->psc_pi;
 382
 383         table_offset = pi->pi_msix.table_offset;
 384         table_count = pi->pi_msix.table_count;
 385         if (offset < table_offset ||
 386             offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
 387                 switch (size) {
 388                 case 1:
 389                         src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
 390                         data = *src8;
 391                         break;
 392                 case 2:
 393                         src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
 394                         data = *src16;
 395                         break;
 396                 case 4:
 397                         src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
 398                         data = *src32;
 399                         break;
 400                 case 8:
 401                         src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
 402                         data = *src64;
 403                         break;
 404                 default:
 405                         return (-1);
 406                 }
 407                 return (data);
 408         }
 409
 410         offset -= table_offset;
 411         index = offset / MSIX_TABLE_ENTRY_SIZE;
 412         assert(index < table_count);
 413
 414         entry = &pi->pi_msix.table[index];
 415         entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
 416
 417         switch (size) {
 418         case 1:
 419                 src8 = (uint8_t *)((uint8_t *)entry + entry_offset);
 420                 data = *src8;
 421                 break;
 422         case 2:
 423                 src16 = (uint16_t *)((uint8_t *)entry + entry_offset);
 424                 data = *src16;
 425                 break;
 426         case 4:
 427                 src32 = (uint32_t *)((uint8_t *)entry + entry_offset);
 428                 data = *src32;
 429                 break;
 430         case 8:
 431                 src64 = (uint64_t *)((uint8_t *)entry + entry_offset);
 432                 data = *src64;
 433                 break;
 434         default:
 435                 return (-1);
 436         }
 437
 438         return (data);
 439 }
 440
 441 static void
 442 msix_table_write(struct vmctx *ctx, struct passthru_softc *sc,
 443                  uint64_t offset, int size, uint64_t data)
 444 {
 445         struct pci_devinst *pi;
 446         struct msix_table_entry *entry;
 447         uint8_t *dest8;
 448         uint16_t *dest16;
 449         uint32_t *dest32;
 450         uint64_t *dest64;
 451         size_t entry_offset;
 452         uint32_t table_offset, vector_control;
 453         int index, table_count;
 454
 455         pi = sc->psc_pi;
 456
 457         table_offset = pi->pi_msix.table_offset;
 458         table_count = pi->pi_msix.table_count;
 459         if (offset < table_offset ||
 460             offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
 461                 switch (size) {
 462                 case 1:
 463                         dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
 464                         *dest8 = data;
 465                         break;
 466                 case 2:
 467                         dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
 468                         *dest16 = data;
 469                         break;
 470                 case 4:
 471                         dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
 472                         *dest32 = data;
 473                         break;
 474                 case 8:
 475                         dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
 476                         *dest64 = data;
 477                         break;
 478                 }
 479                 return;
 480         }
 481
 482         offset -= table_offset;
 483         index = offset / MSIX_TABLE_ENTRY_SIZE;
 484         assert(index < table_count);
 485
 486         entry = &pi->pi_msix.table[index];
 487         entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
 488
 489         /* Only 4 byte naturally-aligned writes are supported */
 490         assert(size == 4);
 491         assert(entry_offset % 4 == 0);
 492
 493         vector_control = entry->vector_control;
 494         dest32 = (uint32_t *)((uint8_t *)entry + entry_offset);
 495         *dest32 = data;
 496         /* If MSI-X hasn't been enabled, do nothing */
 497         if (pi->pi_msix.enabled) {
 498                 /* If the entry is masked, don't set it up */
 499                 if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 ||
 500                     (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
 501                         (void) vm_setup_pptdev_msix(ctx, 0, sc->pptfd,
 502                             index, entry->addr, entry->msg_data,
 503                             entry->vector_control);
 504                 }
 505         }
 506 }
 507
 508 static int
 509 init_msix_table(struct vmctx *ctx __unused, struct passthru_softc *sc)
 510 {
 511         struct pci_devinst *pi = sc->psc_pi;
 512         uint32_t table_size, table_offset;
 513         int i;
 514
 515         i = pci_msix_table_bar(pi);
 516         assert(i >= 0);
 517
 518         /*
 519          * Map the region of the BAR containing the MSI-X table.  This is
 520          * necessary for two reasons:
 521          * 1. The PBA may reside in the first or last page containing the MSI-X
 522          *    table.
 523          * 2. While PCI devices are not supposed to use the page(s) containing
 524          *    the MSI-X table for other purposes, some do in practice.
 525          */
 526
 527         /*
 528          * Mapping pptfd provides access to the BAR containing the MSI-X
 529          * table. See ppt_devmap() in usr/src/uts/intel/io/vmm/io/ppt.c
 530          *
 531          * This maps the whole BAR and then mprotect(PROT_NONE) is used below
 532          * to prevent access to pages that don't contain the MSI-X table.
 533          * When porting this, it was tempting to just map the MSI-X table pages
 534          * but that would mean updating everywhere that assumes that
 535          * pi->pi_msix.mapped_addr points to the start of the BAR. For now,
 536          * keep closer to upstream.
 537          */
 538         pi->pi_msix.mapped_size = sc->psc_bar[i].size;
 539         pi->pi_msix.mapped_addr = (uint8_t *)mmap(NULL, pi->pi_msix.mapped_size,
 540             PROT_READ | PROT_WRITE, MAP_SHARED, sc->pptfd, 0);
 541         if (pi->pi_msix.mapped_addr == MAP_FAILED) {
 542                 warn("Failed to map MSI-X table BAR on %d", sc->pptfd);
 543                 return (-1);
 544         }
 545
 546         table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
 547
 548         table_size = pi->pi_msix.table_offset - table_offset;
 549         table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
 550         table_size = roundup2(table_size, 4096);
 551
 552         /*
 553          * Unmap any pages not containing the table, we do not need to emulate
 554          * accesses to them.  Avoid releasing address space to help ensure that
 555          * a buggy out-of-bounds access causes a crash.
 556          */
 557         if (table_offset != 0)
 558                 if (mprotect((caddr_t)pi->pi_msix.mapped_addr, table_offset,
 559                     PROT_NONE) != 0)
 560                         warn("Failed to unmap MSI-X table BAR region");
 561         if (table_offset + table_size != pi->pi_msix.mapped_size)
 562                 if (mprotect((caddr_t)
 563                     pi->pi_msix.mapped_addr + table_offset + table_size,
 564                     pi->pi_msix.mapped_size - (table_offset + table_size),
 565                     PROT_NONE) != 0)
 566                         warn("Failed to unmap MSI-X table BAR region");
 567
 568         return (0);
 569 }
 570
 571 static int
 572 cfginitbar(struct vmctx *ctx __unused, struct passthru_softc *sc)
 573 {
 574         struct pci_devinst *pi = sc->psc_pi;
 575         uint_t i;
 576
 577         /*
 578          * Initialize BAR registers
 579          */
 580         for (i = 0; i <= PCI_BARMAX; i++) {
 581                 enum pcibar_type bartype;
 582                 uint64_t base, size;
 583                 int error;
 584
 585                 if (passthru_get_bar(sc, i, &bartype, &base, &size) != 0) {
 586                         continue;
 587                 }
 588
 589                 if (bartype != PCIBAR_IO) {
 590                         if (((base | size) & PAGE_MASK) != 0) {
 591                                 warnx("passthru device %d BAR %d: "
 592                                     "base %#lx or size %#lx not page aligned\n",
 593                                     sc->pptfd, i, base, size);
 594                                 return (-1);
 595                         }
 596                 }
 597
 598                 /* Cache information about the "real" BAR */
 599                 sc->psc_bar[i].type = bartype;
 600                 sc->psc_bar[i].size = size;
 601                 sc->psc_bar[i].addr = base;
 602                 sc->psc_bar[i].lobits = 0;
 603
 604                 /* Allocate the BAR in the guest I/O or MMIO space */
 605                 error = pci_emul_alloc_bar(pi, i, bartype, size);
 606                 if (error)
 607                         return (-1);
 608
 609                 /* Use same lobits as physical bar */
 610                 uint8_t lobits = passthru_read_config(sc, PCIR_BAR(i), 0x01);
 611                 if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) {
 612                         lobits &= ~PCIM_BAR_MEM_BASE;
 613                 } else {
 614                         lobits &= ~PCIM_BAR_IO_BASE;
 615                 }
 616                 sc->psc_bar[i].lobits = lobits;
 617                 pi->pi_bar[i].lobits = lobits;
 618
 619                 /*
 620                  * 64-bit BAR takes up two slots so skip the next one.
 621                  */
 622                 if (bartype == PCIBAR_MEM64) {
 623                         i++;
 624                         assert(i <= PCI_BARMAX);
 625                         sc->psc_bar[i].type = PCIBAR_MEMHI64;
 626                 }
 627         }
 628         return (0);
 629 }
 630
 631 static int
 632 cfginit(struct vmctx *ctx, struct passthru_softc *sc)
 633 {
 634         struct pci_devinst *pi = sc->psc_pi;
 635         int error;
 636
 637         if (cfginitmsi(sc) != 0) {
 638                 warnx("failed to initialize MSI for PCI %d", sc->pptfd);
 639                 return (-1);
 640         }
 641
 642         if (cfginitbar(ctx, sc) != 0) {
 643                 warnx("failed to initialize BARs for PCI %d", sc->pptfd);
 644                 return (-1);
 645         }
 646
 647         passthru_write_config(sc, PCIR_COMMAND, 2,
 648             pci_get_cfgdata16(pi, PCIR_COMMAND));
 649
 650         /*
 651         * We need to do this after PCIR_COMMAND got possibly updated, e.g.,
 652         * a BAR was enabled.
 653         */
 654         if (pci_msix_table_bar(pi) >= 0) {
 655                 error = init_msix_table(ctx, sc);
 656                 if (error != 0) {
 657                         warnx("failed to initialize MSI-X table for PCI %d",
 658                             sc->pptfd);
 659                         goto done;
 660                 }
 661         }
 662
 663         error = 0;                              /* success */
 664 done:
 665         return (error);
 666 }
 667
 668 static int
 669 passthru_legacy_config(nvlist_t *nvl, const char *opt)
 670 {
 671         char *config, *name, *tofree, *value;
 672
 673         if (opt == NULL)
 674                 return (0);
 675
 676         config = tofree = strdup(opt);
 677         while ((name = strsep(&config, ",")) != NULL) {
 678                 value = strchr(name, '=');
 679                 if (value != NULL) {
 680                         *value++ = '\0';
 681                         set_config_value_node(nvl, name, value);
 682                 } else {
 683                         if (strncmp(name, "/dev/ppt", 8) != 0) {
 684                                 EPRINTLN("passthru: invalid path \"%s\"", name);
 685                                 free(tofree);
 686                                 return (-1);
 687                         }
 688                         set_config_value_node(nvl, "path", name);
 689                 }
 690         }
 691         free(tofree);
 692         return (0);
 693 }
 694
 695 static int
 696 passthru_init_rom(struct vmctx *const ctx __unused,
 697     struct passthru_softc *const sc, const char *const romfile)
 698 {
 699         if (romfile == NULL) {
 700                 return (0);
 701         }
 702
 703         const int fd = open(romfile, O_RDONLY);
 704         if (fd < 0) {
 705                 warnx("%s: can't open romfile \"%s\"", __func__, romfile);
 706                 return (-1);
 707         }
 708
 709         struct stat sbuf;
 710         if (fstat(fd, &sbuf) < 0) {
 711                 warnx("%s: can't fstat romfile \"%s\"", __func__, romfile);
 712                 close(fd);
 713                 return (-1);
 714         }
 715         const uint64_t rom_size = sbuf.st_size;
 716
 717         void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd,
 718             0);
 719         if (rom_data == MAP_FAILED) {
 720                 warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__,
 721                     romfile, errno);
 722                 close(fd);
 723                 return (-1);
 724         }
 725
 726         void *rom_addr;
 727         int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr);
 728         if (error) {
 729                 warnx("%s: failed to alloc rom segment", __func__);
 730                 munmap(rom_data, rom_size);
 731                 close(fd);
 732                 return (error);
 733         }
 734         memcpy(rom_addr, rom_data, rom_size);
 735
 736         sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM;
 737         sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr;
 738         sc->psc_bar[PCI_ROM_IDX].size = rom_size;
 739
 740         munmap(rom_data, rom_size);
 741         close(fd);
 742
 743         return (0);
 744  }
 745
 746 static int
 747 passthru_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 748 {
 749         int error, memflags, pptfd;
 750         struct passthru_softc *sc;
 751         const char *path;
 752
 753         pptfd = -1;
 754         sc = NULL;
 755         error = 1;
 756
 757         memflags = vm_get_memflags(ctx);
 758         if (!(memflags & VM_MEM_F_WIRED)) {
 759                 warnx("passthru requires guest memory to be wired");
 760                 goto done;
 761         }
 762
 763         path = get_config_value_node(nvl, "path");
 764         if (path == NULL || passthru_dev_open(path, &pptfd) != 0) {
 765                 warnx("invalid passthru options");
 766                 goto done;
 767         }
 768
 769         if (vm_assign_pptdev(ctx, pptfd) != 0) {
 770                 warnx("PCI device at %d is not using the ppt driver", pptfd);
 771                 goto done;
 772         }
 773
 774         sc = calloc(1, sizeof(struct passthru_softc));
 775
 776         pi->pi_arg = sc;
 777         sc->psc_pi = pi;
 778         sc->pptfd = pptfd;
 779
 780         if ((error = vm_get_pptdev_limits(ctx, pptfd, &sc->msi_limit,
 781             &sc->msix_limit)) != 0)
 782                 goto done;
 783
 784         /* initialize config space */
 785         if ((error = cfginit(ctx, sc)) != 0)
 786                 goto done;
 787
 788         /* initialize ROM */
 789         if ((error = passthru_init_rom(ctx, sc,
 790             get_config_value_node(nvl, "rom"))) != 0) {
 791                 goto done;
 792         }
 793
 794 done:
 795         if (error) {
 796                 free(sc);
 797                 if (pptfd != -1)
 798                         vm_unassign_pptdev(ctx, pptfd);
 799         }
 800         return (error);
 801 }
 802
 803 static int
 804 bar_access(int coff)
 805 {
 806         if ((coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1)) ||
 807             coff == PCIR_BIOS)
 808                 return (1);
 809         else
 810                 return (0);
 811 }
 812
 813 static int
 814 msicap_access(struct passthru_softc *sc, int coff)
 815 {
 816         int caplen;
 817
 818         if (sc->psc_msi.capoff == 0)
 819                 return (0);
 820
 821         caplen = msi_caplen(sc->psc_msi.msgctrl);
 822
 823         if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen)
 824                 return (1);
 825         else
 826                 return (0);
 827 }
 828
 829 static int
 830 msixcap_access(struct passthru_softc *sc, int coff)
 831 {
 832         if (sc->psc_msix.capoff == 0)
 833                 return (0);
 834
 835         return (coff >= sc->psc_msix.capoff &&
 836                 coff < sc->psc_msix.capoff + MSIX_CAPLEN);
 837 }
 838
 839 static int
 840 passthru_cfgread(struct vmctx *ctx __unused, struct pci_devinst *pi, int coff,
 841     int bytes, uint32_t *rv)
 842 {
 843         struct passthru_softc *sc;
 844
 845         sc = pi->pi_arg;
 846
 847         /*
 848          * PCI BARs and MSI capability is emulated.
 849          */
 850         if (bar_access(coff) || msicap_access(sc, coff) ||
 851             msixcap_access(sc, coff))
 852                 return (-1);
 853
 854         /*
 855          * MSI-X is also emulated since a limit on interrupts may be imposed by
 856          * the OS, altering the perceived register state.
 857          */
 858         if (msixcap_access(sc, coff))
 859                 return (-1);
 860
 861 #ifdef LEGACY_SUPPORT
 862         /*
 863          * Emulate PCIR_CAP_PTR if this device does not support MSI capability
 864          * natively.
 865          */
 866         if (sc->psc_msi.emulated) {
 867                 if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4)
 868                         return (-1);
 869         }
 870 #endif
 871
 872         /*
 873          * Emulate the command register.  If a single read reads both the
 874          * command and status registers, read the status register from the
 875          * device's config space.
 876          */
 877         if (coff == PCIR_COMMAND) {
 878                 if (bytes <= 2)
 879                         return (-1);
 880                 *rv = passthru_read_config(sc, PCIR_STATUS, 2) << 16 |
 881                     pci_get_cfgdata16(pi, PCIR_COMMAND);
 882                 return (0);
 883         }
 884
 885         /* Everything else just read from the device's config space */
 886         *rv = passthru_read_config(sc, coff, bytes);
 887
 888         return (0);
 889 }
 890
 891 static int
 892 passthru_cfgwrite(struct vmctx *ctx, struct pci_devinst *pi, int coff,
 893     int bytes, uint32_t val)
 894 {
 895         int error, msix_table_entries, i;
 896         struct passthru_softc *sc;
 897         uint16_t cmd_old;
 898
 899         sc = pi->pi_arg;
 900
 901         /*
 902          * PCI BARs are emulated
 903          */
 904         if (bar_access(coff))
 905                 return (-1);
 906
 907         /*
 908          * MSI capability is emulated
 909          */
 910         if (msicap_access(sc, coff)) {
 911                 pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,
 912                     PCIY_MSI);
 913                 error = vm_setup_pptdev_msi(ctx, 0, sc->pptfd,
 914                     pi->pi_msi.addr, pi->pi_msi.msg_data, pi->pi_msi.maxmsgnum);
 915                 if (error != 0)
 916                         err(1, "vm_setup_pptdev_msi");
 917                 return (0);
 918         }
 919
 920         if (msixcap_access(sc, coff)) {
 921                 pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff,
 922                     PCIY_MSIX);
 923                 if (pi->pi_msix.enabled) {
 924                         msix_table_entries = pi->pi_msix.table_count;
 925                         for (i = 0; i < msix_table_entries; i++) {
 926                                 error = vm_setup_pptdev_msix(ctx, 0,
 927                                     sc->pptfd, i,
 928                                     pi->pi_msix.table[i].addr,
 929                                     pi->pi_msix.table[i].msg_data,
 930                                     pi->pi_msix.table[i].vector_control);
 931
 932                                 if (error)
 933                                         err(1, "vm_setup_pptdev_msix");
 934                         }
 935                 } else {
 936                         error = vm_disable_pptdev_msix(ctx, sc->pptfd);
 937                         if (error)
 938                                 err(1, "vm_disable_pptdev_msix");
 939                 }
 940                 return (0);
 941         }
 942
 943 #ifdef LEGACY_SUPPORT
 944         /*
 945          * If this device does not support MSI natively then we cannot let
 946          * the guest disable legacy interrupts from the device. It is the
 947          * legacy interrupt that is triggering the virtual MSI to the guest.
 948          */
 949         if (sc->psc_msi.emulated && pci_msi_enabled(pi)) {
 950                 if (coff == PCIR_COMMAND && bytes == 2)
 951                         val &= ~PCIM_CMD_INTxDIS;
 952         }
 953 #endif
 954
 955         passthru_write_config(sc, coff, bytes, val);
 956         if (coff == PCIR_COMMAND) {
 957                 cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND);
 958                 if (bytes == 1)
 959                         pci_set_cfgdata8(pi, PCIR_COMMAND, val);
 960                 else if (bytes == 2)
 961                         pci_set_cfgdata16(pi, PCIR_COMMAND, val);
 962                 pci_emul_cmd_changed(pi, cmd_old);
 963         }
 964
 965         return (0);
 966 }
 967
 968 static void
 969 passthru_write(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
 970     uint64_t offset, int size, uint64_t value)
 971 {
 972         struct passthru_softc *sc = pi->pi_arg;
 973
 974         if (baridx == pci_msix_table_bar(pi)) {
 975                 msix_table_write(ctx, sc, offset, size, value);
 976         } else {
 977                 struct ppt_bar_io pbi;
 978
 979                 assert(pi->pi_bar[baridx].type == PCIBAR_IO);
 980
 981                 pbi.pbi_bar = baridx;
 982                 pbi.pbi_width = size;
 983                 pbi.pbi_off = offset;
 984                 pbi.pbi_data = value;
 985                 (void) ioctl(sc->pptfd, PPT_BAR_WRITE, &pbi);
 986         }
 987 }
 988
 989 static uint64_t
 990 passthru_read(struct vmctx *ctx __unused, struct pci_devinst *pi, int baridx,
 991     uint64_t offset, int size)
 992 {
 993         struct passthru_softc *sc = pi->pi_arg;
 994         uint64_t val;
 995
 996         if (baridx == pci_msix_table_bar(pi)) {
 997                 val = msix_table_read(sc, offset, size);
 998         } else {
 999                 struct ppt_bar_io pbi;
1000
1001                 assert(pi->pi_bar[baridx].type == PCIBAR_IO);
1002
1003                 pbi.pbi_bar = baridx;
1004                 pbi.pbi_width = size;
1005                 pbi.pbi_off = offset;
1006                 if (ioctl(sc->pptfd, PPT_BAR_READ, &pbi) == 0) {
1007                         val = pbi.pbi_data;
1008                 } else {
1009                         val = 0;
1010                 }
1011         }
1012
1013         return (val);
1014 }
1015
1016 static void
1017 passthru_msix_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
1018                    int enabled, uint64_t address)
1019 {
1020         struct passthru_softc *sc;
1021         size_t remaining;
1022         uint32_t table_size, table_offset;
1023
1024         sc = pi->pi_arg;
1025         table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
1026         if (table_offset > 0) {
1027                 if (!enabled) {
1028                         if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
1029                             table_offset) != 0)
1030                                 warnx("pci_passthru: unmap_pptdev_mmio failed");
1031                 } else {
1032                         if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
1033                             table_offset, sc->psc_bar[baridx].addr) != 0)
1034                                 warnx("pci_passthru: map_pptdev_mmio failed");
1035                 }
1036         }
1037         table_size = pi->pi_msix.table_offset - table_offset;
1038         table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
1039         table_size = roundup2(table_size, 4096);
1040         remaining = pi->pi_bar[baridx].size - table_offset - table_size;
1041         if (remaining > 0) {
1042                 address += table_offset + table_size;
1043                 if (!enabled) {
1044                         if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
1045                             remaining) != 0)
1046                                 warnx("pci_passthru: unmap_pptdev_mmio failed");
1047                 } else {
1048                         if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
1049                             remaining, sc->psc_bar[baridx].addr +
1050                             table_offset + table_size) != 0)
1051                                 warnx("pci_passthru: map_pptdev_mmio failed");
1052                 }
1053         }
1054 }
1055
1056 static void
1057 passthru_mmio_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
1058                    int enabled, uint64_t address)
1059 {
1060         struct passthru_softc *sc;
1061
1062         sc = pi->pi_arg;
1063         if (!enabled) {
1064                 if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
1065                     sc->psc_bar[baridx].size) != 0)
1066                         warnx("pci_passthru: unmap_pptdev_mmio failed");
1067         } else {
1068                 if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
1069                     sc->psc_bar[baridx].size, sc->psc_bar[baridx].addr) != 0)
1070                         warnx("pci_passthru: map_pptdev_mmio failed");
1071         }
1072 }
1073
1074 static void
1075 passthru_addr_rom(struct pci_devinst *const pi, const int idx,
1076     const int enabled)
1077 {
1078         const uint64_t addr = pi->pi_bar[idx].addr;
1079         const uint64_t size = pi->pi_bar[idx].size;
1080
1081         if (!enabled) {
1082                 if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) {
1083                         errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed",
1084                             __func__, addr, addr + size);
1085                 }
1086
1087         } else {
1088                 if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM,
1089                         pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) {
1090                         errx(4, "%s: mmap_memseg @ [%016lx - %016lx]  failed",
1091                             __func__, addr, addr + size);
1092                 }
1093         }
1094 }
1095
1096 static void
1097 passthru_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
1098     int enabled, uint64_t address)
1099 {
1100         switch (pi->pi_bar[baridx].type) {
1101         case PCIBAR_IO:
1102                 /* IO BARs are emulated */
1103                 break;
1104         case PCIBAR_ROM:
1105                 passthru_addr_rom(pi, baridx, enabled);
1106                 break;
1107         case PCIBAR_MEM32:
1108         case PCIBAR_MEM64:
1109                 if (baridx == pci_msix_table_bar(pi))
1110                         passthru_msix_addr(ctx, pi, baridx, enabled, address);
1111                 else
1112                         passthru_mmio_addr(ctx, pi, baridx, enabled, address);
1113                 break;
1114         default:
1115                 errx(4, "%s: invalid BAR type %d", __func__,
1116                     pi->pi_bar[baridx].type);
1117         }
1118 }
1119
1120 static const struct pci_devemu passthru = {
1121         .pe_emu         = "passthru",
1122         .pe_init        = passthru_init,
1123         .pe_legacy_config = passthru_legacy_config,
1124         .pe_cfgwrite    = passthru_cfgwrite,
1125         .pe_cfgread     = passthru_cfgread,
1126         .pe_barwrite    = passthru_write,
1127         .pe_barread     = passthru_read,
1128         .pe_baraddr     = passthru_addr,
1129 };
1130 PCI_EMUL_SET(passthru);