drivers/pci/iov.c

   1 /*
   2  * drivers/pci/iov.c
   3  *
   4  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
   5  *
   6  * PCI Express I/O Virtualization (IOV) support.
   7  *   Single Root IOV 1.0
   8  *   Address Translation Service 1.0
   9  */
  10
  11 #include <linux/pci.h>
  12 #include <linux/mutex.h>
  13 #include <linux/string.h>
  14 #include <linux/delay.h>
  15 #include "pci.h"
  16
  17 #define VIRTFN_ID_LEN   16
  18
  19 static inline u8 virtfn_bus(struct pci_dev *dev, int id)
  20 {
  21         return dev->bus->number + ((dev->devfn + dev->sriov->offset +
  22                                     dev->sriov->stride * id) >> 8);
  23 }
  24
  25 static inline u8 virtfn_devfn(struct pci_dev *dev, int id)
  26 {
  27         return (dev->devfn + dev->sriov->offset +
  28                 dev->sriov->stride * id) & 0xff;
  29 }
  30
  31 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
  32 {
  33         int rc;
  34         struct pci_bus *child;
  35
  36         if (bus->number == busnr)
  37                 return bus;
  38
  39         child = pci_find_bus(pci_domain_nr(bus), busnr);
  40         if (child)
  41                 return child;
  42
  43         child = pci_add_new_bus(bus, NULL, busnr);
  44         if (!child)
  45                 return NULL;
  46
  47         child->subordinate = busnr;
  48         child->dev.parent = bus->bridge;
  49         rc = pci_bus_add_child(child);
  50         if (rc) {
  51                 pci_remove_bus(child);
  52                 return NULL;
  53         }
  54
  55         return child;
  56 }
  57
  58 static void virtfn_remove_bus(struct pci_bus *bus, int busnr)
  59 {
  60         struct pci_bus *child;
  61
  62         if (bus->number == busnr)
  63                 return;
  64
  65         child = pci_find_bus(pci_domain_nr(bus), busnr);
  66         BUG_ON(!child);
  67
  68         if (list_empty(&child->devices))
  69                 pci_remove_bus(child);
  70 }
  71
  72 static int virtfn_add(struct pci_dev *dev, int id, int reset)
  73 {
  74         int i;
  75         int rc;
  76         u64 size;
  77         char buf[VIRTFN_ID_LEN];
  78         struct pci_dev *virtfn;
  79         struct resource *res;
  80         struct pci_sriov *iov = dev->sriov;
  81
  82         virtfn = alloc_pci_dev();
  83         if (!virtfn)
  84                 return -ENOMEM;
  85
  86         mutex_lock(&iov->dev->sriov->lock);
  87         virtfn->bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id));
  88         if (!virtfn->bus) {
  89                 kfree(virtfn);
  90                 mutex_unlock(&iov->dev->sriov->lock);
  91                 return -ENOMEM;
  92         }
  93         virtfn->devfn = virtfn_devfn(dev, id);
  94         virtfn->vendor = dev->vendor;
  95         pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
  96         pci_setup_device(virtfn);
  97         virtfn->dev.parent = dev->dev.parent;
  98
  99         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 100                 res = dev->resource + PCI_IOV_RESOURCES + i;
 101                 if (!res->parent)
 102                         continue;
 103                 virtfn->resource[i].name = pci_name(virtfn);
 104                 virtfn->resource[i].flags = res->flags;
 105                 size = resource_size(res);
 106                 do_div(size, iov->total);
 107                 virtfn->resource[i].start = res->start + size * id;
 108                 virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
 109                 rc = request_resource(res, &virtfn->resource[i]);
 110                 BUG_ON(rc);
 111         }
 112
 113         if (reset)
 114                 __pci_reset_function(virtfn);
 115
 116         pci_device_add(virtfn, virtfn->bus);
 117         mutex_unlock(&iov->dev->sriov->lock);
 118
 119         virtfn->physfn = pci_dev_get(dev);
 120         virtfn->is_virtfn = 1;
 121
 122         rc = pci_bus_add_device(virtfn);
 123         if (rc)
 124                 goto failed1;
 125         sprintf(buf, "virtfn%u", id);
 126         rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
 127         if (rc)
 128                 goto failed1;
 129         rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
 130         if (rc)
 131                 goto failed2;
 132
 133         kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
 134
 135         return 0;
 136
 137 failed2:
 138         sysfs_remove_link(&dev->dev.kobj, buf);
 139 failed1:
 140         pci_dev_put(dev);
 141         mutex_lock(&iov->dev->sriov->lock);
 142         pci_remove_bus_device(virtfn);
 143         virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
 144         mutex_unlock(&iov->dev->sriov->lock);
 145
 146         return rc;
 147 }
 148
 149 static void virtfn_remove(struct pci_dev *dev, int id, int reset)
 150 {
 151         char buf[VIRTFN_ID_LEN];
 152         struct pci_bus *bus;
 153         struct pci_dev *virtfn;
 154         struct pci_sriov *iov = dev->sriov;
 155
 156         bus = pci_find_bus(pci_domain_nr(dev->bus), virtfn_bus(dev, id));
 157         if (!bus)
 158                 return;
 159
 160         virtfn = pci_get_slot(bus, virtfn_devfn(dev, id));
 161         if (!virtfn)
 162                 return;
 163
 164         pci_dev_put(virtfn);
 165
 166         if (reset) {
 167                 device_release_driver(&virtfn->dev);
 168                 __pci_reset_function(virtfn);
 169         }
 170
 171         sprintf(buf, "virtfn%u", id);
 172         sysfs_remove_link(&dev->dev.kobj, buf);
 173         sysfs_remove_link(&virtfn->dev.kobj, "physfn");
 174
 175         mutex_lock(&iov->dev->sriov->lock);
 176         pci_remove_bus_device(virtfn);
 177         virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
 178         mutex_unlock(&iov->dev->sriov->lock);
 179
 180         pci_dev_put(dev);
 181 }
 182
 183 static int sriov_migration(struct pci_dev *dev)
 184 {
 185         u16 status;
 186         struct pci_sriov *iov = dev->sriov;
 187
 188         if (!iov->nr_virtfn)
 189                 return 0;
 190
 191         if (!(iov->cap & PCI_SRIOV_CAP_VFM))
 192                 return 0;
 193
 194         pci_read_config_word(dev, iov->pos + PCI_SRIOV_STATUS, &status);
 195         if (!(status & PCI_SRIOV_STATUS_VFM))
 196                 return 0;
 197
 198         schedule_work(&iov->mtask);
 199
 200         return 1;
 201 }
 202
 203 static void sriov_migration_task(struct work_struct *work)
 204 {
 205         int i;
 206         u8 state;
 207         u16 status;
 208         struct pci_sriov *iov = container_of(work, struct pci_sriov, mtask);
 209
 210         for (i = iov->initial; i < iov->nr_virtfn; i++) {
 211                 state = readb(iov->mstate + i);
 212                 if (state == PCI_SRIOV_VFM_MI) {
 213                         writeb(PCI_SRIOV_VFM_AV, iov->mstate + i);
 214                         state = readb(iov->mstate + i);
 215                         if (state == PCI_SRIOV_VFM_AV)
 216                                 virtfn_add(iov->self, i, 1);
 217                 } else if (state == PCI_SRIOV_VFM_MO) {
 218                         virtfn_remove(iov->self, i, 1);
 219                         writeb(PCI_SRIOV_VFM_UA, iov->mstate + i);
 220                         state = readb(iov->mstate + i);
 221                         if (state == PCI_SRIOV_VFM_AV)
 222                                 virtfn_add(iov->self, i, 0);
 223                 }
 224         }
 225
 226         pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status);
 227         status &= ~PCI_SRIOV_STATUS_VFM;
 228         pci_write_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, status);
 229 }
 230
 231 static int sriov_enable_migration(struct pci_dev *dev, int nr_virtfn)
 232 {
 233         int bir;
 234         u32 table;
 235         resource_size_t pa;
 236         struct pci_sriov *iov = dev->sriov;
 237
 238         if (nr_virtfn <= iov->initial)
 239                 return 0;
 240
 241         pci_read_config_dword(dev, iov->pos + PCI_SRIOV_VFM, &table);
 242         bir = PCI_SRIOV_VFM_BIR(table);
 243         if (bir > PCI_STD_RESOURCE_END)
 244                 return -EIO;
 245
 246         table = PCI_SRIOV_VFM_OFFSET(table);
 247         if (table + nr_virtfn > pci_resource_len(dev, bir))
 248                 return -EIO;
 249
 250         pa = pci_resource_start(dev, bir) + table;
 251         iov->mstate = ioremap(pa, nr_virtfn);
 252         if (!iov->mstate)
 253                 return -ENOMEM;
 254
 255         INIT_WORK(&iov->mtask, sriov_migration_task);
 256
 257         iov->ctrl |= PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR;
 258         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 259
 260         return 0;
 261 }
 262
 263 static void sriov_disable_migration(struct pci_dev *dev)
 264 {
 265         struct pci_sriov *iov = dev->sriov;
 266
 267         iov->ctrl &= ~(PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR);
 268         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 269
 270         cancel_work_sync(&iov->mtask);
 271         iounmap(iov->mstate);
 272 }
 273
 274 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 275 {
 276         int rc;
 277         int i, j;
 278         int nres;
 279         u16 offset, stride, initial;
 280         struct resource *res;
 281         struct pci_dev *pdev;
 282         struct pci_sriov *iov = dev->sriov;
 283
 284         if (!nr_virtfn)
 285                 return 0;
 286
 287         if (iov->nr_virtfn)
 288                 return -EINVAL;
 289
 290         pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
 291         if (initial > iov->total ||
 292             (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total)))
 293                 return -EIO;
 294
 295         if (nr_virtfn < 0 || nr_virtfn > iov->total ||
 296             (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
 297                 return -EINVAL;
 298
 299         pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
 300         pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset);
 301         pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride);
 302         if (!offset || (nr_virtfn > 1 && !stride))
 303                 return -EIO;
 304
 305         nres = 0;
 306         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 307                 res = dev->resource + PCI_IOV_RESOURCES + i;
 308                 if (res->parent)
 309                         nres++;
 310         }
 311         if (nres != iov->nres) {
 312                 dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");
 313                 return -ENOMEM;
 314         }
 315
 316         iov->offset = offset;
 317         iov->stride = stride;
 318
 319         if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->subordinate) {
 320                 dev_err(&dev->dev, "SR-IOV: bus number out of range\n");
 321                 return -ENOMEM;
 322         }
 323
 324         if (iov->link != dev->devfn) {
 325                 pdev = pci_get_slot(dev->bus, iov->link);
 326                 if (!pdev)
 327                         return -ENODEV;
 328
 329                 pci_dev_put(pdev);
 330
 331                 if (!pdev->is_physfn)
 332                         return -ENODEV;
 333
 334                 rc = sysfs_create_link(&dev->dev.kobj,
 335                                         &pdev->dev.kobj, "dep_link");
 336                 if (rc)
 337                         return rc;
 338         }
 339
 340         iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
 341         pci_block_user_cfg_access(dev);
 342         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 343         msleep(100);
 344         pci_unblock_user_cfg_access(dev);
 345
 346         iov->initial = initial;
 347         if (nr_virtfn < initial)
 348                 initial = nr_virtfn;
 349
 350         for (i = 0; i < initial; i++) {
 351                 rc = virtfn_add(dev, i, 0);
 352                 if (rc)
 353                         goto failed;
 354         }
 355
 356         if (iov->cap & PCI_SRIOV_CAP_VFM) {
 357                 rc = sriov_enable_migration(dev, nr_virtfn);
 358                 if (rc)
 359                         goto failed;
 360         }
 361
 362         kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
 363         iov->nr_virtfn = nr_virtfn;
 364
 365         return 0;
 366
 367 failed:
 368         for (j = 0; j < i; j++)
 369                 virtfn_remove(dev, j, 0);
 370
 371         iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
 372         pci_block_user_cfg_access(dev);
 373         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 374         ssleep(1);
 375         pci_unblock_user_cfg_access(dev);
 376
 377         if (iov->link != dev->devfn)
 378                 sysfs_remove_link(&dev->dev.kobj, "dep_link");
 379
 380         return rc;
 381 }
 382
 383 static void sriov_disable(struct pci_dev *dev)
 384 {
 385         int i;
 386         struct pci_sriov *iov = dev->sriov;
 387
 388         if (!iov->nr_virtfn)
 389                 return;
 390
 391         if (iov->cap & PCI_SRIOV_CAP_VFM)
 392                 sriov_disable_migration(dev);
 393
 394         for (i = 0; i < iov->nr_virtfn; i++)
 395                 virtfn_remove(dev, i, 0);
 396
 397         iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
 398         pci_block_user_cfg_access(dev);
 399         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 400         ssleep(1);
 401         pci_unblock_user_cfg_access(dev);
 402
 403         if (iov->link != dev->devfn)
 404                 sysfs_remove_link(&dev->dev.kobj, "dep_link");
 405
 406         iov->nr_virtfn = 0;
 407 }
 408
 409 static int sriov_init(struct pci_dev *dev, int pos)
 410 {
 411         int i;
 412         int rc;
 413         int nres;
 414         u32 pgsz;
 415         u16 ctrl, total, offset, stride;
 416         struct pci_sriov *iov;
 417         struct resource *res;
 418         struct pci_dev *pdev;
 419
 420         if (dev->pcie_type != PCI_EXP_TYPE_RC_END &&
 421             dev->pcie_type != PCI_EXP_TYPE_ENDPOINT)
 422                 return -ENODEV;
 423
 424         pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
 425         if (ctrl & PCI_SRIOV_CTRL_VFE) {
 426                 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
 427                 ssleep(1);
 428         }
 429
 430         pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
 431         if (!total)
 432                 return 0;
 433
 434         ctrl = 0;
 435         list_for_each_entry(pdev, &dev->bus->devices, bus_list)
 436                 if (pdev->is_physfn)
 437                         goto found;
 438
 439         pdev = NULL;
 440         if (pci_ari_enabled(dev->bus))
 441                 ctrl |= PCI_SRIOV_CTRL_ARI;
 442
 443 found:
 444         pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
 445         pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, total);
 446         pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset);
 447         pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride);
 448         if (!offset || (total > 1 && !stride))
 449                 return -EIO;
 450
 451         pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
 452         i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
 453         pgsz &= ~((1 << i) - 1);
 454         if (!pgsz)
 455                 return -EIO;
 456
 457         pgsz &= ~(pgsz - 1);
 458         pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
 459
 460         nres = 0;
 461         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 462                 res = dev->resource + PCI_IOV_RESOURCES + i;
 463                 i += __pci_read_base(dev, pci_bar_unknown, res,
 464                                      pos + PCI_SRIOV_BAR + i * 4);
 465                 if (!res->flags)
 466                         continue;
 467                 if (resource_size(res) & (PAGE_SIZE - 1)) {
 468                         rc = -EIO;
 469                         goto failed;
 470                 }
 471                 res->end = res->start + resource_size(res) * total - 1;
 472                 nres++;
 473         }
 474
 475         iov = kzalloc(sizeof(*iov), GFP_KERNEL);
 476         if (!iov) {
 477                 rc = -ENOMEM;
 478                 goto failed;
 479         }
 480
 481         iov->pos = pos;
 482         iov->nres = nres;
 483         iov->ctrl = ctrl;
 484         iov->total = total;
 485         iov->offset = offset;
 486         iov->stride = stride;
 487         iov->pgsz = pgsz;
 488         iov->self = dev;
 489         pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
 490         pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
 491         if (dev->pcie_type == PCI_EXP_TYPE_RC_END)
 492                 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
 493
 494         if (pdev)
 495                 iov->dev = pci_dev_get(pdev);
 496         else
 497                 iov->dev = dev;
 498
 499         mutex_init(&iov->lock);
 500
 501         dev->sriov = iov;
 502         dev->is_physfn = 1;
 503
 504         return 0;
 505
 506 failed:
 507         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 508                 res = dev->resource + PCI_IOV_RESOURCES + i;
 509                 res->flags = 0;
 510         }
 511
 512         return rc;
 513 }
 514
 515 static void sriov_release(struct pci_dev *dev)
 516 {
 517         BUG_ON(dev->sriov->nr_virtfn);
 518
 519         if (dev != dev->sriov->dev)
 520                 pci_dev_put(dev->sriov->dev);
 521
 522         mutex_destroy(&dev->sriov->lock);
 523
 524         kfree(dev->sriov);
 525         dev->sriov = NULL;
 526 }
 527
 528 static void sriov_restore_state(struct pci_dev *dev)
 529 {
 530         int i;
 531         u16 ctrl;
 532         struct pci_sriov *iov = dev->sriov;
 533
 534         pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
 535         if (ctrl & PCI_SRIOV_CTRL_VFE)
 536                 return;
 537
 538         for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
 539                 pci_update_resource(dev, i);
 540
 541         pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
 542         pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, iov->nr_virtfn);
 543         pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 544         if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
 545                 msleep(100);
 546 }
 547
 548 /**
 549  * pci_iov_init - initialize the IOV capability
 550  * @dev: the PCI device
 551  *
 552  * Returns 0 on success, or negative on failure.
 553  */
 554 int pci_iov_init(struct pci_dev *dev)
 555 {
 556         int pos;
 557
 558         if (!pci_is_pcie(dev))
 559                 return -ENODEV;
 560
 561         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
 562         if (pos)
 563                 return sriov_init(dev, pos);
 564
 565         return -ENODEV;
 566 }
 567
 568 /**
 569  * pci_iov_release - release resources used by the IOV capability
 570  * @dev: the PCI device
 571  */
 572 void pci_iov_release(struct pci_dev *dev)
 573 {
 574         if (dev->is_physfn)
 575                 sriov_release(dev);
 576 }
 577
 578 /**
 579  * pci_iov_resource_bar - get position of the SR-IOV BAR
 580  * @dev: the PCI device
 581  * @resno: the resource number
 582  * @type: the BAR type to be filled in
 583  *
 584  * Returns position of the BAR encapsulated in the SR-IOV capability.
 585  */
 586 int pci_iov_resource_bar(struct pci_dev *dev, int resno,
 587                          enum pci_bar_type *type)
 588 {
 589         if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END)
 590                 return 0;
 591
 592         BUG_ON(!dev->is_physfn);
 593
 594         *type = pci_bar_unknown;
 595
 596         return dev->sriov->pos + PCI_SRIOV_BAR +
 597                 4 * (resno - PCI_IOV_RESOURCES);
 598 }
 599
 600 /**
 601  * pci_sriov_resource_alignment - get resource alignment for VF BAR
 602  * @dev: the PCI device
 603  * @resno: the resource number
 604  *
 605  * Returns the alignment of the VF BAR found in the SR-IOV capability.
 606  * This is not the same as the resource size which is defined as
 607  * the VF BAR size multiplied by the number of VFs.  The alignment
 608  * is just the VF BAR size.
 609  */
 610 int pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
 611 {
 612         struct resource tmp;
 613         enum pci_bar_type type;
 614         int reg = pci_iov_resource_bar(dev, resno, &type);
 615
 616         if (!reg)
 617                 return 0;
 618
 619          __pci_read_base(dev, type, &tmp, reg);
 620         return resource_alignment(&tmp);
 621 }
 622
 623 /**
 624  * pci_restore_iov_state - restore the state of the IOV capability
 625  * @dev: the PCI device
 626  */
 627 void pci_restore_iov_state(struct pci_dev *dev)
 628 {
 629         if (dev->is_physfn)
 630                 sriov_restore_state(dev);
 631 }
 632
 633 /**
 634  * pci_iov_bus_range - find bus range used by Virtual Function
 635  * @bus: the PCI bus
 636  *
 637  * Returns max number of buses (exclude current one) used by Virtual
 638  * Functions.
 639  */
 640 int pci_iov_bus_range(struct pci_bus *bus)
 641 {
 642         int max = 0;
 643         u8 busnr;
 644         struct pci_dev *dev;
 645
 646         list_for_each_entry(dev, &bus->devices, bus_list) {
 647                 if (!dev->is_physfn)
 648                         continue;
 649                 busnr = virtfn_bus(dev, dev->sriov->total - 1);
 650                 if (busnr > max)
 651                         max = busnr;
 652         }
 653
 654         return max ? max - bus->number : 0;
 655 }
 656
 657 /**
 658  * pci_enable_sriov - enable the SR-IOV capability
 659  * @dev: the PCI device
 660  * @nr_virtfn: number of virtual functions to enable
 661  *
 662  * Returns 0 on success, or negative on failure.
 663  */
 664 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 665 {
 666         might_sleep();
 667
 668         if (!dev->is_physfn)
 669                 return -ENODEV;
 670
 671         return sriov_enable(dev, nr_virtfn);
 672 }
 673 EXPORT_SYMBOL_GPL(pci_enable_sriov);
 674
 675 /**
 676  * pci_disable_sriov - disable the SR-IOV capability
 677  * @dev: the PCI device
 678  */
 679 void pci_disable_sriov(struct pci_dev *dev)
 680 {
 681         might_sleep();
 682
 683         if (!dev->is_physfn)
 684                 return;
 685
 686         sriov_disable(dev);
 687 }
 688 EXPORT_SYMBOL_GPL(pci_disable_sriov);
 689
 690 /**
 691  * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration
 692  * @dev: the PCI device
 693  *
 694  * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not.
 695  *
 696  * Physical Function driver is responsible to register IRQ handler using
 697  * VF Migration Interrupt Message Number, and call this function when the
 698  * interrupt is generated by the hardware.
 699  */
 700 irqreturn_t pci_sriov_migration(struct pci_dev *dev)
 701 {
 702         if (!dev->is_physfn)
 703                 return IRQ_NONE;
 704
 705         return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE;
 706 }
 707 EXPORT_SYMBOL_GPL(pci_sriov_migration);
 708
 709 static int ats_alloc_one(struct pci_dev *dev, int ps)
 710 {
 711         int pos;
 712         u16 cap;
 713         struct pci_ats *ats;
 714
 715         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
 716         if (!pos)
 717                 return -ENODEV;
 718
 719         ats = kzalloc(sizeof(*ats), GFP_KERNEL);
 720         if (!ats)
 721                 return -ENOMEM;
 722
 723         ats->pos = pos;
 724         ats->stu = ps;
 725         pci_read_config_word(dev, pos + PCI_ATS_CAP, &cap);
 726         ats->qdep = PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) :
 727                                             PCI_ATS_MAX_QDEP;
 728         dev->ats = ats;
 729
 730         return 0;
 731 }
 732
 733 static void ats_free_one(struct pci_dev *dev)
 734 {
 735         kfree(dev->ats);
 736         dev->ats = NULL;
 737 }
 738
 739 /**
 740  * pci_enable_ats - enable the ATS capability
 741  * @dev: the PCI device
 742  * @ps: the IOMMU page shift
 743  *
 744  * Returns 0 on success, or negative on failure.
 745  */
 746 int pci_enable_ats(struct pci_dev *dev, int ps)
 747 {
 748         int rc;
 749         u16 ctrl;
 750
 751         BUG_ON(dev->ats && dev->ats->is_enabled);
 752
 753         if (ps < PCI_ATS_MIN_STU)
 754                 return -EINVAL;
 755
 756         if (dev->is_physfn || dev->is_virtfn) {
 757                 struct pci_dev *pdev = dev->is_physfn ? dev : dev->physfn;
 758
 759                 mutex_lock(&pdev->sriov->lock);
 760                 if (pdev->ats)
 761                         rc = pdev->ats->stu == ps ? 0 : -EINVAL;
 762                 else
 763                         rc = ats_alloc_one(pdev, ps);
 764
 765                 if (!rc)
 766                         pdev->ats->ref_cnt++;
 767                 mutex_unlock(&pdev->sriov->lock);
 768                 if (rc)
 769                         return rc;
 770         }
 771
 772         if (!dev->is_physfn) {
 773                 rc = ats_alloc_one(dev, ps);
 774                 if (rc)
 775                         return rc;
 776         }
 777
 778         ctrl = PCI_ATS_CTRL_ENABLE;
 779         if (!dev->is_virtfn)
 780                 ctrl |= PCI_ATS_CTRL_STU(ps - PCI_ATS_MIN_STU);
 781         pci_write_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, ctrl);
 782
 783         dev->ats->is_enabled = 1;
 784
 785         return 0;
 786 }
 787
 788 /**
 789  * pci_disable_ats - disable the ATS capability
 790  * @dev: the PCI device
 791  */
 792 void pci_disable_ats(struct pci_dev *dev)
 793 {
 794         u16 ctrl;
 795
 796         BUG_ON(!dev->ats || !dev->ats->is_enabled);
 797
 798         pci_read_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, &ctrl);
 799         ctrl &= ~PCI_ATS_CTRL_ENABLE;
 800         pci_write_config_word(dev, dev->ats->pos + PCI_ATS_CTRL, ctrl);
 801
 802         dev->ats->is_enabled = 0;
 803
 804         if (dev->is_physfn || dev->is_virtfn) {
 805                 struct pci_dev *pdev = dev->is_physfn ? dev : dev->physfn;
 806
 807                 mutex_lock(&pdev->sriov->lock);
 808                 pdev->ats->ref_cnt--;
 809                 if (!pdev->ats->ref_cnt)
 810                         ats_free_one(pdev);
 811                 mutex_unlock(&pdev->sriov->lock);
 812         }
 813
 814         if (!dev->is_physfn)
 815                 ats_free_one(dev);
 816 }
 817
 818 /**
 819  * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
 820  * @dev: the PCI device
 821  *
 822  * Returns the queue depth on success, or negative on failure.
 823  *
 824  * The ATS spec uses 0 in the Invalidate Queue Depth field to
 825  * indicate that the function can accept 32 Invalidate Request.
 826  * But here we use the `real' values (i.e. 1~32) for the Queue
 827  * Depth; and 0 indicates the function shares the Queue with
 828  * other functions (doesn't exclusively own a Queue).
 829  */
 830 int pci_ats_queue_depth(struct pci_dev *dev)
 831 {
 832         int pos;
 833         u16 cap;
 834
 835         if (dev->is_virtfn)
 836                 return 0;
 837
 838         if (dev->ats)
 839                 return dev->ats->qdep;
 840
 841         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
 842         if (!pos)
 843                 return -ENODEV;
 844
 845         pci_read_config_word(dev, pos + PCI_ATS_CAP, &cap);
 846
 847         return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) :
 848                                        PCI_ATS_MAX_QDEP;
 849 }