drivers/virtio/virtio_pci_common.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Virtio PCI driver - common functionality for all device versions
   4  *
   5  * This module allows virtio devices to be used over a virtual PCI device.
   6  * This can be used with QEMU based VMMs like KVM or Xen.
   7  *
   8  * Copyright IBM Corp. 2007
   9  * Copyright Red Hat, Inc. 2014
  10  *
  11  * Authors:
  12  *  Anthony Liguori  <aliguori@us.ibm.com>
  13  *  Rusty Russell <rusty@rustcorp.com.au>
  14  *  Michael S. Tsirkin <mst@redhat.com>
  15  */
  16
  17 #include "virtio_pci_common.h"
  18
  19 static bool force_legacy = false;
  20
  21 #if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
  22 module_param(force_legacy, bool, 0444);
  23 MODULE_PARM_DESC(force_legacy,
  24                  "Force legacy mode for transitional virtio 1 devices");
  25 #endif
  26
  27 /* wait for pending irq handlers */
  28 void vp_synchronize_vectors(struct virtio_device *vdev)
  29 {
  30         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  31         int i;
  32
  33         if (vp_dev->intx_enabled)
  34                 synchronize_irq(vp_dev->pci_dev->irq);
  35
  36         for (i = 0; i < vp_dev->msix_vectors; ++i)
  37                 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
  38 }
  39
  40 /* the notify function used when creating a virt queue */
  41 bool vp_notify(struct virtqueue *vq)
  42 {
  43         /* we write the queue's selector into the notification register to
  44          * signal the other end */
  45         iowrite16(vq->index, (void __iomem *)vq->priv);
  46         return true;
  47 }
  48
  49 /* Notify all slow path virtqueues on an interrupt. */
  50 static void vp_vring_slow_path_interrupt(int irq,
  51                                          struct virtio_pci_device *vp_dev)
  52 {
  53         struct virtio_pci_vq_info *info;
  54         unsigned long flags;
  55
  56         spin_lock_irqsave(&vp_dev->lock, flags);
  57         list_for_each_entry(info, &vp_dev->slow_virtqueues, node)
  58                 vring_interrupt(irq, info->vq);
  59         spin_unlock_irqrestore(&vp_dev->lock, flags);
  60 }
  61
  62 /* Handle a configuration change: Tell driver if it wants to know. */
  63 static irqreturn_t vp_config_changed(int irq, void *opaque)
  64 {
  65         struct virtio_pci_device *vp_dev = opaque;
  66
  67         virtio_config_changed(&vp_dev->vdev);
  68         vp_vring_slow_path_interrupt(irq, vp_dev);
  69         return IRQ_HANDLED;
  70 }
  71
  72 /* Notify all virtqueues on an interrupt. */
  73 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
  74 {
  75         struct virtio_pci_device *vp_dev = opaque;
  76         struct virtio_pci_vq_info *info;
  77         irqreturn_t ret = IRQ_NONE;
  78         unsigned long flags;
  79
  80         spin_lock_irqsave(&vp_dev->lock, flags);
  81         list_for_each_entry(info, &vp_dev->virtqueues, node) {
  82                 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
  83                         ret = IRQ_HANDLED;
  84         }
  85         spin_unlock_irqrestore(&vp_dev->lock, flags);
  86
  87         return ret;
  88 }
  89
  90 /* A small wrapper to also acknowledge the interrupt when it's handled.
  91  * I really need an EIO hook for the vring so I can ack the interrupt once we
  92  * know that we'll be handling the IRQ but before we invoke the callback since
  93  * the callback may notify the host which results in the host attempting to
  94  * raise an interrupt that we would then mask once we acknowledged the
  95  * interrupt. */
  96 static irqreturn_t vp_interrupt(int irq, void *opaque)
  97 {
  98         struct virtio_pci_device *vp_dev = opaque;
  99         u8 isr;
 100
 101         /* reading the ISR has the effect of also clearing it so it's very
 102          * important to save off the value. */
 103         isr = ioread8(vp_dev->isr);
 104
 105         /* It's definitely not us if the ISR was not high */
 106         if (!isr)
 107                 return IRQ_NONE;
 108
 109         /* Configuration change?  Tell driver if it wants to know. */
 110         if (isr & VIRTIO_PCI_ISR_CONFIG)
 111                 vp_config_changed(irq, opaque);
 112
 113         return vp_vring_interrupt(irq, opaque);
 114 }
 115
 116 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 117                                    bool per_vq_vectors, struct irq_affinity *desc)
 118 {
 119         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 120         const char *name = dev_name(&vp_dev->vdev.dev);
 121         unsigned int flags = PCI_IRQ_MSIX;
 122         unsigned int i, v;
 123         int err = -ENOMEM;
 124
 125         vp_dev->msix_vectors = nvectors;
 126
 127         vp_dev->msix_names = kmalloc_array(nvectors,
 128                                            sizeof(*vp_dev->msix_names),
 129                                            GFP_KERNEL);
 130         if (!vp_dev->msix_names)
 131                 goto error;
 132         vp_dev->msix_affinity_masks
 133                 = kcalloc(nvectors, sizeof(*vp_dev->msix_affinity_masks),
 134                           GFP_KERNEL);
 135         if (!vp_dev->msix_affinity_masks)
 136                 goto error;
 137         for (i = 0; i < nvectors; ++i)
 138                 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
 139                                         GFP_KERNEL))
 140                         goto error;
 141
 142         if (!per_vq_vectors)
 143                 desc = NULL;
 144
 145         if (desc) {
 146                 flags |= PCI_IRQ_AFFINITY;
 147                 desc->pre_vectors++; /* virtio config vector */
 148         }
 149
 150         err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,
 151                                              nvectors, flags, desc);
 152         if (err < 0)
 153                 goto error;
 154         vp_dev->msix_enabled = 1;
 155
 156         /* Set the vector used for configuration */
 157         v = vp_dev->msix_used_vectors;
 158         snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 159                  "%s-config", name);
 160         err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
 161                           vp_config_changed, 0, vp_dev->msix_names[v],
 162                           vp_dev);
 163         if (err)
 164                 goto error;
 165         ++vp_dev->msix_used_vectors;
 166
 167         v = vp_dev->config_vector(vp_dev, v);
 168         /* Verify we had enough resources to assign the vector */
 169         if (v == VIRTIO_MSI_NO_VECTOR) {
 170                 err = -EBUSY;
 171                 goto error;
 172         }
 173
 174         if (!per_vq_vectors) {
 175                 /* Shared vector for all VQs */
 176                 v = vp_dev->msix_used_vectors;
 177                 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 178                          "%s-virtqueues", name);
 179                 err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
 180                                   vp_vring_interrupt, 0, vp_dev->msix_names[v],
 181                                   vp_dev);
 182                 if (err)
 183                         goto error;
 184                 ++vp_dev->msix_used_vectors;
 185         }
 186         return 0;
 187 error:
 188         return err;
 189 }
 190
 191 static bool vp_is_slow_path_vector(u16 msix_vec)
 192 {
 193         return msix_vec == VP_MSIX_CONFIG_VECTOR;
 194 }
 195
 196 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index,
 197                                      void (*callback)(struct virtqueue *vq),
 198                                      const char *name,
 199                                      bool ctx,
 200                                      u16 msix_vec,
 201                                      struct virtio_pci_vq_info **p_info)
 202 {
 203         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 204         struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
 205         struct virtqueue *vq;
 206         unsigned long flags;
 207
 208         /* fill out our structure that represents an active queue */
 209         if (!info)
 210                 return ERR_PTR(-ENOMEM);
 211
 212         vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx,
 213                               msix_vec);
 214         if (IS_ERR(vq))
 215                 goto out_info;
 216
 217         info->vq = vq;
 218         if (callback) {
 219                 spin_lock_irqsave(&vp_dev->lock, flags);
 220                 if (!vp_is_slow_path_vector(msix_vec))
 221                         list_add(&info->node, &vp_dev->virtqueues);
 222                 else
 223                         list_add(&info->node, &vp_dev->slow_virtqueues);
 224                 spin_unlock_irqrestore(&vp_dev->lock, flags);
 225         } else {
 226                 INIT_LIST_HEAD(&info->node);
 227         }
 228
 229         *p_info = info;
 230         return vq;
 231
 232 out_info:
 233         kfree(info);
 234         return vq;
 235 }
 236
 237 static void vp_del_vq(struct virtqueue *vq)
 238 {
 239         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
 240         struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 241         unsigned long flags;
 242
 243         /*
 244          * If it fails during re-enable reset vq. This way we won't rejoin
 245          * info->node to the queue. Prevent unexpected irqs.
 246          */
 247         if (!vq->reset) {
 248                 spin_lock_irqsave(&vp_dev->lock, flags);
 249                 list_del(&info->node);
 250                 spin_unlock_irqrestore(&vp_dev->lock, flags);
 251         }
 252
 253         vp_dev->del_vq(info);
 254         kfree(info);
 255 }
 256
 257 /* the config->del_vqs() implementation */
 258 void vp_del_vqs(struct virtio_device *vdev)
 259 {
 260         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 261         struct virtqueue *vq, *n;
 262         int i;
 263
 264         list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
 265                 if (vp_dev->per_vq_vectors) {
 266                         int v = vp_dev->vqs[vq->index]->msix_vector;
 267
 268                         if (v != VIRTIO_MSI_NO_VECTOR &&
 269                             !vp_is_slow_path_vector(v)) {
 270                                 int irq = pci_irq_vector(vp_dev->pci_dev, v);
 271
 272                                 irq_update_affinity_hint(irq, NULL);
 273                                 free_irq(irq, vq);
 274                         }
 275                 }
 276                 vp_del_vq(vq);
 277         }
 278         vp_dev->per_vq_vectors = false;
 279
 280         if (vp_dev->intx_enabled) {
 281                 free_irq(vp_dev->pci_dev->irq, vp_dev);
 282                 vp_dev->intx_enabled = 0;
 283         }
 284
 285         for (i = 0; i < vp_dev->msix_used_vectors; ++i)
 286                 free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);
 287
 288         if (vp_dev->msix_affinity_masks) {
 289                 for (i = 0; i < vp_dev->msix_vectors; i++)
 290                         free_cpumask_var(vp_dev->msix_affinity_masks[i]);
 291         }
 292
 293         if (vp_dev->msix_enabled) {
 294                 /* Disable the vector used for configuration */
 295                 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
 296
 297                 pci_free_irq_vectors(vp_dev->pci_dev);
 298                 vp_dev->msix_enabled = 0;
 299         }
 300
 301         vp_dev->msix_vectors = 0;
 302         vp_dev->msix_used_vectors = 0;
 303         kfree(vp_dev->msix_names);
 304         vp_dev->msix_names = NULL;
 305         kfree(vp_dev->msix_affinity_masks);
 306         vp_dev->msix_affinity_masks = NULL;
 307         kfree(vp_dev->vqs);
 308         vp_dev->vqs = NULL;
 309 }
 310
 311 enum vp_vq_vector_policy {
 312         VP_VQ_VECTOR_POLICY_EACH,
 313         VP_VQ_VECTOR_POLICY_SHARED_SLOW,
 314         VP_VQ_VECTOR_POLICY_SHARED,
 315 };
 316
 317 static struct virtqueue *
 318 vp_find_one_vq_msix(struct virtio_device *vdev, int queue_idx,
 319                     vq_callback_t *callback, const char *name, bool ctx,
 320                     bool slow_path, int *allocated_vectors,
 321                     enum vp_vq_vector_policy vector_policy,
 322                     struct virtio_pci_vq_info **p_info)
 323 {
 324         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 325         struct virtqueue *vq;
 326         u16 msix_vec;
 327         int err;
 328
 329         if (!callback)
 330                 msix_vec = VIRTIO_MSI_NO_VECTOR;
 331         else if (vector_policy == VP_VQ_VECTOR_POLICY_EACH ||
 332                  (vector_policy == VP_VQ_VECTOR_POLICY_SHARED_SLOW &&
 333                  !slow_path))
 334                 msix_vec = (*allocated_vectors)++;
 335         else if (vector_policy != VP_VQ_VECTOR_POLICY_EACH &&
 336                  slow_path)
 337                 msix_vec = VP_MSIX_CONFIG_VECTOR;
 338         else
 339                 msix_vec = VP_MSIX_VQ_VECTOR;
 340         vq = vp_setup_vq(vdev, queue_idx, callback, name, ctx, msix_vec,
 341                          p_info);
 342         if (IS_ERR(vq))
 343                 return vq;
 344
 345         if (vector_policy == VP_VQ_VECTOR_POLICY_SHARED ||
 346             msix_vec == VIRTIO_MSI_NO_VECTOR ||
 347             vp_is_slow_path_vector(msix_vec))
 348                 return vq;
 349
 350         /* allocate per-vq irq if available and necessary */
 351         snprintf(vp_dev->msix_names[msix_vec], sizeof(*vp_dev->msix_names),
 352                  "%s-%s", dev_name(&vp_dev->vdev.dev), name);
 353         err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
 354                           vring_interrupt, 0,
 355                           vp_dev->msix_names[msix_vec], vq);
 356         if (err) {
 357                 vp_del_vq(vq);
 358                 return ERR_PTR(err);
 359         }
 360
 361         return vq;
 362 }
 363
 364 static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
 365                             struct virtqueue *vqs[],
 366                             struct virtqueue_info vqs_info[],
 367                             enum vp_vq_vector_policy vector_policy,
 368                             struct irq_affinity *desc)
 369 {
 370         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 371         struct virtio_pci_admin_vq *avq = &vp_dev->admin_vq;
 372         struct virtqueue_info *vqi;
 373         int i, err, nvectors, allocated_vectors, queue_idx = 0;
 374         struct virtqueue *vq;
 375         bool per_vq_vectors;
 376         u16 avq_num = 0;
 377
 378         vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
 379         if (!vp_dev->vqs)
 380                 return -ENOMEM;
 381
 382         if (vp_dev->avq_index) {
 383                 err = vp_dev->avq_index(vdev, &avq->vq_index, &avq_num);
 384                 if (err)
 385                         goto error_find;
 386         }
 387
 388         per_vq_vectors = vector_policy != VP_VQ_VECTOR_POLICY_SHARED;
 389
 390         if (per_vq_vectors) {
 391                 /* Best option: one for change interrupt, one per vq. */
 392                 nvectors = 1;
 393                 for (i = 0; i < nvqs; ++i) {
 394                         vqi = &vqs_info[i];
 395                         if (vqi->name && vqi->callback)
 396                                 ++nvectors;
 397                 }
 398                 if (avq_num && vector_policy == VP_VQ_VECTOR_POLICY_EACH)
 399                         ++nvectors;
 400         } else {
 401                 /* Second best: one for change, shared for all vqs. */
 402                 nvectors = 2;
 403         }
 404
 405         err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors, desc);
 406         if (err)
 407                 goto error_find;
 408
 409         vp_dev->per_vq_vectors = per_vq_vectors;
 410         allocated_vectors = vp_dev->msix_used_vectors;
 411         for (i = 0; i < nvqs; ++i) {
 412                 vqi = &vqs_info[i];
 413                 if (!vqi->name) {
 414                         vqs[i] = NULL;
 415                         continue;
 416                 }
 417                 vqs[i] = vp_find_one_vq_msix(vdev, queue_idx++, vqi->callback,
 418                                              vqi->name, vqi->ctx, false,
 419                                              &allocated_vectors, vector_policy,
 420                                              &vp_dev->vqs[i]);
 421                 if (IS_ERR(vqs[i])) {
 422                         err = PTR_ERR(vqs[i]);
 423                         goto error_find;
 424                 }
 425         }
 426
 427         if (!avq_num)
 428                 return 0;
 429         sprintf(avq->name, "avq.%u", avq->vq_index);
 430         vq = vp_find_one_vq_msix(vdev, avq->vq_index, vp_modern_avq_done,
 431                                  avq->name, false, true, &allocated_vectors,
 432                                  vector_policy, &vp_dev->admin_vq.info);
 433         if (IS_ERR(vq)) {
 434                 err = PTR_ERR(vq);
 435                 goto error_find;
 436         }
 437
 438         return 0;
 439
 440 error_find:
 441         vp_del_vqs(vdev);
 442         return err;
 443 }
 444
 445 static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs,
 446                             struct virtqueue *vqs[],
 447                             struct virtqueue_info vqs_info[])
 448 {
 449         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 450         struct virtio_pci_admin_vq *avq = &vp_dev->admin_vq;
 451         int i, err, queue_idx = 0;
 452         struct virtqueue *vq;
 453         u16 avq_num = 0;
 454
 455         vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
 456         if (!vp_dev->vqs)
 457                 return -ENOMEM;
 458
 459         if (vp_dev->avq_index) {
 460                 err = vp_dev->avq_index(vdev, &avq->vq_index, &avq_num);
 461                 if (err)
 462                         goto out_del_vqs;
 463         }
 464
 465         err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
 466                         dev_name(&vdev->dev), vp_dev);
 467         if (err)
 468                 goto out_del_vqs;
 469
 470         vp_dev->intx_enabled = 1;
 471         vp_dev->per_vq_vectors = false;
 472         for (i = 0; i < nvqs; ++i) {
 473                 struct virtqueue_info *vqi = &vqs_info[i];
 474
 475                 if (!vqi->name) {
 476                         vqs[i] = NULL;
 477                         continue;
 478                 }
 479                 vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback,
 480                                      vqi->name, vqi->ctx,
 481                                      VIRTIO_MSI_NO_VECTOR, &vp_dev->vqs[i]);
 482                 if (IS_ERR(vqs[i])) {
 483                         err = PTR_ERR(vqs[i]);
 484                         goto out_del_vqs;
 485                 }
 486         }
 487
 488         if (!avq_num)
 489                 return 0;
 490         sprintf(avq->name, "avq.%u", avq->vq_index);
 491         vq = vp_setup_vq(vdev, queue_idx++, vp_modern_avq_done, avq->name,
 492                          false, VIRTIO_MSI_NO_VECTOR,
 493                          &vp_dev->admin_vq.info);
 494         if (IS_ERR(vq)) {
 495                 err = PTR_ERR(vq);
 496                 goto out_del_vqs;
 497         }
 498
 499         return 0;
 500 out_del_vqs:
 501         vp_del_vqs(vdev);
 502         return err;
 503 }
 504
 505 /* the config->find_vqs() implementation */
 506 int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 507                 struct virtqueue *vqs[], struct virtqueue_info vqs_info[],
 508                 struct irq_affinity *desc)
 509 {
 510         int err;
 511
 512         /* Try MSI-X with one vector per queue. */
 513         err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info,
 514                                VP_VQ_VECTOR_POLICY_EACH, desc);
 515         if (!err)
 516                 return 0;
 517         /* Fallback: MSI-X with one shared vector for config and
 518          * slow path queues, one vector per queue for the rest.
 519          */
 520         err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info,
 521                                VP_VQ_VECTOR_POLICY_SHARED_SLOW, desc);
 522         if (!err)
 523                 return 0;
 524         /* Fallback: MSI-X with one vector for config, one shared for queues. */
 525         err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info,
 526                                VP_VQ_VECTOR_POLICY_SHARED, desc);
 527         if (!err)
 528                 return 0;
 529         /* Is there an interrupt? If not give up. */
 530         if (!(to_vp_device(vdev)->pci_dev->irq))
 531                 return err;
 532         /* Finally fall back to regular interrupts. */
 533         return vp_find_vqs_intx(vdev, nvqs, vqs, vqs_info);
 534 }
 535
 536 const char *vp_bus_name(struct virtio_device *vdev)
 537 {
 538         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 539
 540         return pci_name(vp_dev->pci_dev);
 541 }
 542
 543 /* Setup the affinity for a virtqueue:
 544  * - force the affinity for per vq vector
 545  * - OR over all affinities for shared MSI
 546  * - ignore the affinity request if we're using INTX
 547  */
 548 int vp_set_vq_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask)
 549 {
 550         struct virtio_device *vdev = vq->vdev;
 551         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 552         struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 553         struct cpumask *mask;
 554         unsigned int irq;
 555
 556         if (!vq->callback)
 557                 return -EINVAL;
 558
 559         if (vp_dev->msix_enabled) {
 560                 mask = vp_dev->msix_affinity_masks[info->msix_vector];
 561                 irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector);
 562                 if (!cpu_mask)
 563                         irq_update_affinity_hint(irq, NULL);
 564                 else {
 565                         cpumask_copy(mask, cpu_mask);
 566                         irq_set_affinity_and_hint(irq, mask);
 567                 }
 568         }
 569         return 0;
 570 }
 571
 572 const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index)
 573 {
 574         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 575
 576         if (!vp_dev->per_vq_vectors ||
 577             vp_dev->vqs[index]->msix_vector == VIRTIO_MSI_NO_VECTOR ||
 578             vp_is_slow_path_vector(vp_dev->vqs[index]->msix_vector))
 579                 return NULL;
 580
 581         return pci_irq_get_affinity(vp_dev->pci_dev,
 582                                     vp_dev->vqs[index]->msix_vector);
 583 }
 584
 585 #ifdef CONFIG_PM_SLEEP
 586 static int virtio_pci_freeze(struct device *dev)
 587 {
 588         struct pci_dev *pci_dev = to_pci_dev(dev);
 589         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 590         int ret;
 591
 592         ret = virtio_device_freeze(&vp_dev->vdev);
 593
 594         if (!ret)
 595                 pci_disable_device(pci_dev);
 596         return ret;
 597 }
 598
 599 static int virtio_pci_restore(struct device *dev)
 600 {
 601         struct pci_dev *pci_dev = to_pci_dev(dev);
 602         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 603         int ret;
 604
 605         ret = pci_enable_device(pci_dev);
 606         if (ret)
 607                 return ret;
 608
 609         pci_set_master(pci_dev);
 610         return virtio_device_restore(&vp_dev->vdev);
 611 }
 612
 613 static bool vp_supports_pm_no_reset(struct device *dev)
 614 {
 615         struct pci_dev *pci_dev = to_pci_dev(dev);
 616         u16 pmcsr;
 617
 618         if (!pci_dev->pm_cap)
 619                 return false;
 620
 621         pci_read_config_word(pci_dev, pci_dev->pm_cap + PCI_PM_CTRL, &pmcsr);
 622         if (PCI_POSSIBLE_ERROR(pmcsr)) {
 623                 dev_err(dev, "Unable to query pmcsr");
 624                 return false;
 625         }
 626
 627         return pmcsr & PCI_PM_CTRL_NO_SOFT_RESET;
 628 }
 629
 630 static int virtio_pci_suspend(struct device *dev)
 631 {
 632         return vp_supports_pm_no_reset(dev) ? 0 : virtio_pci_freeze(dev);
 633 }
 634
 635 static int virtio_pci_resume(struct device *dev)
 636 {
 637         return vp_supports_pm_no_reset(dev) ? 0 : virtio_pci_restore(dev);
 638 }
 639
 640 static const struct dev_pm_ops virtio_pci_pm_ops = {
 641         .suspend = virtio_pci_suspend,
 642         .resume = virtio_pci_resume,
 643         .freeze = virtio_pci_freeze,
 644         .thaw = virtio_pci_restore,
 645         .poweroff = virtio_pci_freeze,
 646         .restore = virtio_pci_restore,
 647 };
 648 #endif
 649
 650
 651 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
 652 static const struct pci_device_id virtio_pci_id_table[] = {
 653         { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) },
 654         { 0 }
 655 };
 656
 657 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
 658
 659 static void virtio_pci_release_dev(struct device *_d)
 660 {
 661         struct virtio_device *vdev = dev_to_virtio(_d);
 662         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 663
 664         /* As struct device is a kobject, it's not safe to
 665          * free the memory (including the reference counter itself)
 666          * until it's release callback. */
 667         kfree(vp_dev);
 668 }
 669
 670 static int virtio_pci_probe(struct pci_dev *pci_dev,
 671                             const struct pci_device_id *id)
 672 {
 673         struct virtio_pci_device *vp_dev, *reg_dev = NULL;
 674         int rc;
 675
 676         /* allocate our structure and fill it out */
 677         vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
 678         if (!vp_dev)
 679                 return -ENOMEM;
 680
 681         pci_set_drvdata(pci_dev, vp_dev);
 682         vp_dev->vdev.dev.parent = &pci_dev->dev;
 683         vp_dev->vdev.dev.release = virtio_pci_release_dev;
 684         vp_dev->pci_dev = pci_dev;
 685         INIT_LIST_HEAD(&vp_dev->virtqueues);
 686         INIT_LIST_HEAD(&vp_dev->slow_virtqueues);
 687         spin_lock_init(&vp_dev->lock);
 688
 689         /* enable the device */
 690         rc = pci_enable_device(pci_dev);
 691         if (rc)
 692                 goto err_enable_device;
 693
 694         if (force_legacy) {
 695                 rc = virtio_pci_legacy_probe(vp_dev);
 696                 /* Also try modern mode if we can't map BAR0 (no IO space). */
 697                 if (rc == -ENODEV || rc == -ENOMEM)
 698                         rc = virtio_pci_modern_probe(vp_dev);
 699                 if (rc)
 700                         goto err_probe;
 701         } else {
 702                 rc = virtio_pci_modern_probe(vp_dev);
 703                 if (rc == -ENODEV)
 704                         rc = virtio_pci_legacy_probe(vp_dev);
 705                 if (rc)
 706                         goto err_probe;
 707         }
 708
 709         pci_set_master(pci_dev);
 710
 711         rc = register_virtio_device(&vp_dev->vdev);
 712         reg_dev = vp_dev;
 713         if (rc)
 714                 goto err_register;
 715
 716         return 0;
 717
 718 err_register:
 719         if (vp_dev->is_legacy)
 720                 virtio_pci_legacy_remove(vp_dev);
 721         else
 722                 virtio_pci_modern_remove(vp_dev);
 723 err_probe:
 724         pci_disable_device(pci_dev);
 725 err_enable_device:
 726         if (reg_dev)
 727                 put_device(&vp_dev->vdev.dev);
 728         else
 729                 kfree(vp_dev);
 730         return rc;
 731 }
 732
 733 static void virtio_pci_remove(struct pci_dev *pci_dev)
 734 {
 735         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 736         struct device *dev = get_device(&vp_dev->vdev.dev);
 737
 738         /*
 739          * Device is marked broken on surprise removal so that virtio upper
 740          * layers can abort any ongoing operation.
 741          */
 742         if (!pci_device_is_present(pci_dev))
 743                 virtio_break_device(&vp_dev->vdev);
 744
 745         pci_disable_sriov(pci_dev);
 746
 747         unregister_virtio_device(&vp_dev->vdev);
 748
 749         if (vp_dev->is_legacy)
 750                 virtio_pci_legacy_remove(vp_dev);
 751         else
 752                 virtio_pci_modern_remove(vp_dev);
 753
 754         pci_disable_device(pci_dev);
 755         put_device(dev);
 756 }
 757
 758 static int virtio_pci_sriov_configure(struct pci_dev *pci_dev, int num_vfs)
 759 {
 760         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 761         struct virtio_device *vdev = &vp_dev->vdev;
 762         int ret;
 763
 764         if (!(vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_DRIVER_OK))
 765                 return -EBUSY;
 766
 767         if (!__virtio_test_bit(vdev, VIRTIO_F_SR_IOV))
 768                 return -EINVAL;
 769
 770         if (pci_vfs_assigned(pci_dev))
 771                 return -EPERM;
 772
 773         if (num_vfs == 0) {
 774                 pci_disable_sriov(pci_dev);
 775                 return 0;
 776         }
 777
 778         ret = pci_enable_sriov(pci_dev, num_vfs);
 779         if (ret < 0)
 780                 return ret;
 781
 782         return num_vfs;
 783 }
 784
 785 static struct pci_driver virtio_pci_driver = {
 786         .name           = "virtio-pci",
 787         .id_table       = virtio_pci_id_table,
 788         .probe          = virtio_pci_probe,
 789         .remove         = virtio_pci_remove,
 790 #ifdef CONFIG_PM_SLEEP
 791         .driver.pm      = &virtio_pci_pm_ops,
 792 #endif
 793         .sriov_configure = virtio_pci_sriov_configure,
 794 };
 795
 796 struct virtio_device *virtio_pci_vf_get_pf_dev(struct pci_dev *pdev)
 797 {
 798         struct virtio_pci_device *pf_vp_dev;
 799
 800         pf_vp_dev = pci_iov_get_pf_drvdata(pdev, &virtio_pci_driver);
 801         if (IS_ERR(pf_vp_dev))
 802                 return NULL;
 803
 804         return &pf_vp_dev->vdev;
 805 }
 806
 807 module_pci_driver(virtio_pci_driver);
 808
 809 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
 810 MODULE_DESCRIPTION("virtio-pci");
 811 MODULE_LICENSE("GPL");
 812 MODULE_VERSION("1");