drivers/pci/pcie/aer/aerdrv_core.c

   1 /*
   2  * drivers/pci/pcie/aer/aerdrv_core.c
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * This file implements the core part of PCI-Express AER. When an pci-express
   9  * error is delivered, an error message will be collected and printed to
  10  * console, then, an error recovery procedure will be executed by following
  11  * the pci error recovery rules.
  12  *
  13  * Copyright (C) 2006 Intel Corp.
  14  *      Tom Long Nguyen (tom.l.nguyen@intel.com)
  15  *      Zhang Yanmin (yanmin.zhang@intel.com)
  16  *
  17  */
  18
  19 #include <linux/module.h>
  20 #include <linux/pci.h>
  21 #include <linux/kernel.h>
  22 #include <linux/errno.h>
  23 #include <linux/pm.h>
  24 #include <linux/suspend.h>
  25 #include <linux/delay.h>
  26 #include "aerdrv.h"
  27
  28 static int forceload;
  29 module_param(forceload, bool, 0);
  30
  31 #define PCI_CFG_SPACE_SIZE      (0x100)
  32 int pci_find_aer_capability(struct pci_dev *dev)
  33 {
  34         int pos;
  35         u32 reg32 = 0;
  36
  37         /* Check if it's a pci-express device */
  38         pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
  39         if (!pos)
  40                 return 0;
  41
  42         /* Check if it supports pci-express AER */
  43         pos = PCI_CFG_SPACE_SIZE;
  44         while (pos) {
  45                 if (pci_read_config_dword(dev, pos, &reg32))
  46                         return 0;
  47
  48                 /* some broken boards return ~0 */
  49                 if (reg32 == 0xffffffff)
  50                         return 0;
  51
  52                 if (PCI_EXT_CAP_ID(reg32) == PCI_EXT_CAP_ID_ERR)
  53                         break;
  54
  55                 pos = reg32 >> 20;
  56         }
  57
  58         return pos;
  59 }
  60
  61 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
  62 {
  63         u16 reg16 = 0;
  64         int pos;
  65
  66         pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
  67         if (!pos)
  68                 return -EIO;
  69
  70         pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
  71         reg16 = reg16 |
  72                 PCI_EXP_DEVCTL_CERE |
  73                 PCI_EXP_DEVCTL_NFERE |
  74                 PCI_EXP_DEVCTL_FERE |
  75                 PCI_EXP_DEVCTL_URRE;
  76         pci_write_config_word(dev, pos+PCI_EXP_DEVCTL,
  77                         reg16);
  78         return 0;
  79 }
  80
  81 int pci_disable_pcie_error_reporting(struct pci_dev *dev)
  82 {
  83         u16 reg16 = 0;
  84         int pos;
  85
  86         pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
  87         if (!pos)
  88                 return -EIO;
  89
  90         pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
  91         reg16 = reg16 & ~(PCI_EXP_DEVCTL_CERE |
  92                         PCI_EXP_DEVCTL_NFERE |
  93                         PCI_EXP_DEVCTL_FERE |
  94                         PCI_EXP_DEVCTL_URRE);
  95         pci_write_config_word(dev, pos+PCI_EXP_DEVCTL,
  96                         reg16);
  97         return 0;
  98 }
  99
 100 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 101 {
 102         int pos;
 103         u32 status, mask;
 104
 105         pos = pci_find_aer_capability(dev);
 106         if (!pos)
 107                 return -EIO;
 108
 109         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 110         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 111         if (dev->error_state == pci_channel_io_normal)
 112                 status &= ~mask; /* Clear corresponding nonfatal bits */
 113         else
 114                 status &= mask; /* Clear corresponding fatal bits */
 115         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 116
 117         return 0;
 118 }
 119
 120 #if 0
 121 int pci_cleanup_aer_correct_error_status(struct pci_dev *dev)
 122 {
 123         int pos;
 124         u32 status;
 125
 126         pos = pci_find_aer_capability(dev);
 127         if (!pos)
 128                 return -EIO;
 129
 130         pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
 131         pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status);
 132
 133         return 0;
 134 }
 135 #endif  /*  0  */
 136
 137 static int find_device_iter(struct device *device, void *data)
 138 {
 139         struct pci_dev *dev;
 140         u16 id = *(unsigned long *)data;
 141         u8 secondary, subordinate, d_bus = id >> 8;
 142
 143         if (device->bus == &pci_bus_type) {
 144                 dev = to_pci_dev(device);
 145                 if (id == ((dev->bus->number << 8) | dev->devfn)) {
 146                         /*
 147                          * Device ID match
 148                          */
 149                         *(unsigned long*)data = (unsigned long)device;
 150                         return 1;
 151                 }
 152
 153                 /*
 154                  * If device is P2P, check if it is an upstream?
 155                  */
 156                 if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
 157                         pci_read_config_byte(dev, PCI_SECONDARY_BUS,
 158                                 &secondary);
 159                         pci_read_config_byte(dev, PCI_SUBORDINATE_BUS,
 160                                 &subordinate);
 161                         if (d_bus >= secondary && d_bus <= subordinate) {
 162                                 *(unsigned long*)data = (unsigned long)device;
 163                                 return 1;
 164                         }
 165                 }
 166         }
 167
 168         return 0;
 169 }
 170
 171 /**
 172  * find_source_device - search through device hierarchy for source device
 173  * @parent: pointer to Root Port pci_dev data structure
 174  * @id: device ID of agent who sends an error message to this Root Port
 175  *
 176  * Invoked when error is detected at the Root Port.
 177  */
 178 static struct device* find_source_device(struct pci_dev *parent, u16 id)
 179 {
 180         struct pci_dev *dev = parent;
 181         struct device *device;
 182         unsigned long device_addr;
 183         int status;
 184
 185         /* Is Root Port an agent that sends error message? */
 186         if (id == ((dev->bus->number << 8) | dev->devfn))
 187                 return &dev->dev;
 188
 189         do {
 190                 device_addr = id;
 191                 if ((status = device_for_each_child(&dev->dev,
 192                         &device_addr, find_device_iter))) {
 193                         device = (struct device*)device_addr;
 194                         dev = to_pci_dev(device);
 195                         if (id == ((dev->bus->number << 8) | dev->devfn))
 196                                 return device;
 197                 }
 198         }while (status);
 199
 200         return NULL;
 201 }
 202
 203 static void report_error_detected(struct pci_dev *dev, void *data)
 204 {
 205         pci_ers_result_t vote;
 206         struct pci_error_handlers *err_handler;
 207         struct aer_broadcast_data *result_data;
 208         result_data = (struct aer_broadcast_data *) data;
 209
 210         dev->error_state = result_data->state;
 211
 212         if (!dev->driver ||
 213                 !dev->driver->err_handler ||
 214                 !dev->driver->err_handler->error_detected) {
 215                 if (result_data->state == pci_channel_io_frozen &&
 216                         !(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) {
 217                         /*
 218                          * In case of fatal recovery, if one of down-
 219                          * stream device has no driver. We might be
 220                          * unable to recover because a later insmod
 221                          * of a driver for this device is unaware of
 222                          * its hw state.
 223                          */
 224                         dev_printk(KERN_DEBUG, &dev->dev, "device has %s\n",
 225                                    dev->driver ?
 226                                    "no AER-aware driver" : "no driver");
 227                 }
 228                 return;
 229         }
 230
 231         err_handler = dev->driver->err_handler;
 232         vote = err_handler->error_detected(dev, result_data->state);
 233         result_data->result = merge_result(result_data->result, vote);
 234         return;
 235 }
 236
 237 static void report_mmio_enabled(struct pci_dev *dev, void *data)
 238 {
 239         pci_ers_result_t vote;
 240         struct pci_error_handlers *err_handler;
 241         struct aer_broadcast_data *result_data;
 242         result_data = (struct aer_broadcast_data *) data;
 243
 244         if (!dev->driver ||
 245                 !dev->driver->err_handler ||
 246                 !dev->driver->err_handler->mmio_enabled)
 247                 return;
 248
 249         err_handler = dev->driver->err_handler;
 250         vote = err_handler->mmio_enabled(dev);
 251         result_data->result = merge_result(result_data->result, vote);
 252         return;
 253 }
 254
 255 static void report_slot_reset(struct pci_dev *dev, void *data)
 256 {
 257         pci_ers_result_t vote;
 258         struct pci_error_handlers *err_handler;
 259         struct aer_broadcast_data *result_data;
 260         result_data = (struct aer_broadcast_data *) data;
 261
 262         if (!dev->driver ||
 263                 !dev->driver->err_handler ||
 264                 !dev->driver->err_handler->slot_reset)
 265                 return;
 266
 267         err_handler = dev->driver->err_handler;
 268         vote = err_handler->slot_reset(dev);
 269         result_data->result = merge_result(result_data->result, vote);
 270         return;
 271 }
 272
 273 static void report_resume(struct pci_dev *dev, void *data)
 274 {
 275         struct pci_error_handlers *err_handler;
 276
 277         dev->error_state = pci_channel_io_normal;
 278
 279         if (!dev->driver ||
 280                 !dev->driver->err_handler ||
 281                 !dev->driver->err_handler->slot_reset)
 282                 return;
 283
 284         err_handler = dev->driver->err_handler;
 285         err_handler->resume(dev);
 286         return;
 287 }
 288
 289 /**
 290  * broadcast_error_message - handle message broadcast to downstream drivers
 291  * @dev: pointer to from where in a hierarchy message is broadcasted down
 292  * @state: error state
 293  * @error_mesg: message to print
 294  * @cb: callback to be broadcasted
 295  *
 296  * Invoked during error recovery process. Once being invoked, the content
 297  * of error severity will be broadcasted to all downstream drivers in a
 298  * hierarchy in question.
 299  */
 300 static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
 301         enum pci_channel_state state,
 302         char *error_mesg,
 303         void (*cb)(struct pci_dev *, void *))
 304 {
 305         struct aer_broadcast_data result_data;
 306
 307         dev_printk(KERN_DEBUG, &dev->dev, "broadcast %s message\n", error_mesg);
 308         result_data.state = state;
 309         if (cb == report_error_detected)
 310                 result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
 311         else
 312                 result_data.result = PCI_ERS_RESULT_RECOVERED;
 313
 314         if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
 315                 /*
 316                  * If the error is reported by a bridge, we think this error
 317                  * is related to the downstream link of the bridge, so we
 318                  * do error recovery on all subordinates of the bridge instead
 319                  * of the bridge and clear the error status of the bridge.
 320                  */
 321                 if (cb == report_error_detected)
 322                         dev->error_state = state;
 323                 pci_walk_bus(dev->subordinate, cb, &result_data);
 324                 if (cb == report_resume) {
 325                         pci_cleanup_aer_uncorrect_error_status(dev);
 326                         dev->error_state = pci_channel_io_normal;
 327                 }
 328         }
 329         else {
 330                 /*
 331                  * If the error is reported by an end point, we think this
 332                  * error is related to the upstream link of the end point.
 333                  */
 334                 pci_walk_bus(dev->bus, cb, &result_data);
 335         }
 336
 337         return result_data.result;
 338 }
 339
 340 struct find_aer_service_data {
 341         struct pcie_port_service_driver *aer_driver;
 342         int is_downstream;
 343 };
 344
 345 static int find_aer_service_iter(struct device *device, void *data)
 346 {
 347         struct device_driver *driver;
 348         struct pcie_port_service_driver *service_driver;
 349         struct pcie_device *pcie_dev;
 350         struct find_aer_service_data *result;
 351
 352         result = (struct find_aer_service_data *) data;
 353
 354         if (device->bus == &pcie_port_bus_type) {
 355                 pcie_dev = to_pcie_device(device);
 356                 if (pcie_dev->id.port_type == PCIE_SW_DOWNSTREAM_PORT)
 357                         result->is_downstream = 1;
 358
 359                 driver = device->driver;
 360                 if (driver) {
 361                         service_driver = to_service_driver(driver);
 362                         if (service_driver->id_table->service_type ==
 363                                         PCIE_PORT_SERVICE_AER) {
 364                                 result->aer_driver = service_driver;
 365                                 return 1;
 366                         }
 367                 }
 368         }
 369
 370         return 0;
 371 }
 372
 373 static void find_aer_service(struct pci_dev *dev,
 374                 struct find_aer_service_data *data)
 375 {
 376         int retval;
 377         retval = device_for_each_child(&dev->dev, data, find_aer_service_iter);
 378 }
 379
 380 static pci_ers_result_t reset_link(struct pcie_device *aerdev,
 381                 struct pci_dev *dev)
 382 {
 383         struct pci_dev *udev;
 384         pci_ers_result_t status;
 385         struct find_aer_service_data data;
 386
 387         if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)
 388                 udev = dev;
 389         else
 390                 udev= dev->bus->self;
 391
 392         data.is_downstream = 0;
 393         data.aer_driver = NULL;
 394         find_aer_service(udev, &data);
 395
 396         /*
 397          * Use the aer driver of the error agent firstly.
 398          * If it hasn't the aer driver, use the root port's
 399          */
 400         if (!data.aer_driver || !data.aer_driver->reset_link) {
 401                 if (data.is_downstream &&
 402                         aerdev->device.driver &&
 403                         to_service_driver(aerdev->device.driver)->reset_link) {
 404                         data.aer_driver =
 405                                 to_service_driver(aerdev->device.driver);
 406                 } else {
 407                         dev_printk(KERN_DEBUG, &dev->dev, "no link-reset "
 408                                    "support\n");
 409                         return PCI_ERS_RESULT_DISCONNECT;
 410                 }
 411         }
 412
 413         status = data.aer_driver->reset_link(udev);
 414         if (status != PCI_ERS_RESULT_RECOVERED) {
 415                 dev_printk(KERN_DEBUG, &dev->dev, "link reset at upstream "
 416                            "device %s failed\n", pci_name(udev));
 417                 return PCI_ERS_RESULT_DISCONNECT;
 418         }
 419
 420         return status;
 421 }
 422
 423 /**
 424  * do_recovery - handle nonfatal/fatal error recovery process
 425  * @aerdev: pointer to a pcie_device data structure of root port
 426  * @dev: pointer to a pci_dev data structure of agent detecting an error
 427  * @severity: error severity type
 428  *
 429  * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
 430  * error detected message to all downstream drivers within a hierarchy in
 431  * question and return the returned code.
 432  */
 433 static pci_ers_result_t do_recovery(struct pcie_device *aerdev,
 434                 struct pci_dev *dev,
 435                 int severity)
 436 {
 437         pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED;
 438         enum pci_channel_state state;
 439
 440         if (severity == AER_FATAL)
 441                 state = pci_channel_io_frozen;
 442         else
 443                 state = pci_channel_io_normal;
 444
 445         status = broadcast_error_message(dev,
 446                         state,
 447                         "error_detected",
 448                         report_error_detected);
 449
 450         if (severity == AER_FATAL) {
 451                 result = reset_link(aerdev, dev);
 452                 if (result != PCI_ERS_RESULT_RECOVERED) {
 453                         /* TODO: Should panic here? */
 454                         return result;
 455                 }
 456         }
 457
 458         if (status == PCI_ERS_RESULT_CAN_RECOVER)
 459                 status = broadcast_error_message(dev,
 460                                 state,
 461                                 "mmio_enabled",
 462                                 report_mmio_enabled);
 463
 464         if (status == PCI_ERS_RESULT_NEED_RESET) {
 465                 /*
 466                  * TODO: Should call platform-specific
 467                  * functions to reset slot before calling
 468                  * drivers' slot_reset callbacks?
 469                  */
 470                 status = broadcast_error_message(dev,
 471                                 state,
 472                                 "slot_reset",
 473                                 report_slot_reset);
 474         }
 475
 476         if (status == PCI_ERS_RESULT_RECOVERED)
 477                 broadcast_error_message(dev,
 478                                 state,
 479                                 "resume",
 480                                 report_resume);
 481
 482         return status;
 483 }
 484
 485 /**
 486  * handle_error_source - handle logging error into an event log
 487  * @aerdev: pointer to pcie_device data structure of the root port
 488  * @dev: pointer to pci_dev data structure of error source device
 489  * @info: comprehensive error information
 490  *
 491  * Invoked when an error being detected by Root Port.
 492  */
 493 static void handle_error_source(struct pcie_device * aerdev,
 494         struct pci_dev *dev,
 495         struct aer_err_info info)
 496 {
 497         pci_ers_result_t status = 0;
 498         int pos;
 499
 500         if (info.severity == AER_CORRECTABLE) {
 501                 /*
 502                  * Correctable error does not need software intevention.
 503                  * No need to go through error recovery process.
 504                  */
 505                 pos = pci_find_aer_capability(dev);
 506                 if (pos)
 507                         pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 508                                         info.status);
 509         } else {
 510                 status = do_recovery(aerdev, dev, info.severity);
 511                 if (status == PCI_ERS_RESULT_RECOVERED) {
 512                         dev_printk(KERN_DEBUG, &dev->dev, "AER driver "
 513                                    "successfully recovered\n");
 514                 } else {
 515                         /* TODO: Should kernel panic here? */
 516                         dev_printk(KERN_DEBUG, &dev->dev, "AER driver didn't "
 517                                    "recover\n");
 518                 }
 519         }
 520 }
 521
 522 /**
 523  * aer_enable_rootport - enable Root Port's interrupts when receiving messages
 524  * @rpc: pointer to a Root Port data structure
 525  *
 526  * Invoked when PCIE bus loads AER service driver.
 527  */
 528 void aer_enable_rootport(struct aer_rpc *rpc)
 529 {
 530         struct pci_dev *pdev = rpc->rpd->port;
 531         int pos, aer_pos;
 532         u16 reg16;
 533         u32 reg32;
 534
 535         pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
 536         /* Clear PCIE Capability's Device Status */
 537         pci_read_config_word(pdev, pos+PCI_EXP_DEVSTA, &reg16);
 538         pci_write_config_word(pdev, pos+PCI_EXP_DEVSTA, reg16);
 539
 540         /* Disable system error generation in response to error messages */
 541         pci_read_config_word(pdev, pos + PCI_EXP_RTCTL, &reg16);
 542         reg16 &= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK);
 543         pci_write_config_word(pdev, pos + PCI_EXP_RTCTL, reg16);
 544
 545         aer_pos = pci_find_aer_capability(pdev);
 546         /* Clear error status */
 547         pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, &reg32);
 548         pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
 549         pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, &reg32);
 550         pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32);
 551         pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, &reg32);
 552         pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32);
 553
 554         /* Enable Root Port device reporting error itself */
 555         pci_read_config_word(pdev, pos+PCI_EXP_DEVCTL, &reg16);
 556         reg16 = reg16 |
 557                 PCI_EXP_DEVCTL_CERE |
 558                 PCI_EXP_DEVCTL_NFERE |
 559                 PCI_EXP_DEVCTL_FERE |
 560                 PCI_EXP_DEVCTL_URRE;
 561         pci_write_config_word(pdev, pos+PCI_EXP_DEVCTL,
 562                 reg16);
 563
 564         /* Enable Root Port's interrupt in response to error messages */
 565         pci_write_config_dword(pdev,
 566                 aer_pos + PCI_ERR_ROOT_COMMAND,
 567                 ROOT_PORT_INTR_ON_MESG_MASK);
 568 }
 569
 570 /**
 571  * disable_root_aer - disable Root Port's interrupts when receiving messages
 572  * @rpc: pointer to a Root Port data structure
 573  *
 574  * Invoked when PCIE bus unloads AER service driver.
 575  */
 576 static void disable_root_aer(struct aer_rpc *rpc)
 577 {
 578         struct pci_dev *pdev = rpc->rpd->port;
 579         u32 reg32;
 580         int pos;
 581
 582         pos = pci_find_aer_capability(pdev);
 583         /* Disable Root's interrupt in response to error messages */
 584         pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, 0);
 585
 586         /* Clear Root's error status reg */
 587         pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, &reg32);
 588         pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32);
 589 }
 590
 591 /**
 592  * get_e_source - retrieve an error source
 593  * @rpc: pointer to the root port which holds an error
 594  *
 595  * Invoked by DPC handler to consume an error.
 596  */
 597 static struct aer_err_source* get_e_source(struct aer_rpc *rpc)
 598 {
 599         struct aer_err_source *e_source;
 600         unsigned long flags;
 601
 602         /* Lock access to Root error producer/consumer index */
 603         spin_lock_irqsave(&rpc->e_lock, flags);
 604         if (rpc->prod_idx == rpc->cons_idx) {
 605                 spin_unlock_irqrestore(&rpc->e_lock, flags);
 606                 return NULL;
 607         }
 608         e_source = &rpc->e_sources[rpc->cons_idx];
 609         rpc->cons_idx++;
 610         if (rpc->cons_idx == AER_ERROR_SOURCES_MAX)
 611                 rpc->cons_idx = 0;
 612         spin_unlock_irqrestore(&rpc->e_lock, flags);
 613
 614         return e_source;
 615 }
 616
 617 static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 618 {
 619         int pos;
 620
 621         pos = pci_find_aer_capability(dev);
 622
 623         /* The device might not support AER */
 624         if (!pos)
 625                 return AER_SUCCESS;
 626
 627         if (info->severity == AER_CORRECTABLE) {
 628                 pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 629                         &info->status);
 630                 if (!(info->status & ERR_CORRECTABLE_ERROR_MASK))
 631                         return AER_UNSUCCESS;
 632         } else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE ||
 633                 info->severity == AER_NONFATAL) {
 634
 635                 /* Link is still healthy for IO reads */
 636                 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
 637                         &info->status);
 638                 if (!(info->status & ERR_UNCORRECTABLE_ERROR_MASK))
 639                         return AER_UNSUCCESS;
 640
 641                 if (info->status & AER_LOG_TLP_MASKS) {
 642                         info->flags |= AER_TLP_HEADER_VALID_FLAG;
 643                         pci_read_config_dword(dev,
 644                                 pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
 645                         pci_read_config_dword(dev,
 646                                 pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
 647                         pci_read_config_dword(dev,
 648                                 pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
 649                         pci_read_config_dword(dev,
 650                                 pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
 651                 }
 652         }
 653
 654         return AER_SUCCESS;
 655 }
 656
 657 /**
 658  * aer_isr_one_error - consume an error detected by root port
 659  * @p_device: pointer to error root port service device
 660  * @e_src: pointer to an error source
 661  */
 662 static void aer_isr_one_error(struct pcie_device *p_device,
 663                 struct aer_err_source *e_src)
 664 {
 665         struct device *s_device;
 666         struct aer_err_info e_info = {0, 0, 0,};
 667         int i;
 668         u16 id;
 669
 670         /*
 671          * There is a possibility that both correctable error and
 672          * uncorrectable error being logged. Report correctable error first.
 673          */
 674         for (i = 1; i & ROOT_ERR_STATUS_MASKS ; i <<= 2) {
 675                 if (i > 4)
 676                         break;
 677                 if (!(e_src->status & i))
 678                         continue;
 679
 680                 /* Init comprehensive error information */
 681                 if (i & PCI_ERR_ROOT_COR_RCV) {
 682                         id = ERR_COR_ID(e_src->id);
 683                         e_info.severity = AER_CORRECTABLE;
 684                 } else {
 685                         id = ERR_UNCOR_ID(e_src->id);
 686                         e_info.severity = ((e_src->status >> 6) & 1);
 687                 }
 688                 if (e_src->status &
 689                         (PCI_ERR_ROOT_MULTI_COR_RCV |
 690                          PCI_ERR_ROOT_MULTI_UNCOR_RCV))
 691                         e_info.flags |= AER_MULTI_ERROR_VALID_FLAG;
 692                 if (!(s_device = find_source_device(p_device->port, id))) {
 693                         printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
 694                                 __func__, id);
 695                         continue;
 696                 }
 697                 if (get_device_error_info(to_pci_dev(s_device), &e_info) ==
 698                                 AER_SUCCESS) {
 699                         aer_print_error(to_pci_dev(s_device), &e_info);
 700                         handle_error_source(p_device,
 701                                 to_pci_dev(s_device),
 702                                 e_info);
 703                 }
 704         }
 705 }
 706
 707 /**
 708  * aer_isr - consume errors detected by root port
 709  * @work: definition of this work item
 710  *
 711  * Invoked, as DPC, when root port records new detected error
 712  */
 713 void aer_isr(struct work_struct *work)
 714 {
 715         struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler);
 716         struct pcie_device *p_device = rpc->rpd;
 717         struct aer_err_source *e_src;
 718
 719         mutex_lock(&rpc->rpc_mutex);
 720         e_src = get_e_source(rpc);
 721         while (e_src) {
 722                 aer_isr_one_error(p_device, e_src);
 723                 e_src = get_e_source(rpc);
 724         }
 725         mutex_unlock(&rpc->rpc_mutex);
 726
 727         wake_up(&rpc->wait_release);
 728 }
 729
 730 /**
 731  * aer_delete_rootport - disable root port aer and delete service data
 732  * @rpc: pointer to a root port device being deleted
 733  *
 734  * Invoked when AER service unloaded on a specific Root Port
 735  */
 736 void aer_delete_rootport(struct aer_rpc *rpc)
 737 {
 738         /* Disable root port AER itself */
 739         disable_root_aer(rpc);
 740
 741         kfree(rpc);
 742 }
 743
 744 /**
 745  * aer_init - provide AER initialization
 746  * @dev: pointer to AER pcie device
 747  *
 748  * Invoked when AER service driver is loaded.
 749  */
 750 int aer_init(struct pcie_device *dev)
 751 {
 752         if (aer_osc_setup(dev) && !forceload)
 753                 return -ENXIO;
 754
 755         return AER_SUCCESS;
 756 }
 757
 758 EXPORT_SYMBOL_GPL(pci_find_aer_capability);
 759 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
 760 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting);
 761 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status);
 762