2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/acpi.h>
26 #include <linux/pci-acpi.h>
27 #include <linux/delay.h>
31 module_param(forceload
, bool, 0);
33 #define PCI_CFG_SPACE_SIZE (0x100)
34 int pci_find_aer_capability(struct pci_dev
*dev
)
39 /* Check if it's a pci-express device */
40 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
44 /* Check if it supports pci-express AER */
45 pos
= PCI_CFG_SPACE_SIZE
;
47 if (pci_read_config_dword(dev
, pos
, ®32
))
50 /* some broken boards return ~0 */
51 if (reg32
== 0xffffffff)
54 if (PCI_EXT_CAP_ID(reg32
) == PCI_EXT_CAP_ID_ERR
)
63 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
68 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
72 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
75 PCI_EXP_DEVCTL_NFERE
|
78 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
83 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
88 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
92 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
93 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
94 PCI_EXP_DEVCTL_NFERE
|
97 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
102 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
107 pos
= pci_find_aer_capability(dev
);
111 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
112 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
113 if (dev
->error_state
== pci_channel_io_normal
)
114 status
&= ~mask
; /* Clear corresponding nonfatal bits */
116 status
&= mask
; /* Clear corresponding fatal bits */
117 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
122 static int find_device_iter(struct device
*device
, void *data
)
125 u16 id
= *(unsigned long *)data
;
126 u8 secondary
, subordinate
, d_bus
= id
>> 8;
128 if (device
->bus
== &pci_bus_type
) {
129 dev
= to_pci_dev(device
);
130 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
134 *(unsigned long*)data
= (unsigned long)device
;
139 * If device is P2P, check if it is an upstream?
141 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
142 pci_read_config_byte(dev
, PCI_SECONDARY_BUS
,
144 pci_read_config_byte(dev
, PCI_SUBORDINATE_BUS
,
146 if (d_bus
>= secondary
&& d_bus
<= subordinate
) {
147 *(unsigned long*)data
= (unsigned long)device
;
157 * find_source_device - search through device hierarchy for source device
158 * @p_dev: pointer to Root Port pci_dev data structure
159 * @id: device ID of agent who sends an error message to this Root Port
161 * Invoked when error is detected at the Root Port.
163 static struct device
* find_source_device(struct pci_dev
*parent
, u16 id
)
165 struct pci_dev
*dev
= parent
;
166 struct device
*device
;
167 unsigned long device_addr
;
170 /* Is Root Port an agent that sends error message? */
171 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
176 if ((status
= device_for_each_child(&dev
->dev
,
177 &device_addr
, find_device_iter
))) {
178 device
= (struct device
*)device_addr
;
179 dev
= to_pci_dev(device
);
180 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
188 static void report_error_detected(struct pci_dev
*dev
, void *data
)
190 pci_ers_result_t vote
;
191 struct pci_error_handlers
*err_handler
;
192 struct aer_broadcast_data
*result_data
;
193 result_data
= (struct aer_broadcast_data
*) data
;
195 dev
->error_state
= result_data
->state
;
198 !dev
->driver
->err_handler
||
199 !dev
->driver
->err_handler
->error_detected
) {
200 if (result_data
->state
== pci_channel_io_frozen
&&
201 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
203 * In case of fatal recovery, if one of down-
204 * stream device has no driver. We might be
205 * unable to recover because a later insmod
206 * of a driver for this device is unaware of
209 printk(KERN_DEBUG
"Device ID[%s] has %s\n",
210 dev
->dev
.bus_id
, (dev
->driver
) ?
211 "no AER-aware driver" : "no driver");
216 err_handler
= dev
->driver
->err_handler
;
217 vote
= err_handler
->error_detected(dev
, result_data
->state
);
218 result_data
->result
= merge_result(result_data
->result
, vote
);
222 static void report_mmio_enabled(struct pci_dev
*dev
, void *data
)
224 pci_ers_result_t vote
;
225 struct pci_error_handlers
*err_handler
;
226 struct aer_broadcast_data
*result_data
;
227 result_data
= (struct aer_broadcast_data
*) data
;
230 !dev
->driver
->err_handler
||
231 !dev
->driver
->err_handler
->mmio_enabled
)
234 err_handler
= dev
->driver
->err_handler
;
235 vote
= err_handler
->mmio_enabled(dev
);
236 result_data
->result
= merge_result(result_data
->result
, vote
);
240 static void report_slot_reset(struct pci_dev
*dev
, void *data
)
242 pci_ers_result_t vote
;
243 struct pci_error_handlers
*err_handler
;
244 struct aer_broadcast_data
*result_data
;
245 result_data
= (struct aer_broadcast_data
*) data
;
248 !dev
->driver
->err_handler
||
249 !dev
->driver
->err_handler
->slot_reset
)
252 err_handler
= dev
->driver
->err_handler
;
253 vote
= err_handler
->slot_reset(dev
);
254 result_data
->result
= merge_result(result_data
->result
, vote
);
258 static void report_resume(struct pci_dev
*dev
, void *data
)
260 struct pci_error_handlers
*err_handler
;
262 dev
->error_state
= pci_channel_io_normal
;
265 !dev
->driver
->err_handler
||
266 !dev
->driver
->err_handler
->slot_reset
)
269 err_handler
= dev
->driver
->err_handler
;
270 err_handler
->resume(dev
);
275 * broadcast_error_message - handle message broadcast to downstream drivers
276 * @device: pointer to from where in a hierarchy message is broadcasted down
277 * @api: callback to be broadcasted
278 * @state: error state
280 * Invoked during error recovery process. Once being invoked, the content
281 * of error severity will be broadcasted to all downstream drivers in a
282 * hierarchy in question.
284 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
285 enum pci_channel_state state
,
287 void (*cb
)(struct pci_dev
*, void *))
289 struct aer_broadcast_data result_data
;
291 printk(KERN_DEBUG
"Broadcast %s message\n", error_mesg
);
292 result_data
.state
= state
;
293 if (cb
== report_error_detected
)
294 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
296 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
298 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
300 * If the error is reported by a bridge, we think this error
301 * is related to the downstream link of the bridge, so we
302 * do error recovery on all subordinates of the bridge instead
303 * of the bridge and clear the error status of the bridge.
305 if (cb
== report_error_detected
)
306 dev
->error_state
= state
;
307 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
308 if (cb
== report_resume
) {
309 pci_cleanup_aer_uncorrect_error_status(dev
);
310 dev
->error_state
= pci_channel_io_normal
;
315 * If the error is reported by an end point, we think this
316 * error is related to the upstream link of the end point.
318 pci_walk_bus(dev
->bus
, cb
, &result_data
);
321 return result_data
.result
;
324 struct find_aer_service_data
{
325 struct pcie_port_service_driver
*aer_driver
;
329 static int find_aer_service_iter(struct device
*device
, void *data
)
331 struct device_driver
*driver
;
332 struct pcie_port_service_driver
*service_driver
;
333 struct pcie_device
*pcie_dev
;
334 struct find_aer_service_data
*result
;
336 result
= (struct find_aer_service_data
*) data
;
338 if (device
->bus
== &pcie_port_bus_type
) {
339 pcie_dev
= to_pcie_device(device
);
340 if (pcie_dev
->id
.port_type
== PCIE_SW_DOWNSTREAM_PORT
)
341 result
->is_downstream
= 1;
343 driver
= device
->driver
;
345 service_driver
= to_service_driver(driver
);
346 if (service_driver
->id_table
->service_type
==
347 PCIE_PORT_SERVICE_AER
) {
348 result
->aer_driver
= service_driver
;
357 static void find_aer_service(struct pci_dev
*dev
,
358 struct find_aer_service_data
*data
)
361 retval
= device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
364 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
367 struct pci_dev
*udev
;
368 pci_ers_result_t status
;
369 struct find_aer_service_data data
;
371 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
374 udev
= dev
->bus
->self
;
376 data
.is_downstream
= 0;
377 data
.aer_driver
= NULL
;
378 find_aer_service(udev
, &data
);
381 * Use the aer driver of the error agent firstly.
382 * If it hasn't the aer driver, use the root port's
384 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
385 if (data
.is_downstream
&&
386 aerdev
->device
.driver
&&
387 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
389 to_service_driver(aerdev
->device
.driver
);
391 printk(KERN_DEBUG
"No link-reset support to Device ID"
394 return PCI_ERS_RESULT_DISCONNECT
;
398 status
= data
.aer_driver
->reset_link(udev
);
399 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
400 printk(KERN_DEBUG
"Link reset at upstream Device ID"
403 return PCI_ERS_RESULT_DISCONNECT
;
410 * do_recovery - handle nonfatal/fatal error recovery process
411 * @aerdev: pointer to a pcie_device data structure of root port
412 * @dev: pointer to a pci_dev data structure of agent detecting an error
413 * @severity: error severity type
415 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
416 * error detected message to all downstream drivers within a hierarchy in
417 * question and return the returned code.
419 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
423 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
424 enum pci_channel_state state
;
426 if (severity
== AER_FATAL
)
427 state
= pci_channel_io_frozen
;
429 state
= pci_channel_io_normal
;
431 status
= broadcast_error_message(dev
,
434 report_error_detected
);
436 if (severity
== AER_FATAL
) {
437 result
= reset_link(aerdev
, dev
);
438 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
439 /* TODO: Should panic here? */
444 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
445 status
= broadcast_error_message(dev
,
448 report_mmio_enabled
);
450 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
452 * TODO: Should call platform-specific
453 * functions to reset slot before calling
454 * drivers' slot_reset callbacks?
456 status
= broadcast_error_message(dev
,
462 if (status
== PCI_ERS_RESULT_RECOVERED
)
463 broadcast_error_message(dev
,
472 * handle_error_source - handle logging error into an event log
473 * @aerdev: pointer to pcie_device data structure of the root port
474 * @dev: pointer to pci_dev data structure of error source device
475 * @info: comprehensive error information
477 * Invoked when an error being detected by Root Port.
479 static void handle_error_source(struct pcie_device
* aerdev
,
481 struct aer_err_info info
)
483 pci_ers_result_t status
= 0;
486 if (info
.severity
== AER_CORRECTABLE
) {
488 * Correctable error does not need software intevention.
489 * No need to go through error recovery process.
491 pos
= pci_find_aer_capability(dev
);
493 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
496 status
= do_recovery(aerdev
, dev
, info
.severity
);
497 if (status
== PCI_ERS_RESULT_RECOVERED
) {
498 printk(KERN_DEBUG
"AER driver successfully recovered\n");
500 /* TODO: Should kernel panic here? */
501 printk(KERN_DEBUG
"AER driver didn't recover\n");
507 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
508 * @rpc: pointer to a Root Port data structure
510 * Invoked when PCIE bus loads AER service driver.
512 void aer_enable_rootport(struct aer_rpc
*rpc
)
514 struct pci_dev
*pdev
= rpc
->rpd
->port
;
519 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
520 /* Clear PCIE Capability's Device Status */
521 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, ®16
);
522 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, reg16
);
524 /* Disable system error generation in response to error messages */
525 pci_read_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, ®16
);
526 reg16
&= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK
);
527 pci_write_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, reg16
);
529 aer_pos
= pci_find_aer_capability(pdev
);
530 /* Clear error status */
531 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
532 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
533 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, ®32
);
534 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, reg32
);
535 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, ®32
);
536 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, reg32
);
538 /* Enable Root Port device reporting error itself */
539 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVCTL
, ®16
);
541 PCI_EXP_DEVCTL_CERE
|
542 PCI_EXP_DEVCTL_NFERE
|
543 PCI_EXP_DEVCTL_FERE
|
545 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVCTL
,
548 /* Enable Root Port's interrupt in response to error messages */
549 pci_write_config_dword(pdev
,
550 aer_pos
+ PCI_ERR_ROOT_COMMAND
,
551 ROOT_PORT_INTR_ON_MESG_MASK
);
555 * disable_root_aer - disable Root Port's interrupts when receiving messages
556 * @rpc: pointer to a Root Port data structure
558 * Invoked when PCIE bus unloads AER service driver.
560 static void disable_root_aer(struct aer_rpc
*rpc
)
562 struct pci_dev
*pdev
= rpc
->rpd
->port
;
566 pos
= pci_find_aer_capability(pdev
);
567 /* Disable Root's interrupt in response to error messages */
568 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_COMMAND
, 0);
570 /* Clear Root's error status reg */
571 pci_read_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
572 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
576 * get_e_source - retrieve an error source
577 * @rpc: pointer to the root port which holds an error
579 * Invoked by DPC handler to consume an error.
581 static struct aer_err_source
* get_e_source(struct aer_rpc
*rpc
)
583 struct aer_err_source
*e_source
;
586 /* Lock access to Root error producer/consumer index */
587 spin_lock_irqsave(&rpc
->e_lock
, flags
);
588 if (rpc
->prod_idx
== rpc
->cons_idx
) {
589 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
592 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
594 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
596 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
601 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
605 pos
= pci_find_aer_capability(dev
);
607 /* The device might not support AER */
611 if (info
->severity
== AER_CORRECTABLE
) {
612 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
614 if (!(info
->status
& ERR_CORRECTABLE_ERROR_MASK
))
615 return AER_UNSUCCESS
;
616 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
617 info
->severity
== AER_NONFATAL
) {
619 /* Link is still healthy for IO reads */
620 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
622 if (!(info
->status
& ERR_UNCORRECTABLE_ERROR_MASK
))
623 return AER_UNSUCCESS
;
625 if (info
->status
& AER_LOG_TLP_MASKS
) {
626 info
->flags
|= AER_TLP_HEADER_VALID_FLAG
;
627 pci_read_config_dword(dev
,
628 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
629 pci_read_config_dword(dev
,
630 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
631 pci_read_config_dword(dev
,
632 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
633 pci_read_config_dword(dev
,
634 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
642 * aer_isr_one_error - consume an error detected by root port
643 * @p_device: pointer to error root port service device
644 * @e_src: pointer to an error source
646 static void aer_isr_one_error(struct pcie_device
*p_device
,
647 struct aer_err_source
*e_src
)
649 struct device
*s_device
;
650 struct aer_err_info e_info
= {0, 0, 0,};
655 * There is a possibility that both correctable error and
656 * uncorrectable error being logged. Report correctable error first.
658 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
661 if (!(e_src
->status
& i
))
664 /* Init comprehensive error information */
665 if (i
& PCI_ERR_ROOT_COR_RCV
) {
666 id
= ERR_COR_ID(e_src
->id
);
667 e_info
.severity
= AER_CORRECTABLE
;
669 id
= ERR_UNCOR_ID(e_src
->id
);
670 e_info
.severity
= ((e_src
->status
>> 6) & 1);
673 (PCI_ERR_ROOT_MULTI_COR_RCV
|
674 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
675 e_info
.flags
|= AER_MULTI_ERROR_VALID_FLAG
;
676 if (!(s_device
= find_source_device(p_device
->port
, id
))) {
677 printk(KERN_DEBUG
"%s->can't find device of ID%04x\n",
681 if (get_device_error_info(to_pci_dev(s_device
), &e_info
) ==
683 aer_print_error(to_pci_dev(s_device
), &e_info
);
684 handle_error_source(p_device
,
685 to_pci_dev(s_device
),
692 * aer_isr - consume errors detected by root port
693 * @work: definition of this work item
695 * Invoked, as DPC, when root port records new detected error
697 void aer_isr(struct work_struct
*work
)
699 struct aer_rpc
*rpc
= container_of(work
, struct aer_rpc
, dpc_handler
);
700 struct pcie_device
*p_device
= rpc
->rpd
;
701 struct aer_err_source
*e_src
;
703 mutex_lock(&rpc
->rpc_mutex
);
704 e_src
= get_e_source(rpc
);
706 aer_isr_one_error(p_device
, e_src
);
707 e_src
= get_e_source(rpc
);
709 mutex_unlock(&rpc
->rpc_mutex
);
711 wake_up(&rpc
->wait_release
);
715 * aer_delete_rootport - disable root port aer and delete service data
716 * @rpc: pointer to a root port device being deleted
718 * Invoked when AER service unloaded on a specific Root Port
720 void aer_delete_rootport(struct aer_rpc
*rpc
)
722 /* Disable root port AER itself */
723 disable_root_aer(rpc
);
729 * aer_init - provide AER initialization
730 * @dev: pointer to AER pcie device
732 * Invoked when AER service driver is loaded.
734 int aer_init(struct pcie_device
*dev
)
738 /* Run _OSC Method */
739 status
= aer_osc_setup(dev
->port
);
741 if(status
!= OSC_METHOD_RUN_SUCCESS
) {
742 printk(KERN_DEBUG
"%s: AER service init fails - %s\n",
744 (status
== OSC_METHOD_NOT_SUPPORTED
) ?
745 "No ACPI _OSC support" : "Run ACPI _OSC fails");
754 EXPORT_SYMBOL_GPL(pci_find_aer_capability
);
755 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
756 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
757 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);