2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/delay.h>
26 #include <linux/slab.h>
30 static int nosourceid
;
31 module_param(forceload
, bool, 0);
32 module_param(nosourceid
, bool, 0);
34 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
39 if (dev
->aer_firmware_first
)
42 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
46 pos
= pci_pcie_cap(dev
);
50 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
53 PCI_EXP_DEVCTL_NFERE
|
56 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
, reg16
);
60 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
62 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
67 if (dev
->aer_firmware_first
)
70 pos
= pci_pcie_cap(dev
);
74 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
75 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
76 PCI_EXP_DEVCTL_NFERE
|
79 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
, reg16
);
83 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
85 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
90 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
94 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
96 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
100 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);
102 static inline int compare_device_id(struct pci_dev
*dev
,
103 struct aer_err_info
*e_info
)
105 if (e_info
->id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
115 static int add_error_device(struct aer_err_info
*e_info
, struct pci_dev
*dev
)
117 if (e_info
->error_dev_num
< AER_MAX_MULTI_ERR_DEVICES
) {
118 e_info
->dev
[e_info
->error_dev_num
] = dev
;
119 e_info
->error_dev_num
++;
127 #define PCI_BUS(x) (((x) >> 8) & 0xff)
129 static int find_device_iter(struct pci_dev
*dev
, void *data
)
136 struct aer_err_info
*e_info
= (struct aer_err_info
*)data
;
139 * When bus id is equal to 0, it might be a bad id
140 * reported by root port.
142 if (!nosourceid
&& (PCI_BUS(e_info
->id
) != 0)) {
143 result
= compare_device_id(dev
, e_info
);
145 add_error_device(e_info
, dev
);
148 * If there is no multiple error, we stop
149 * or continue based on the id comparing.
151 if (!e_info
->multi_error_valid
)
155 * If there are multiple errors and id does match,
156 * We need continue to search other devices under
157 * the root port. Return 0 means that.
166 * 2) bus id is equal to 0. Some ports might lose the bus
167 * id of error source id;
168 * 3) There are multiple errors and prior id comparing fails;
169 * We check AER status registers to find the initial reporter.
171 if (atomic_read(&dev
->enable_cnt
) == 0)
173 pos
= pci_pcie_cap(dev
);
176 /* Check if AER is enabled */
177 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
179 PCI_EXP_DEVCTL_CERE
|
180 PCI_EXP_DEVCTL_NFERE
|
181 PCI_EXP_DEVCTL_FERE
|
182 PCI_EXP_DEVCTL_URRE
)))
184 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
190 if (e_info
->severity
== AER_CORRECTABLE
) {
191 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, &status
);
192 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_MASK
, &mask
);
193 if (status
& ~mask
) {
194 add_error_device(e_info
, dev
);
198 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
199 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_MASK
, &mask
);
200 if (status
& ~mask
) {
201 add_error_device(e_info
, dev
);
209 if (e_info
->multi_error_valid
)
216 * find_source_device - search through device hierarchy for source device
217 * @parent: pointer to Root Port pci_dev data structure
218 * @e_info: including detailed error information such like id
220 * Return true if found.
222 * Invoked by DPC when error is detected at the Root Port.
224 static bool find_source_device(struct pci_dev
*parent
,
225 struct aer_err_info
*e_info
)
227 struct pci_dev
*dev
= parent
;
230 /* Is Root Port an agent that sends error message? */
231 result
= find_device_iter(dev
, e_info
);
235 pci_walk_bus(parent
->subordinate
, find_device_iter
, e_info
);
237 if (!e_info
->error_dev_num
) {
238 dev_printk(KERN_DEBUG
, &parent
->dev
,
239 "can't find device of ID%04x\n",
246 static int report_error_detected(struct pci_dev
*dev
, void *data
)
248 pci_ers_result_t vote
;
249 struct pci_error_handlers
*err_handler
;
250 struct aer_broadcast_data
*result_data
;
251 result_data
= (struct aer_broadcast_data
*) data
;
253 dev
->error_state
= result_data
->state
;
256 !dev
->driver
->err_handler
||
257 !dev
->driver
->err_handler
->error_detected
) {
258 if (result_data
->state
== pci_channel_io_frozen
&&
259 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
261 * In case of fatal recovery, if one of down-
262 * stream device has no driver. We might be
263 * unable to recover because a later insmod
264 * of a driver for this device is unaware of
267 dev_printk(KERN_DEBUG
, &dev
->dev
, "device has %s\n",
269 "no AER-aware driver" : "no driver");
274 err_handler
= dev
->driver
->err_handler
;
275 vote
= err_handler
->error_detected(dev
, result_data
->state
);
276 result_data
->result
= merge_result(result_data
->result
, vote
);
280 static int report_mmio_enabled(struct pci_dev
*dev
, void *data
)
282 pci_ers_result_t vote
;
283 struct pci_error_handlers
*err_handler
;
284 struct aer_broadcast_data
*result_data
;
285 result_data
= (struct aer_broadcast_data
*) data
;
288 !dev
->driver
->err_handler
||
289 !dev
->driver
->err_handler
->mmio_enabled
)
292 err_handler
= dev
->driver
->err_handler
;
293 vote
= err_handler
->mmio_enabled(dev
);
294 result_data
->result
= merge_result(result_data
->result
, vote
);
298 static int report_slot_reset(struct pci_dev
*dev
, void *data
)
300 pci_ers_result_t vote
;
301 struct pci_error_handlers
*err_handler
;
302 struct aer_broadcast_data
*result_data
;
303 result_data
= (struct aer_broadcast_data
*) data
;
306 !dev
->driver
->err_handler
||
307 !dev
->driver
->err_handler
->slot_reset
)
310 err_handler
= dev
->driver
->err_handler
;
311 vote
= err_handler
->slot_reset(dev
);
312 result_data
->result
= merge_result(result_data
->result
, vote
);
316 static int report_resume(struct pci_dev
*dev
, void *data
)
318 struct pci_error_handlers
*err_handler
;
320 dev
->error_state
= pci_channel_io_normal
;
323 !dev
->driver
->err_handler
||
324 !dev
->driver
->err_handler
->resume
)
327 err_handler
= dev
->driver
->err_handler
;
328 err_handler
->resume(dev
);
333 * broadcast_error_message - handle message broadcast to downstream drivers
334 * @dev: pointer to from where in a hierarchy message is broadcasted down
335 * @state: error state
336 * @error_mesg: message to print
337 * @cb: callback to be broadcasted
339 * Invoked during error recovery process. Once being invoked, the content
340 * of error severity will be broadcasted to all downstream drivers in a
341 * hierarchy in question.
343 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
344 enum pci_channel_state state
,
346 int (*cb
)(struct pci_dev
*, void *))
348 struct aer_broadcast_data result_data
;
350 dev_printk(KERN_DEBUG
, &dev
->dev
, "broadcast %s message\n", error_mesg
);
351 result_data
.state
= state
;
352 if (cb
== report_error_detected
)
353 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
355 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
357 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
359 * If the error is reported by a bridge, we think this error
360 * is related to the downstream link of the bridge, so we
361 * do error recovery on all subordinates of the bridge instead
362 * of the bridge and clear the error status of the bridge.
364 if (cb
== report_error_detected
)
365 dev
->error_state
= state
;
366 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
367 if (cb
== report_resume
) {
368 pci_cleanup_aer_uncorrect_error_status(dev
);
369 dev
->error_state
= pci_channel_io_normal
;
373 * If the error is reported by an end point, we think this
374 * error is related to the upstream link of the end point.
376 pci_walk_bus(dev
->bus
, cb
, &result_data
);
379 return result_data
.result
;
382 struct find_aer_service_data
{
383 struct pcie_port_service_driver
*aer_driver
;
387 static int find_aer_service_iter(struct device
*device
, void *data
)
389 struct device_driver
*driver
;
390 struct pcie_port_service_driver
*service_driver
;
391 struct find_aer_service_data
*result
;
393 result
= (struct find_aer_service_data
*) data
;
395 if (device
->bus
== &pcie_port_bus_type
) {
396 struct pcie_device
*pcie
= to_pcie_device(device
);
398 if (pcie
->port
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
)
399 result
->is_downstream
= 1;
401 driver
= device
->driver
;
403 service_driver
= to_service_driver(driver
);
404 if (service_driver
->service
== PCIE_PORT_SERVICE_AER
) {
405 result
->aer_driver
= service_driver
;
414 static void find_aer_service(struct pci_dev
*dev
,
415 struct find_aer_service_data
*data
)
418 retval
= device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
421 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
424 struct pci_dev
*udev
;
425 pci_ers_result_t status
;
426 struct find_aer_service_data data
;
428 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
431 udev
= dev
->bus
->self
;
433 data
.is_downstream
= 0;
434 data
.aer_driver
= NULL
;
435 find_aer_service(udev
, &data
);
438 * Use the aer driver of the error agent firstly.
439 * If it hasn't the aer driver, use the root port's
441 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
442 if (data
.is_downstream
&&
443 aerdev
->device
.driver
&&
444 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
446 to_service_driver(aerdev
->device
.driver
);
448 dev_printk(KERN_DEBUG
, &dev
->dev
, "no link-reset "
450 return PCI_ERS_RESULT_DISCONNECT
;
454 status
= data
.aer_driver
->reset_link(udev
);
455 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
456 dev_printk(KERN_DEBUG
, &dev
->dev
, "link reset at upstream "
457 "device %s failed\n", pci_name(udev
));
458 return PCI_ERS_RESULT_DISCONNECT
;
465 * do_recovery - handle nonfatal/fatal error recovery process
466 * @aerdev: pointer to a pcie_device data structure of root port
467 * @dev: pointer to a pci_dev data structure of agent detecting an error
468 * @severity: error severity type
470 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
471 * error detected message to all downstream drivers within a hierarchy in
472 * question and return the returned code.
474 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
478 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
479 enum pci_channel_state state
;
481 if (severity
== AER_FATAL
)
482 state
= pci_channel_io_frozen
;
484 state
= pci_channel_io_normal
;
486 status
= broadcast_error_message(dev
,
489 report_error_detected
);
491 if (severity
== AER_FATAL
) {
492 result
= reset_link(aerdev
, dev
);
493 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
494 /* TODO: Should panic here? */
499 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
500 status
= broadcast_error_message(dev
,
503 report_mmio_enabled
);
505 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
507 * TODO: Should call platform-specific
508 * functions to reset slot before calling
509 * drivers' slot_reset callbacks?
511 status
= broadcast_error_message(dev
,
517 if (status
== PCI_ERS_RESULT_RECOVERED
)
518 broadcast_error_message(dev
,
527 * handle_error_source - handle logging error into an event log
528 * @aerdev: pointer to pcie_device data structure of the root port
529 * @dev: pointer to pci_dev data structure of error source device
530 * @info: comprehensive error information
532 * Invoked when an error being detected by Root Port.
534 static void handle_error_source(struct pcie_device
*aerdev
,
536 struct aer_err_info
*info
)
538 pci_ers_result_t status
= 0;
541 if (info
->severity
== AER_CORRECTABLE
) {
543 * Correctable error does not need software intevention.
544 * No need to go through error recovery process.
546 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
548 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
551 status
= do_recovery(aerdev
, dev
, info
->severity
);
552 if (status
== PCI_ERS_RESULT_RECOVERED
) {
553 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver "
554 "successfully recovered\n");
556 /* TODO: Should kernel panic here? */
557 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver didn't "
564 * get_e_source - retrieve an error source
565 * @rpc: pointer to the root port which holds an error
567 * Invoked by DPC handler to consume an error.
569 static struct aer_err_source
*get_e_source(struct aer_rpc
*rpc
)
571 struct aer_err_source
*e_source
;
574 /* Lock access to Root error producer/consumer index */
575 spin_lock_irqsave(&rpc
->e_lock
, flags
);
576 if (rpc
->prod_idx
== rpc
->cons_idx
) {
577 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
580 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
582 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
584 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
590 * get_device_error_info - read error status from dev and store it to info
591 * @dev: pointer to the device expected to have a error record
592 * @info: pointer to structure to store the error record
594 * Return 1 on success, 0 on error.
596 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
601 info
->tlp_header_valid
= 0;
603 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
605 /* The device might not support AER */
609 if (info
->severity
== AER_CORRECTABLE
) {
610 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
612 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_MASK
,
614 if (!(info
->status
& ~info
->mask
))
616 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
617 info
->severity
== AER_NONFATAL
) {
619 /* Link is still healthy for IO reads */
620 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
622 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_MASK
,
624 if (!(info
->status
& ~info
->mask
))
627 /* Get First Error Pointer */
628 pci_read_config_dword(dev
, pos
+ PCI_ERR_CAP
, &temp
);
629 info
->first_error
= PCI_ERR_CAP_FEP(temp
);
631 if (info
->status
& AER_LOG_TLP_MASKS
) {
632 info
->tlp_header_valid
= 1;
633 pci_read_config_dword(dev
,
634 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
635 pci_read_config_dword(dev
,
636 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
637 pci_read_config_dword(dev
,
638 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
639 pci_read_config_dword(dev
,
640 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
647 static inline void aer_process_err_devices(struct pcie_device
*p_device
,
648 struct aer_err_info
*e_info
)
652 /* Report all before handle them, not to lost records by reset etc. */
653 for (i
= 0; i
< e_info
->error_dev_num
&& e_info
->dev
[i
]; i
++) {
654 if (get_device_error_info(e_info
->dev
[i
], e_info
))
655 aer_print_error(e_info
->dev
[i
], e_info
);
657 for (i
= 0; i
< e_info
->error_dev_num
&& e_info
->dev
[i
]; i
++) {
658 if (get_device_error_info(e_info
->dev
[i
], e_info
))
659 handle_error_source(p_device
, e_info
->dev
[i
], e_info
);
664 * aer_isr_one_error - consume an error detected by root port
665 * @p_device: pointer to error root port service device
666 * @e_src: pointer to an error source
668 static void aer_isr_one_error(struct pcie_device
*p_device
,
669 struct aer_err_source
*e_src
)
671 struct aer_err_info
*e_info
;
674 /* struct aer_err_info might be big, so we allocate it with slab */
675 e_info
= kmalloc(sizeof(struct aer_err_info
), GFP_KERNEL
);
676 if (e_info
== NULL
) {
677 dev_printk(KERN_DEBUG
, &p_device
->port
->dev
,
678 "Can't allocate mem when processing AER errors\n");
683 * There is a possibility that both correctable error and
684 * uncorrectable error being logged. Report correctable error first.
686 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
689 if (!(e_src
->status
& i
))
692 memset(e_info
, 0, sizeof(struct aer_err_info
));
694 /* Init comprehensive error information */
695 if (i
& PCI_ERR_ROOT_COR_RCV
) {
696 e_info
->id
= ERR_COR_ID(e_src
->id
);
697 e_info
->severity
= AER_CORRECTABLE
;
699 e_info
->id
= ERR_UNCOR_ID(e_src
->id
);
700 e_info
->severity
= ((e_src
->status
>> 6) & 1);
703 (PCI_ERR_ROOT_MULTI_COR_RCV
|
704 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
705 e_info
->multi_error_valid
= 1;
707 aer_print_port_info(p_device
->port
, e_info
);
709 if (find_source_device(p_device
->port
, e_info
))
710 aer_process_err_devices(p_device
, e_info
);
717 * aer_isr - consume errors detected by root port
718 * @work: definition of this work item
720 * Invoked, as DPC, when root port records new detected error
722 void aer_isr(struct work_struct
*work
)
724 struct aer_rpc
*rpc
= container_of(work
, struct aer_rpc
, dpc_handler
);
725 struct pcie_device
*p_device
= rpc
->rpd
;
726 struct aer_err_source
*e_src
;
728 mutex_lock(&rpc
->rpc_mutex
);
729 e_src
= get_e_source(rpc
);
731 aer_isr_one_error(p_device
, e_src
);
732 e_src
= get_e_source(rpc
);
734 mutex_unlock(&rpc
->rpc_mutex
);
736 wake_up(&rpc
->wait_release
);
740 * aer_init - provide AER initialization
741 * @dev: pointer to AER pcie device
743 * Invoked when AER service driver is loaded.
745 int aer_init(struct pcie_device
*dev
)
747 if (dev
->port
->aer_firmware_first
) {
748 dev_printk(KERN_DEBUG
, &dev
->device
,
749 "PCIe errors handled by platform firmware.\n");
753 if (aer_osc_setup(dev
))
759 dev_printk(KERN_DEBUG
, &dev
->device
,
760 "aerdrv forceload requested.\n");
761 dev
->port
->aer_firmware_first
= 0;