2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/delay.h>
29 module_param(forceload
, bool, 0);
31 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
36 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
40 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
44 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
47 PCI_EXP_DEVCTL_NFERE
|
50 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
55 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
60 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
64 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
65 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
66 PCI_EXP_DEVCTL_NFERE
|
69 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
74 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
79 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
83 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
84 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
85 if (dev
->error_state
== pci_channel_io_normal
)
86 status
&= ~mask
; /* Clear corresponding nonfatal bits */
88 status
&= mask
; /* Clear corresponding fatal bits */
89 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
95 int pci_cleanup_aer_correct_error_status(struct pci_dev
*dev
)
100 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
104 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, &status
);
105 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, status
);
111 static int find_device_iter(struct device
*device
, void *data
)
114 u16 id
= *(unsigned long *)data
;
115 u8 secondary
, subordinate
, d_bus
= id
>> 8;
117 if (device
->bus
== &pci_bus_type
) {
118 dev
= to_pci_dev(device
);
119 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
123 *(unsigned long*)data
= (unsigned long)device
;
128 * If device is P2P, check if it is an upstream?
130 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
131 pci_read_config_byte(dev
, PCI_SECONDARY_BUS
,
133 pci_read_config_byte(dev
, PCI_SUBORDINATE_BUS
,
135 if (d_bus
>= secondary
&& d_bus
<= subordinate
) {
136 *(unsigned long*)data
= (unsigned long)device
;
146 * find_source_device - search through device hierarchy for source device
147 * @parent: pointer to Root Port pci_dev data structure
148 * @id: device ID of agent who sends an error message to this Root Port
150 * Invoked when error is detected at the Root Port.
152 static struct device
* find_source_device(struct pci_dev
*parent
, u16 id
)
154 struct pci_dev
*dev
= parent
;
155 struct device
*device
;
156 unsigned long device_addr
;
159 /* Is Root Port an agent that sends error message? */
160 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
165 if ((status
= device_for_each_child(&dev
->dev
,
166 &device_addr
, find_device_iter
))) {
167 device
= (struct device
*)device_addr
;
168 dev
= to_pci_dev(device
);
169 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
177 static void report_error_detected(struct pci_dev
*dev
, void *data
)
179 pci_ers_result_t vote
;
180 struct pci_error_handlers
*err_handler
;
181 struct aer_broadcast_data
*result_data
;
182 result_data
= (struct aer_broadcast_data
*) data
;
184 dev
->error_state
= result_data
->state
;
187 !dev
->driver
->err_handler
||
188 !dev
->driver
->err_handler
->error_detected
) {
189 if (result_data
->state
== pci_channel_io_frozen
&&
190 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
192 * In case of fatal recovery, if one of down-
193 * stream device has no driver. We might be
194 * unable to recover because a later insmod
195 * of a driver for this device is unaware of
198 dev_printk(KERN_DEBUG
, &dev
->dev
, "device has %s\n",
200 "no AER-aware driver" : "no driver");
205 err_handler
= dev
->driver
->err_handler
;
206 vote
= err_handler
->error_detected(dev
, result_data
->state
);
207 result_data
->result
= merge_result(result_data
->result
, vote
);
211 static void report_mmio_enabled(struct pci_dev
*dev
, void *data
)
213 pci_ers_result_t vote
;
214 struct pci_error_handlers
*err_handler
;
215 struct aer_broadcast_data
*result_data
;
216 result_data
= (struct aer_broadcast_data
*) data
;
219 !dev
->driver
->err_handler
||
220 !dev
->driver
->err_handler
->mmio_enabled
)
223 err_handler
= dev
->driver
->err_handler
;
224 vote
= err_handler
->mmio_enabled(dev
);
225 result_data
->result
= merge_result(result_data
->result
, vote
);
229 static void report_slot_reset(struct pci_dev
*dev
, void *data
)
231 pci_ers_result_t vote
;
232 struct pci_error_handlers
*err_handler
;
233 struct aer_broadcast_data
*result_data
;
234 result_data
= (struct aer_broadcast_data
*) data
;
237 !dev
->driver
->err_handler
||
238 !dev
->driver
->err_handler
->slot_reset
)
241 err_handler
= dev
->driver
->err_handler
;
242 vote
= err_handler
->slot_reset(dev
);
243 result_data
->result
= merge_result(result_data
->result
, vote
);
247 static void report_resume(struct pci_dev
*dev
, void *data
)
249 struct pci_error_handlers
*err_handler
;
251 dev
->error_state
= pci_channel_io_normal
;
254 !dev
->driver
->err_handler
||
255 !dev
->driver
->err_handler
->slot_reset
)
258 err_handler
= dev
->driver
->err_handler
;
259 err_handler
->resume(dev
);
264 * broadcast_error_message - handle message broadcast to downstream drivers
265 * @dev: pointer to from where in a hierarchy message is broadcasted down
266 * @state: error state
267 * @error_mesg: message to print
268 * @cb: callback to be broadcasted
270 * Invoked during error recovery process. Once being invoked, the content
271 * of error severity will be broadcasted to all downstream drivers in a
272 * hierarchy in question.
274 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
275 enum pci_channel_state state
,
277 void (*cb
)(struct pci_dev
*, void *))
279 struct aer_broadcast_data result_data
;
281 dev_printk(KERN_DEBUG
, &dev
->dev
, "broadcast %s message\n", error_mesg
);
282 result_data
.state
= state
;
283 if (cb
== report_error_detected
)
284 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
286 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
288 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
290 * If the error is reported by a bridge, we think this error
291 * is related to the downstream link of the bridge, so we
292 * do error recovery on all subordinates of the bridge instead
293 * of the bridge and clear the error status of the bridge.
295 if (cb
== report_error_detected
)
296 dev
->error_state
= state
;
297 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
298 if (cb
== report_resume
) {
299 pci_cleanup_aer_uncorrect_error_status(dev
);
300 dev
->error_state
= pci_channel_io_normal
;
305 * If the error is reported by an end point, we think this
306 * error is related to the upstream link of the end point.
308 pci_walk_bus(dev
->bus
, cb
, &result_data
);
311 return result_data
.result
;
314 struct find_aer_service_data
{
315 struct pcie_port_service_driver
*aer_driver
;
319 static int find_aer_service_iter(struct device
*device
, void *data
)
321 struct device_driver
*driver
;
322 struct pcie_port_service_driver
*service_driver
;
323 struct pcie_device
*pcie_dev
;
324 struct find_aer_service_data
*result
;
326 result
= (struct find_aer_service_data
*) data
;
328 if (device
->bus
== &pcie_port_bus_type
) {
329 pcie_dev
= to_pcie_device(device
);
330 if (pcie_dev
->id
.port_type
== PCIE_SW_DOWNSTREAM_PORT
)
331 result
->is_downstream
= 1;
333 driver
= device
->driver
;
335 service_driver
= to_service_driver(driver
);
336 if (service_driver
->id_table
->service_type
==
337 PCIE_PORT_SERVICE_AER
) {
338 result
->aer_driver
= service_driver
;
347 static void find_aer_service(struct pci_dev
*dev
,
348 struct find_aer_service_data
*data
)
351 retval
= device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
354 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
357 struct pci_dev
*udev
;
358 pci_ers_result_t status
;
359 struct find_aer_service_data data
;
361 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
364 udev
= dev
->bus
->self
;
366 data
.is_downstream
= 0;
367 data
.aer_driver
= NULL
;
368 find_aer_service(udev
, &data
);
371 * Use the aer driver of the error agent firstly.
372 * If it hasn't the aer driver, use the root port's
374 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
375 if (data
.is_downstream
&&
376 aerdev
->device
.driver
&&
377 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
379 to_service_driver(aerdev
->device
.driver
);
381 dev_printk(KERN_DEBUG
, &dev
->dev
, "no link-reset "
383 return PCI_ERS_RESULT_DISCONNECT
;
387 status
= data
.aer_driver
->reset_link(udev
);
388 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
389 dev_printk(KERN_DEBUG
, &dev
->dev
, "link reset at upstream "
390 "device %s failed\n", pci_name(udev
));
391 return PCI_ERS_RESULT_DISCONNECT
;
398 * do_recovery - handle nonfatal/fatal error recovery process
399 * @aerdev: pointer to a pcie_device data structure of root port
400 * @dev: pointer to a pci_dev data structure of agent detecting an error
401 * @severity: error severity type
403 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
404 * error detected message to all downstream drivers within a hierarchy in
405 * question and return the returned code.
407 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
411 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
412 enum pci_channel_state state
;
414 if (severity
== AER_FATAL
)
415 state
= pci_channel_io_frozen
;
417 state
= pci_channel_io_normal
;
419 status
= broadcast_error_message(dev
,
422 report_error_detected
);
424 if (severity
== AER_FATAL
) {
425 result
= reset_link(aerdev
, dev
);
426 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
427 /* TODO: Should panic here? */
432 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
433 status
= broadcast_error_message(dev
,
436 report_mmio_enabled
);
438 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
440 * TODO: Should call platform-specific
441 * functions to reset slot before calling
442 * drivers' slot_reset callbacks?
444 status
= broadcast_error_message(dev
,
450 if (status
== PCI_ERS_RESULT_RECOVERED
)
451 broadcast_error_message(dev
,
460 * handle_error_source - handle logging error into an event log
461 * @aerdev: pointer to pcie_device data structure of the root port
462 * @dev: pointer to pci_dev data structure of error source device
463 * @info: comprehensive error information
465 * Invoked when an error being detected by Root Port.
467 static void handle_error_source(struct pcie_device
* aerdev
,
469 struct aer_err_info info
)
471 pci_ers_result_t status
= 0;
474 if (info
.severity
== AER_CORRECTABLE
) {
476 * Correctable error does not need software intevention.
477 * No need to go through error recovery process.
479 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
481 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
484 status
= do_recovery(aerdev
, dev
, info
.severity
);
485 if (status
== PCI_ERS_RESULT_RECOVERED
) {
486 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver "
487 "successfully recovered\n");
489 /* TODO: Should kernel panic here? */
490 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver didn't "
497 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
498 * @rpc: pointer to a Root Port data structure
500 * Invoked when PCIE bus loads AER service driver.
502 void aer_enable_rootport(struct aer_rpc
*rpc
)
504 struct pci_dev
*pdev
= rpc
->rpd
->port
;
509 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
510 /* Clear PCIE Capability's Device Status */
511 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, ®16
);
512 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, reg16
);
514 /* Disable system error generation in response to error messages */
515 pci_read_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, ®16
);
516 reg16
&= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK
);
517 pci_write_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, reg16
);
519 aer_pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
520 /* Clear error status */
521 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
522 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
523 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, ®32
);
524 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, reg32
);
525 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, ®32
);
526 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, reg32
);
528 /* Enable Root Port device reporting error itself */
529 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVCTL
, ®16
);
531 PCI_EXP_DEVCTL_CERE
|
532 PCI_EXP_DEVCTL_NFERE
|
533 PCI_EXP_DEVCTL_FERE
|
535 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVCTL
,
538 /* Enable Root Port's interrupt in response to error messages */
539 pci_write_config_dword(pdev
,
540 aer_pos
+ PCI_ERR_ROOT_COMMAND
,
541 ROOT_PORT_INTR_ON_MESG_MASK
);
545 * disable_root_aer - disable Root Port's interrupts when receiving messages
546 * @rpc: pointer to a Root Port data structure
548 * Invoked when PCIE bus unloads AER service driver.
550 static void disable_root_aer(struct aer_rpc
*rpc
)
552 struct pci_dev
*pdev
= rpc
->rpd
->port
;
556 pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
557 /* Disable Root's interrupt in response to error messages */
558 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_COMMAND
, 0);
560 /* Clear Root's error status reg */
561 pci_read_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
562 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
566 * get_e_source - retrieve an error source
567 * @rpc: pointer to the root port which holds an error
569 * Invoked by DPC handler to consume an error.
571 static struct aer_err_source
* get_e_source(struct aer_rpc
*rpc
)
573 struct aer_err_source
*e_source
;
576 /* Lock access to Root error producer/consumer index */
577 spin_lock_irqsave(&rpc
->e_lock
, flags
);
578 if (rpc
->prod_idx
== rpc
->cons_idx
) {
579 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
582 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
584 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
586 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
591 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
595 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
597 /* The device might not support AER */
601 if (info
->severity
== AER_CORRECTABLE
) {
602 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
604 if (!(info
->status
& ERR_CORRECTABLE_ERROR_MASK
))
605 return AER_UNSUCCESS
;
606 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
607 info
->severity
== AER_NONFATAL
) {
609 /* Link is still healthy for IO reads */
610 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
612 if (!(info
->status
& ERR_UNCORRECTABLE_ERROR_MASK
))
613 return AER_UNSUCCESS
;
615 if (info
->status
& AER_LOG_TLP_MASKS
) {
616 info
->flags
|= AER_TLP_HEADER_VALID_FLAG
;
617 pci_read_config_dword(dev
,
618 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
619 pci_read_config_dword(dev
,
620 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
621 pci_read_config_dword(dev
,
622 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
623 pci_read_config_dword(dev
,
624 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
632 * aer_isr_one_error - consume an error detected by root port
633 * @p_device: pointer to error root port service device
634 * @e_src: pointer to an error source
636 static void aer_isr_one_error(struct pcie_device
*p_device
,
637 struct aer_err_source
*e_src
)
639 struct device
*s_device
;
640 struct aer_err_info e_info
= {0, 0, 0,};
645 * There is a possibility that both correctable error and
646 * uncorrectable error being logged. Report correctable error first.
648 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
651 if (!(e_src
->status
& i
))
654 /* Init comprehensive error information */
655 if (i
& PCI_ERR_ROOT_COR_RCV
) {
656 id
= ERR_COR_ID(e_src
->id
);
657 e_info
.severity
= AER_CORRECTABLE
;
659 id
= ERR_UNCOR_ID(e_src
->id
);
660 e_info
.severity
= ((e_src
->status
>> 6) & 1);
663 (PCI_ERR_ROOT_MULTI_COR_RCV
|
664 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
665 e_info
.flags
|= AER_MULTI_ERROR_VALID_FLAG
;
666 if (!(s_device
= find_source_device(p_device
->port
, id
))) {
667 printk(KERN_DEBUG
"%s->can't find device of ID%04x\n",
671 if (get_device_error_info(to_pci_dev(s_device
), &e_info
) ==
673 aer_print_error(to_pci_dev(s_device
), &e_info
);
674 handle_error_source(p_device
,
675 to_pci_dev(s_device
),
682 * aer_isr - consume errors detected by root port
683 * @work: definition of this work item
685 * Invoked, as DPC, when root port records new detected error
687 void aer_isr(struct work_struct
*work
)
689 struct aer_rpc
*rpc
= container_of(work
, struct aer_rpc
, dpc_handler
);
690 struct pcie_device
*p_device
= rpc
->rpd
;
691 struct aer_err_source
*e_src
;
693 mutex_lock(&rpc
->rpc_mutex
);
694 e_src
= get_e_source(rpc
);
696 aer_isr_one_error(p_device
, e_src
);
697 e_src
= get_e_source(rpc
);
699 mutex_unlock(&rpc
->rpc_mutex
);
701 wake_up(&rpc
->wait_release
);
705 * aer_delete_rootport - disable root port aer and delete service data
706 * @rpc: pointer to a root port device being deleted
708 * Invoked when AER service unloaded on a specific Root Port
710 void aer_delete_rootport(struct aer_rpc
*rpc
)
712 /* Disable root port AER itself */
713 disable_root_aer(rpc
);
719 * aer_init - provide AER initialization
720 * @dev: pointer to AER pcie device
722 * Invoked when AER service driver is loaded.
724 int aer_init(struct pcie_device
*dev
)
726 if (aer_osc_setup(dev
) && !forceload
)
732 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
733 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
734 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);