2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIE link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
30 u32 l0s
; /* L0s latency (nsec) */
31 u32 l1
; /* L1 latency (nsec) */
34 struct pcie_link_state
{
35 struct pci_dev
*pdev
; /* Upstream component of the Link */
36 struct pcie_link_state
*root
; /* pointer to the root port link */
37 struct pcie_link_state
*parent
; /* pointer to the parent Link state */
38 struct list_head sibling
; /* node in link_list */
39 struct list_head children
; /* list of child link states */
40 struct list_head link
; /* node in parent's children list */
43 u32 aspm_support
:2; /* Supported ASPM state */
44 u32 aspm_enabled
:2; /* Enabled ASPM state */
45 u32 aspm_capable
:2; /* Capable ASPM state with latency */
46 u32 aspm_default
:2; /* Default ASPM state by BIOS */
47 u32 aspm_disable
:2; /* Disabled ASPM state */
50 u32 clkpm_capable
:1; /* Clock PM capable? */
51 u32 clkpm_enabled
:1; /* Current Clock PM state */
52 u32 clkpm_default
:1; /* Default Clock PM state by BIOS */
55 struct aspm_latency latency
; /* Exit latency */
57 * Endpoint acceptable latencies. A pcie downstream port only
58 * has one slot under it, so at most there are 8 functions.
60 struct aspm_latency acceptable
[8];
63 static int aspm_disabled
, aspm_force
;
64 static DEFINE_MUTEX(aspm_lock
);
65 static LIST_HEAD(link_list
);
67 #define POLICY_DEFAULT 0 /* BIOS default setting */
68 #define POLICY_PERFORMANCE 1 /* high performance */
69 #define POLICY_POWERSAVE 2 /* high power saving */
70 static int aspm_policy
;
71 static const char *policy_str
[] = {
72 [POLICY_DEFAULT
] = "default",
73 [POLICY_PERFORMANCE
] = "performance",
74 [POLICY_POWERSAVE
] = "powersave"
77 #define LINK_RETRAIN_TIMEOUT HZ
79 static int policy_to_aspm_state(struct pcie_link_state
*link
)
81 switch (aspm_policy
) {
82 case POLICY_PERFORMANCE
:
83 /* Disable ASPM and Clock PM */
85 case POLICY_POWERSAVE
:
86 /* Enable ASPM L0s/L1 */
87 return PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
89 return link
->aspm_default
;
94 static int policy_to_clkpm_state(struct pcie_link_state
*link
)
96 switch (aspm_policy
) {
97 case POLICY_PERFORMANCE
:
98 /* Disable ASPM and Clock PM */
100 case POLICY_POWERSAVE
:
101 /* Disable Clock PM */
104 return link
->clkpm_default
;
109 static void pcie_set_clkpm_nocheck(struct pcie_link_state
*link
, int enable
)
113 struct pci_dev
*child
;
114 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
116 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
117 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
120 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
122 reg16
|= PCI_EXP_LNKCTL_CLKREQ_EN
;
124 reg16
&= ~PCI_EXP_LNKCTL_CLKREQ_EN
;
125 pci_write_config_word(child
, pos
+ PCI_EXP_LNKCTL
, reg16
);
127 link
->clkpm_enabled
= !!enable
;
130 static void pcie_set_clkpm(struct pcie_link_state
*link
, int enable
)
132 /* Don't enable Clock PM if the link is not Clock PM capable */
133 if (!link
->clkpm_capable
&& enable
)
135 /* Need nothing if the specified equals to current state */
136 if (link
->clkpm_enabled
== enable
)
138 pcie_set_clkpm_nocheck(link
, enable
);
141 static void pcie_clkpm_cap_init(struct pcie_link_state
*link
, int blacklist
)
143 int pos
, capable
= 1, enabled
= 1;
146 struct pci_dev
*child
;
147 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
149 /* All functions should have the same cap and state, take the worst */
150 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
151 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
154 pci_read_config_dword(child
, pos
+ PCI_EXP_LNKCAP
, ®32
);
155 if (!(reg32
& PCI_EXP_LNKCAP_CLKPM
)) {
160 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
161 if (!(reg16
& PCI_EXP_LNKCTL_CLKREQ_EN
))
164 link
->clkpm_enabled
= enabled
;
165 link
->clkpm_default
= enabled
;
166 link
->clkpm_capable
= (blacklist
) ? 0 : capable
;
169 static bool pcie_aspm_downstream_has_switch(struct pcie_link_state
*link
)
171 struct pci_dev
*child
;
172 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
174 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
175 if (child
->pcie_type
== PCI_EXP_TYPE_UPSTREAM
)
182 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
183 * could use common clock. If they are, configure them to use the
184 * common clock. That will reduce the ASPM state exit latency.
186 static void pcie_aspm_configure_common_clock(struct pcie_link_state
*link
)
188 int ppos
, cpos
, same_clock
= 1;
189 u16 reg16
, parent_reg
, child_reg
[8];
190 unsigned long start_jiffies
;
191 struct pci_dev
*child
, *parent
= link
->pdev
;
192 struct pci_bus
*linkbus
= parent
->subordinate
;
194 * All functions of a slot should have the same Slot Clock
195 * Configuration, so just check one function
197 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
198 BUG_ON(!child
->is_pcie
);
200 /* Check downstream component if bit Slot Clock Configuration is 1 */
201 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
202 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKSTA
, ®16
);
203 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
206 /* Check upstream component if bit Slot Clock Configuration is 1 */
207 ppos
= pci_find_capability(parent
, PCI_CAP_ID_EXP
);
208 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
209 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
212 /* Configure downstream component, all functions */
213 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
214 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
215 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, ®16
);
216 child_reg
[PCI_FUNC(child
->devfn
)] = reg16
;
218 reg16
|= PCI_EXP_LNKCTL_CCC
;
220 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
221 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, reg16
);
224 /* Configure upstream component */
225 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, ®16
);
228 reg16
|= PCI_EXP_LNKCTL_CCC
;
230 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
231 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
234 reg16
|= PCI_EXP_LNKCTL_RL
;
235 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
237 /* Wait for link training end. Break out after waiting for timeout */
238 start_jiffies
= jiffies
;
240 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
241 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
243 if (time_after(jiffies
, start_jiffies
+ LINK_RETRAIN_TIMEOUT
))
247 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
250 /* Training failed. Restore common clock configurations */
251 dev_printk(KERN_ERR
, &parent
->dev
,
252 "ASPM: Could not configure common clock\n");
253 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
254 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
255 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
,
256 child_reg
[PCI_FUNC(child
->devfn
)]);
258 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, parent_reg
);
261 /* Convert L0s latency encoding to ns */
262 static u32
calc_l0s_latency(u32 encoding
)
265 return (5 * 1000); /* > 4us */
266 return (64 << encoding
);
269 /* Convert L0s acceptable latency encoding to ns */
270 static u32
calc_l0s_acceptable(u32 encoding
)
274 return (64 << encoding
);
277 /* Convert L1 latency encoding to ns */
278 static u32
calc_l1_latency(u32 encoding
)
281 return (65 * 1000); /* > 64us */
282 return (1000 << encoding
);
285 /* Convert L1 acceptable latency encoding to ns */
286 static u32
calc_l1_acceptable(u32 encoding
)
290 return (1000 << encoding
);
293 static void pcie_aspm_get_cap_device(struct pci_dev
*pdev
, u32
*state
,
294 u32
*l0s
, u32
*l1
, u32
*enabled
)
300 *l0s
= *l1
= *enabled
= 0;
301 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
302 pci_read_config_dword(pdev
, pos
+ PCI_EXP_LNKCAP
, ®32
);
303 *state
= (reg32
& PCI_EXP_LNKCAP_ASPMS
) >> 10;
304 if (*state
!= PCIE_LINK_STATE_L0S
&&
305 *state
!= (PCIE_LINK_STATE_L1
| PCIE_LINK_STATE_L0S
))
310 encoding
= (reg32
& PCI_EXP_LNKCAP_L0SEL
) >> 12;
311 *l0s
= calc_l0s_latency(encoding
);
312 if (*state
& PCIE_LINK_STATE_L1
) {
313 encoding
= (reg32
& PCI_EXP_LNKCAP_L1EL
) >> 15;
314 *l1
= calc_l1_latency(encoding
);
316 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
317 *enabled
= reg16
& (PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
);
320 static void pcie_aspm_check_latency(struct pci_dev
*endpoint
)
322 u32 l1_switch_latency
= 0;
323 struct aspm_latency
*acceptable
;
324 struct pcie_link_state
*link
;
326 /* Device not in D0 doesn't need latency check */
327 if ((endpoint
->current_state
!= PCI_D0
) &&
328 (endpoint
->current_state
!= PCI_UNKNOWN
))
331 link
= endpoint
->bus
->self
->link_state
;
332 acceptable
= &link
->acceptable
[PCI_FUNC(endpoint
->devfn
)];
335 /* Check L0s latency */
336 if ((link
->aspm_capable
& PCIE_LINK_STATE_L0S
) &&
337 (link
->latency
.l0s
> acceptable
->l0s
))
338 link
->aspm_capable
&= ~PCIE_LINK_STATE_L0S
;
341 * Every switch on the path to root complex need 1
342 * more microsecond for L1. Spec doesn't mention L0s.
344 if ((link
->aspm_capable
& PCIE_LINK_STATE_L1
) &&
345 (link
->latency
.l1
+ l1_switch_latency
> acceptable
->l1
))
346 link
->aspm_capable
&= ~PCIE_LINK_STATE_L1
;
347 l1_switch_latency
+= 1000;
353 static void pcie_aspm_cap_init(struct pcie_link_state
*link
, int blacklist
)
355 u32 support
, l0s
, l1
, enabled
;
356 struct pci_dev
*child
, *parent
= link
->pdev
;
357 struct pci_bus
*linkbus
= parent
->subordinate
;
360 /* Set enabled/disable so that we will disable ASPM later */
361 link
->aspm_enabled
= PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
362 link
->aspm_disable
= PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
366 /* Configure common clock before checking latencies */
367 pcie_aspm_configure_common_clock(link
);
369 /* upstream component states */
370 pcie_aspm_get_cap_device(parent
, &support
, &l0s
, &l1
, &enabled
);
371 link
->aspm_support
= support
;
372 link
->latency
.l0s
= l0s
;
373 link
->latency
.l1
= l1
;
374 link
->aspm_enabled
= enabled
;
376 /* downstream component states, all functions have the same setting */
377 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
378 pcie_aspm_get_cap_device(child
, &support
, &l0s
, &l1
, &enabled
);
379 link
->aspm_support
&= support
;
380 link
->latency
.l0s
= max_t(u32
, link
->latency
.l0s
, l0s
);
381 link
->latency
.l1
= max_t(u32
, link
->latency
.l1
, l1
);
383 /* Save default state */
384 link
->aspm_default
= link
->aspm_enabled
;
386 /* Setup initial capable state. Will be updated later */
387 link
->aspm_capable
= link
->aspm_support
;
389 * If the downstream component has pci bridge function, don't
392 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
393 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
) {
395 PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
400 if (!link
->aspm_support
)
404 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
407 struct aspm_latency
*acceptable
=
408 &link
->acceptable
[PCI_FUNC(child
->devfn
)];
410 if (child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
411 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
)
414 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
415 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
416 /* Calculate endpoint L0s acceptable latency */
417 encoding
= (reg32
& PCI_EXP_DEVCAP_L0S
) >> 6;
418 acceptable
->l0s
= calc_l0s_acceptable(encoding
);
419 /* Calculate endpoint L1 acceptable latency */
420 encoding
= (reg32
& PCI_EXP_DEVCAP_L1
) >> 9;
421 acceptable
->l1
= calc_l1_acceptable(encoding
);
423 pcie_aspm_check_latency(child
);
428 * __pcie_aspm_check_state_one - check latency for endpoint device.
429 * @endpoint: pointer to the struct pci_dev of endpoint device
431 * TBD: The latency from the endpoint to root complex vary per switch's
432 * upstream link state above the device. Here we just do a simple check
433 * which assumes all links above the device can be in L1 state, that
434 * is we just consider the worst case. If switch's upstream link can't
435 * be put into L0S/L1, then our check is too strictly.
437 static u32
__pcie_aspm_check_state_one(struct pci_dev
*endpoint
, u32 state
)
439 struct pcie_link_state
*link
= endpoint
->bus
->self
->link_state
;
440 while (link
&& state
) {
441 state
&= link
->aspm_capable
;
447 static u32
pcie_aspm_check_state(struct pcie_link_state
*link
, u32 state
)
449 pci_power_t power_state
;
450 struct pci_dev
*child
;
451 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
453 /* If no child, ignore the link */
454 if (list_empty(&linkbus
->devices
))
457 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
459 * If downstream component of a link is pci bridge, we
460 * disable ASPM for now for the link
462 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
)
465 if ((child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
466 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
))
468 /* Device not in D0 doesn't need check latency */
469 power_state
= child
->current_state
;
470 if (power_state
== PCI_D1
|| power_state
== PCI_D2
||
471 power_state
== PCI_D3hot
|| power_state
== PCI_D3cold
)
473 state
= __pcie_aspm_check_state_one(child
, state
);
478 static void __pcie_aspm_config_one_dev(struct pci_dev
*pdev
, unsigned int state
)
481 int pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
483 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
486 pci_write_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, reg16
);
489 static void __pcie_aspm_config_link(struct pcie_link_state
*link
, u32 state
)
491 struct pci_dev
*child
, *parent
= link
->pdev
;
492 struct pci_bus
*linkbus
= parent
->subordinate
;
494 state
&= ~link
->aspm_disable
;
495 /* Nothing to do if the link is already in the requested state */
496 if (link
->aspm_enabled
== state
)
499 * Spec 2.0 suggests all functions should be configured the
500 * same setting for ASPM. Enabling ASPM L1 should be done in
501 * upstream component first and then downstream, and vice
502 * versa for disabling ASPM L1. Spec doesn't mention L0S.
504 if (state
& PCIE_LINK_STATE_L1
)
505 __pcie_aspm_config_one_dev(parent
, state
);
507 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
508 __pcie_aspm_config_one_dev(child
, state
);
510 if (!(state
& PCIE_LINK_STATE_L1
))
511 __pcie_aspm_config_one_dev(parent
, state
);
513 link
->aspm_enabled
= state
;
516 /* Check the whole hierarchy, and configure each link in the hierarchy */
517 static void __pcie_aspm_configure_link_state(struct pcie_link_state
*link
,
520 struct pcie_link_state
*leaf
, *root
= link
->root
;
522 state
&= (PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
);
524 /* Check all links who have specific root port link */
525 list_for_each_entry(leaf
, &link_list
, sibling
) {
526 if (!list_empty(&leaf
->children
) || (leaf
->root
!= root
))
528 state
= pcie_aspm_check_state(leaf
, state
);
530 /* Check root port link too in case it hasn't children */
531 state
= pcie_aspm_check_state(root
, state
);
532 if (link
->aspm_enabled
== state
)
535 * We must change the hierarchy. See comments in
536 * __pcie_aspm_config_link for the order
538 if (state
& PCIE_LINK_STATE_L1
) {
539 list_for_each_entry(leaf
, &link_list
, sibling
) {
540 if (leaf
->root
== root
)
541 __pcie_aspm_config_link(leaf
, state
);
544 list_for_each_entry_reverse(leaf
, &link_list
, sibling
) {
545 if (leaf
->root
== root
)
546 __pcie_aspm_config_link(leaf
, state
);
552 * pcie_aspm_configure_link_state: enable/disable PCI express link state
553 * @pdev: the root port or switch downstream port
555 static void pcie_aspm_configure_link_state(struct pcie_link_state
*link
,
558 down_read(&pci_bus_sem
);
559 mutex_lock(&aspm_lock
);
560 __pcie_aspm_configure_link_state(link
, state
);
561 mutex_unlock(&aspm_lock
);
562 up_read(&pci_bus_sem
);
565 static void free_link_state(struct pcie_link_state
*link
)
567 link
->pdev
->link_state
= NULL
;
571 static int pcie_aspm_sanity_check(struct pci_dev
*pdev
)
573 struct pci_dev
*child
;
577 * Some functions in a slot might not all be PCIE functions,
578 * very strange. Disable ASPM for the whole slot
580 list_for_each_entry(child
, &pdev
->subordinate
->devices
, bus_list
) {
581 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
585 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
586 * RBER bit to determine if a function is 1.1 version device
588 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
589 if (!(reg32
& PCI_EXP_DEVCAP_RBER
) && !aspm_force
) {
590 dev_printk(KERN_INFO
, &child
->dev
, "disabling ASPM"
591 " on pre-1.1 PCIe device. You can enable it"
592 " with 'pcie_aspm=force'\n");
599 static struct pcie_link_state
*pcie_aspm_setup_link_state(struct pci_dev
*pdev
)
601 struct pcie_link_state
*link
;
602 int blacklist
= !!pcie_aspm_sanity_check(pdev
);
604 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
607 INIT_LIST_HEAD(&link
->sibling
);
608 INIT_LIST_HEAD(&link
->children
);
609 INIT_LIST_HEAD(&link
->link
);
611 if (pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
) {
612 struct pcie_link_state
*parent
;
613 parent
= pdev
->bus
->parent
->self
->link_state
;
618 link
->parent
= parent
;
619 list_add(&link
->link
, &parent
->children
);
621 /* Setup a pointer to the root port link */
625 link
->root
= link
->parent
->root
;
627 list_add(&link
->sibling
, &link_list
);
629 pdev
->link_state
= link
;
631 /* Check ASPM capability */
632 pcie_aspm_cap_init(link
, blacklist
);
634 /* Check Clock PM capability */
635 pcie_clkpm_cap_init(link
, blacklist
);
641 * pcie_aspm_init_link_state: Initiate PCI express link state.
642 * It is called after the pcie and its children devices are scaned.
643 * @pdev: the root port or switch downstream port
645 void pcie_aspm_init_link_state(struct pci_dev
*pdev
)
648 struct pcie_link_state
*link
;
650 if (aspm_disabled
|| !pdev
->is_pcie
|| pdev
->link_state
)
652 if (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
653 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
656 /* VIA has a strange chipset, root port is under a bridge */
657 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
&&
661 down_read(&pci_bus_sem
);
662 if (list_empty(&pdev
->subordinate
->devices
))
665 mutex_lock(&aspm_lock
);
666 link
= pcie_aspm_setup_link_state(pdev
);
670 * Setup initial ASPM state
672 * If link has switch, delay the link config. The leaf link
673 * initialization will config the whole hierarchy. But we must
674 * make sure BIOS doesn't set unsupported link state.
676 if (pcie_aspm_downstream_has_switch(link
)) {
677 state
= pcie_aspm_check_state(link
, link
->aspm_default
);
678 __pcie_aspm_config_link(link
, state
);
680 state
= policy_to_aspm_state(link
);
681 __pcie_aspm_configure_link_state(link
, state
);
684 /* Setup initial Clock PM state */
685 state
= (link
->clkpm_capable
) ? policy_to_clkpm_state(link
) : 0;
686 pcie_set_clkpm(link
, state
);
688 mutex_unlock(&aspm_lock
);
690 up_read(&pci_bus_sem
);
693 /* Recheck latencies and update aspm_capable for links under the root */
694 static void pcie_update_aspm_capable(struct pcie_link_state
*root
)
696 struct pcie_link_state
*link
;
697 BUG_ON(root
->parent
);
698 list_for_each_entry(link
, &link_list
, sibling
) {
699 if (link
->root
!= root
)
701 link
->aspm_capable
= link
->aspm_support
;
703 list_for_each_entry(link
, &link_list
, sibling
) {
704 struct pci_dev
*child
;
705 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
706 if (link
->root
!= root
)
708 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
709 if ((child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
) &&
710 (child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
))
712 pcie_aspm_check_latency(child
);
717 /* @pdev: the endpoint device */
718 void pcie_aspm_exit_link_state(struct pci_dev
*pdev
)
720 struct pci_dev
*parent
= pdev
->bus
->self
;
721 struct pcie_link_state
*link
, *root
;
723 if (aspm_disabled
|| !pdev
->is_pcie
|| !parent
|| !parent
->link_state
)
725 if (parent
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
726 parent
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
729 down_read(&pci_bus_sem
);
730 mutex_lock(&aspm_lock
);
732 * All PCIe functions are in one slot, remove one function will remove
733 * the whole slot, so just wait until we are the last function left.
735 if (!list_is_last(&pdev
->bus_list
, &parent
->subordinate
->devices
))
738 link
= parent
->link_state
;
741 /* All functions are removed, so just disable ASPM for the link */
742 __pcie_aspm_config_one_dev(parent
, 0);
743 list_del(&link
->sibling
);
744 list_del(&link
->link
);
745 /* Clock PM is for endpoint device */
746 free_link_state(link
);
748 /* Recheck latencies and configure upstream links */
749 pcie_update_aspm_capable(root
);
751 mutex_unlock(&aspm_lock
);
752 up_read(&pci_bus_sem
);
755 /* @pdev: the root port or switch downstream port */
756 void pcie_aspm_pm_state_change(struct pci_dev
*pdev
)
758 struct pcie_link_state
*link
= pdev
->link_state
;
760 if (aspm_disabled
|| !pdev
->is_pcie
|| !link
)
762 if ((pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
763 (pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
766 * Devices changed PM state, we should recheck if latency
767 * meets all functions' requirement
769 down_read(&pci_bus_sem
);
770 mutex_lock(&aspm_lock
);
771 pcie_update_aspm_capable(link
->root
);
772 __pcie_aspm_configure_link_state(link
, link
->aspm_enabled
);
773 mutex_unlock(&aspm_lock
);
774 up_read(&pci_bus_sem
);
778 * pci_disable_link_state - disable pci device's link state, so the link will
779 * never enter specific states
781 void pci_disable_link_state(struct pci_dev
*pdev
, int state
)
783 struct pci_dev
*parent
= pdev
->bus
->self
;
784 struct pcie_link_state
*link
;
786 if (aspm_disabled
|| !pdev
->is_pcie
)
788 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
||
789 pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
)
791 if (!parent
|| !parent
->link_state
)
794 down_read(&pci_bus_sem
);
795 mutex_lock(&aspm_lock
);
796 link
= parent
->link_state
;
797 link
->aspm_disable
|= state
;
798 __pcie_aspm_configure_link_state(link
, link
->aspm_enabled
);
799 if (state
& PCIE_LINK_STATE_CLKPM
) {
800 link
->clkpm_capable
= 0;
801 pcie_set_clkpm(link
, 0);
803 mutex_unlock(&aspm_lock
);
804 up_read(&pci_bus_sem
);
806 EXPORT_SYMBOL(pci_disable_link_state
);
808 static int pcie_aspm_set_policy(const char *val
, struct kernel_param
*kp
)
811 struct pcie_link_state
*link_state
;
813 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
814 if (!strncmp(val
, policy_str
[i
], strlen(policy_str
[i
])))
816 if (i
>= ARRAY_SIZE(policy_str
))
818 if (i
== aspm_policy
)
821 down_read(&pci_bus_sem
);
822 mutex_lock(&aspm_lock
);
824 list_for_each_entry(link_state
, &link_list
, sibling
) {
825 __pcie_aspm_configure_link_state(link_state
,
826 policy_to_aspm_state(link_state
));
827 pcie_set_clkpm(link_state
, policy_to_clkpm_state(link_state
));
829 mutex_unlock(&aspm_lock
);
830 up_read(&pci_bus_sem
);
834 static int pcie_aspm_get_policy(char *buffer
, struct kernel_param
*kp
)
837 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
838 if (i
== aspm_policy
)
839 cnt
+= sprintf(buffer
+ cnt
, "[%s] ", policy_str
[i
]);
841 cnt
+= sprintf(buffer
+ cnt
, "%s ", policy_str
[i
]);
845 module_param_call(policy
, pcie_aspm_set_policy
, pcie_aspm_get_policy
,
848 #ifdef CONFIG_PCIEASPM_DEBUG
849 static ssize_t
link_state_show(struct device
*dev
,
850 struct device_attribute
*attr
,
853 struct pci_dev
*pci_device
= to_pci_dev(dev
);
854 struct pcie_link_state
*link_state
= pci_device
->link_state
;
856 return sprintf(buf
, "%d\n", link_state
->aspm_enabled
);
859 static ssize_t
link_state_store(struct device
*dev
,
860 struct device_attribute
*attr
,
864 struct pci_dev
*pdev
= to_pci_dev(dev
);
870 if (state
>= 0 && state
<= 3) {
871 /* setup link aspm state */
872 pcie_aspm_configure_link_state(pdev
->link_state
, state
);
879 static ssize_t
clk_ctl_show(struct device
*dev
,
880 struct device_attribute
*attr
,
883 struct pci_dev
*pci_device
= to_pci_dev(dev
);
884 struct pcie_link_state
*link_state
= pci_device
->link_state
;
886 return sprintf(buf
, "%d\n", link_state
->clkpm_enabled
);
889 static ssize_t
clk_ctl_store(struct device
*dev
,
890 struct device_attribute
*attr
,
894 struct pci_dev
*pdev
= to_pci_dev(dev
);
901 down_read(&pci_bus_sem
);
902 mutex_lock(&aspm_lock
);
903 pcie_set_clkpm_nocheck(pdev
->link_state
, !!state
);
904 mutex_unlock(&aspm_lock
);
905 up_read(&pci_bus_sem
);
910 static DEVICE_ATTR(link_state
, 0644, link_state_show
, link_state_store
);
911 static DEVICE_ATTR(clk_ctl
, 0644, clk_ctl_show
, clk_ctl_store
);
913 static char power_group
[] = "power";
914 void pcie_aspm_create_sysfs_dev_files(struct pci_dev
*pdev
)
916 struct pcie_link_state
*link_state
= pdev
->link_state
;
918 if (!pdev
->is_pcie
|| (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
919 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
922 if (link_state
->aspm_support
)
923 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
924 &dev_attr_link_state
.attr
, power_group
);
925 if (link_state
->clkpm_capable
)
926 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
927 &dev_attr_clk_ctl
.attr
, power_group
);
930 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev
*pdev
)
932 struct pcie_link_state
*link_state
= pdev
->link_state
;
934 if (!pdev
->is_pcie
|| (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
935 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
938 if (link_state
->aspm_support
)
939 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
940 &dev_attr_link_state
.attr
, power_group
);
941 if (link_state
->clkpm_capable
)
942 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
943 &dev_attr_clk_ctl
.attr
, power_group
);
947 static int __init
pcie_aspm_disable(char *str
)
949 if (!strcmp(str
, "off")) {
951 printk(KERN_INFO
"PCIe ASPM is disabled\n");
952 } else if (!strcmp(str
, "force")) {
954 printk(KERN_INFO
"PCIe ASPM is forcedly enabled\n");
959 __setup("pcie_aspm=", pcie_aspm_disable
);
961 void pcie_no_aspm(void)
968 * pcie_aspm_enabled - is PCIe ASPM enabled?
970 * Returns true if ASPM has not been disabled by the command-line option
973 int pcie_aspm_enabled(void)
975 return !aspm_disabled
;
977 EXPORT_SYMBOL(pcie_aspm_enabled
);