2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIE link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
30 u32 l0s
; /* L0s latency (nsec) */
31 u32 l1
; /* L1 latency (nsec) */
34 struct pcie_link_state
{
35 struct pci_dev
*pdev
; /* Upstream component of the Link */
36 struct pcie_link_state
*parent
; /* pointer to the parent Link state */
37 struct list_head sibling
; /* node in link_list */
38 struct list_head children
; /* list of child link states */
39 struct list_head link
; /* node in parent's children list */
42 u32 aspm_support
:2; /* Supported ASPM state */
43 u32 aspm_enabled
:2; /* Enabled ASPM state */
44 u32 aspm_default
:2; /* Default ASPM state by BIOS */
47 u32 clkpm_capable
:1; /* Clock PM capable? */
48 u32 clkpm_enabled
:1; /* Current Clock PM state */
49 u32 clkpm_default
:1; /* Default Clock PM state by BIOS */
51 u32 has_switch
:1; /* Downstream has switches? */
54 struct aspm_latency latency
; /* Exit latency */
56 * Endpoint acceptable latencies. A pcie downstream port only
57 * has one slot under it, so at most there are 8 functions.
59 struct aspm_latency acceptable
[8];
62 static int aspm_disabled
, aspm_force
;
63 static DEFINE_MUTEX(aspm_lock
);
64 static LIST_HEAD(link_list
);
66 #define POLICY_DEFAULT 0 /* BIOS default setting */
67 #define POLICY_PERFORMANCE 1 /* high performance */
68 #define POLICY_POWERSAVE 2 /* high power saving */
69 static int aspm_policy
;
70 static const char *policy_str
[] = {
71 [POLICY_DEFAULT
] = "default",
72 [POLICY_PERFORMANCE
] = "performance",
73 [POLICY_POWERSAVE
] = "powersave"
76 #define LINK_RETRAIN_TIMEOUT HZ
78 static int policy_to_aspm_state(struct pcie_link_state
*link
)
80 switch (aspm_policy
) {
81 case POLICY_PERFORMANCE
:
82 /* Disable ASPM and Clock PM */
84 case POLICY_POWERSAVE
:
85 /* Enable ASPM L0s/L1 */
86 return PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
88 return link
->aspm_default
;
93 static int policy_to_clkpm_state(struct pcie_link_state
*link
)
95 switch (aspm_policy
) {
96 case POLICY_PERFORMANCE
:
97 /* Disable ASPM and Clock PM */
99 case POLICY_POWERSAVE
:
100 /* Disable Clock PM */
103 return link
->clkpm_default
;
108 static void pcie_set_clkpm_nocheck(struct pcie_link_state
*link
, int enable
)
112 struct pci_dev
*child
;
113 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
115 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
116 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
119 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
121 reg16
|= PCI_EXP_LNKCTL_CLKREQ_EN
;
123 reg16
&= ~PCI_EXP_LNKCTL_CLKREQ_EN
;
124 pci_write_config_word(child
, pos
+ PCI_EXP_LNKCTL
, reg16
);
126 link
->clkpm_enabled
= !!enable
;
129 static void pcie_set_clkpm(struct pcie_link_state
*link
, int enable
)
131 /* Don't enable Clock PM if the link is not Clock PM capable */
132 if (!link
->clkpm_capable
&& enable
)
134 /* Need nothing if the specified equals to current state */
135 if (link
->clkpm_enabled
== enable
)
137 pcie_set_clkpm_nocheck(link
, enable
);
140 static void pcie_clkpm_cap_init(struct pcie_link_state
*link
, int blacklist
)
142 int pos
, capable
= 1, enabled
= 1;
145 struct pci_dev
*child
;
146 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
148 /* All functions should have the same cap and state, take the worst */
149 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
150 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
153 pci_read_config_dword(child
, pos
+ PCI_EXP_LNKCAP
, ®32
);
154 if (!(reg32
& PCI_EXP_LNKCAP_CLKPM
)) {
159 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
160 if (!(reg16
& PCI_EXP_LNKCTL_CLKREQ_EN
))
163 link
->clkpm_enabled
= enabled
;
164 link
->clkpm_default
= enabled
;
165 link
->clkpm_capable
= (blacklist
) ? 0 : capable
;
168 static bool pcie_aspm_downstream_has_switch(struct pcie_link_state
*link
)
170 struct pci_dev
*child
;
171 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
173 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
174 if (child
->pcie_type
== PCI_EXP_TYPE_UPSTREAM
)
181 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
182 * could use common clock. If they are, configure them to use the
183 * common clock. That will reduce the ASPM state exit latency.
185 static void pcie_aspm_configure_common_clock(struct pcie_link_state
*link
)
187 int ppos
, cpos
, same_clock
= 1;
188 u16 reg16
, parent_reg
, child_reg
[8];
189 unsigned long start_jiffies
;
190 struct pci_dev
*child
, *parent
= link
->pdev
;
191 struct pci_bus
*linkbus
= parent
->subordinate
;
193 * All functions of a slot should have the same Slot Clock
194 * Configuration, so just check one function
196 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
197 BUG_ON(!child
->is_pcie
);
199 /* Check downstream component if bit Slot Clock Configuration is 1 */
200 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
201 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKSTA
, ®16
);
202 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
205 /* Check upstream component if bit Slot Clock Configuration is 1 */
206 ppos
= pci_find_capability(parent
, PCI_CAP_ID_EXP
);
207 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
208 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
211 /* Configure downstream component, all functions */
212 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
213 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
214 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, ®16
);
215 child_reg
[PCI_FUNC(child
->devfn
)] = reg16
;
217 reg16
|= PCI_EXP_LNKCTL_CCC
;
219 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
220 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, reg16
);
223 /* Configure upstream component */
224 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, ®16
);
227 reg16
|= PCI_EXP_LNKCTL_CCC
;
229 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
230 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
233 reg16
|= PCI_EXP_LNKCTL_RL
;
234 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
236 /* Wait for link training end. Break out after waiting for timeout */
237 start_jiffies
= jiffies
;
239 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
240 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
242 if (time_after(jiffies
, start_jiffies
+ LINK_RETRAIN_TIMEOUT
))
246 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
249 /* Training failed. Restore common clock configurations */
250 dev_printk(KERN_ERR
, &parent
->dev
,
251 "ASPM: Could not configure common clock\n");
252 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
253 cpos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
254 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
,
255 child_reg
[PCI_FUNC(child
->devfn
)]);
257 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, parent_reg
);
261 * calc_L0S_latency: Convert L0s latency encoding to ns
263 static unsigned int calc_L0S_latency(unsigned int latency_encoding
, int ac
)
265 unsigned int ns
= 64;
267 if (latency_encoding
== 0x7) {
271 ns
= 5*1000; /* > 4us */
273 ns
*= (1 << latency_encoding
);
278 * calc_L1_latency: Convert L1 latency encoding to ns
280 static unsigned int calc_L1_latency(unsigned int latency_encoding
, int ac
)
282 unsigned int ns
= 1000;
284 if (latency_encoding
== 0x7) {
288 ns
= 65*1000; /* > 64us */
290 ns
*= (1 << latency_encoding
);
294 static void pcie_aspm_get_cap_device(struct pci_dev
*pdev
, u32
*state
,
295 unsigned int *l0s
, unsigned int *l1
, unsigned int *enabled
)
300 unsigned int latency
;
302 *l0s
= *l1
= *enabled
= 0;
303 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
304 pci_read_config_dword(pdev
, pos
+ PCI_EXP_LNKCAP
, ®32
);
305 *state
= (reg32
& PCI_EXP_LNKCAP_ASPMS
) >> 10;
306 if (*state
!= PCIE_LINK_STATE_L0S
&&
307 *state
!= (PCIE_LINK_STATE_L1
|PCIE_LINK_STATE_L0S
))
312 latency
= (reg32
& PCI_EXP_LNKCAP_L0SEL
) >> 12;
313 *l0s
= calc_L0S_latency(latency
, 0);
314 if (*state
& PCIE_LINK_STATE_L1
) {
315 latency
= (reg32
& PCI_EXP_LNKCAP_L1EL
) >> 15;
316 *l1
= calc_L1_latency(latency
, 0);
318 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
319 *enabled
= reg16
& (PCIE_LINK_STATE_L0S
|PCIE_LINK_STATE_L1
);
322 static void pcie_aspm_cap_init(struct pcie_link_state
*link
, int blacklist
)
324 u32 support
, l0s
, l1
, enabled
;
325 struct pci_dev
*child
, *parent
= link
->pdev
;
326 struct pci_bus
*linkbus
= parent
->subordinate
;
329 /* Set support state to 0, so we will disable ASPM later */
330 link
->aspm_support
= 0;
331 link
->aspm_default
= 0;
332 link
->aspm_enabled
= PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
;
336 /* Configure common clock before checking latencies */
337 pcie_aspm_configure_common_clock(link
);
339 /* upstream component states */
340 pcie_aspm_get_cap_device(parent
, &support
, &l0s
, &l1
, &enabled
);
341 link
->aspm_support
= support
;
342 link
->latency
.l0s
= l0s
;
343 link
->latency
.l1
= l1
;
344 link
->aspm_enabled
= enabled
;
346 /* downstream component states, all functions have the same setting */
347 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
348 pcie_aspm_get_cap_device(child
, &support
, &l0s
, &l1
, &enabled
);
349 link
->aspm_support
&= support
;
350 link
->latency
.l0s
= max_t(u32
, link
->latency
.l0s
, l0s
);
351 link
->latency
.l1
= max_t(u32
, link
->latency
.l1
, l1
);
353 if (!link
->aspm_support
)
356 link
->aspm_enabled
&= link
->aspm_support
;
357 link
->aspm_default
= link
->aspm_enabled
;
360 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
363 unsigned int latency
;
364 struct aspm_latency
*acceptable
=
365 &link
->acceptable
[PCI_FUNC(child
->devfn
)];
367 if (child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
368 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
)
371 pos
= pci_find_capability(child
, PCI_CAP_ID_EXP
);
372 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
373 latency
= (reg32
& PCI_EXP_DEVCAP_L0S
) >> 6;
374 latency
= calc_L0S_latency(latency
, 1);
375 acceptable
->l0s
= latency
;
376 if (link
->aspm_support
& PCIE_LINK_STATE_L1
) {
377 latency
= (reg32
& PCI_EXP_DEVCAP_L1
) >> 9;
378 latency
= calc_L1_latency(latency
, 1);
379 acceptable
->l1
= latency
;
385 * __pcie_aspm_check_state_one - check latency for endpoint device.
386 * @endpoint: pointer to the struct pci_dev of endpoint device
388 * TBD: The latency from the endpoint to root complex vary per switch's
389 * upstream link state above the device. Here we just do a simple check
390 * which assumes all links above the device can be in L1 state, that
391 * is we just consider the worst case. If switch's upstream link can't
392 * be put into L0S/L1, then our check is too strictly.
394 static u32
__pcie_aspm_check_state_one(struct pci_dev
*endpoint
, u32 state
)
396 u32 l1_switch_latency
= 0;
397 struct aspm_latency
*acceptable
;
398 struct pcie_link_state
*link
;
400 link
= endpoint
->bus
->self
->link_state
;
401 state
&= link
->aspm_support
;
402 acceptable
= &link
->acceptable
[PCI_FUNC(endpoint
->devfn
)];
404 while (link
&& state
) {
405 if ((state
& PCIE_LINK_STATE_L0S
) &&
406 (link
->latency
.l0s
> acceptable
->l0s
))
407 state
&= ~PCIE_LINK_STATE_L0S
;
408 if ((state
& PCIE_LINK_STATE_L1
) &&
409 (link
->latency
.l1
+ l1_switch_latency
> acceptable
->l1
))
410 state
&= ~PCIE_LINK_STATE_L1
;
413 * Every switch on the path to root complex need 1
414 * more microsecond for L1. Spec doesn't mention L0s.
416 l1_switch_latency
+= 1000;
421 static u32
pcie_aspm_check_state(struct pcie_link_state
*link
, u32 state
)
423 pci_power_t power_state
;
424 struct pci_dev
*child
;
425 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
427 /* If no child, ignore the link */
428 if (list_empty(&linkbus
->devices
))
431 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
433 * If downstream component of a link is pci bridge, we
434 * disable ASPM for now for the link
436 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
)
439 if ((child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
440 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
))
442 /* Device not in D0 doesn't need check latency */
443 power_state
= child
->current_state
;
444 if (power_state
== PCI_D1
|| power_state
== PCI_D2
||
445 power_state
== PCI_D3hot
|| power_state
== PCI_D3cold
)
447 state
= __pcie_aspm_check_state_one(child
, state
);
452 static void __pcie_aspm_config_one_dev(struct pci_dev
*pdev
, unsigned int state
)
455 int pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
457 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
460 pci_write_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, reg16
);
463 static void __pcie_aspm_config_link(struct pcie_link_state
*link
, u32 state
)
465 struct pci_dev
*child
, *parent
= link
->pdev
;
466 struct pci_bus
*linkbus
= parent
->subordinate
;
468 /* If no child, disable the link */
469 if (list_empty(&linkbus
->devices
))
472 * If the downstream component has pci bridge function, don't
475 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
476 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
)
480 * Spec 2.0 suggests all functions should be configured the
481 * same setting for ASPM. Enabling ASPM L1 should be done in
482 * upstream component first and then downstream, and vice
483 * versa for disabling ASPM L1. Spec doesn't mention L0S.
485 if (state
& PCIE_LINK_STATE_L1
)
486 __pcie_aspm_config_one_dev(parent
, state
);
488 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
489 __pcie_aspm_config_one_dev(child
, state
);
491 if (!(state
& PCIE_LINK_STATE_L1
))
492 __pcie_aspm_config_one_dev(parent
, state
);
494 link
->aspm_enabled
= state
;
497 static struct pcie_link_state
*get_root_port_link(struct pcie_link_state
*link
)
499 struct pcie_link_state
*root_port_link
= link
;
500 while (root_port_link
->parent
)
501 root_port_link
= root_port_link
->parent
;
502 return root_port_link
;
505 /* Check the whole hierarchy, and configure each link in the hierarchy */
506 static void __pcie_aspm_configure_link_state(struct pcie_link_state
*link
,
509 struct pcie_link_state
*leaf
, *root
= get_root_port_link(link
);
511 state
&= (PCIE_LINK_STATE_L0S
| PCIE_LINK_STATE_L1
);
513 /* Check all links who have specific root port link */
514 list_for_each_entry(leaf
, &link_list
, sibling
) {
515 if (!list_empty(&leaf
->children
) ||
516 get_root_port_link(leaf
) != root
)
518 state
= pcie_aspm_check_state(leaf
, state
);
520 /* Check root port link too in case it hasn't children */
521 state
= pcie_aspm_check_state(root
, state
);
522 if (link
->aspm_enabled
== state
)
525 * We must change the hierarchy. See comments in
526 * __pcie_aspm_config_link for the order
528 if (state
& PCIE_LINK_STATE_L1
) {
529 list_for_each_entry(leaf
, &link_list
, sibling
) {
530 if (get_root_port_link(leaf
) == root
)
531 __pcie_aspm_config_link(leaf
, state
);
534 list_for_each_entry_reverse(leaf
, &link_list
, sibling
) {
535 if (get_root_port_link(leaf
) == root
)
536 __pcie_aspm_config_link(leaf
, state
);
542 * pcie_aspm_configure_link_state: enable/disable PCI express link state
543 * @pdev: the root port or switch downstream port
545 static void pcie_aspm_configure_link_state(struct pcie_link_state
*link
,
548 down_read(&pci_bus_sem
);
549 mutex_lock(&aspm_lock
);
550 __pcie_aspm_configure_link_state(link
, state
);
551 mutex_unlock(&aspm_lock
);
552 up_read(&pci_bus_sem
);
555 static void free_link_state(struct pcie_link_state
*link
)
557 link
->pdev
->link_state
= NULL
;
561 static int pcie_aspm_sanity_check(struct pci_dev
*pdev
)
563 struct pci_dev
*child_dev
;
568 * Some functions in a slot might not all be PCIE functions, very
569 * strange. Disable ASPM for the whole slot
571 list_for_each_entry(child_dev
, &pdev
->subordinate
->devices
, bus_list
) {
572 child_pos
= pci_find_capability(child_dev
, PCI_CAP_ID_EXP
);
577 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
578 * RBER bit to determine if a function is 1.1 version device
580 pci_read_config_dword(child_dev
, child_pos
+ PCI_EXP_DEVCAP
,
582 if (!(reg32
& PCI_EXP_DEVCAP_RBER
) && !aspm_force
) {
583 dev_printk(KERN_INFO
, &child_dev
->dev
, "disabling ASPM"
584 " on pre-1.1 PCIe device. You can enable it"
585 " with 'pcie_aspm=force'\n");
592 static struct pcie_link_state
*pcie_aspm_setup_link_state(struct pci_dev
*pdev
)
594 struct pcie_link_state
*link
;
595 int blacklist
= !!pcie_aspm_sanity_check(pdev
);
597 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
600 INIT_LIST_HEAD(&link
->sibling
);
601 INIT_LIST_HEAD(&link
->children
);
602 INIT_LIST_HEAD(&link
->link
);
604 link
->has_switch
= pcie_aspm_downstream_has_switch(link
);
605 if (pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
) {
606 struct pcie_link_state
*parent
;
607 parent
= pdev
->bus
->parent
->self
->link_state
;
612 link
->parent
= parent
;
613 list_add(&link
->link
, &parent
->children
);
615 list_add(&link
->sibling
, &link_list
);
617 pdev
->link_state
= link
;
619 /* Check ASPM capability */
620 pcie_aspm_cap_init(link
, blacklist
);
622 /* Check Clock PM capability */
623 pcie_clkpm_cap_init(link
, blacklist
);
629 * pcie_aspm_init_link_state: Initiate PCI express link state.
630 * It is called after the pcie and its children devices are scaned.
631 * @pdev: the root port or switch downstream port
633 void pcie_aspm_init_link_state(struct pci_dev
*pdev
)
636 struct pcie_link_state
*link
;
638 if (aspm_disabled
|| !pdev
->is_pcie
|| pdev
->link_state
)
640 if (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
641 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
644 /* VIA has a strange chipset, root port is under a bridge */
645 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
&&
649 down_read(&pci_bus_sem
);
650 if (list_empty(&pdev
->subordinate
->devices
))
653 mutex_lock(&aspm_lock
);
654 link
= pcie_aspm_setup_link_state(pdev
);
658 * Setup initial ASPM state
660 * If link has switch, delay the link config. The leaf link
661 * initialization will config the whole hierarchy. But we must
662 * make sure BIOS doesn't set unsupported link state.
664 if (link
->has_switch
) {
665 state
= pcie_aspm_check_state(link
, link
->aspm_default
);
666 __pcie_aspm_config_link(link
, state
);
668 state
= policy_to_aspm_state(link
);
669 __pcie_aspm_configure_link_state(link
, state
);
672 /* Setup initial Clock PM state */
673 state
= (link
->clkpm_capable
) ? policy_to_clkpm_state(link
) : 0;
674 pcie_set_clkpm(link
, state
);
676 mutex_unlock(&aspm_lock
);
678 up_read(&pci_bus_sem
);
681 /* @pdev: the endpoint device */
682 void pcie_aspm_exit_link_state(struct pci_dev
*pdev
)
684 struct pci_dev
*parent
= pdev
->bus
->self
;
685 struct pcie_link_state
*link_state
= parent
->link_state
;
687 if (aspm_disabled
|| !pdev
->is_pcie
|| !parent
|| !link_state
)
689 if (parent
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
690 parent
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
692 down_read(&pci_bus_sem
);
693 mutex_lock(&aspm_lock
);
696 * All PCIe functions are in one slot, remove one function will remove
697 * the whole slot, so just wait until we are the last function left.
699 if (!list_is_last(&pdev
->bus_list
, &parent
->subordinate
->devices
))
702 /* All functions are removed, so just disable ASPM for the link */
703 __pcie_aspm_config_one_dev(parent
, 0);
704 list_del(&link_state
->sibling
);
705 list_del(&link_state
->link
);
706 /* Clock PM is for endpoint device */
708 free_link_state(link_state
);
710 mutex_unlock(&aspm_lock
);
711 up_read(&pci_bus_sem
);
714 /* @pdev: the root port or switch downstream port */
715 void pcie_aspm_pm_state_change(struct pci_dev
*pdev
)
717 struct pcie_link_state
*link_state
= pdev
->link_state
;
719 if (aspm_disabled
|| !pdev
->is_pcie
|| !pdev
->link_state
)
721 if (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
722 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
725 * devices changed PM state, we should recheck if latency meets all
726 * functions' requirement
728 pcie_aspm_configure_link_state(link_state
, link_state
->aspm_enabled
);
732 * pci_disable_link_state - disable pci device's link state, so the link will
733 * never enter specific states
735 void pci_disable_link_state(struct pci_dev
*pdev
, int state
)
737 struct pci_dev
*parent
= pdev
->bus
->self
;
738 struct pcie_link_state
*link_state
;
740 if (aspm_disabled
|| !pdev
->is_pcie
)
742 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
||
743 pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
)
745 if (!parent
|| !parent
->link_state
)
748 down_read(&pci_bus_sem
);
749 mutex_lock(&aspm_lock
);
750 link_state
= parent
->link_state
;
751 link_state
->aspm_support
&= ~state
;
752 __pcie_aspm_configure_link_state(link_state
, link_state
->aspm_enabled
);
753 if (state
& PCIE_LINK_STATE_CLKPM
) {
754 link_state
->clkpm_capable
= 0;
755 pcie_set_clkpm(link_state
, 0);
757 mutex_unlock(&aspm_lock
);
758 up_read(&pci_bus_sem
);
760 EXPORT_SYMBOL(pci_disable_link_state
);
762 static int pcie_aspm_set_policy(const char *val
, struct kernel_param
*kp
)
765 struct pcie_link_state
*link_state
;
767 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
768 if (!strncmp(val
, policy_str
[i
], strlen(policy_str
[i
])))
770 if (i
>= ARRAY_SIZE(policy_str
))
772 if (i
== aspm_policy
)
775 down_read(&pci_bus_sem
);
776 mutex_lock(&aspm_lock
);
778 list_for_each_entry(link_state
, &link_list
, sibling
) {
779 __pcie_aspm_configure_link_state(link_state
,
780 policy_to_aspm_state(link_state
));
781 pcie_set_clkpm(link_state
, policy_to_clkpm_state(link_state
));
783 mutex_unlock(&aspm_lock
);
784 up_read(&pci_bus_sem
);
788 static int pcie_aspm_get_policy(char *buffer
, struct kernel_param
*kp
)
791 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
792 if (i
== aspm_policy
)
793 cnt
+= sprintf(buffer
+ cnt
, "[%s] ", policy_str
[i
]);
795 cnt
+= sprintf(buffer
+ cnt
, "%s ", policy_str
[i
]);
799 module_param_call(policy
, pcie_aspm_set_policy
, pcie_aspm_get_policy
,
802 #ifdef CONFIG_PCIEASPM_DEBUG
803 static ssize_t
link_state_show(struct device
*dev
,
804 struct device_attribute
*attr
,
807 struct pci_dev
*pci_device
= to_pci_dev(dev
);
808 struct pcie_link_state
*link_state
= pci_device
->link_state
;
810 return sprintf(buf
, "%d\n", link_state
->aspm_enabled
);
813 static ssize_t
link_state_store(struct device
*dev
,
814 struct device_attribute
*attr
,
818 struct pci_dev
*pdev
= to_pci_dev(dev
);
824 if (state
>= 0 && state
<= 3) {
825 /* setup link aspm state */
826 pcie_aspm_configure_link_state(pdev
->link_state
, state
);
833 static ssize_t
clk_ctl_show(struct device
*dev
,
834 struct device_attribute
*attr
,
837 struct pci_dev
*pci_device
= to_pci_dev(dev
);
838 struct pcie_link_state
*link_state
= pci_device
->link_state
;
840 return sprintf(buf
, "%d\n", link_state
->clkpm_enabled
);
843 static ssize_t
clk_ctl_store(struct device
*dev
,
844 struct device_attribute
*attr
,
848 struct pci_dev
*pdev
= to_pci_dev(dev
);
855 down_read(&pci_bus_sem
);
856 mutex_lock(&aspm_lock
);
857 pcie_set_clkpm_nocheck(pdev
->link_state
, !!state
);
858 mutex_unlock(&aspm_lock
);
859 up_read(&pci_bus_sem
);
864 static DEVICE_ATTR(link_state
, 0644, link_state_show
, link_state_store
);
865 static DEVICE_ATTR(clk_ctl
, 0644, clk_ctl_show
, clk_ctl_store
);
867 static char power_group
[] = "power";
868 void pcie_aspm_create_sysfs_dev_files(struct pci_dev
*pdev
)
870 struct pcie_link_state
*link_state
= pdev
->link_state
;
872 if (!pdev
->is_pcie
|| (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
873 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
876 if (link_state
->aspm_support
)
877 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
878 &dev_attr_link_state
.attr
, power_group
);
879 if (link_state
->clkpm_capable
)
880 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
881 &dev_attr_clk_ctl
.attr
, power_group
);
884 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev
*pdev
)
886 struct pcie_link_state
*link_state
= pdev
->link_state
;
888 if (!pdev
->is_pcie
|| (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
889 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
892 if (link_state
->aspm_support
)
893 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
894 &dev_attr_link_state
.attr
, power_group
);
895 if (link_state
->clkpm_capable
)
896 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
897 &dev_attr_clk_ctl
.attr
, power_group
);
901 static int __init
pcie_aspm_disable(char *str
)
903 if (!strcmp(str
, "off")) {
905 printk(KERN_INFO
"PCIe ASPM is disabled\n");
906 } else if (!strcmp(str
, "force")) {
908 printk(KERN_INFO
"PCIe ASPM is forcedly enabled\n");
913 __setup("pcie_aspm=", pcie_aspm_disable
);
915 void pcie_no_aspm(void)
922 * pcie_aspm_enabled - is PCIe ASPM enabled?
924 * Returns true if ASPM has not been disabled by the command-line option
927 int pcie_aspm_enabled(void)
929 return !aspm_disabled
;
931 EXPORT_SYMBOL(pcie_aspm_enabled
);