2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIE link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
30 u32 l0s; /* L0s latency (nsec) */
31 u32 l1; /* L1 latency (nsec) */
34 struct pcie_link_state {
35 struct pci_dev *pdev; /* Upstream component of the Link */
36 struct pcie_link_state *parent; /* pointer to the parent Link state */
37 struct list_head sibling; /* node in link_list */
38 struct list_head children; /* list of child link states */
39 struct list_head link; /* node in parent's children list */
42 u32 aspm_support:2; /* Supported ASPM state */
43 u32 aspm_enabled:2; /* Enabled ASPM state */
44 u32 aspm_default:2; /* Default ASPM state by BIOS */
47 u32 clkpm_capable:1; /* Clock PM capable? */
48 u32 clkpm_enabled:1; /* Current Clock PM state */
49 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
51 u32 has_switch:1; /* Downstream has switches? */
54 struct aspm_latency latency; /* Exit latency */
56 * Endpoint acceptable latencies. A pcie downstream port only
57 * has one slot under it, so at most there are 8 functions.
59 struct aspm_latency acceptable[8];
62 static int aspm_disabled, aspm_force;
63 static DEFINE_MUTEX(aspm_lock);
64 static LIST_HEAD(link_list);
66 #define POLICY_DEFAULT 0 /* BIOS default setting */
67 #define POLICY_PERFORMANCE 1 /* high performance */
68 #define POLICY_POWERSAVE 2 /* high power saving */
69 static int aspm_policy;
70 static const char *policy_str[] = {
71 [POLICY_DEFAULT] = "default",
72 [POLICY_PERFORMANCE] = "performance",
73 [POLICY_POWERSAVE] = "powersave"
76 #define LINK_RETRAIN_TIMEOUT HZ
78 static int policy_to_aspm_state(struct pcie_link_state *link)
80 switch (aspm_policy) {
81 case POLICY_PERFORMANCE:
82 /* Disable ASPM and Clock PM */
84 case POLICY_POWERSAVE:
85 /* Enable ASPM L0s/L1 */
86 return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
88 return link->aspm_default;
93 static int policy_to_clkpm_state(struct pcie_link_state *link)
95 switch (aspm_policy) {
96 case POLICY_PERFORMANCE:
97 /* Disable ASPM and Clock PM */
99 case POLICY_POWERSAVE:
100 /* Disable Clock PM */
103 return link->clkpm_default;
108 static void pcie_set_clock_pm(struct pcie_link_state *link, int enable)
112 struct pci_dev *child;
113 struct pci_bus *linkbus = link->pdev->subordinate;
115 list_for_each_entry(child, &linkbus->devices, bus_list) {
116 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
119 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
121 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
123 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
124 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
126 link->clkpm_enabled = !!enable;
129 static void pcie_check_clock_pm(struct pcie_link_state *link, int blacklist)
131 int pos, capable = 1, enabled = 1;
134 struct pci_dev *child;
135 struct pci_bus *linkbus = link->pdev->subordinate;
137 /* All functions should have the same cap and state, take the worst */
138 list_for_each_entry(child, &linkbus->devices, bus_list) {
139 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
142 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32);
143 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
148 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
149 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
152 link->clkpm_enabled = enabled;
153 link->clkpm_default = enabled;
155 link->clkpm_capable = capable;
156 pcie_set_clock_pm(link, policy_to_clkpm_state(link));
158 link->clkpm_capable = 0;
159 pcie_set_clock_pm(link, 0);
163 static bool pcie_aspm_downstream_has_switch(struct pcie_link_state *link)
165 struct pci_dev *child;
166 struct pci_bus *linkbus = link->pdev->subordinate;
168 list_for_each_entry(child, &linkbus->devices, bus_list) {
169 if (child->pcie_type == PCI_EXP_TYPE_UPSTREAM)
176 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
177 * could use common clock. If they are, configure them to use the
178 * common clock. That will reduce the ASPM state exit latency.
180 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
182 int ppos, cpos, same_clock = 1;
183 u16 reg16, parent_reg, child_reg[8];
184 unsigned long start_jiffies;
185 struct pci_dev *child, *parent = link->pdev;
186 struct pci_bus *linkbus = parent->subordinate;
188 * All functions of a slot should have the same Slot Clock
189 * Configuration, so just check one function
191 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
192 BUG_ON(!child->is_pcie);
194 /* Check downstream component if bit Slot Clock Configuration is 1 */
195 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
196 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16);
197 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
200 /* Check upstream component if bit Slot Clock Configuration is 1 */
201 ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
202 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
203 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
206 /* Configure downstream component, all functions */
207 list_for_each_entry(child, &linkbus->devices, bus_list) {
208 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
209 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16);
210 child_reg[PCI_FUNC(child->devfn)] = reg16;
212 reg16 |= PCI_EXP_LNKCTL_CCC;
214 reg16 &= ~PCI_EXP_LNKCTL_CCC;
215 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
218 /* Configure upstream component */
219 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16);
222 reg16 |= PCI_EXP_LNKCTL_CCC;
224 reg16 &= ~PCI_EXP_LNKCTL_CCC;
225 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
228 reg16 |= PCI_EXP_LNKCTL_RL;
229 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
231 /* Wait for link training end. Break out after waiting for timeout */
232 start_jiffies = jiffies;
234 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
235 if (!(reg16 & PCI_EXP_LNKSTA_LT))
237 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
241 if (!(reg16 & PCI_EXP_LNKSTA_LT))
244 /* Training failed. Restore common clock configurations */
245 dev_printk(KERN_ERR, &parent->dev,
246 "ASPM: Could not configure common clock\n");
247 list_for_each_entry(child, &linkbus->devices, bus_list) {
248 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
249 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
250 child_reg[PCI_FUNC(child->devfn)]);
252 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
256 * calc_L0S_latency: Convert L0s latency encoding to ns
258 static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
260 unsigned int ns = 64;
262 if (latency_encoding == 0x7) {
266 ns = 5*1000; /* > 4us */
268 ns *= (1 << latency_encoding);
273 * calc_L1_latency: Convert L1 latency encoding to ns
275 static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
277 unsigned int ns = 1000;
279 if (latency_encoding == 0x7) {
283 ns = 65*1000; /* > 64us */
285 ns *= (1 << latency_encoding);
289 static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
290 unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
295 unsigned int latency;
297 *l0s = *l1 = *enabled = 0;
298 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
299 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
300 *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
301 if (*state != PCIE_LINK_STATE_L0S &&
302 *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
307 latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
308 *l0s = calc_L0S_latency(latency, 0);
309 if (*state & PCIE_LINK_STATE_L1) {
310 latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
311 *l1 = calc_L1_latency(latency, 0);
313 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
314 *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
317 static void pcie_aspm_cap_init(struct pcie_link_state *link)
319 u32 support, l0s, l1, enabled;
320 struct pci_dev *child, *parent = link->pdev;
321 struct pci_bus *linkbus = parent->subordinate;
323 /* upstream component states */
324 pcie_aspm_get_cap_device(parent, &support, &l0s, &l1, &enabled);
325 link->aspm_support = support;
326 link->latency.l0s = l0s;
327 link->latency.l1 = l1;
328 link->aspm_enabled = enabled;
330 /* downstream component states, all functions have the same setting */
331 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
332 pcie_aspm_get_cap_device(child, &support, &l0s, &l1, &enabled);
333 link->aspm_support &= support;
334 link->latency.l0s = max_t(u32, link->latency.l0s, l0s);
335 link->latency.l1 = max_t(u32, link->latency.l1, l1);
337 if (!link->aspm_support)
340 link->aspm_enabled &= link->aspm_support;
341 link->aspm_default = link->aspm_enabled;
344 list_for_each_entry(child, &linkbus->devices, bus_list) {
347 unsigned int latency;
348 struct aspm_latency *acceptable =
349 &link->acceptable[PCI_FUNC(child->devfn)];
351 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
352 child->pcie_type != PCI_EXP_TYPE_LEG_END)
355 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
356 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
357 latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
358 latency = calc_L0S_latency(latency, 1);
359 acceptable->l0s = latency;
360 if (link->aspm_support & PCIE_LINK_STATE_L1) {
361 latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
362 latency = calc_L1_latency(latency, 1);
363 acceptable->l1 = latency;
368 static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
371 struct pci_dev *parent_dev, *tmp_dev;
372 unsigned int l1_latency = 0;
373 struct pcie_link_state *link_state;
374 struct aspm_latency *acceptable;
376 parent_dev = pdev->bus->self;
377 link_state = parent_dev->link_state;
378 state &= link_state->aspm_support;
381 acceptable = &link_state->acceptable[PCI_FUNC(pdev->devfn)];
384 * Check latency for endpoint device.
385 * TBD: The latency from the endpoint to root complex vary per
386 * switch's upstream link state above the device. Here we just do a
387 * simple check which assumes all links above the device can be in L1
388 * state, that is we just consider the worst case. If switch's upstream
389 * link can't be put into L0S/L1, then our check is too strictly.
392 while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
393 parent_dev = tmp_dev->bus->self;
394 link_state = parent_dev->link_state;
395 if ((state & PCIE_LINK_STATE_L0S) &&
396 (link_state->latency.l0s > acceptable->l0s))
397 state &= ~PCIE_LINK_STATE_L0S;
399 if ((state & PCIE_LINK_STATE_L1) &&
400 (link_state->latency.l1 + l1_latency > acceptable->l1))
401 state &= ~PCIE_LINK_STATE_L1;
403 if (!parent_dev->bus->self) /* parent_dev is a root port */
407 * parent_dev is the downstream port of a switch, make
408 * tmp_dev the upstream port of the switch
410 tmp_dev = parent_dev->bus->self;
412 * every switch on the path to root complex need 1 more
413 * microsecond for L1. Spec doesn't mention L0S.
415 if (state & PCIE_LINK_STATE_L1)
422 static u32 pcie_aspm_check_state(struct pcie_link_state *link, u32 state)
424 pci_power_t power_state;
425 struct pci_dev *child;
426 struct pci_bus *linkbus = link->pdev->subordinate;
428 /* If no child, ignore the link */
429 if (list_empty(&linkbus->devices))
432 list_for_each_entry(child, &linkbus->devices, bus_list) {
434 * If downstream component of a link is pci bridge, we
435 * disable ASPM for now for the link
437 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
440 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
441 child->pcie_type != PCI_EXP_TYPE_LEG_END))
443 /* Device not in D0 doesn't need check latency */
444 power_state = child->current_state;
445 if (power_state == PCI_D1 || power_state == PCI_D2 ||
446 power_state == PCI_D3hot || power_state == PCI_D3cold)
448 state = __pcie_aspm_check_state_one(child, state);
453 static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
456 int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
458 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
461 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
464 static void __pcie_aspm_config_link(struct pcie_link_state *link, u32 state)
466 struct pci_dev *child, *parent = link->pdev;
467 struct pci_bus *linkbus = parent->subordinate;
469 /* If no child, disable the link */
470 if (list_empty(&linkbus->devices))
473 * If the downstream component has pci bridge function, don't
476 list_for_each_entry(child, &linkbus->devices, bus_list) {
477 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
481 * Spec 2.0 suggests all functions should be configured the
482 * same setting for ASPM. Enabling ASPM L1 should be done in
483 * upstream component first and then downstream, and vice
484 * versa for disabling ASPM L1. Spec doesn't mention L0S.
486 if (state & PCIE_LINK_STATE_L1)
487 __pcie_aspm_config_one_dev(parent, state);
489 list_for_each_entry(child, &linkbus->devices, bus_list)
490 __pcie_aspm_config_one_dev(child, state);
492 if (!(state & PCIE_LINK_STATE_L1))
493 __pcie_aspm_config_one_dev(parent, state);
495 link->aspm_enabled = state;
498 static struct pcie_link_state *get_root_port_link(struct pcie_link_state *link)
500 struct pcie_link_state *root_port_link = link;
501 while (root_port_link->parent)
502 root_port_link = root_port_link->parent;
503 return root_port_link;
506 /* Check the whole hierarchy, and configure each link in the hierarchy */
507 static void __pcie_aspm_configure_link_state(struct pcie_link_state *link,
510 struct pcie_link_state *leaf, *root = get_root_port_link(link);
512 state &= (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
514 /* Check all links who have specific root port link */
515 list_for_each_entry(leaf, &link_list, sibling) {
516 if (!list_empty(&leaf->children) ||
517 get_root_port_link(leaf) != root)
519 state = pcie_aspm_check_state(leaf, state);
521 /* Check root port link too in case it hasn't children */
522 state = pcie_aspm_check_state(root, state);
523 if (link->aspm_enabled == state)
526 * We must change the hierarchy. See comments in
527 * __pcie_aspm_config_link for the order
529 if (state & PCIE_LINK_STATE_L1) {
530 list_for_each_entry(leaf, &link_list, sibling) {
531 if (get_root_port_link(leaf) == root)
532 __pcie_aspm_config_link(leaf, state);
535 list_for_each_entry_reverse(leaf, &link_list, sibling) {
536 if (get_root_port_link(leaf) == root)
537 __pcie_aspm_config_link(leaf, state);
543 * pcie_aspm_configure_link_state: enable/disable PCI express link state
544 * @pdev: the root port or switch downstream port
546 static void pcie_aspm_configure_link_state(struct pcie_link_state *link,
549 down_read(&pci_bus_sem);
550 mutex_lock(&aspm_lock);
551 __pcie_aspm_configure_link_state(link, state);
552 mutex_unlock(&aspm_lock);
553 up_read(&pci_bus_sem);
556 static void free_link_state(struct pcie_link_state *link)
558 link->pdev->link_state = NULL;
562 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
564 struct pci_dev *child_dev;
569 * Some functions in a slot might not all be PCIE functions, very
570 * strange. Disable ASPM for the whole slot
572 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
573 child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
578 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
579 * RBER bit to determine if a function is 1.1 version device
581 pci_read_config_dword(child_dev, child_pos + PCI_EXP_DEVCAP,
583 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
584 dev_printk(KERN_INFO, &child_dev->dev, "disabling ASPM"
585 " on pre-1.1 PCIe device. You can enable it"
586 " with 'pcie_aspm=force'\n");
594 * pcie_aspm_init_link_state: Initiate PCI express link state.
595 * It is called after the pcie and its children devices are scaned.
596 * @pdev: the root port or switch downstream port
598 void pcie_aspm_init_link_state(struct pci_dev *pdev)
601 struct pcie_link_state *link_state;
605 if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
607 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
608 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
610 /* VIA has a strange chipset, root port is under a bridge */
611 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
614 down_read(&pci_bus_sem);
615 if (list_empty(&pdev->subordinate->devices))
618 blacklist = !!pcie_aspm_sanity_check(pdev);
620 mutex_lock(&aspm_lock);
622 link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
626 INIT_LIST_HEAD(&link_state->children);
627 INIT_LIST_HEAD(&link_state->link);
628 if (pdev->bus->self) {/* this is a switch */
629 struct pcie_link_state *parent_link_state;
631 parent_link_state = pdev->bus->parent->self->link_state;
632 if (!parent_link_state) {
636 list_add(&link_state->link, &parent_link_state->children);
637 link_state->parent = parent_link_state;
639 link_state->pdev = pdev;
640 link_state->has_switch = pcie_aspm_downstream_has_switch(link_state);
641 pdev->link_state = link_state;
644 pcie_aspm_configure_common_clock(link_state);
645 pcie_aspm_cap_init(link_state);
647 link_state->aspm_enabled =
648 (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
649 link_state->aspm_default = 0;
650 /* Set support state to 0, so we will disable ASPM later */
651 link_state->aspm_support = 0;
654 list_add(&link_state->sibling, &link_list);
656 if (link_state->has_switch) {
658 * If link has switch, delay the link config. The leaf link
659 * initialization will config the whole hierarchy. but we must
660 * make sure BIOS doesn't set unsupported link state
662 state = pcie_aspm_check_state(link_state,
663 link_state->aspm_default);
664 __pcie_aspm_config_link(link_state, state);
666 __pcie_aspm_configure_link_state(link_state,
667 policy_to_aspm_state(link_state));
669 pcie_check_clock_pm(link_state, blacklist);
673 free_link_state(link_state);
674 mutex_unlock(&aspm_lock);
676 up_read(&pci_bus_sem);
679 /* @pdev: the endpoint device */
680 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
682 struct pci_dev *parent = pdev->bus->self;
683 struct pcie_link_state *link_state = parent->link_state;
685 if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
687 if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
688 parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
690 down_read(&pci_bus_sem);
691 mutex_lock(&aspm_lock);
694 * All PCIe functions are in one slot, remove one function will remove
695 * the whole slot, so just wait until we are the last function left.
697 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
700 /* All functions are removed, so just disable ASPM for the link */
701 __pcie_aspm_config_one_dev(parent, 0);
702 list_del(&link_state->sibling);
703 list_del(&link_state->link);
704 /* Clock PM is for endpoint device */
706 free_link_state(link_state);
708 mutex_unlock(&aspm_lock);
709 up_read(&pci_bus_sem);
712 /* @pdev: the root port or switch downstream port */
713 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
715 struct pcie_link_state *link_state = pdev->link_state;
717 if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
719 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
720 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
723 * devices changed PM state, we should recheck if latency meets all
724 * functions' requirement
726 pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
730 * pci_disable_link_state - disable pci device's link state, so the link will
731 * never enter specific states
733 void pci_disable_link_state(struct pci_dev *pdev, int state)
735 struct pci_dev *parent = pdev->bus->self;
736 struct pcie_link_state *link_state;
738 if (aspm_disabled || !pdev->is_pcie)
740 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
741 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
743 if (!parent || !parent->link_state)
746 down_read(&pci_bus_sem);
747 mutex_lock(&aspm_lock);
748 link_state = parent->link_state;
749 link_state->aspm_support &= ~state;
750 if (state & PCIE_LINK_STATE_CLKPM)
751 link_state->clkpm_capable = 0;
753 __pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
754 if (!link_state->clkpm_capable && link_state->clkpm_enabled)
755 pcie_set_clock_pm(link_state, 0);
756 mutex_unlock(&aspm_lock);
757 up_read(&pci_bus_sem);
759 EXPORT_SYMBOL(pci_disable_link_state);
761 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
764 struct pcie_link_state *link_state;
766 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
767 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
769 if (i >= ARRAY_SIZE(policy_str))
771 if (i == aspm_policy)
774 down_read(&pci_bus_sem);
775 mutex_lock(&aspm_lock);
777 list_for_each_entry(link_state, &link_list, sibling) {
778 __pcie_aspm_configure_link_state(link_state,
779 policy_to_aspm_state(link_state));
780 if (link_state->clkpm_capable &&
781 link_state->clkpm_enabled != policy_to_clkpm_state(link_state))
782 pcie_set_clock_pm(link_state,
783 policy_to_clkpm_state(link_state));
786 mutex_unlock(&aspm_lock);
787 up_read(&pci_bus_sem);
791 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
794 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
795 if (i == aspm_policy)
796 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
798 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
802 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
805 #ifdef CONFIG_PCIEASPM_DEBUG
806 static ssize_t link_state_show(struct device *dev,
807 struct device_attribute *attr,
810 struct pci_dev *pci_device = to_pci_dev(dev);
811 struct pcie_link_state *link_state = pci_device->link_state;
813 return sprintf(buf, "%d\n", link_state->aspm_enabled);
816 static ssize_t link_state_store(struct device *dev,
817 struct device_attribute *attr,
821 struct pci_dev *pdev = to_pci_dev(dev);
827 if (state >= 0 && state <= 3) {
828 /* setup link aspm state */
829 pcie_aspm_configure_link_state(pdev->link_state, state);
836 static ssize_t clk_ctl_show(struct device *dev,
837 struct device_attribute *attr,
840 struct pci_dev *pci_device = to_pci_dev(dev);
841 struct pcie_link_state *link_state = pci_device->link_state;
843 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
846 static ssize_t clk_ctl_store(struct device *dev,
847 struct device_attribute *attr,
851 struct pci_dev *pci_device = to_pci_dev(dev);
858 down_read(&pci_bus_sem);
859 mutex_lock(&aspm_lock);
860 pcie_set_clock_pm(pci_device->link_state, !!state);
861 mutex_unlock(&aspm_lock);
862 up_read(&pci_bus_sem);
867 static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
868 static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
870 static char power_group[] = "power";
871 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
873 struct pcie_link_state *link_state = pdev->link_state;
875 if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
876 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
879 if (link_state->aspm_support)
880 sysfs_add_file_to_group(&pdev->dev.kobj,
881 &dev_attr_link_state.attr, power_group);
882 if (link_state->clkpm_capable)
883 sysfs_add_file_to_group(&pdev->dev.kobj,
884 &dev_attr_clk_ctl.attr, power_group);
887 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
889 struct pcie_link_state *link_state = pdev->link_state;
891 if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
892 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
895 if (link_state->aspm_support)
896 sysfs_remove_file_from_group(&pdev->dev.kobj,
897 &dev_attr_link_state.attr, power_group);
898 if (link_state->clkpm_capable)
899 sysfs_remove_file_from_group(&pdev->dev.kobj,
900 &dev_attr_clk_ctl.attr, power_group);
904 static int __init pcie_aspm_disable(char *str)
906 if (!strcmp(str, "off")) {
908 printk(KERN_INFO "PCIe ASPM is disabled\n");
909 } else if (!strcmp(str, "force")) {
911 printk(KERN_INFO "PCIe ASPM is forcedly enabled\n");
916 __setup("pcie_aspm=", pcie_aspm_disable);
918 void pcie_no_aspm(void)
925 * pcie_aspm_enabled - is PCIe ASPM enabled?
927 * Returns true if ASPM has not been disabled by the command-line option
930 int pcie_aspm_enabled(void)
932 return !aspm_disabled;
934 EXPORT_SYMBOL(pcie_aspm_enabled);