2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIE link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
30 u32 l0s; /* L0s latency (nsec) */
31 u32 l1; /* L1 latency (nsec) */
34 struct pcie_link_state {
35 struct pci_dev *pdev; /* Upstream component of the Link */
36 struct pcie_link_state *parent; /* pointer to the parent Link state */
37 struct list_head sibling; /* node in link_list */
38 struct list_head children; /* list of child link states */
39 struct list_head link; /* node in parent's children list */
42 u32 aspm_support:2; /* Supported ASPM state */
43 u32 aspm_enabled:2; /* Enabled ASPM state */
44 u32 aspm_default:2; /* Default ASPM state by BIOS */
47 u32 clkpm_capable:1; /* Clock PM capable? */
48 u32 clkpm_enabled:1; /* Current Clock PM state */
49 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
51 u32 has_switch:1; /* Downstream has switches? */
54 struct aspm_latency latency; /* Exit latency */
56 * Endpoint acceptable latencies. A pcie downstream port only
57 * has one slot under it, so at most there are 8 functions.
59 struct aspm_latency acceptable[8];
62 static int aspm_disabled, aspm_force;
63 static DEFINE_MUTEX(aspm_lock);
64 static LIST_HEAD(link_list);
66 #define POLICY_DEFAULT 0 /* BIOS default setting */
67 #define POLICY_PERFORMANCE 1 /* high performance */
68 #define POLICY_POWERSAVE 2 /* high power saving */
69 static int aspm_policy;
70 static const char *policy_str[] = {
71 [POLICY_DEFAULT] = "default",
72 [POLICY_PERFORMANCE] = "performance",
73 [POLICY_POWERSAVE] = "powersave"
76 #define LINK_RETRAIN_TIMEOUT HZ
78 static int policy_to_aspm_state(struct pcie_link_state *link)
80 switch (aspm_policy) {
81 case POLICY_PERFORMANCE:
82 /* Disable ASPM and Clock PM */
84 case POLICY_POWERSAVE:
85 /* Enable ASPM L0s/L1 */
86 return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
88 return link->aspm_default;
93 static int policy_to_clkpm_state(struct pcie_link_state *link)
95 switch (aspm_policy) {
96 case POLICY_PERFORMANCE:
97 /* Disable ASPM and Clock PM */
99 case POLICY_POWERSAVE:
100 /* Disable Clock PM */
103 return link->clkpm_default;
108 static void pcie_set_clock_pm(struct pcie_link_state *link, int enable)
112 struct pci_dev *child;
113 struct pci_bus *linkbus = link->pdev->subordinate;
115 list_for_each_entry(child, &linkbus->devices, bus_list) {
116 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
119 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
121 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
123 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
124 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
126 link->clkpm_enabled = !!enable;
129 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
131 int pos, capable = 1, enabled = 1;
134 struct pci_dev *child;
135 struct pci_bus *linkbus = link->pdev->subordinate;
137 /* All functions should have the same cap and state, take the worst */
138 list_for_each_entry(child, &linkbus->devices, bus_list) {
139 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
142 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32);
143 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
148 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
149 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
152 link->clkpm_enabled = enabled;
153 link->clkpm_default = enabled;
154 link->clkpm_capable = (blacklist) ? 0 : capable;
157 static bool pcie_aspm_downstream_has_switch(struct pcie_link_state *link)
159 struct pci_dev *child;
160 struct pci_bus *linkbus = link->pdev->subordinate;
162 list_for_each_entry(child, &linkbus->devices, bus_list) {
163 if (child->pcie_type == PCI_EXP_TYPE_UPSTREAM)
170 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
171 * could use common clock. If they are, configure them to use the
172 * common clock. That will reduce the ASPM state exit latency.
174 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
176 int ppos, cpos, same_clock = 1;
177 u16 reg16, parent_reg, child_reg[8];
178 unsigned long start_jiffies;
179 struct pci_dev *child, *parent = link->pdev;
180 struct pci_bus *linkbus = parent->subordinate;
182 * All functions of a slot should have the same Slot Clock
183 * Configuration, so just check one function
185 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
186 BUG_ON(!child->is_pcie);
188 /* Check downstream component if bit Slot Clock Configuration is 1 */
189 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
190 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16);
191 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
194 /* Check upstream component if bit Slot Clock Configuration is 1 */
195 ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
196 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
197 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
200 /* Configure downstream component, all functions */
201 list_for_each_entry(child, &linkbus->devices, bus_list) {
202 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
203 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16);
204 child_reg[PCI_FUNC(child->devfn)] = reg16;
206 reg16 |= PCI_EXP_LNKCTL_CCC;
208 reg16 &= ~PCI_EXP_LNKCTL_CCC;
209 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
212 /* Configure upstream component */
213 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16);
216 reg16 |= PCI_EXP_LNKCTL_CCC;
218 reg16 &= ~PCI_EXP_LNKCTL_CCC;
219 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
222 reg16 |= PCI_EXP_LNKCTL_RL;
223 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
225 /* Wait for link training end. Break out after waiting for timeout */
226 start_jiffies = jiffies;
228 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
229 if (!(reg16 & PCI_EXP_LNKSTA_LT))
231 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
235 if (!(reg16 & PCI_EXP_LNKSTA_LT))
238 /* Training failed. Restore common clock configurations */
239 dev_printk(KERN_ERR, &parent->dev,
240 "ASPM: Could not configure common clock\n");
241 list_for_each_entry(child, &linkbus->devices, bus_list) {
242 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
243 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
244 child_reg[PCI_FUNC(child->devfn)]);
246 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
250 * calc_L0S_latency: Convert L0s latency encoding to ns
252 static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
254 unsigned int ns = 64;
256 if (latency_encoding == 0x7) {
260 ns = 5*1000; /* > 4us */
262 ns *= (1 << latency_encoding);
267 * calc_L1_latency: Convert L1 latency encoding to ns
269 static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
271 unsigned int ns = 1000;
273 if (latency_encoding == 0x7) {
277 ns = 65*1000; /* > 64us */
279 ns *= (1 << latency_encoding);
283 static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
284 unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
289 unsigned int latency;
291 *l0s = *l1 = *enabled = 0;
292 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
293 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
294 *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
295 if (*state != PCIE_LINK_STATE_L0S &&
296 *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
301 latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
302 *l0s = calc_L0S_latency(latency, 0);
303 if (*state & PCIE_LINK_STATE_L1) {
304 latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
305 *l1 = calc_L1_latency(latency, 0);
307 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
308 *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
311 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
313 u32 support, l0s, l1, enabled;
314 struct pci_dev *child, *parent = link->pdev;
315 struct pci_bus *linkbus = parent->subordinate;
318 /* Set support state to 0, so we will disable ASPM later */
319 link->aspm_support = 0;
320 link->aspm_default = 0;
321 link->aspm_enabled = PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
325 /* Configure common clock before checking latencies */
326 pcie_aspm_configure_common_clock(link);
328 /* upstream component states */
329 pcie_aspm_get_cap_device(parent, &support, &l0s, &l1, &enabled);
330 link->aspm_support = support;
331 link->latency.l0s = l0s;
332 link->latency.l1 = l1;
333 link->aspm_enabled = enabled;
335 /* downstream component states, all functions have the same setting */
336 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
337 pcie_aspm_get_cap_device(child, &support, &l0s, &l1, &enabled);
338 link->aspm_support &= support;
339 link->latency.l0s = max_t(u32, link->latency.l0s, l0s);
340 link->latency.l1 = max_t(u32, link->latency.l1, l1);
342 if (!link->aspm_support)
345 link->aspm_enabled &= link->aspm_support;
346 link->aspm_default = link->aspm_enabled;
349 list_for_each_entry(child, &linkbus->devices, bus_list) {
352 unsigned int latency;
353 struct aspm_latency *acceptable =
354 &link->acceptable[PCI_FUNC(child->devfn)];
356 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
357 child->pcie_type != PCI_EXP_TYPE_LEG_END)
360 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
361 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
362 latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
363 latency = calc_L0S_latency(latency, 1);
364 acceptable->l0s = latency;
365 if (link->aspm_support & PCIE_LINK_STATE_L1) {
366 latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
367 latency = calc_L1_latency(latency, 1);
368 acceptable->l1 = latency;
374 * __pcie_aspm_check_state_one - check latency for endpoint device.
375 * @endpoint: pointer to the struct pci_dev of endpoint device
377 * TBD: The latency from the endpoint to root complex vary per switch's
378 * upstream link state above the device. Here we just do a simple check
379 * which assumes all links above the device can be in L1 state, that
380 * is we just consider the worst case. If switch's upstream link can't
381 * be put into L0S/L1, then our check is too strictly.
383 static u32 __pcie_aspm_check_state_one(struct pci_dev *endpoint, u32 state)
385 u32 l1_switch_latency = 0;
386 struct aspm_latency *acceptable;
387 struct pcie_link_state *link;
389 link = endpoint->bus->self->link_state;
390 state &= link->aspm_support;
391 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
393 while (link && state) {
394 if ((state & PCIE_LINK_STATE_L0S) &&
395 (link->latency.l0s > acceptable->l0s))
396 state &= ~PCIE_LINK_STATE_L0S;
397 if ((state & PCIE_LINK_STATE_L1) &&
398 (link->latency.l1 + l1_switch_latency > acceptable->l1))
399 state &= ~PCIE_LINK_STATE_L1;
402 * Every switch on the path to root complex need 1
403 * more microsecond for L1. Spec doesn't mention L0s.
405 l1_switch_latency += 1000;
410 static u32 pcie_aspm_check_state(struct pcie_link_state *link, u32 state)
412 pci_power_t power_state;
413 struct pci_dev *child;
414 struct pci_bus *linkbus = link->pdev->subordinate;
416 /* If no child, ignore the link */
417 if (list_empty(&linkbus->devices))
420 list_for_each_entry(child, &linkbus->devices, bus_list) {
422 * If downstream component of a link is pci bridge, we
423 * disable ASPM for now for the link
425 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
428 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
429 child->pcie_type != PCI_EXP_TYPE_LEG_END))
431 /* Device not in D0 doesn't need check latency */
432 power_state = child->current_state;
433 if (power_state == PCI_D1 || power_state == PCI_D2 ||
434 power_state == PCI_D3hot || power_state == PCI_D3cold)
436 state = __pcie_aspm_check_state_one(child, state);
441 static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
444 int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
446 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
449 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
452 static void __pcie_aspm_config_link(struct pcie_link_state *link, u32 state)
454 struct pci_dev *child, *parent = link->pdev;
455 struct pci_bus *linkbus = parent->subordinate;
457 /* If no child, disable the link */
458 if (list_empty(&linkbus->devices))
461 * If the downstream component has pci bridge function, don't
464 list_for_each_entry(child, &linkbus->devices, bus_list) {
465 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
469 * Spec 2.0 suggests all functions should be configured the
470 * same setting for ASPM. Enabling ASPM L1 should be done in
471 * upstream component first and then downstream, and vice
472 * versa for disabling ASPM L1. Spec doesn't mention L0S.
474 if (state & PCIE_LINK_STATE_L1)
475 __pcie_aspm_config_one_dev(parent, state);
477 list_for_each_entry(child, &linkbus->devices, bus_list)
478 __pcie_aspm_config_one_dev(child, state);
480 if (!(state & PCIE_LINK_STATE_L1))
481 __pcie_aspm_config_one_dev(parent, state);
483 link->aspm_enabled = state;
486 static struct pcie_link_state *get_root_port_link(struct pcie_link_state *link)
488 struct pcie_link_state *root_port_link = link;
489 while (root_port_link->parent)
490 root_port_link = root_port_link->parent;
491 return root_port_link;
494 /* Check the whole hierarchy, and configure each link in the hierarchy */
495 static void __pcie_aspm_configure_link_state(struct pcie_link_state *link,
498 struct pcie_link_state *leaf, *root = get_root_port_link(link);
500 state &= (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
502 /* Check all links who have specific root port link */
503 list_for_each_entry(leaf, &link_list, sibling) {
504 if (!list_empty(&leaf->children) ||
505 get_root_port_link(leaf) != root)
507 state = pcie_aspm_check_state(leaf, state);
509 /* Check root port link too in case it hasn't children */
510 state = pcie_aspm_check_state(root, state);
511 if (link->aspm_enabled == state)
514 * We must change the hierarchy. See comments in
515 * __pcie_aspm_config_link for the order
517 if (state & PCIE_LINK_STATE_L1) {
518 list_for_each_entry(leaf, &link_list, sibling) {
519 if (get_root_port_link(leaf) == root)
520 __pcie_aspm_config_link(leaf, state);
523 list_for_each_entry_reverse(leaf, &link_list, sibling) {
524 if (get_root_port_link(leaf) == root)
525 __pcie_aspm_config_link(leaf, state);
531 * pcie_aspm_configure_link_state: enable/disable PCI express link state
532 * @pdev: the root port or switch downstream port
534 static void pcie_aspm_configure_link_state(struct pcie_link_state *link,
537 down_read(&pci_bus_sem);
538 mutex_lock(&aspm_lock);
539 __pcie_aspm_configure_link_state(link, state);
540 mutex_unlock(&aspm_lock);
541 up_read(&pci_bus_sem);
544 static void free_link_state(struct pcie_link_state *link)
546 link->pdev->link_state = NULL;
550 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
552 struct pci_dev *child_dev;
557 * Some functions in a slot might not all be PCIE functions, very
558 * strange. Disable ASPM for the whole slot
560 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
561 child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
566 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
567 * RBER bit to determine if a function is 1.1 version device
569 pci_read_config_dword(child_dev, child_pos + PCI_EXP_DEVCAP,
571 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
572 dev_printk(KERN_INFO, &child_dev->dev, "disabling ASPM"
573 " on pre-1.1 PCIe device. You can enable it"
574 " with 'pcie_aspm=force'\n");
581 static struct pcie_link_state *pcie_aspm_setup_link_state(struct pci_dev *pdev)
583 struct pcie_link_state *link;
584 int blacklist = !!pcie_aspm_sanity_check(pdev);
586 link = kzalloc(sizeof(*link), GFP_KERNEL);
589 INIT_LIST_HEAD(&link->sibling);
590 INIT_LIST_HEAD(&link->children);
591 INIT_LIST_HEAD(&link->link);
593 link->has_switch = pcie_aspm_downstream_has_switch(link);
594 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
595 struct pcie_link_state *parent;
596 parent = pdev->bus->parent->self->link_state;
601 link->parent = parent;
602 list_add(&link->link, &parent->children);
604 list_add(&link->sibling, &link_list);
606 pdev->link_state = link;
608 /* Check ASPM capability */
609 pcie_aspm_cap_init(link, blacklist);
611 /* Check Clock PM capability */
612 pcie_clkpm_cap_init(link, blacklist);
618 * pcie_aspm_init_link_state: Initiate PCI express link state.
619 * It is called after the pcie and its children devices are scaned.
620 * @pdev: the root port or switch downstream port
622 void pcie_aspm_init_link_state(struct pci_dev *pdev)
625 struct pcie_link_state *link;
627 if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
629 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
630 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
633 /* VIA has a strange chipset, root port is under a bridge */
634 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
638 down_read(&pci_bus_sem);
639 if (list_empty(&pdev->subordinate->devices))
642 mutex_lock(&aspm_lock);
643 link = pcie_aspm_setup_link_state(pdev);
647 * Setup initial ASPM state
649 * If link has switch, delay the link config. The leaf link
650 * initialization will config the whole hierarchy. But we must
651 * make sure BIOS doesn't set unsupported link state.
653 if (link->has_switch) {
654 state = pcie_aspm_check_state(link, link->aspm_default);
655 __pcie_aspm_config_link(link, state);
657 state = policy_to_aspm_state(link);
658 __pcie_aspm_configure_link_state(link, state);
661 /* Setup initial Clock PM state */
662 state = (link->clkpm_capable) ? policy_to_clkpm_state(link) : 0;
663 pcie_set_clock_pm(link, state);
665 mutex_unlock(&aspm_lock);
667 up_read(&pci_bus_sem);
670 /* @pdev: the endpoint device */
671 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
673 struct pci_dev *parent = pdev->bus->self;
674 struct pcie_link_state *link_state = parent->link_state;
676 if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
678 if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
679 parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
681 down_read(&pci_bus_sem);
682 mutex_lock(&aspm_lock);
685 * All PCIe functions are in one slot, remove one function will remove
686 * the whole slot, so just wait until we are the last function left.
688 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
691 /* All functions are removed, so just disable ASPM for the link */
692 __pcie_aspm_config_one_dev(parent, 0);
693 list_del(&link_state->sibling);
694 list_del(&link_state->link);
695 /* Clock PM is for endpoint device */
697 free_link_state(link_state);
699 mutex_unlock(&aspm_lock);
700 up_read(&pci_bus_sem);
703 /* @pdev: the root port or switch downstream port */
704 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
706 struct pcie_link_state *link_state = pdev->link_state;
708 if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
710 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
711 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
714 * devices changed PM state, we should recheck if latency meets all
715 * functions' requirement
717 pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
721 * pci_disable_link_state - disable pci device's link state, so the link will
722 * never enter specific states
724 void pci_disable_link_state(struct pci_dev *pdev, int state)
726 struct pci_dev *parent = pdev->bus->self;
727 struct pcie_link_state *link_state;
729 if (aspm_disabled || !pdev->is_pcie)
731 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
732 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
734 if (!parent || !parent->link_state)
737 down_read(&pci_bus_sem);
738 mutex_lock(&aspm_lock);
739 link_state = parent->link_state;
740 link_state->aspm_support &= ~state;
741 if (state & PCIE_LINK_STATE_CLKPM)
742 link_state->clkpm_capable = 0;
744 __pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
745 if (!link_state->clkpm_capable && link_state->clkpm_enabled)
746 pcie_set_clock_pm(link_state, 0);
747 mutex_unlock(&aspm_lock);
748 up_read(&pci_bus_sem);
750 EXPORT_SYMBOL(pci_disable_link_state);
752 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
755 struct pcie_link_state *link_state;
757 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
758 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
760 if (i >= ARRAY_SIZE(policy_str))
762 if (i == aspm_policy)
765 down_read(&pci_bus_sem);
766 mutex_lock(&aspm_lock);
768 list_for_each_entry(link_state, &link_list, sibling) {
769 __pcie_aspm_configure_link_state(link_state,
770 policy_to_aspm_state(link_state));
771 if (link_state->clkpm_capable &&
772 link_state->clkpm_enabled != policy_to_clkpm_state(link_state))
773 pcie_set_clock_pm(link_state,
774 policy_to_clkpm_state(link_state));
777 mutex_unlock(&aspm_lock);
778 up_read(&pci_bus_sem);
782 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
785 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
786 if (i == aspm_policy)
787 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
789 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
793 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
796 #ifdef CONFIG_PCIEASPM_DEBUG
797 static ssize_t link_state_show(struct device *dev,
798 struct device_attribute *attr,
801 struct pci_dev *pci_device = to_pci_dev(dev);
802 struct pcie_link_state *link_state = pci_device->link_state;
804 return sprintf(buf, "%d\n", link_state->aspm_enabled);
807 static ssize_t link_state_store(struct device *dev,
808 struct device_attribute *attr,
812 struct pci_dev *pdev = to_pci_dev(dev);
818 if (state >= 0 && state <= 3) {
819 /* setup link aspm state */
820 pcie_aspm_configure_link_state(pdev->link_state, state);
827 static ssize_t clk_ctl_show(struct device *dev,
828 struct device_attribute *attr,
831 struct pci_dev *pci_device = to_pci_dev(dev);
832 struct pcie_link_state *link_state = pci_device->link_state;
834 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
837 static ssize_t clk_ctl_store(struct device *dev,
838 struct device_attribute *attr,
842 struct pci_dev *pci_device = to_pci_dev(dev);
849 down_read(&pci_bus_sem);
850 mutex_lock(&aspm_lock);
851 pcie_set_clock_pm(pci_device->link_state, !!state);
852 mutex_unlock(&aspm_lock);
853 up_read(&pci_bus_sem);
858 static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
859 static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
861 static char power_group[] = "power";
862 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
864 struct pcie_link_state *link_state = pdev->link_state;
866 if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
867 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
870 if (link_state->aspm_support)
871 sysfs_add_file_to_group(&pdev->dev.kobj,
872 &dev_attr_link_state.attr, power_group);
873 if (link_state->clkpm_capable)
874 sysfs_add_file_to_group(&pdev->dev.kobj,
875 &dev_attr_clk_ctl.attr, power_group);
878 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
880 struct pcie_link_state *link_state = pdev->link_state;
882 if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
883 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
886 if (link_state->aspm_support)
887 sysfs_remove_file_from_group(&pdev->dev.kobj,
888 &dev_attr_link_state.attr, power_group);
889 if (link_state->clkpm_capable)
890 sysfs_remove_file_from_group(&pdev->dev.kobj,
891 &dev_attr_clk_ctl.attr, power_group);
895 static int __init pcie_aspm_disable(char *str)
897 if (!strcmp(str, "off")) {
899 printk(KERN_INFO "PCIe ASPM is disabled\n");
900 } else if (!strcmp(str, "force")) {
902 printk(KERN_INFO "PCIe ASPM is forcedly enabled\n");
907 __setup("pcie_aspm=", pcie_aspm_disable);
909 void pcie_no_aspm(void)
916 * pcie_aspm_enabled - is PCIe ASPM enabled?
918 * Returns true if ASPM has not been disabled by the command-line option
921 int pcie_aspm_enabled(void)
923 return !aspm_disabled;
925 EXPORT_SYMBOL(pcie_aspm_enabled);