1 /* 2 * File: drivers/pci/pcie/aspm.c 3 * Enabling PCIe link L0s/L1 state and Clock Power Management 4 * 5 * Copyright (C) 2007 Intel 6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 7 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/moduleparam.h> 13 #include <linux/pci.h> 14 #include <linux/pci_regs.h> 15 #include <linux/errno.h> 16 #include <linux/pm.h> 17 #include <linux/init.h> 18 #include <linux/slab.h> 19 #include <linux/jiffies.h> 20 #include <linux/delay.h> 21 #include <linux/pci-aspm.h> 22 #include "../pci.h" 23 24 #ifdef MODULE_PARAM_PREFIX 25 #undef MODULE_PARAM_PREFIX 26 #endif 27 #define MODULE_PARAM_PREFIX "pcie_aspm." 28 29 /* Note: those are not register definitions */ 30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32 #define ASPM_STATE_L1 (4) /* L1 state */ 33 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ 34 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ 35 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ 36 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ 37 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) 38 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) 39 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ 40 ASPM_STATE_L1_2_MASK) 41 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 42 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ 43 ASPM_STATE_L1SS) 44 45 /* 46 * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter 47 * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details). 48 * Not sure is there is a way to "calculate" this on the fly, but maybe we 49 * could turn it into a parameter in future. This value has been taken from 50 * the following files from Intel's coreboot (which is the only code I found 51 * to have used this): 52 * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html 53 * https://review.coreboot.org/#/c/8832/ 54 */ 55 #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30)) 56 57 struct aspm_latency { 58 u32 l0s; /* L0s latency (nsec) */ 59 u32 l1; /* L1 latency (nsec) */ 60 }; 61 62 struct pcie_link_state { 63 struct pci_dev *pdev; /* Upstream component of the Link */ 64 struct pci_dev *downstream; /* Downstream component, function 0 */ 65 struct pcie_link_state *root; /* pointer to the root port link */ 66 struct pcie_link_state *parent; /* pointer to the parent Link state */ 67 struct list_head sibling; /* node in link_list */ 68 struct list_head children; /* list of child link states */ 69 struct list_head link; /* node in parent's children list */ 70 71 /* ASPM state */ 72 u32 aspm_support:7; /* Supported ASPM state */ 73 u32 aspm_enabled:7; /* Enabled ASPM state */ 74 u32 aspm_capable:7; /* Capable ASPM state with latency */ 75 u32 aspm_default:7; /* Default ASPM state by BIOS */ 76 u32 aspm_disable:7; /* Disabled ASPM state */ 77 78 /* Clock PM state */ 79 u32 clkpm_capable:1; /* Clock PM capable? */ 80 u32 clkpm_enabled:1; /* Current Clock PM state */ 81 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 82 83 /* Exit latencies */ 84 struct aspm_latency latency_up; /* Upstream direction exit latency */ 85 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 86 /* 87 * Endpoint acceptable latencies. A pcie downstream port only 88 * has one slot under it, so at most there are 8 functions. 89 */ 90 struct aspm_latency acceptable[8]; 91 92 /* L1 PM Substate info */ 93 struct { 94 u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ 95 u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ 96 u32 ctl1; /* value to be programmed in ctl1 */ 97 u32 ctl2; /* value to be programmed in ctl2 */ 98 } l1ss; 99 }; 100 101 static int aspm_disabled, aspm_force; 102 static bool aspm_support_enabled = true; 103 static DEFINE_MUTEX(aspm_lock); 104 static LIST_HEAD(link_list); 105 106 #define POLICY_DEFAULT 0 /* BIOS default setting */ 107 #define POLICY_PERFORMANCE 1 /* high performance */ 108 #define POLICY_POWERSAVE 2 /* high power saving */ 109 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ 110 111 #ifdef CONFIG_PCIEASPM_PERFORMANCE 112 static int aspm_policy = POLICY_PERFORMANCE; 113 #elif defined CONFIG_PCIEASPM_POWERSAVE 114 static int aspm_policy = POLICY_POWERSAVE; 115 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE 116 static int aspm_policy = POLICY_POWER_SUPERSAVE; 117 #else 118 static int aspm_policy; 119 #endif 120 121 static const char *policy_str[] = { 122 [POLICY_DEFAULT] = "default", 123 [POLICY_PERFORMANCE] = "performance", 124 [POLICY_POWERSAVE] = "powersave", 125 [POLICY_POWER_SUPERSAVE] = "powersupersave" 126 }; 127 128 #define LINK_RETRAIN_TIMEOUT HZ 129 130 static int policy_to_aspm_state(struct pcie_link_state *link) 131 { 132 switch (aspm_policy) { 133 case POLICY_PERFORMANCE: 134 /* Disable ASPM and Clock PM */ 135 return 0; 136 case POLICY_POWERSAVE: 137 /* Enable ASPM L0s/L1 */ 138 return (ASPM_STATE_L0S | ASPM_STATE_L1); 139 case POLICY_POWER_SUPERSAVE: 140 /* Enable Everything */ 141 return ASPM_STATE_ALL; 142 case POLICY_DEFAULT: 143 return link->aspm_default; 144 } 145 return 0; 146 } 147 148 static int policy_to_clkpm_state(struct pcie_link_state *link) 149 { 150 switch (aspm_policy) { 151 case POLICY_PERFORMANCE: 152 /* Disable ASPM and Clock PM */ 153 return 0; 154 case POLICY_POWERSAVE: 155 case POLICY_POWER_SUPERSAVE: 156 /* Enable Clock PM */ 157 return 1; 158 case POLICY_DEFAULT: 159 return link->clkpm_default; 160 } 161 return 0; 162 } 163 164 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 165 { 166 struct pci_dev *child; 167 struct pci_bus *linkbus = link->pdev->subordinate; 168 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; 169 170 list_for_each_entry(child, &linkbus->devices, bus_list) 171 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 172 PCI_EXP_LNKCTL_CLKREQ_EN, 173 val); 174 link->clkpm_enabled = !!enable; 175 } 176 177 static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 178 { 179 /* Don't enable Clock PM if the link is not Clock PM capable */ 180 if (!link->clkpm_capable) 181 enable = 0; 182 /* Need nothing if the specified equals to current state */ 183 if (link->clkpm_enabled == enable) 184 return; 185 pcie_set_clkpm_nocheck(link, enable); 186 } 187 188 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 189 { 190 int capable = 1, enabled = 1; 191 u32 reg32; 192 u16 reg16; 193 struct pci_dev *child; 194 struct pci_bus *linkbus = link->pdev->subordinate; 195 196 /* All functions should have the same cap and state, take the worst */ 197 list_for_each_entry(child, &linkbus->devices, bus_list) { 198 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 199 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 200 capable = 0; 201 enabled = 0; 202 break; 203 } 204 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 205 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 206 enabled = 0; 207 } 208 link->clkpm_enabled = enabled; 209 link->clkpm_default = enabled; 210 link->clkpm_capable = (blacklist) ? 0 : capable; 211 } 212 213 /* 214 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 215 * could use common clock. If they are, configure them to use the 216 * common clock. That will reduce the ASPM state exit latency. 217 */ 218 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 219 { 220 int same_clock = 1; 221 u16 reg16, parent_reg, child_reg[8]; 222 unsigned long start_jiffies; 223 struct pci_dev *child, *parent = link->pdev; 224 struct pci_bus *linkbus = parent->subordinate; 225 /* 226 * All functions of a slot should have the same Slot Clock 227 * Configuration, so just check one function 228 */ 229 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 230 BUG_ON(!pci_is_pcie(child)); 231 232 /* Check downstream component if bit Slot Clock Configuration is 1 */ 233 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 234 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 235 same_clock = 0; 236 237 /* Check upstream component if bit Slot Clock Configuration is 1 */ 238 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 239 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 240 same_clock = 0; 241 242 /* Configure downstream component, all functions */ 243 list_for_each_entry(child, &linkbus->devices, bus_list) { 244 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 245 child_reg[PCI_FUNC(child->devfn)] = reg16; 246 if (same_clock) 247 reg16 |= PCI_EXP_LNKCTL_CCC; 248 else 249 reg16 &= ~PCI_EXP_LNKCTL_CCC; 250 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); 251 } 252 253 /* Configure upstream component */ 254 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 255 parent_reg = reg16; 256 if (same_clock) 257 reg16 |= PCI_EXP_LNKCTL_CCC; 258 else 259 reg16 &= ~PCI_EXP_LNKCTL_CCC; 260 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 261 262 /* Retrain link */ 263 reg16 |= PCI_EXP_LNKCTL_RL; 264 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 265 266 /* Wait for link training end. Break out after waiting for timeout */ 267 start_jiffies = jiffies; 268 for (;;) { 269 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 270 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 271 break; 272 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 273 break; 274 msleep(1); 275 } 276 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 277 return; 278 279 /* Training failed. Restore common clock configurations */ 280 dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); 281 list_for_each_entry(child, &linkbus->devices, bus_list) 282 pcie_capability_write_word(child, PCI_EXP_LNKCTL, 283 child_reg[PCI_FUNC(child->devfn)]); 284 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); 285 } 286 287 /* Convert L0s latency encoding to ns */ 288 static u32 calc_l0s_latency(u32 encoding) 289 { 290 if (encoding == 0x7) 291 return (5 * 1000); /* > 4us */ 292 return (64 << encoding); 293 } 294 295 /* Convert L0s acceptable latency encoding to ns */ 296 static u32 calc_l0s_acceptable(u32 encoding) 297 { 298 if (encoding == 0x7) 299 return -1U; 300 return (64 << encoding); 301 } 302 303 /* Convert L1 latency encoding to ns */ 304 static u32 calc_l1_latency(u32 encoding) 305 { 306 if (encoding == 0x7) 307 return (65 * 1000); /* > 64us */ 308 return (1000 << encoding); 309 } 310 311 /* Convert L1 acceptable latency encoding to ns */ 312 static u32 calc_l1_acceptable(u32 encoding) 313 { 314 if (encoding == 0x7) 315 return -1U; 316 return (1000 << encoding); 317 } 318 319 /* Convert L1SS T_pwr encoding to usec */ 320 static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) 321 { 322 switch (scale) { 323 case 0: 324 return val * 2; 325 case 1: 326 return val * 10; 327 case 2: 328 return val * 100; 329 } 330 dev_err(&pdev->dev, "%s: Invalid T_PwrOn scale: %u\n", 331 __func__, scale); 332 return 0; 333 } 334 335 struct aspm_register_info { 336 u32 support:2; 337 u32 enabled:2; 338 u32 latency_encoding_l0s; 339 u32 latency_encoding_l1; 340 341 /* L1 substates */ 342 u32 l1ss_cap_ptr; 343 u32 l1ss_cap; 344 u32 l1ss_ctl1; 345 u32 l1ss_ctl2; 346 }; 347 348 static void pcie_get_aspm_reg(struct pci_dev *pdev, 349 struct aspm_register_info *info) 350 { 351 u16 reg16; 352 u32 reg32; 353 354 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); 355 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 356 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 357 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 358 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); 359 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 360 361 /* Read L1 PM substate capabilities */ 362 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; 363 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); 364 if (!info->l1ss_cap_ptr) 365 return; 366 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, 367 &info->l1ss_cap); 368 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { 369 info->l1ss_cap = 0; 370 return; 371 } 372 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, 373 &info->l1ss_ctl1); 374 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, 375 &info->l1ss_ctl2); 376 } 377 378 static void pcie_aspm_check_latency(struct pci_dev *endpoint) 379 { 380 u32 latency, l1_switch_latency = 0; 381 struct aspm_latency *acceptable; 382 struct pcie_link_state *link; 383 384 /* Device not in D0 doesn't need latency check */ 385 if ((endpoint->current_state != PCI_D0) && 386 (endpoint->current_state != PCI_UNKNOWN)) 387 return; 388 389 link = endpoint->bus->self->link_state; 390 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 391 392 while (link) { 393 /* Check upstream direction L0s latency */ 394 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 395 (link->latency_up.l0s > acceptable->l0s)) 396 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 397 398 /* Check downstream direction L0s latency */ 399 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 400 (link->latency_dw.l0s > acceptable->l0s)) 401 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 402 /* 403 * Check L1 latency. 404 * Every switch on the path to root complex need 1 405 * more microsecond for L1. Spec doesn't mention L0s. 406 * 407 * The exit latencies for L1 substates are not advertised 408 * by a device. Since the spec also doesn't mention a way 409 * to determine max latencies introduced by enabling L1 410 * substates on the components, it is not clear how to do 411 * a L1 substate exit latency check. We assume that the 412 * L1 exit latencies advertised by a device include L1 413 * substate latencies (and hence do not do any check). 414 */ 415 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 416 if ((link->aspm_capable & ASPM_STATE_L1) && 417 (latency + l1_switch_latency > acceptable->l1)) 418 link->aspm_capable &= ~ASPM_STATE_L1; 419 l1_switch_latency += 1000; 420 421 link = link->parent; 422 } 423 } 424 425 /* 426 * The L1 PM substate capability is only implemented in function 0 in a 427 * multi function device. 428 */ 429 static struct pci_dev *pci_function_0(struct pci_bus *linkbus) 430 { 431 struct pci_dev *child; 432 433 list_for_each_entry(child, &linkbus->devices, bus_list) 434 if (PCI_FUNC(child->devfn) == 0) 435 return child; 436 return NULL; 437 } 438 439 /* Calculate L1.2 PM substate timing parameters */ 440 static void aspm_calc_l1ss_info(struct pcie_link_state *link, 441 struct aspm_register_info *upreg, 442 struct aspm_register_info *dwreg) 443 { 444 u32 val1, val2, scale1, scale2; 445 446 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; 447 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; 448 link->l1ss.ctl1 = link->l1ss.ctl2 = 0; 449 450 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) 451 return; 452 453 /* Choose the greater of the two T_cmn_mode_rstr_time */ 454 val1 = (upreg->l1ss_cap >> 8) & 0xFF; 455 val2 = (upreg->l1ss_cap >> 8) & 0xFF; 456 if (val1 > val2) 457 link->l1ss.ctl1 |= val1 << 8; 458 else 459 link->l1ss.ctl1 |= val2 << 8; 460 /* 461 * We currently use LTR L1.2 threshold to be fixed constant picked from 462 * Intel's coreboot. 463 */ 464 link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS; 465 466 /* Choose the greater of the two T_pwr_on */ 467 val1 = (upreg->l1ss_cap >> 19) & 0x1F; 468 scale1 = (upreg->l1ss_cap >> 16) & 0x03; 469 val2 = (dwreg->l1ss_cap >> 19) & 0x1F; 470 scale2 = (dwreg->l1ss_cap >> 16) & 0x03; 471 472 if (calc_l1ss_pwron(link->pdev, scale1, val1) > 473 calc_l1ss_pwron(link->downstream, scale2, val2)) 474 link->l1ss.ctl2 |= scale1 | (val1 << 3); 475 else 476 link->l1ss.ctl2 |= scale2 | (val2 << 3); 477 } 478 479 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 480 { 481 struct pci_dev *child = link->downstream, *parent = link->pdev; 482 struct pci_bus *linkbus = parent->subordinate; 483 struct aspm_register_info upreg, dwreg; 484 485 if (blacklist) { 486 /* Set enabled/disable so that we will disable ASPM later */ 487 link->aspm_enabled = ASPM_STATE_ALL; 488 link->aspm_disable = ASPM_STATE_ALL; 489 return; 490 } 491 492 /* Get upstream/downstream components' register state */ 493 pcie_get_aspm_reg(parent, &upreg); 494 pcie_get_aspm_reg(child, &dwreg); 495 496 /* 497 * If ASPM not supported, don't mess with the clocks and link, 498 * bail out now. 499 */ 500 if (!(upreg.support & dwreg.support)) 501 return; 502 503 /* Configure common clock before checking latencies */ 504 pcie_aspm_configure_common_clock(link); 505 506 /* 507 * Re-read upstream/downstream components' register state 508 * after clock configuration 509 */ 510 pcie_get_aspm_reg(parent, &upreg); 511 pcie_get_aspm_reg(child, &dwreg); 512 513 /* 514 * Setup L0s state 515 * 516 * Note that we must not enable L0s in either direction on a 517 * given link unless components on both sides of the link each 518 * support L0s. 519 */ 520 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 521 link->aspm_support |= ASPM_STATE_L0S; 522 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 523 link->aspm_enabled |= ASPM_STATE_L0S_UP; 524 if (upreg.enabled & PCIE_LINK_STATE_L0S) 525 link->aspm_enabled |= ASPM_STATE_L0S_DW; 526 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 527 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 528 529 /* Setup L1 state */ 530 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 531 link->aspm_support |= ASPM_STATE_L1; 532 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 533 link->aspm_enabled |= ASPM_STATE_L1; 534 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 535 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 536 537 /* Setup L1 substate */ 538 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) 539 link->aspm_support |= ASPM_STATE_L1_1; 540 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) 541 link->aspm_support |= ASPM_STATE_L1_2; 542 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) 543 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; 544 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) 545 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; 546 547 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) 548 link->aspm_enabled |= ASPM_STATE_L1_1; 549 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) 550 link->aspm_enabled |= ASPM_STATE_L1_2; 551 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) 552 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; 553 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) 554 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; 555 556 if (link->aspm_support & ASPM_STATE_L1SS) 557 aspm_calc_l1ss_info(link, &upreg, &dwreg); 558 559 /* Save default state */ 560 link->aspm_default = link->aspm_enabled; 561 562 /* Setup initial capable state. Will be updated later */ 563 link->aspm_capable = link->aspm_support; 564 /* 565 * If the downstream component has pci bridge function, don't 566 * do ASPM for now. 567 */ 568 list_for_each_entry(child, &linkbus->devices, bus_list) { 569 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { 570 link->aspm_disable = ASPM_STATE_ALL; 571 break; 572 } 573 } 574 575 /* Get and check endpoint acceptable latencies */ 576 list_for_each_entry(child, &linkbus->devices, bus_list) { 577 u32 reg32, encoding; 578 struct aspm_latency *acceptable = 579 &link->acceptable[PCI_FUNC(child->devfn)]; 580 581 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 582 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 583 continue; 584 585 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 586 /* Calculate endpoint L0s acceptable latency */ 587 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 588 acceptable->l0s = calc_l0s_acceptable(encoding); 589 /* Calculate endpoint L1 acceptable latency */ 590 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 591 acceptable->l1 = calc_l1_acceptable(encoding); 592 593 pcie_aspm_check_latency(child); 594 } 595 } 596 597 static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, 598 u32 clear, u32 set) 599 { 600 u32 val; 601 602 pci_read_config_dword(pdev, pos, &val); 603 val &= ~clear; 604 val |= set; 605 pci_write_config_dword(pdev, pos, val); 606 } 607 608 /* Configure the ASPM L1 substates */ 609 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) 610 { 611 u32 val, enable_req; 612 struct pci_dev *child = link->downstream, *parent = link->pdev; 613 u32 up_cap_ptr = link->l1ss.up_cap_ptr; 614 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; 615 616 enable_req = (link->aspm_enabled ^ state) & state; 617 618 /* 619 * Here are the rules specified in the PCIe spec for enabling L1SS: 620 * - When enabling L1.x, enable bit at parent first, then at child 621 * - When disabling L1.x, disable bit at child first, then at parent 622 * - When enabling ASPM L1.x, need to disable L1 623 * (at child followed by parent). 624 * - The ASPM/PCIPM L1.2 must be disabled while programming timing 625 * parameters 626 * 627 * To keep it simple, disable all L1SS bits first, and later enable 628 * what is needed. 629 */ 630 631 /* Disable all L1 substates */ 632 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 633 PCI_L1SS_CTL1_L1SS_MASK, 0); 634 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 635 PCI_L1SS_CTL1_L1SS_MASK, 0); 636 /* 637 * If needed, disable L1, and it gets enabled later 638 * in pcie_config_aspm_link(). 639 */ 640 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { 641 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 642 PCI_EXP_LNKCTL_ASPM_L1, 0); 643 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 644 PCI_EXP_LNKCTL_ASPM_L1, 0); 645 } 646 647 if (enable_req & ASPM_STATE_L1_2_MASK) { 648 649 /* Program T_pwr_on in both ports */ 650 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, 651 link->l1ss.ctl2); 652 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, 653 link->l1ss.ctl2); 654 655 /* Program T_cmn_mode in parent */ 656 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 657 0xFF00, link->l1ss.ctl1); 658 659 /* Program LTR L1.2 threshold in both ports */ 660 pci_clear_and_set_dword(parent, dw_cap_ptr + PCI_L1SS_CTL1, 661 0xE3FF0000, link->l1ss.ctl1); 662 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 663 0xE3FF0000, link->l1ss.ctl1); 664 } 665 666 val = 0; 667 if (state & ASPM_STATE_L1_1) 668 val |= PCI_L1SS_CTL1_ASPM_L1_1; 669 if (state & ASPM_STATE_L1_2) 670 val |= PCI_L1SS_CTL1_ASPM_L1_2; 671 if (state & ASPM_STATE_L1_1_PCIPM) 672 val |= PCI_L1SS_CTL1_PCIPM_L1_1; 673 if (state & ASPM_STATE_L1_2_PCIPM) 674 val |= PCI_L1SS_CTL1_PCIPM_L1_2; 675 676 /* Enable what we need to enable */ 677 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 678 PCI_L1SS_CAP_L1_PM_SS, val); 679 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 680 PCI_L1SS_CAP_L1_PM_SS, val); 681 } 682 683 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 684 { 685 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 686 PCI_EXP_LNKCTL_ASPMC, val); 687 } 688 689 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 690 { 691 u32 upstream = 0, dwstream = 0; 692 struct pci_dev *child = link->downstream, *parent = link->pdev; 693 struct pci_bus *linkbus = parent->subordinate; 694 695 /* Enable only the states that were not explicitly disabled */ 696 state &= (link->aspm_capable & ~link->aspm_disable); 697 698 /* Can't enable any substates if L1 is not enabled */ 699 if (!(state & ASPM_STATE_L1)) 700 state &= ~ASPM_STATE_L1SS; 701 702 /* Spec says both ports must be in D0 before enabling PCI PM substates*/ 703 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { 704 state &= ~ASPM_STATE_L1_SS_PCIPM; 705 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); 706 } 707 708 /* Nothing to do if the link is already in the requested state */ 709 if (link->aspm_enabled == state) 710 return; 711 /* Convert ASPM state to upstream/downstream ASPM register state */ 712 if (state & ASPM_STATE_L0S_UP) 713 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 714 if (state & ASPM_STATE_L0S_DW) 715 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 716 if (state & ASPM_STATE_L1) { 717 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 718 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 719 } 720 721 if (link->aspm_capable & ASPM_STATE_L1SS) 722 pcie_config_aspm_l1ss(link, state); 723 724 /* 725 * Spec 2.0 suggests all functions should be configured the 726 * same setting for ASPM. Enabling ASPM L1 should be done in 727 * upstream component first and then downstream, and vice 728 * versa for disabling ASPM L1. Spec doesn't mention L0S. 729 */ 730 if (state & ASPM_STATE_L1) 731 pcie_config_aspm_dev(parent, upstream); 732 list_for_each_entry(child, &linkbus->devices, bus_list) 733 pcie_config_aspm_dev(child, dwstream); 734 if (!(state & ASPM_STATE_L1)) 735 pcie_config_aspm_dev(parent, upstream); 736 737 link->aspm_enabled = state; 738 } 739 740 static void pcie_config_aspm_path(struct pcie_link_state *link) 741 { 742 while (link) { 743 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 744 link = link->parent; 745 } 746 } 747 748 static void free_link_state(struct pcie_link_state *link) 749 { 750 link->pdev->link_state = NULL; 751 kfree(link); 752 } 753 754 static int pcie_aspm_sanity_check(struct pci_dev *pdev) 755 { 756 struct pci_dev *child; 757 u32 reg32; 758 759 /* 760 * Some functions in a slot might not all be PCIe functions, 761 * very strange. Disable ASPM for the whole slot 762 */ 763 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 764 if (!pci_is_pcie(child)) 765 return -EINVAL; 766 767 /* 768 * If ASPM is disabled then we're not going to change 769 * the BIOS state. It's safe to continue even if it's a 770 * pre-1.1 device 771 */ 772 773 if (aspm_disabled) 774 continue; 775 776 /* 777 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 778 * RBER bit to determine if a function is 1.1 version device 779 */ 780 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 781 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 782 dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 783 return -EINVAL; 784 } 785 } 786 return 0; 787 } 788 789 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 790 { 791 struct pcie_link_state *link; 792 793 link = kzalloc(sizeof(*link), GFP_KERNEL); 794 if (!link) 795 return NULL; 796 797 INIT_LIST_HEAD(&link->sibling); 798 INIT_LIST_HEAD(&link->children); 799 INIT_LIST_HEAD(&link->link); 800 link->pdev = pdev; 801 link->downstream = pci_function_0(pdev->subordinate); 802 803 /* 804 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe 805 * hierarchies. 806 */ 807 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || 808 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE) { 809 link->root = link; 810 } else { 811 struct pcie_link_state *parent; 812 813 parent = pdev->bus->parent->self->link_state; 814 if (!parent) { 815 kfree(link); 816 return NULL; 817 } 818 819 link->parent = parent; 820 link->root = link->parent->root; 821 list_add(&link->link, &parent->children); 822 } 823 824 list_add(&link->sibling, &link_list); 825 pdev->link_state = link; 826 return link; 827 } 828 829 /* 830 * pcie_aspm_init_link_state: Initiate PCI express link state. 831 * It is called after the pcie and its children devices are scanned. 832 * @pdev: the root port or switch downstream port 833 */ 834 void pcie_aspm_init_link_state(struct pci_dev *pdev) 835 { 836 struct pcie_link_state *link; 837 int blacklist = !!pcie_aspm_sanity_check(pdev); 838 839 if (!aspm_support_enabled) 840 return; 841 842 if (pdev->link_state) 843 return; 844 845 /* 846 * We allocate pcie_link_state for the component on the upstream 847 * end of a Link, so there's nothing to do unless this device has a 848 * Link on its secondary side. 849 */ 850 if (!pdev->has_secondary_link) 851 return; 852 853 /* VIA has a strange chipset, root port is under a bridge */ 854 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 855 pdev->bus->self) 856 return; 857 858 down_read(&pci_bus_sem); 859 if (list_empty(&pdev->subordinate->devices)) 860 goto out; 861 862 mutex_lock(&aspm_lock); 863 link = alloc_pcie_link_state(pdev); 864 if (!link) 865 goto unlock; 866 /* 867 * Setup initial ASPM state. Note that we need to configure 868 * upstream links also because capable state of them can be 869 * update through pcie_aspm_cap_init(). 870 */ 871 pcie_aspm_cap_init(link, blacklist); 872 873 /* Setup initial Clock PM state */ 874 pcie_clkpm_cap_init(link, blacklist); 875 876 /* 877 * At this stage drivers haven't had an opportunity to change the 878 * link policy setting. Enabling ASPM on broken hardware can cripple 879 * it even before the driver has had a chance to disable ASPM, so 880 * default to a safe level right now. If we're enabling ASPM beyond 881 * the BIOS's expectation, we'll do so once pci_enable_device() is 882 * called. 883 */ 884 if (aspm_policy != POLICY_POWERSAVE && 885 aspm_policy != POLICY_POWER_SUPERSAVE) { 886 pcie_config_aspm_path(link); 887 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 888 } 889 890 unlock: 891 mutex_unlock(&aspm_lock); 892 out: 893 up_read(&pci_bus_sem); 894 } 895 896 /* Recheck latencies and update aspm_capable for links under the root */ 897 static void pcie_update_aspm_capable(struct pcie_link_state *root) 898 { 899 struct pcie_link_state *link; 900 BUG_ON(root->parent); 901 list_for_each_entry(link, &link_list, sibling) { 902 if (link->root != root) 903 continue; 904 link->aspm_capable = link->aspm_support; 905 } 906 list_for_each_entry(link, &link_list, sibling) { 907 struct pci_dev *child; 908 struct pci_bus *linkbus = link->pdev->subordinate; 909 if (link->root != root) 910 continue; 911 list_for_each_entry(child, &linkbus->devices, bus_list) { 912 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 913 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 914 continue; 915 pcie_aspm_check_latency(child); 916 } 917 } 918 } 919 920 /* @pdev: the endpoint device */ 921 void pcie_aspm_exit_link_state(struct pci_dev *pdev) 922 { 923 struct pci_dev *parent = pdev->bus->self; 924 struct pcie_link_state *link, *root, *parent_link; 925 926 if (!parent || !parent->link_state) 927 return; 928 929 down_read(&pci_bus_sem); 930 mutex_lock(&aspm_lock); 931 /* 932 * All PCIe functions are in one slot, remove one function will remove 933 * the whole slot, so just wait until we are the last function left. 934 */ 935 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 936 goto out; 937 938 link = parent->link_state; 939 root = link->root; 940 parent_link = link->parent; 941 942 /* All functions are removed, so just disable ASPM for the link */ 943 pcie_config_aspm_link(link, 0); 944 list_del(&link->sibling); 945 list_del(&link->link); 946 /* Clock PM is for endpoint device */ 947 free_link_state(link); 948 949 /* Recheck latencies and configure upstream links */ 950 if (parent_link) { 951 pcie_update_aspm_capable(root); 952 pcie_config_aspm_path(parent_link); 953 } 954 out: 955 mutex_unlock(&aspm_lock); 956 up_read(&pci_bus_sem); 957 } 958 959 /* @pdev: the root port or switch downstream port */ 960 void pcie_aspm_pm_state_change(struct pci_dev *pdev) 961 { 962 struct pcie_link_state *link = pdev->link_state; 963 964 if (aspm_disabled || !link) 965 return; 966 /* 967 * Devices changed PM state, we should recheck if latency 968 * meets all functions' requirement 969 */ 970 down_read(&pci_bus_sem); 971 mutex_lock(&aspm_lock); 972 pcie_update_aspm_capable(link->root); 973 pcie_config_aspm_path(link); 974 mutex_unlock(&aspm_lock); 975 up_read(&pci_bus_sem); 976 } 977 978 void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 979 { 980 struct pcie_link_state *link = pdev->link_state; 981 982 if (aspm_disabled || !link) 983 return; 984 985 if (aspm_policy != POLICY_POWERSAVE && 986 aspm_policy != POLICY_POWER_SUPERSAVE) 987 return; 988 989 down_read(&pci_bus_sem); 990 mutex_lock(&aspm_lock); 991 pcie_config_aspm_path(link); 992 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 993 mutex_unlock(&aspm_lock); 994 up_read(&pci_bus_sem); 995 } 996 997 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) 998 { 999 struct pci_dev *parent = pdev->bus->self; 1000 struct pcie_link_state *link; 1001 1002 if (!pci_is_pcie(pdev)) 1003 return; 1004 1005 if (pdev->has_secondary_link) 1006 parent = pdev; 1007 if (!parent || !parent->link_state) 1008 return; 1009 1010 /* 1011 * A driver requested that ASPM be disabled on this device, but 1012 * if we don't have permission to manage ASPM (e.g., on ACPI 1013 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 1014 * the _OSC method), we can't honor that request. Windows has 1015 * a similar mechanism using "PciASPMOptOut", which is also 1016 * ignored in this situation. 1017 */ 1018 if (aspm_disabled) { 1019 dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); 1020 return; 1021 } 1022 1023 if (sem) 1024 down_read(&pci_bus_sem); 1025 mutex_lock(&aspm_lock); 1026 link = parent->link_state; 1027 if (state & PCIE_LINK_STATE_L0S) 1028 link->aspm_disable |= ASPM_STATE_L0S; 1029 if (state & PCIE_LINK_STATE_L1) 1030 link->aspm_disable |= ASPM_STATE_L1; 1031 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1032 1033 if (state & PCIE_LINK_STATE_CLKPM) { 1034 link->clkpm_capable = 0; 1035 pcie_set_clkpm(link, 0); 1036 } 1037 mutex_unlock(&aspm_lock); 1038 if (sem) 1039 up_read(&pci_bus_sem); 1040 } 1041 1042 void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 1043 { 1044 __pci_disable_link_state(pdev, state, false); 1045 } 1046 EXPORT_SYMBOL(pci_disable_link_state_locked); 1047 1048 /** 1049 * pci_disable_link_state - Disable device's link state, so the link will 1050 * never enter specific states. Note that if the BIOS didn't grant ASPM 1051 * control to the OS, this does nothing because we can't touch the LNKCTL 1052 * register. 1053 * 1054 * @pdev: PCI device 1055 * @state: ASPM link state to disable 1056 */ 1057 void pci_disable_link_state(struct pci_dev *pdev, int state) 1058 { 1059 __pci_disable_link_state(pdev, state, true); 1060 } 1061 EXPORT_SYMBOL(pci_disable_link_state); 1062 1063 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 1064 { 1065 int i; 1066 struct pcie_link_state *link; 1067 1068 if (aspm_disabled) 1069 return -EPERM; 1070 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1071 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 1072 break; 1073 if (i >= ARRAY_SIZE(policy_str)) 1074 return -EINVAL; 1075 if (i == aspm_policy) 1076 return 0; 1077 1078 down_read(&pci_bus_sem); 1079 mutex_lock(&aspm_lock); 1080 aspm_policy = i; 1081 list_for_each_entry(link, &link_list, sibling) { 1082 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1083 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1084 } 1085 mutex_unlock(&aspm_lock); 1086 up_read(&pci_bus_sem); 1087 return 0; 1088 } 1089 1090 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 1091 { 1092 int i, cnt = 0; 1093 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1094 if (i == aspm_policy) 1095 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 1096 else 1097 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 1098 return cnt; 1099 } 1100 1101 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 1102 NULL, 0644); 1103 1104 #ifdef CONFIG_PCIEASPM_DEBUG 1105 static ssize_t link_state_show(struct device *dev, 1106 struct device_attribute *attr, 1107 char *buf) 1108 { 1109 struct pci_dev *pci_device = to_pci_dev(dev); 1110 struct pcie_link_state *link_state = pci_device->link_state; 1111 1112 return sprintf(buf, "%d\n", link_state->aspm_enabled); 1113 } 1114 1115 static ssize_t link_state_store(struct device *dev, 1116 struct device_attribute *attr, 1117 const char *buf, 1118 size_t n) 1119 { 1120 struct pci_dev *pdev = to_pci_dev(dev); 1121 struct pcie_link_state *link, *root = pdev->link_state->root; 1122 u32 state; 1123 1124 if (aspm_disabled) 1125 return -EPERM; 1126 1127 if (kstrtouint(buf, 10, &state)) 1128 return -EINVAL; 1129 if ((state & ~ASPM_STATE_ALL) != 0) 1130 return -EINVAL; 1131 1132 down_read(&pci_bus_sem); 1133 mutex_lock(&aspm_lock); 1134 list_for_each_entry(link, &link_list, sibling) { 1135 if (link->root != root) 1136 continue; 1137 pcie_config_aspm_link(link, state); 1138 } 1139 mutex_unlock(&aspm_lock); 1140 up_read(&pci_bus_sem); 1141 return n; 1142 } 1143 1144 static ssize_t clk_ctl_show(struct device *dev, 1145 struct device_attribute *attr, 1146 char *buf) 1147 { 1148 struct pci_dev *pci_device = to_pci_dev(dev); 1149 struct pcie_link_state *link_state = pci_device->link_state; 1150 1151 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 1152 } 1153 1154 static ssize_t clk_ctl_store(struct device *dev, 1155 struct device_attribute *attr, 1156 const char *buf, 1157 size_t n) 1158 { 1159 struct pci_dev *pdev = to_pci_dev(dev); 1160 bool state; 1161 1162 if (strtobool(buf, &state)) 1163 return -EINVAL; 1164 1165 down_read(&pci_bus_sem); 1166 mutex_lock(&aspm_lock); 1167 pcie_set_clkpm_nocheck(pdev->link_state, state); 1168 mutex_unlock(&aspm_lock); 1169 up_read(&pci_bus_sem); 1170 1171 return n; 1172 } 1173 1174 static DEVICE_ATTR_RW(link_state); 1175 static DEVICE_ATTR_RW(clk_ctl); 1176 1177 static char power_group[] = "power"; 1178 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 1179 { 1180 struct pcie_link_state *link_state = pdev->link_state; 1181 1182 if (!link_state) 1183 return; 1184 1185 if (link_state->aspm_support) 1186 sysfs_add_file_to_group(&pdev->dev.kobj, 1187 &dev_attr_link_state.attr, power_group); 1188 if (link_state->clkpm_capable) 1189 sysfs_add_file_to_group(&pdev->dev.kobj, 1190 &dev_attr_clk_ctl.attr, power_group); 1191 } 1192 1193 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 1194 { 1195 struct pcie_link_state *link_state = pdev->link_state; 1196 1197 if (!link_state) 1198 return; 1199 1200 if (link_state->aspm_support) 1201 sysfs_remove_file_from_group(&pdev->dev.kobj, 1202 &dev_attr_link_state.attr, power_group); 1203 if (link_state->clkpm_capable) 1204 sysfs_remove_file_from_group(&pdev->dev.kobj, 1205 &dev_attr_clk_ctl.attr, power_group); 1206 } 1207 #endif 1208 1209 static int __init pcie_aspm_disable(char *str) 1210 { 1211 if (!strcmp(str, "off")) { 1212 aspm_policy = POLICY_DEFAULT; 1213 aspm_disabled = 1; 1214 aspm_support_enabled = false; 1215 printk(KERN_INFO "PCIe ASPM is disabled\n"); 1216 } else if (!strcmp(str, "force")) { 1217 aspm_force = 1; 1218 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 1219 } 1220 return 1; 1221 } 1222 1223 __setup("pcie_aspm=", pcie_aspm_disable); 1224 1225 void pcie_no_aspm(void) 1226 { 1227 /* 1228 * Disabling ASPM is intended to prevent the kernel from modifying 1229 * existing hardware state, not to clear existing state. To that end: 1230 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 1231 * (b) prevent userspace from changing policy 1232 */ 1233 if (!aspm_force) { 1234 aspm_policy = POLICY_DEFAULT; 1235 aspm_disabled = 1; 1236 } 1237 } 1238 1239 bool pcie_aspm_support_enabled(void) 1240 { 1241 return aspm_support_enabled; 1242 } 1243 EXPORT_SYMBOL(pcie_aspm_support_enabled); 1244