1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Enable PCIe link L0s/L1 state and Clock Power Management 4 * 5 * Copyright (C) 2007 Intel 6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 7 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/math.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/pci.h> 15 #include <linux/pci_regs.h> 16 #include <linux/errno.h> 17 #include <linux/pm.h> 18 #include <linux/init.h> 19 #include <linux/slab.h> 20 #include <linux/jiffies.h> 21 #include <linux/delay.h> 22 #include "../pci.h" 23 24 #ifdef MODULE_PARAM_PREFIX 25 #undef MODULE_PARAM_PREFIX 26 #endif 27 #define MODULE_PARAM_PREFIX "pcie_aspm." 28 29 /* Note: those are not register definitions */ 30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32 #define ASPM_STATE_L1 (4) /* L1 state */ 33 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ 34 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ 35 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ 36 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ 37 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) 38 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) 39 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ 40 ASPM_STATE_L1_2_MASK) 41 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 42 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ 43 ASPM_STATE_L1SS) 44 45 struct pcie_link_state { 46 struct pci_dev *pdev; /* Upstream component of the Link */ 47 struct pci_dev *downstream; /* Downstream component, function 0 */ 48 struct pcie_link_state *root; /* pointer to the root port link */ 49 struct pcie_link_state *parent; /* pointer to the parent Link state */ 50 struct list_head sibling; /* node in link_list */ 51 52 /* ASPM state */ 53 u32 aspm_support:7; /* Supported ASPM state */ 54 u32 aspm_enabled:7; /* Enabled ASPM state */ 55 u32 aspm_capable:7; /* Capable ASPM state with latency */ 56 u32 aspm_default:7; /* Default ASPM state by BIOS */ 57 u32 aspm_disable:7; /* Disabled ASPM state */ 58 59 /* Clock PM state */ 60 u32 clkpm_capable:1; /* Clock PM capable? */ 61 u32 clkpm_enabled:1; /* Current Clock PM state */ 62 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 63 u32 clkpm_disable:1; /* Clock PM disabled */ 64 }; 65 66 static int aspm_disabled, aspm_force; 67 static bool aspm_support_enabled = true; 68 static DEFINE_MUTEX(aspm_lock); 69 static LIST_HEAD(link_list); 70 71 #define POLICY_DEFAULT 0 /* BIOS default setting */ 72 #define POLICY_PERFORMANCE 1 /* high performance */ 73 #define POLICY_POWERSAVE 2 /* high power saving */ 74 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ 75 76 #ifdef CONFIG_PCIEASPM_PERFORMANCE 77 static int aspm_policy = POLICY_PERFORMANCE; 78 #elif defined CONFIG_PCIEASPM_POWERSAVE 79 static int aspm_policy = POLICY_POWERSAVE; 80 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE 81 static int aspm_policy = POLICY_POWER_SUPERSAVE; 82 #else 83 static int aspm_policy; 84 #endif 85 86 static const char *policy_str[] = { 87 [POLICY_DEFAULT] = "default", 88 [POLICY_PERFORMANCE] = "performance", 89 [POLICY_POWERSAVE] = "powersave", 90 [POLICY_POWER_SUPERSAVE] = "powersupersave" 91 }; 92 93 /* 94 * The L1 PM substate capability is only implemented in function 0 in a 95 * multi function device. 96 */ 97 static struct pci_dev *pci_function_0(struct pci_bus *linkbus) 98 { 99 struct pci_dev *child; 100 101 list_for_each_entry(child, &linkbus->devices, bus_list) 102 if (PCI_FUNC(child->devfn) == 0) 103 return child; 104 return NULL; 105 } 106 107 static int policy_to_aspm_state(struct pcie_link_state *link) 108 { 109 switch (aspm_policy) { 110 case POLICY_PERFORMANCE: 111 /* Disable ASPM and Clock PM */ 112 return 0; 113 case POLICY_POWERSAVE: 114 /* Enable ASPM L0s/L1 */ 115 return (ASPM_STATE_L0S | ASPM_STATE_L1); 116 case POLICY_POWER_SUPERSAVE: 117 /* Enable Everything */ 118 return ASPM_STATE_ALL; 119 case POLICY_DEFAULT: 120 return link->aspm_default; 121 } 122 return 0; 123 } 124 125 static int policy_to_clkpm_state(struct pcie_link_state *link) 126 { 127 switch (aspm_policy) { 128 case POLICY_PERFORMANCE: 129 /* Disable ASPM and Clock PM */ 130 return 0; 131 case POLICY_POWERSAVE: 132 case POLICY_POWER_SUPERSAVE: 133 /* Enable Clock PM */ 134 return 1; 135 case POLICY_DEFAULT: 136 return link->clkpm_default; 137 } 138 return 0; 139 } 140 141 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 142 { 143 struct pci_dev *child; 144 struct pci_bus *linkbus = link->pdev->subordinate; 145 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; 146 147 list_for_each_entry(child, &linkbus->devices, bus_list) 148 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 149 PCI_EXP_LNKCTL_CLKREQ_EN, 150 val); 151 link->clkpm_enabled = !!enable; 152 } 153 154 static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 155 { 156 /* 157 * Don't enable Clock PM if the link is not Clock PM capable 158 * or Clock PM is disabled 159 */ 160 if (!link->clkpm_capable || link->clkpm_disable) 161 enable = 0; 162 /* Need nothing if the specified equals to current state */ 163 if (link->clkpm_enabled == enable) 164 return; 165 pcie_set_clkpm_nocheck(link, enable); 166 } 167 168 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 169 { 170 int capable = 1, enabled = 1; 171 u32 reg32; 172 u16 reg16; 173 struct pci_dev *child; 174 struct pci_bus *linkbus = link->pdev->subordinate; 175 176 /* All functions should have the same cap and state, take the worst */ 177 list_for_each_entry(child, &linkbus->devices, bus_list) { 178 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 179 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 180 capable = 0; 181 enabled = 0; 182 break; 183 } 184 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 185 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 186 enabled = 0; 187 } 188 link->clkpm_enabled = enabled; 189 link->clkpm_default = enabled; 190 link->clkpm_capable = capable; 191 link->clkpm_disable = blacklist ? 1 : 0; 192 } 193 194 /* 195 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 196 * could use common clock. If they are, configure them to use the 197 * common clock. That will reduce the ASPM state exit latency. 198 */ 199 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 200 { 201 int same_clock = 1; 202 u16 reg16, ccc, parent_old_ccc, child_old_ccc[8]; 203 struct pci_dev *child, *parent = link->pdev; 204 struct pci_bus *linkbus = parent->subordinate; 205 /* 206 * All functions of a slot should have the same Slot Clock 207 * Configuration, so just check one function 208 */ 209 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 210 BUG_ON(!pci_is_pcie(child)); 211 212 /* Check downstream component if bit Slot Clock Configuration is 1 */ 213 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 214 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 215 same_clock = 0; 216 217 /* Check upstream component if bit Slot Clock Configuration is 1 */ 218 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 219 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 220 same_clock = 0; 221 222 /* Port might be already in common clock mode */ 223 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 224 parent_old_ccc = reg16 & PCI_EXP_LNKCTL_CCC; 225 if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) { 226 bool consistent = true; 227 228 list_for_each_entry(child, &linkbus->devices, bus_list) { 229 pcie_capability_read_word(child, PCI_EXP_LNKCTL, 230 ®16); 231 if (!(reg16 & PCI_EXP_LNKCTL_CCC)) { 232 consistent = false; 233 break; 234 } 235 } 236 if (consistent) 237 return; 238 pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n"); 239 } 240 241 ccc = same_clock ? PCI_EXP_LNKCTL_CCC : 0; 242 /* Configure downstream component, all functions */ 243 list_for_each_entry(child, &linkbus->devices, bus_list) { 244 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 245 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC; 246 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 247 PCI_EXP_LNKCTL_CCC, ccc); 248 } 249 250 /* Configure upstream component */ 251 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 252 PCI_EXP_LNKCTL_CCC, ccc); 253 254 if (pcie_retrain_link(link->pdev, true)) { 255 256 /* Training failed. Restore common clock configurations */ 257 pci_err(parent, "ASPM: Could not configure common clock\n"); 258 list_for_each_entry(child, &linkbus->devices, bus_list) 259 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 260 PCI_EXP_LNKCTL_CCC, 261 child_old_ccc[PCI_FUNC(child->devfn)]); 262 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 263 PCI_EXP_LNKCTL_CCC, parent_old_ccc); 264 } 265 } 266 267 /* Convert L0s latency encoding to ns */ 268 static u32 calc_l0s_latency(u32 lnkcap) 269 { 270 u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12; 271 272 if (encoding == 0x7) 273 return (5 * 1000); /* > 4us */ 274 return (64 << encoding); 275 } 276 277 /* Convert L0s acceptable latency encoding to ns */ 278 static u32 calc_l0s_acceptable(u32 encoding) 279 { 280 if (encoding == 0x7) 281 return -1U; 282 return (64 << encoding); 283 } 284 285 /* Convert L1 latency encoding to ns */ 286 static u32 calc_l1_latency(u32 lnkcap) 287 { 288 u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15; 289 290 if (encoding == 0x7) 291 return (65 * 1000); /* > 64us */ 292 return (1000 << encoding); 293 } 294 295 /* Convert L1 acceptable latency encoding to ns */ 296 static u32 calc_l1_acceptable(u32 encoding) 297 { 298 if (encoding == 0x7) 299 return -1U; 300 return (1000 << encoding); 301 } 302 303 /* Convert L1SS T_pwr encoding to usec */ 304 static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val) 305 { 306 switch (scale) { 307 case 0: 308 return val * 2; 309 case 1: 310 return val * 10; 311 case 2: 312 return val * 100; 313 } 314 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale); 315 return 0; 316 } 317 318 /* 319 * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1 320 * register. Ports enter L1.2 when the most recent LTR value is greater 321 * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we 322 * don't enter L1.2 too aggressively. 323 * 324 * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3. 325 */ 326 static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value) 327 { 328 u64 threshold_ns = (u64) threshold_us * 1000; 329 330 /* 331 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max 332 * value of 0x3ff. 333 */ 334 if (threshold_ns <= 0x3ff * 1) { 335 *scale = 0; /* Value times 1ns */ 336 *value = threshold_ns; 337 } else if (threshold_ns <= 0x3ff * 32) { 338 *scale = 1; /* Value times 32ns */ 339 *value = roundup(threshold_ns, 32) / 32; 340 } else if (threshold_ns <= 0x3ff * 1024) { 341 *scale = 2; /* Value times 1024ns */ 342 *value = roundup(threshold_ns, 1024) / 1024; 343 } else if (threshold_ns <= 0x3ff * 32768) { 344 *scale = 3; /* Value times 32768ns */ 345 *value = roundup(threshold_ns, 32768) / 32768; 346 } else if (threshold_ns <= 0x3ff * 1048576) { 347 *scale = 4; /* Value times 1048576ns */ 348 *value = roundup(threshold_ns, 1048576) / 1048576; 349 } else if (threshold_ns <= 0x3ff * (u64) 33554432) { 350 *scale = 5; /* Value times 33554432ns */ 351 *value = roundup(threshold_ns, 33554432) / 33554432; 352 } else { 353 *scale = 5; 354 *value = 0x3ff; /* Max representable value */ 355 } 356 } 357 358 static void pcie_aspm_check_latency(struct pci_dev *endpoint) 359 { 360 u32 latency, encoding, lnkcap_up, lnkcap_dw; 361 u32 l1_switch_latency = 0, latency_up_l0s; 362 u32 latency_up_l1, latency_dw_l0s, latency_dw_l1; 363 u32 acceptable_l0s, acceptable_l1; 364 struct pcie_link_state *link; 365 366 /* Device not in D0 doesn't need latency check */ 367 if ((endpoint->current_state != PCI_D0) && 368 (endpoint->current_state != PCI_UNKNOWN)) 369 return; 370 371 link = endpoint->bus->self->link_state; 372 373 /* Calculate endpoint L0s acceptable latency */ 374 encoding = (endpoint->devcap & PCI_EXP_DEVCAP_L0S) >> 6; 375 acceptable_l0s = calc_l0s_acceptable(encoding); 376 377 /* Calculate endpoint L1 acceptable latency */ 378 encoding = (endpoint->devcap & PCI_EXP_DEVCAP_L1) >> 9; 379 acceptable_l1 = calc_l1_acceptable(encoding); 380 381 while (link) { 382 struct pci_dev *dev = pci_function_0(link->pdev->subordinate); 383 384 /* Read direction exit latencies */ 385 pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP, 386 &lnkcap_up); 387 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, 388 &lnkcap_dw); 389 latency_up_l0s = calc_l0s_latency(lnkcap_up); 390 latency_up_l1 = calc_l1_latency(lnkcap_up); 391 latency_dw_l0s = calc_l0s_latency(lnkcap_dw); 392 latency_dw_l1 = calc_l1_latency(lnkcap_dw); 393 394 /* Check upstream direction L0s latency */ 395 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 396 (latency_up_l0s > acceptable_l0s)) 397 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 398 399 /* Check downstream direction L0s latency */ 400 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 401 (latency_dw_l0s > acceptable_l0s)) 402 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 403 /* 404 * Check L1 latency. 405 * Every switch on the path to root complex need 1 406 * more microsecond for L1. Spec doesn't mention L0s. 407 * 408 * The exit latencies for L1 substates are not advertised 409 * by a device. Since the spec also doesn't mention a way 410 * to determine max latencies introduced by enabling L1 411 * substates on the components, it is not clear how to do 412 * a L1 substate exit latency check. We assume that the 413 * L1 exit latencies advertised by a device include L1 414 * substate latencies (and hence do not do any check). 415 */ 416 latency = max_t(u32, latency_up_l1, latency_dw_l1); 417 if ((link->aspm_capable & ASPM_STATE_L1) && 418 (latency + l1_switch_latency > acceptable_l1)) 419 link->aspm_capable &= ~ASPM_STATE_L1; 420 l1_switch_latency += 1000; 421 422 link = link->parent; 423 } 424 } 425 426 static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, 427 u32 clear, u32 set) 428 { 429 u32 val; 430 431 pci_read_config_dword(pdev, pos, &val); 432 val &= ~clear; 433 val |= set; 434 pci_write_config_dword(pdev, pos, val); 435 } 436 437 /* Calculate L1.2 PM substate timing parameters */ 438 static void aspm_calc_l12_info(struct pcie_link_state *link, 439 u32 parent_l1ss_cap, u32 child_l1ss_cap) 440 { 441 struct pci_dev *child = link->downstream, *parent = link->pdev; 442 u32 val1, val2, scale1, scale2; 443 u32 t_common_mode, t_power_on, l1_2_threshold, scale, value; 444 u32 ctl1 = 0, ctl2 = 0; 445 u32 pctl1, pctl2, cctl1, cctl2; 446 u32 pl1_2_enables, cl1_2_enables; 447 448 /* Choose the greater of the two Port Common_Mode_Restore_Times */ 449 val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; 450 val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; 451 t_common_mode = max(val1, val2); 452 453 /* Choose the greater of the two Port T_POWER_ON times */ 454 val1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; 455 scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; 456 val2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; 457 scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; 458 459 if (calc_l12_pwron(parent, scale1, val1) > 460 calc_l12_pwron(child, scale2, val2)) { 461 ctl2 |= scale1 | (val1 << 3); 462 t_power_on = calc_l12_pwron(parent, scale1, val1); 463 } else { 464 ctl2 |= scale2 | (val2 << 3); 465 t_power_on = calc_l12_pwron(child, scale2, val2); 466 } 467 468 /* 469 * Set LTR_L1.2_THRESHOLD to the time required to transition the 470 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if 471 * downstream devices report (via LTR) that they can tolerate at 472 * least that much latency. 473 * 474 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and 475 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at 476 * least 4us. 477 */ 478 l1_2_threshold = 2 + 4 + t_common_mode + t_power_on; 479 encode_l12_threshold(l1_2_threshold, &scale, &value); 480 ctl1 |= t_common_mode << 8 | scale << 29 | value << 16; 481 482 /* Some broken devices only support dword access to L1 SS */ 483 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1); 484 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2); 485 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1); 486 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2); 487 488 if (ctl1 == pctl1 && ctl1 == cctl1 && 489 ctl2 == pctl2 && ctl2 == cctl2) 490 return; 491 492 /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */ 493 pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK; 494 cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK; 495 496 if (pl1_2_enables || cl1_2_enables) { 497 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 498 PCI_L1SS_CTL1_L1_2_MASK, 0); 499 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 500 PCI_L1SS_CTL1_L1_2_MASK, 0); 501 } 502 503 /* Program T_POWER_ON times in both ports */ 504 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); 505 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); 506 507 /* Program Common_Mode_Restore_Time in upstream device */ 508 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 509 PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1); 510 511 /* Program LTR_L1.2_THRESHOLD time in both ports */ 512 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 513 PCI_L1SS_CTL1_LTR_L12_TH_VALUE | 514 PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); 515 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 516 PCI_L1SS_CTL1_LTR_L12_TH_VALUE | 517 PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); 518 519 if (pl1_2_enables || cl1_2_enables) { 520 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, 521 pl1_2_enables); 522 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, 523 cl1_2_enables); 524 } 525 } 526 527 static void aspm_l1ss_init(struct pcie_link_state *link) 528 { 529 struct pci_dev *child = link->downstream, *parent = link->pdev; 530 u32 parent_l1ss_cap, child_l1ss_cap; 531 u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; 532 533 if (!parent->l1ss || !child->l1ss) 534 return; 535 536 /* Setup L1 substate */ 537 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, 538 &parent_l1ss_cap); 539 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, 540 &child_l1ss_cap); 541 542 if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) 543 parent_l1ss_cap = 0; 544 if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) 545 child_l1ss_cap = 0; 546 547 /* 548 * If we don't have LTR for the entire path from the Root Complex 549 * to this device, we can't use ASPM L1.2 because it relies on the 550 * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. 551 */ 552 if (!child->ltr_path) 553 child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; 554 555 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) 556 link->aspm_support |= ASPM_STATE_L1_1; 557 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) 558 link->aspm_support |= ASPM_STATE_L1_2; 559 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) 560 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; 561 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) 562 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; 563 564 if (parent_l1ss_cap) 565 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 566 &parent_l1ss_ctl1); 567 if (child_l1ss_cap) 568 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, 569 &child_l1ss_ctl1); 570 571 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) 572 link->aspm_enabled |= ASPM_STATE_L1_1; 573 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) 574 link->aspm_enabled |= ASPM_STATE_L1_2; 575 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) 576 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; 577 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) 578 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; 579 580 if (link->aspm_support & ASPM_STATE_L1_2_MASK) 581 aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap); 582 } 583 584 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 585 { 586 struct pci_dev *child = link->downstream, *parent = link->pdev; 587 u32 parent_lnkcap, child_lnkcap; 588 u16 parent_lnkctl, child_lnkctl; 589 struct pci_bus *linkbus = parent->subordinate; 590 591 if (blacklist) { 592 /* Set enabled/disable so that we will disable ASPM later */ 593 link->aspm_enabled = ASPM_STATE_ALL; 594 link->aspm_disable = ASPM_STATE_ALL; 595 return; 596 } 597 598 /* 599 * If ASPM not supported, don't mess with the clocks and link, 600 * bail out now. 601 */ 602 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap); 603 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap); 604 if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS)) 605 return; 606 607 /* Configure common clock before checking latencies */ 608 pcie_aspm_configure_common_clock(link); 609 610 /* 611 * Re-read upstream/downstream components' register state after 612 * clock configuration. L0s & L1 exit latencies in the otherwise 613 * read-only Link Capabilities may change depending on common clock 614 * configuration (PCIe r5.0, sec 7.5.3.6). 615 */ 616 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap); 617 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap); 618 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl); 619 pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl); 620 621 /* 622 * Setup L0s state 623 * 624 * Note that we must not enable L0s in either direction on a 625 * given link unless components on both sides of the link each 626 * support L0s. 627 */ 628 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S) 629 link->aspm_support |= ASPM_STATE_L0S; 630 631 if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) 632 link->aspm_enabled |= ASPM_STATE_L0S_UP; 633 if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) 634 link->aspm_enabled |= ASPM_STATE_L0S_DW; 635 636 /* Setup L1 state */ 637 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1) 638 link->aspm_support |= ASPM_STATE_L1; 639 640 if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1) 641 link->aspm_enabled |= ASPM_STATE_L1; 642 643 aspm_l1ss_init(link); 644 645 /* Save default state */ 646 link->aspm_default = link->aspm_enabled; 647 648 /* Setup initial capable state. Will be updated later */ 649 link->aspm_capable = link->aspm_support; 650 651 /* Get and check endpoint acceptable latencies */ 652 list_for_each_entry(child, &linkbus->devices, bus_list) { 653 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 654 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 655 continue; 656 657 pcie_aspm_check_latency(child); 658 } 659 } 660 661 /* Configure the ASPM L1 substates */ 662 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) 663 { 664 u32 val, enable_req; 665 struct pci_dev *child = link->downstream, *parent = link->pdev; 666 667 enable_req = (link->aspm_enabled ^ state) & state; 668 669 /* 670 * Here are the rules specified in the PCIe spec for enabling L1SS: 671 * - When enabling L1.x, enable bit at parent first, then at child 672 * - When disabling L1.x, disable bit at child first, then at parent 673 * - When enabling ASPM L1.x, need to disable L1 674 * (at child followed by parent). 675 * - The ASPM/PCIPM L1.2 must be disabled while programming timing 676 * parameters 677 * 678 * To keep it simple, disable all L1SS bits first, and later enable 679 * what is needed. 680 */ 681 682 /* Disable all L1 substates */ 683 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 684 PCI_L1SS_CTL1_L1SS_MASK, 0); 685 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 686 PCI_L1SS_CTL1_L1SS_MASK, 0); 687 /* 688 * If needed, disable L1, and it gets enabled later 689 * in pcie_config_aspm_link(). 690 */ 691 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { 692 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 693 PCI_EXP_LNKCTL_ASPM_L1, 0); 694 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 695 PCI_EXP_LNKCTL_ASPM_L1, 0); 696 } 697 698 val = 0; 699 if (state & ASPM_STATE_L1_1) 700 val |= PCI_L1SS_CTL1_ASPM_L1_1; 701 if (state & ASPM_STATE_L1_2) 702 val |= PCI_L1SS_CTL1_ASPM_L1_2; 703 if (state & ASPM_STATE_L1_1_PCIPM) 704 val |= PCI_L1SS_CTL1_PCIPM_L1_1; 705 if (state & ASPM_STATE_L1_2_PCIPM) 706 val |= PCI_L1SS_CTL1_PCIPM_L1_2; 707 708 /* Enable what we need to enable */ 709 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 710 PCI_L1SS_CTL1_L1SS_MASK, val); 711 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 712 PCI_L1SS_CTL1_L1SS_MASK, val); 713 } 714 715 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 716 { 717 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 718 PCI_EXP_LNKCTL_ASPMC, val); 719 } 720 721 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 722 { 723 u32 upstream = 0, dwstream = 0; 724 struct pci_dev *child = link->downstream, *parent = link->pdev; 725 struct pci_bus *linkbus = parent->subordinate; 726 727 /* Enable only the states that were not explicitly disabled */ 728 state &= (link->aspm_capable & ~link->aspm_disable); 729 730 /* Can't enable any substates if L1 is not enabled */ 731 if (!(state & ASPM_STATE_L1)) 732 state &= ~ASPM_STATE_L1SS; 733 734 /* Spec says both ports must be in D0 before enabling PCI PM substates*/ 735 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { 736 state &= ~ASPM_STATE_L1_SS_PCIPM; 737 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); 738 } 739 740 /* Nothing to do if the link is already in the requested state */ 741 if (link->aspm_enabled == state) 742 return; 743 /* Convert ASPM state to upstream/downstream ASPM register state */ 744 if (state & ASPM_STATE_L0S_UP) 745 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 746 if (state & ASPM_STATE_L0S_DW) 747 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 748 if (state & ASPM_STATE_L1) { 749 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 750 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 751 } 752 753 if (link->aspm_capable & ASPM_STATE_L1SS) 754 pcie_config_aspm_l1ss(link, state); 755 756 /* 757 * Spec 2.0 suggests all functions should be configured the 758 * same setting for ASPM. Enabling ASPM L1 should be done in 759 * upstream component first and then downstream, and vice 760 * versa for disabling ASPM L1. Spec doesn't mention L0S. 761 */ 762 if (state & ASPM_STATE_L1) 763 pcie_config_aspm_dev(parent, upstream); 764 list_for_each_entry(child, &linkbus->devices, bus_list) 765 pcie_config_aspm_dev(child, dwstream); 766 if (!(state & ASPM_STATE_L1)) 767 pcie_config_aspm_dev(parent, upstream); 768 769 link->aspm_enabled = state; 770 } 771 772 static void pcie_config_aspm_path(struct pcie_link_state *link) 773 { 774 while (link) { 775 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 776 link = link->parent; 777 } 778 } 779 780 static void free_link_state(struct pcie_link_state *link) 781 { 782 link->pdev->link_state = NULL; 783 kfree(link); 784 } 785 786 static int pcie_aspm_sanity_check(struct pci_dev *pdev) 787 { 788 struct pci_dev *child; 789 u32 reg32; 790 791 /* 792 * Some functions in a slot might not all be PCIe functions, 793 * very strange. Disable ASPM for the whole slot 794 */ 795 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 796 if (!pci_is_pcie(child)) 797 return -EINVAL; 798 799 /* 800 * If ASPM is disabled then we're not going to change 801 * the BIOS state. It's safe to continue even if it's a 802 * pre-1.1 device 803 */ 804 805 if (aspm_disabled) 806 continue; 807 808 /* 809 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 810 * RBER bit to determine if a function is 1.1 version device 811 */ 812 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 813 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 814 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 815 return -EINVAL; 816 } 817 } 818 return 0; 819 } 820 821 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 822 { 823 struct pcie_link_state *link; 824 825 link = kzalloc(sizeof(*link), GFP_KERNEL); 826 if (!link) 827 return NULL; 828 829 INIT_LIST_HEAD(&link->sibling); 830 link->pdev = pdev; 831 link->downstream = pci_function_0(pdev->subordinate); 832 833 /* 834 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe 835 * hierarchies. Note that some PCIe host implementations omit 836 * the root ports entirely, in which case a downstream port on 837 * a switch may become the root of the link state chain for all 838 * its subordinate endpoints. 839 */ 840 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || 841 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE || 842 !pdev->bus->parent->self) { 843 link->root = link; 844 } else { 845 struct pcie_link_state *parent; 846 847 parent = pdev->bus->parent->self->link_state; 848 if (!parent) { 849 kfree(link); 850 return NULL; 851 } 852 853 link->parent = parent; 854 link->root = link->parent->root; 855 } 856 857 list_add(&link->sibling, &link_list); 858 pdev->link_state = link; 859 return link; 860 } 861 862 static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev) 863 { 864 struct pci_dev *child; 865 866 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) 867 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group); 868 } 869 870 /* 871 * pcie_aspm_init_link_state: Initiate PCI express link state. 872 * It is called after the pcie and its children devices are scanned. 873 * @pdev: the root port or switch downstream port 874 */ 875 void pcie_aspm_init_link_state(struct pci_dev *pdev) 876 { 877 struct pcie_link_state *link; 878 int blacklist = !!pcie_aspm_sanity_check(pdev); 879 880 if (!aspm_support_enabled) 881 return; 882 883 if (pdev->link_state) 884 return; 885 886 /* 887 * We allocate pcie_link_state for the component on the upstream 888 * end of a Link, so there's nothing to do unless this device is 889 * downstream port. 890 */ 891 if (!pcie_downstream_port(pdev)) 892 return; 893 894 /* VIA has a strange chipset, root port is under a bridge */ 895 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 896 pdev->bus->self) 897 return; 898 899 down_read(&pci_bus_sem); 900 if (list_empty(&pdev->subordinate->devices)) 901 goto out; 902 903 mutex_lock(&aspm_lock); 904 link = alloc_pcie_link_state(pdev); 905 if (!link) 906 goto unlock; 907 /* 908 * Setup initial ASPM state. Note that we need to configure 909 * upstream links also because capable state of them can be 910 * update through pcie_aspm_cap_init(). 911 */ 912 pcie_aspm_cap_init(link, blacklist); 913 914 /* Setup initial Clock PM state */ 915 pcie_clkpm_cap_init(link, blacklist); 916 917 /* 918 * At this stage drivers haven't had an opportunity to change the 919 * link policy setting. Enabling ASPM on broken hardware can cripple 920 * it even before the driver has had a chance to disable ASPM, so 921 * default to a safe level right now. If we're enabling ASPM beyond 922 * the BIOS's expectation, we'll do so once pci_enable_device() is 923 * called. 924 */ 925 if (aspm_policy != POLICY_POWERSAVE && 926 aspm_policy != POLICY_POWER_SUPERSAVE) { 927 pcie_config_aspm_path(link); 928 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 929 } 930 931 pcie_aspm_update_sysfs_visibility(pdev); 932 933 unlock: 934 mutex_unlock(&aspm_lock); 935 out: 936 up_read(&pci_bus_sem); 937 } 938 939 /* Recheck latencies and update aspm_capable for links under the root */ 940 static void pcie_update_aspm_capable(struct pcie_link_state *root) 941 { 942 struct pcie_link_state *link; 943 BUG_ON(root->parent); 944 list_for_each_entry(link, &link_list, sibling) { 945 if (link->root != root) 946 continue; 947 link->aspm_capable = link->aspm_support; 948 } 949 list_for_each_entry(link, &link_list, sibling) { 950 struct pci_dev *child; 951 struct pci_bus *linkbus = link->pdev->subordinate; 952 if (link->root != root) 953 continue; 954 list_for_each_entry(child, &linkbus->devices, bus_list) { 955 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 956 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 957 continue; 958 pcie_aspm_check_latency(child); 959 } 960 } 961 } 962 963 /* @pdev: the endpoint device */ 964 void pcie_aspm_exit_link_state(struct pci_dev *pdev) 965 { 966 struct pci_dev *parent = pdev->bus->self; 967 struct pcie_link_state *link, *root, *parent_link; 968 969 if (!parent || !parent->link_state) 970 return; 971 972 down_read(&pci_bus_sem); 973 mutex_lock(&aspm_lock); 974 975 link = parent->link_state; 976 root = link->root; 977 parent_link = link->parent; 978 979 /* 980 * link->downstream is a pointer to the pci_dev of function 0. If 981 * we remove that function, the pci_dev is about to be deallocated, 982 * so we can't use link->downstream again. Free the link state to 983 * avoid this. 984 * 985 * If we're removing a non-0 function, it's possible we could 986 * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends 987 * programming the same ASPM Control value for all functions of 988 * multi-function devices, so disable ASPM for all of them. 989 */ 990 pcie_config_aspm_link(link, 0); 991 list_del(&link->sibling); 992 free_link_state(link); 993 994 /* Recheck latencies and configure upstream links */ 995 if (parent_link) { 996 pcie_update_aspm_capable(root); 997 pcie_config_aspm_path(parent_link); 998 } 999 1000 mutex_unlock(&aspm_lock); 1001 up_read(&pci_bus_sem); 1002 } 1003 1004 void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 1005 { 1006 struct pcie_link_state *link = pdev->link_state; 1007 1008 if (aspm_disabled || !link) 1009 return; 1010 1011 if (aspm_policy != POLICY_POWERSAVE && 1012 aspm_policy != POLICY_POWER_SUPERSAVE) 1013 return; 1014 1015 down_read(&pci_bus_sem); 1016 mutex_lock(&aspm_lock); 1017 pcie_config_aspm_path(link); 1018 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1019 mutex_unlock(&aspm_lock); 1020 up_read(&pci_bus_sem); 1021 } 1022 1023 static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev) 1024 { 1025 struct pci_dev *bridge; 1026 1027 if (!pci_is_pcie(pdev)) 1028 return NULL; 1029 1030 bridge = pci_upstream_bridge(pdev); 1031 if (!bridge || !pci_is_pcie(bridge)) 1032 return NULL; 1033 1034 return bridge->link_state; 1035 } 1036 1037 static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) 1038 { 1039 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1040 1041 if (!link) 1042 return -EINVAL; 1043 /* 1044 * A driver requested that ASPM be disabled on this device, but 1045 * if we don't have permission to manage ASPM (e.g., on ACPI 1046 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 1047 * the _OSC method), we can't honor that request. Windows has 1048 * a similar mechanism using "PciASPMOptOut", which is also 1049 * ignored in this situation. 1050 */ 1051 if (aspm_disabled) { 1052 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n"); 1053 return -EPERM; 1054 } 1055 1056 if (sem) 1057 down_read(&pci_bus_sem); 1058 mutex_lock(&aspm_lock); 1059 if (state & PCIE_LINK_STATE_L0S) 1060 link->aspm_disable |= ASPM_STATE_L0S; 1061 if (state & PCIE_LINK_STATE_L1) 1062 link->aspm_disable |= ASPM_STATE_L1; 1063 if (state & PCIE_LINK_STATE_L1_1) 1064 link->aspm_disable |= ASPM_STATE_L1_1; 1065 if (state & PCIE_LINK_STATE_L1_2) 1066 link->aspm_disable |= ASPM_STATE_L1_2; 1067 if (state & PCIE_LINK_STATE_L1_1_PCIPM) 1068 link->aspm_disable |= ASPM_STATE_L1_1_PCIPM; 1069 if (state & PCIE_LINK_STATE_L1_2_PCIPM) 1070 link->aspm_disable |= ASPM_STATE_L1_2_PCIPM; 1071 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1072 1073 if (state & PCIE_LINK_STATE_CLKPM) 1074 link->clkpm_disable = 1; 1075 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1076 mutex_unlock(&aspm_lock); 1077 if (sem) 1078 up_read(&pci_bus_sem); 1079 1080 return 0; 1081 } 1082 1083 int pci_disable_link_state_locked(struct pci_dev *pdev, int state) 1084 { 1085 return __pci_disable_link_state(pdev, state, false); 1086 } 1087 EXPORT_SYMBOL(pci_disable_link_state_locked); 1088 1089 /** 1090 * pci_disable_link_state - Disable device's link state, so the link will 1091 * never enter specific states. Note that if the BIOS didn't grant ASPM 1092 * control to the OS, this does nothing because we can't touch the LNKCTL 1093 * register. Returns 0 or a negative errno. 1094 * 1095 * @pdev: PCI device 1096 * @state: ASPM link state to disable 1097 */ 1098 int pci_disable_link_state(struct pci_dev *pdev, int state) 1099 { 1100 return __pci_disable_link_state(pdev, state, true); 1101 } 1102 EXPORT_SYMBOL(pci_disable_link_state); 1103 1104 /** 1105 * pci_enable_link_state - Clear and set the default device link state so that 1106 * the link may be allowed to enter the specified states. Note that if the 1107 * BIOS didn't grant ASPM control to the OS, this does nothing because we can't 1108 * touch the LNKCTL register. Also note that this does not enable states 1109 * disabled by pci_disable_link_state(). Return 0 or a negative errno. 1110 * 1111 * @pdev: PCI device 1112 * @state: Mask of ASPM link states to enable 1113 */ 1114 int pci_enable_link_state(struct pci_dev *pdev, int state) 1115 { 1116 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1117 1118 if (!link) 1119 return -EINVAL; 1120 /* 1121 * A driver requested that ASPM be enabled on this device, but 1122 * if we don't have permission to manage ASPM (e.g., on ACPI 1123 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 1124 * the _OSC method), we can't honor that request. 1125 */ 1126 if (aspm_disabled) { 1127 pci_warn(pdev, "can't override BIOS ASPM; OS doesn't have ASPM control\n"); 1128 return -EPERM; 1129 } 1130 1131 down_read(&pci_bus_sem); 1132 mutex_lock(&aspm_lock); 1133 link->aspm_default = 0; 1134 if (state & PCIE_LINK_STATE_L0S) 1135 link->aspm_default |= ASPM_STATE_L0S; 1136 if (state & PCIE_LINK_STATE_L1) 1137 link->aspm_default |= ASPM_STATE_L1; 1138 /* L1 PM substates require L1 */ 1139 if (state & PCIE_LINK_STATE_L1_1) 1140 link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1; 1141 if (state & PCIE_LINK_STATE_L1_2) 1142 link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1; 1143 if (state & PCIE_LINK_STATE_L1_1_PCIPM) 1144 link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1; 1145 if (state & PCIE_LINK_STATE_L1_2_PCIPM) 1146 link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1; 1147 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1148 1149 link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0; 1150 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1151 mutex_unlock(&aspm_lock); 1152 up_read(&pci_bus_sem); 1153 1154 return 0; 1155 } 1156 EXPORT_SYMBOL(pci_enable_link_state); 1157 1158 static int pcie_aspm_set_policy(const char *val, 1159 const struct kernel_param *kp) 1160 { 1161 int i; 1162 struct pcie_link_state *link; 1163 1164 if (aspm_disabled) 1165 return -EPERM; 1166 i = sysfs_match_string(policy_str, val); 1167 if (i < 0) 1168 return i; 1169 if (i == aspm_policy) 1170 return 0; 1171 1172 down_read(&pci_bus_sem); 1173 mutex_lock(&aspm_lock); 1174 aspm_policy = i; 1175 list_for_each_entry(link, &link_list, sibling) { 1176 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1177 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1178 } 1179 mutex_unlock(&aspm_lock); 1180 up_read(&pci_bus_sem); 1181 return 0; 1182 } 1183 1184 static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp) 1185 { 1186 int i, cnt = 0; 1187 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1188 if (i == aspm_policy) 1189 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 1190 else 1191 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 1192 cnt += sprintf(buffer + cnt, "\n"); 1193 return cnt; 1194 } 1195 1196 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 1197 NULL, 0644); 1198 1199 /** 1200 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device. 1201 * @pdev: Target device. 1202 * 1203 * Relies on the upstream bridge's link_state being valid. The link_state 1204 * is deallocated only when the last child of the bridge (i.e., @pdev or a 1205 * sibling) is removed, and the caller should be holding a reference to 1206 * @pdev, so this should be safe. 1207 */ 1208 bool pcie_aspm_enabled(struct pci_dev *pdev) 1209 { 1210 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1211 1212 if (!link) 1213 return false; 1214 1215 return link->aspm_enabled; 1216 } 1217 EXPORT_SYMBOL_GPL(pcie_aspm_enabled); 1218 1219 static ssize_t aspm_attr_show_common(struct device *dev, 1220 struct device_attribute *attr, 1221 char *buf, u8 state) 1222 { 1223 struct pci_dev *pdev = to_pci_dev(dev); 1224 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1225 1226 return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); 1227 } 1228 1229 static ssize_t aspm_attr_store_common(struct device *dev, 1230 struct device_attribute *attr, 1231 const char *buf, size_t len, u8 state) 1232 { 1233 struct pci_dev *pdev = to_pci_dev(dev); 1234 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1235 bool state_enable; 1236 1237 if (kstrtobool(buf, &state_enable) < 0) 1238 return -EINVAL; 1239 1240 down_read(&pci_bus_sem); 1241 mutex_lock(&aspm_lock); 1242 1243 if (state_enable) { 1244 link->aspm_disable &= ~state; 1245 /* need to enable L1 for substates */ 1246 if (state & ASPM_STATE_L1SS) 1247 link->aspm_disable &= ~ASPM_STATE_L1; 1248 } else { 1249 link->aspm_disable |= state; 1250 } 1251 1252 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1253 1254 mutex_unlock(&aspm_lock); 1255 up_read(&pci_bus_sem); 1256 1257 return len; 1258 } 1259 1260 #define ASPM_ATTR(_f, _s) \ 1261 static ssize_t _f##_show(struct device *dev, \ 1262 struct device_attribute *attr, char *buf) \ 1263 { return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \ 1264 \ 1265 static ssize_t _f##_store(struct device *dev, \ 1266 struct device_attribute *attr, \ 1267 const char *buf, size_t len) \ 1268 { return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); } 1269 1270 ASPM_ATTR(l0s_aspm, L0S) 1271 ASPM_ATTR(l1_aspm, L1) 1272 ASPM_ATTR(l1_1_aspm, L1_1) 1273 ASPM_ATTR(l1_2_aspm, L1_2) 1274 ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM) 1275 ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM) 1276 1277 static ssize_t clkpm_show(struct device *dev, 1278 struct device_attribute *attr, char *buf) 1279 { 1280 struct pci_dev *pdev = to_pci_dev(dev); 1281 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1282 1283 return sysfs_emit(buf, "%d\n", link->clkpm_enabled); 1284 } 1285 1286 static ssize_t clkpm_store(struct device *dev, 1287 struct device_attribute *attr, 1288 const char *buf, size_t len) 1289 { 1290 struct pci_dev *pdev = to_pci_dev(dev); 1291 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1292 bool state_enable; 1293 1294 if (kstrtobool(buf, &state_enable) < 0) 1295 return -EINVAL; 1296 1297 down_read(&pci_bus_sem); 1298 mutex_lock(&aspm_lock); 1299 1300 link->clkpm_disable = !state_enable; 1301 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1302 1303 mutex_unlock(&aspm_lock); 1304 up_read(&pci_bus_sem); 1305 1306 return len; 1307 } 1308 1309 static DEVICE_ATTR_RW(clkpm); 1310 static DEVICE_ATTR_RW(l0s_aspm); 1311 static DEVICE_ATTR_RW(l1_aspm); 1312 static DEVICE_ATTR_RW(l1_1_aspm); 1313 static DEVICE_ATTR_RW(l1_2_aspm); 1314 static DEVICE_ATTR_RW(l1_1_pcipm); 1315 static DEVICE_ATTR_RW(l1_2_pcipm); 1316 1317 static struct attribute *aspm_ctrl_attrs[] = { 1318 &dev_attr_clkpm.attr, 1319 &dev_attr_l0s_aspm.attr, 1320 &dev_attr_l1_aspm.attr, 1321 &dev_attr_l1_1_aspm.attr, 1322 &dev_attr_l1_2_aspm.attr, 1323 &dev_attr_l1_1_pcipm.attr, 1324 &dev_attr_l1_2_pcipm.attr, 1325 NULL 1326 }; 1327 1328 static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj, 1329 struct attribute *a, int n) 1330 { 1331 struct device *dev = kobj_to_dev(kobj); 1332 struct pci_dev *pdev = to_pci_dev(dev); 1333 struct pcie_link_state *link = pcie_aspm_get_link(pdev); 1334 static const u8 aspm_state_map[] = { 1335 ASPM_STATE_L0S, 1336 ASPM_STATE_L1, 1337 ASPM_STATE_L1_1, 1338 ASPM_STATE_L1_2, 1339 ASPM_STATE_L1_1_PCIPM, 1340 ASPM_STATE_L1_2_PCIPM, 1341 }; 1342 1343 if (aspm_disabled || !link) 1344 return 0; 1345 1346 if (n == 0) 1347 return link->clkpm_capable ? a->mode : 0; 1348 1349 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0; 1350 } 1351 1352 const struct attribute_group aspm_ctrl_attr_group = { 1353 .name = "link", 1354 .attrs = aspm_ctrl_attrs, 1355 .is_visible = aspm_ctrl_attrs_are_visible, 1356 }; 1357 1358 static int __init pcie_aspm_disable(char *str) 1359 { 1360 if (!strcmp(str, "off")) { 1361 aspm_policy = POLICY_DEFAULT; 1362 aspm_disabled = 1; 1363 aspm_support_enabled = false; 1364 printk(KERN_INFO "PCIe ASPM is disabled\n"); 1365 } else if (!strcmp(str, "force")) { 1366 aspm_force = 1; 1367 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 1368 } 1369 return 1; 1370 } 1371 1372 __setup("pcie_aspm=", pcie_aspm_disable); 1373 1374 void pcie_no_aspm(void) 1375 { 1376 /* 1377 * Disabling ASPM is intended to prevent the kernel from modifying 1378 * existing hardware state, not to clear existing state. To that end: 1379 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 1380 * (b) prevent userspace from changing policy 1381 */ 1382 if (!aspm_force) { 1383 aspm_policy = POLICY_DEFAULT; 1384 aspm_disabled = 1; 1385 } 1386 } 1387 1388 bool pcie_aspm_support_enabled(void) 1389 { 1390 return aspm_support_enabled; 1391 } 1392