Lines Matching +full:lock +full:- +full:latency +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0
49 cap = &save_state->cap.data[0]; in pci_save_ltr_state()
65 cap = &save_state->cap.data[0]; in pci_restore_ltr_state()
73 pdev->l1ss = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); in pci_configure_aspm_l1ss()
84 struct pci_dev *parent = pdev->bus->self; in pci_save_aspm_l1ss_state()
96 if (!pdev->l1ss || !parent->l1ss) in pci_save_aspm_l1ss_state()
107 cap = &save_state->cap.data[0]; in pci_save_aspm_l1ss_state()
108 pci_read_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL2, cap++); in pci_save_aspm_l1ss_state()
109 pci_read_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, cap++); in pci_save_aspm_l1ss_state()
119 cap = &save_state->cap.data[0]; in pci_save_aspm_l1ss_state()
120 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, cap++); in pci_save_aspm_l1ss_state()
121 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, cap++); in pci_save_aspm_l1ss_state()
127 struct pci_dev *parent = pdev->bus->self; in pci_restore_aspm_l1ss_state()
140 if (!pdev->l1ss || !parent->l1ss) in pci_restore_aspm_l1ss_state()
148 cap = &cl_save_state->cap.data[0]; in pci_restore_aspm_l1ss_state()
151 cap = &pl_save_state->cap.data[0]; in pci_restore_aspm_l1ss_state()
170 pci_clear_and_set_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, in pci_restore_aspm_l1ss_state()
172 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pci_restore_aspm_l1ss_state()
186 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, pl_ctl2); in pci_restore_aspm_l1ss_state()
187 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL2, cl_ctl2); in pci_restore_aspm_l1ss_state()
188 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, pl_ctl1); in pci_restore_aspm_l1ss_state()
189 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, cl_ctl1); in pci_restore_aspm_l1ss_state()
193 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pci_restore_aspm_l1ss_state()
195 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, in pci_restore_aspm_l1ss_state()
237 u32 aspm_capable:7; /* Capable ASPM state with latency */
283 list_for_each_entry(child, &linkbus->devices, bus_list) in pci_function_0()
284 if (PCI_FUNC(child->devfn) == 0) in pci_function_0()
302 return link->aspm_default; in policy_to_aspm_state()
318 return link->clkpm_default; in policy_to_clkpm_state()
343 cap = (u16 *)&save_state->cap.data[0]; in pci_update_aspm_saved_state()
350 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_set_clkpm_nocheck()
353 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_set_clkpm_nocheck()
359 link->clkpm_enabled = !!enable; in pcie_set_clkpm_nocheck()
368 if (!link->clkpm_capable || link->clkpm_disable) in pcie_set_clkpm()
371 if (link->clkpm_enabled == enable) in pcie_set_clkpm()
382 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_clkpm_cap_init()
385 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_clkpm_cap_init()
396 link->clkpm_enabled = enabled; in pcie_clkpm_cap_init()
397 link->clkpm_default = enabled; in pcie_clkpm_cap_init()
398 link->clkpm_capable = capable; in pcie_clkpm_cap_init()
399 link->clkpm_disable = blacklist ? 1 : 0; in pcie_clkpm_cap_init()
405 * common clock. That will reduce the ASPM state exit latency.
411 struct pci_dev *child, *parent = link->pdev; in pcie_aspm_configure_common_clock()
412 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_configure_common_clock()
417 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); in pcie_aspm_configure_common_clock()
436 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
451 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
453 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC; in pcie_aspm_configure_common_clock()
462 if (pcie_retrain_link(link->pdev, true)) { in pcie_aspm_configure_common_clock()
466 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_aspm_configure_common_clock()
469 child_old_ccc[PCI_FUNC(child->devfn)]); in pcie_aspm_configure_common_clock()
475 /* Convert L0s latency encoding to ns */
485 /* Convert L0s acceptable latency encoding to ns */
493 /* Convert L1 latency encoding to ns */
503 /* Convert L1 acceptable latency encoding to ns */
539 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max in encode_l12_threshold()
543 *scale = 0; /* Value times 1ns */ in encode_l12_threshold()
546 *scale = 1; /* Value times 32ns */ in encode_l12_threshold()
549 *scale = 2; /* Value times 1024ns */ in encode_l12_threshold()
552 *scale = 3; /* Value times 32768ns */ in encode_l12_threshold()
555 *scale = 4; /* Value times 1048576ns */ in encode_l12_threshold()
558 *scale = 5; /* Value times 33554432ns */ in encode_l12_threshold()
568 u32 latency, encoding, lnkcap_up, lnkcap_dw; in pcie_aspm_check_latency() local
574 /* Device not in D0 doesn't need latency check */ in pcie_aspm_check_latency()
575 if ((endpoint->current_state != PCI_D0) && in pcie_aspm_check_latency()
576 (endpoint->current_state != PCI_UNKNOWN)) in pcie_aspm_check_latency()
579 link = endpoint->bus->self->link_state; in pcie_aspm_check_latency()
581 /* Calculate endpoint L0s acceptable latency */ in pcie_aspm_check_latency()
582 encoding = FIELD_GET(PCI_EXP_DEVCAP_L0S, endpoint->devcap); in pcie_aspm_check_latency()
585 /* Calculate endpoint L1 acceptable latency */ in pcie_aspm_check_latency()
586 encoding = FIELD_GET(PCI_EXP_DEVCAP_L1, endpoint->devcap); in pcie_aspm_check_latency()
590 struct pci_dev *dev = pci_function_0(link->pdev->subordinate); in pcie_aspm_check_latency()
593 pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP, in pcie_aspm_check_latency()
602 /* Check upstream direction L0s latency */ in pcie_aspm_check_latency()
603 if ((link->aspm_capable & PCIE_LINK_STATE_L0S_UP) && in pcie_aspm_check_latency()
605 link->aspm_capable &= ~PCIE_LINK_STATE_L0S_UP; in pcie_aspm_check_latency()
607 /* Check downstream direction L0s latency */ in pcie_aspm_check_latency()
608 if ((link->aspm_capable & PCIE_LINK_STATE_L0S_DW) && in pcie_aspm_check_latency()
610 link->aspm_capable &= ~PCIE_LINK_STATE_L0S_DW; in pcie_aspm_check_latency()
612 * Check L1 latency. in pcie_aspm_check_latency()
620 * a L1 substate exit latency check. We assume that the in pcie_aspm_check_latency()
624 latency = max_t(u32, latency_up_l1, latency_dw_l1); in pcie_aspm_check_latency()
625 if ((link->aspm_capable & PCIE_LINK_STATE_L1) && in pcie_aspm_check_latency()
626 (latency + l1_switch_latency > acceptable_l1)) in pcie_aspm_check_latency()
627 link->aspm_capable &= ~PCIE_LINK_STATE_L1; in pcie_aspm_check_latency()
630 link = link->parent; in pcie_aspm_check_latency()
638 struct pci_dev *child = link->downstream, *parent = link->pdev; in aspm_calc_l12_info()
671 * least that much latency. in aspm_calc_l12_info()
673 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and in aspm_calc_l12_info()
674 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at in aspm_calc_l12_info()
684 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1); in aspm_calc_l12_info()
685 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2); in aspm_calc_l12_info()
686 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1); in aspm_calc_l12_info()
687 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2); in aspm_calc_l12_info()
699 child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
702 parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
707 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l12_info()
708 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l12_info()
711 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
715 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
719 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
726 parent->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l12_info()
729 child->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l12_info()
736 struct pci_dev *child = link->downstream, *parent = link->pdev; in aspm_l1ss_init()
740 if (!parent->l1ss || !child->l1ss) in aspm_l1ss_init()
744 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, in aspm_l1ss_init()
746 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, in aspm_l1ss_init()
759 if (!child->ltr_path) in aspm_l1ss_init()
763 link->aspm_support |= PCIE_LINK_STATE_L1_1; in aspm_l1ss_init()
765 link->aspm_support |= PCIE_LINK_STATE_L1_2; in aspm_l1ss_init()
767 link->aspm_support |= PCIE_LINK_STATE_L1_1_PCIPM; in aspm_l1ss_init()
769 link->aspm_support |= PCIE_LINK_STATE_L1_2_PCIPM; in aspm_l1ss_init()
772 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_l1ss_init()
775 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_l1ss_init()
779 link->aspm_enabled |= PCIE_LINK_STATE_L1_1; in aspm_l1ss_init()
781 link->aspm_enabled |= PCIE_LINK_STATE_L1_2; in aspm_l1ss_init()
783 link->aspm_enabled |= PCIE_LINK_STATE_L1_1_PCIPM; in aspm_l1ss_init()
785 link->aspm_enabled |= PCIE_LINK_STATE_L1_2_PCIPM; in aspm_l1ss_init()
787 if (link->aspm_support & PCIE_LINK_STATE_L1_2_MASK) in aspm_l1ss_init()
793 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_aspm_cap_init()
796 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_cap_init()
800 link->aspm_enabled = PCIE_LINK_STATE_ASPM_ALL; in pcie_aspm_cap_init()
801 link->aspm_disable = PCIE_LINK_STATE_ASPM_ALL; in pcie_aspm_cap_init()
818 * Re-read upstream/downstream components' register state after in pcie_aspm_cap_init()
820 * read-only Link Capabilities may change depending on common clock in pcie_aspm_cap_init()
845 link->aspm_support |= PCIE_LINK_STATE_L0S; in pcie_aspm_cap_init()
848 link->aspm_enabled |= PCIE_LINK_STATE_L0S_UP; in pcie_aspm_cap_init()
850 link->aspm_enabled |= PCIE_LINK_STATE_L0S_DW; in pcie_aspm_cap_init()
854 link->aspm_support |= PCIE_LINK_STATE_L1; in pcie_aspm_cap_init()
857 link->aspm_enabled |= PCIE_LINK_STATE_L1; in pcie_aspm_cap_init()
869 link->aspm_default = link->aspm_enabled; in pcie_aspm_cap_init()
872 link->aspm_capable = link->aspm_support; in pcie_aspm_cap_init()
875 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_cap_init()
888 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_l1ss()
902 * - Clear L1.x enable bits at child first, then at parent in pcie_config_aspm_l1ss()
903 * - Set L1.x enable bits at parent first, then at child in pcie_config_aspm_l1ss()
904 * - ASPM/PCIPM L1.2 must be disabled while programming timing in pcie_config_aspm_l1ss()
909 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
911 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
915 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
917 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
930 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_link()
931 struct pci_bus *linkbus = parent->subordinate; in pcie_config_aspm_link()
934 state &= (link->aspm_capable & ~link->aspm_disable); in pcie_config_aspm_link()
941 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { in pcie_config_aspm_link()
943 state |= (link->aspm_enabled & PCIE_LINK_STATE_L1_SS_PCIPM); in pcie_config_aspm_link()
947 if (link->aspm_enabled == state) in pcie_config_aspm_link()
971 * value for all functions of a multi-function device. in pcie_config_aspm_link()
973 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_config_aspm_link()
977 if (link->aspm_capable & PCIE_LINK_STATE_L1SS) in pcie_config_aspm_link()
981 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_config_aspm_link()
984 link->aspm_enabled = state; in pcie_config_aspm_link()
987 pci_save_aspm_l1ss_state(link->downstream); in pcie_config_aspm_link()
988 pci_update_aspm_saved_state(link->downstream); in pcie_config_aspm_link()
997 link = link->parent; in pcie_config_aspm_path()
1003 link->pdev->link_state = NULL; in free_link_state()
1016 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { in pcie_aspm_sanity_check()
1018 return -EINVAL; in pcie_aspm_sanity_check()
1023 * pre-1.1 device in pcie_aspm_sanity_check()
1030 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use in pcie_aspm_sanity_check()
1035 …pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\… in pcie_aspm_sanity_check()
1036 return -EINVAL; in pcie_aspm_sanity_check()
1050 INIT_LIST_HEAD(&link->sibling); in alloc_pcie_link_state()
1051 link->pdev = pdev; in alloc_pcie_link_state()
1052 link->downstream = pci_function_0(pdev->subordinate); in alloc_pcie_link_state()
1055 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe in alloc_pcie_link_state()
1063 !pdev->bus->parent->self) { in alloc_pcie_link_state()
1064 link->root = link; in alloc_pcie_link_state()
1068 parent = pdev->bus->parent->self->link_state; in alloc_pcie_link_state()
1074 link->parent = parent; in alloc_pcie_link_state()
1075 link->root = link->parent->root; in alloc_pcie_link_state()
1078 list_add(&link->sibling, &link_list); in alloc_pcie_link_state()
1079 pdev->link_state = link; in alloc_pcie_link_state()
1087 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) in pcie_aspm_update_sysfs_visibility()
1088 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group); in pcie_aspm_update_sysfs_visibility()
1104 if (pdev->link_state) in pcie_aspm_init_link_state()
1117 pdev->bus->self) in pcie_aspm_init_link_state()
1121 if (list_empty(&pdev->subordinate->devices)) in pcie_aspm_init_link_state()
1166 if (bridge && bridge->ltr_path) { in pci_bridge_reconfigure_ltr()
1169 pci_dbg(bridge, "re-enabling LTR\n"); in pci_bridge_reconfigure_ltr()
1178 struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus); in pci_configure_ltr()
1192 pdev->ltr_path = 1; in pci_configure_ltr()
1197 if (bridge && bridge->ltr_path) in pci_configure_ltr()
1198 pdev->ltr_path = 1; in pci_configure_ltr()
1203 if (!host->native_ltr) in pci_configure_ltr()
1214 pdev->ltr_path = 1; in pci_configure_ltr()
1219 * If we're configuring a hot-added device, LTR was likely in pci_configure_ltr()
1220 * disabled in the upstream bridge, so re-enable it before enabling in pci_configure_ltr()
1224 if (bridge && bridge->ltr_path) { in pci_configure_ltr()
1228 pdev->ltr_path = 1; in pci_configure_ltr()
1236 BUG_ON(root->parent); in pcie_update_aspm_capable()
1238 if (link->root != root) in pcie_update_aspm_capable()
1240 link->aspm_capable = link->aspm_support; in pcie_update_aspm_capable()
1244 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_update_aspm_capable()
1245 if (link->root != root) in pcie_update_aspm_capable()
1247 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_update_aspm_capable()
1259 struct pci_dev *parent = pdev->bus->self; in pcie_aspm_exit_link_state()
1262 if (!parent || !parent->link_state) in pcie_aspm_exit_link_state()
1268 link = parent->link_state; in pcie_aspm_exit_link_state()
1269 root = link->root; in pcie_aspm_exit_link_state()
1270 parent_link = link->parent; in pcie_aspm_exit_link_state()
1273 * link->downstream is a pointer to the pci_dev of function 0. If in pcie_aspm_exit_link_state()
1275 * so we can't use link->downstream again. Free the link state to in pcie_aspm_exit_link_state()
1278 * If we're removing a non-0 function, it's possible we could in pcie_aspm_exit_link_state()
1281 * multi-function devices, so disable ASPM for all of them. in pcie_aspm_exit_link_state()
1284 list_del(&link->sibling); in pcie_aspm_exit_link_state()
1303 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_pm_state_change()
1308 * Devices changed PM state, we should recheck if latency in pcie_aspm_pm_state_change()
1314 pcie_update_aspm_capable(link->root); in pcie_aspm_pm_state_change()
1323 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_powersave_config_link()
1351 return bridge->link_state; in pcie_aspm_get_link()
1381 return -EINVAL; in __pci_disable_link_state()
1392 return -EPERM; in __pci_disable_link_state()
1398 link->aspm_disable |= pci_calc_aspm_disable_mask(state); in __pci_disable_link_state()
1402 link->clkpm_disable = 1; in __pci_disable_link_state()
1420 * pci_disable_link_state - Disable device's link state, so the link will
1439 return -EINVAL; in __pci_enable_link_state()
1448 return -EPERM; in __pci_enable_link_state()
1454 link->aspm_default = pci_calc_aspm_enable_mask(state); in __pci_enable_link_state()
1457 link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0; in __pci_enable_link_state()
1467 * pci_enable_link_state - Clear and set the default device link state so that
1473 * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per
1486 * pci_enable_link_state_locked - Clear and set the default device link state
1492 * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per
1498 * Context: Caller holds pci_bus_sem read lock.
1515 return -EPERM; in pcie_aspm_set_policy()
1550 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1565 return link->aspm_enabled; in pcie_aspm_enabled()
1576 return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); in aspm_attr_show_common()
1588 return -EINVAL; in aspm_attr_store_common()
1594 link->aspm_disable &= ~state; in aspm_attr_store_common()
1597 link->aspm_disable &= ~PCIE_LINK_STATE_L1; in aspm_attr_store_common()
1599 link->aspm_disable |= state; in aspm_attr_store_common()
1601 link->aspm_disable |= PCIE_LINK_STATE_L1SS; in aspm_attr_store_common()
1635 return sysfs_emit(buf, "%d\n", link->clkpm_enabled); in ASPM_ATTR()
1647 return -EINVAL; in clkpm_store()
1652 link->clkpm_disable = !state_enable; in clkpm_store()
1699 return link->clkpm_capable ? a->mode : 0; in aspm_ctrl_attrs_are_visible()
1701 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0; in aspm_ctrl_attrs_are_visible()