3  * Enable PCIe link L0s/L1 state and Clock Power Management
228 	struct pci_dev *pdev;		/* Upstream component of the Link */
230 	struct pcie_link_state *root;	/* pointer to the root port link */
231 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
289 static int policy_to_aspm_state(struct pcie_link_state *link)  in policy_to_aspm_state()  argument
302 		return link->aspm_default;  in policy_to_aspm_state()
307 static int policy_to_clkpm_state(struct pcie_link_state *link)  in policy_to_clkpm_state()  argument
318 		return link->clkpm_default;  in policy_to_clkpm_state()
347 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)  in pcie_set_clkpm_nocheck()  argument
350 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_set_clkpm_nocheck()
359 	link->clkpm_enabled = !!enable;  in pcie_set_clkpm_nocheck()
362 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)  in pcie_set_clkpm()  argument
365 	 * Don't enable Clock PM if the link is not Clock PM capable  in pcie_set_clkpm()
368 	if (!link->clkpm_capable || link->clkpm_disable)  in pcie_set_clkpm()
371 	if (link->clkpm_enabled == enable)  in pcie_set_clkpm()
373 	pcie_set_clkpm_nocheck(link, enable);  in pcie_set_clkpm()
376 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_clkpm_cap_init()  argument
382 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_clkpm_cap_init()
396 	link->clkpm_enabled = enabled;  in pcie_clkpm_cap_init()
397 	link->clkpm_default = enabled;  in pcie_clkpm_cap_init()
398 	link->clkpm_capable = capable;  in pcie_clkpm_cap_init()
399 	link->clkpm_disable = blacklist ? 1 : 0;  in pcie_clkpm_cap_init()
403  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
407 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)  in pcie_aspm_configure_common_clock()  argument
411 	struct pci_dev *child, *parent = link->pdev;  in pcie_aspm_configure_common_clock()
462 	if (pcie_retrain_link(link->pdev, true)) {  in pcie_aspm_configure_common_clock()
572 	struct pcie_link_state *link;  in pcie_aspm_check_latency()  local
579 	link = endpoint->bus->self->link_state;  in pcie_aspm_check_latency()
589 	while (link) {  in pcie_aspm_check_latency()
590 		struct pci_dev *dev = pci_function_0(link->pdev->subordinate);  in pcie_aspm_check_latency()
593 		pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP,  in pcie_aspm_check_latency()
603 		if ((link->aspm_capable & PCIE_LINK_STATE_L0S_UP) &&  in pcie_aspm_check_latency()
605 			link->aspm_capable &= ~PCIE_LINK_STATE_L0S_UP;  in pcie_aspm_check_latency()
608 		if ((link->aspm_capable & PCIE_LINK_STATE_L0S_DW) &&  in pcie_aspm_check_latency()
610 			link->aspm_capable &= ~PCIE_LINK_STATE_L0S_DW;  in pcie_aspm_check_latency()
625 		if ((link->aspm_capable & PCIE_LINK_STATE_L1) &&  in pcie_aspm_check_latency()
627 			link->aspm_capable &= ~PCIE_LINK_STATE_L1;  in pcie_aspm_check_latency()
630 		link = link->parent;  in pcie_aspm_check_latency()
635 static void aspm_calc_l12_info(struct pcie_link_state *link,  in aspm_calc_l12_info()  argument
638 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_calc_l12_info()
669 	 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if  in aspm_calc_l12_info()
734 static void aspm_l1ss_init(struct pcie_link_state *link)  in aspm_l1ss_init()  argument
736 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_l1ss_init()
763 		link->aspm_support |= PCIE_LINK_STATE_L1_1;  in aspm_l1ss_init()
765 		link->aspm_support |= PCIE_LINK_STATE_L1_2;  in aspm_l1ss_init()
767 		link->aspm_support |= PCIE_LINK_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
769 		link->aspm_support |= PCIE_LINK_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
779 		link->aspm_enabled |= PCIE_LINK_STATE_L1_1;  in aspm_l1ss_init()
781 		link->aspm_enabled |= PCIE_LINK_STATE_L1_2;  in aspm_l1ss_init()
783 		link->aspm_enabled |= PCIE_LINK_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
785 		link->aspm_enabled |= PCIE_LINK_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
787 	if (link->aspm_support & PCIE_LINK_STATE_L1_2_MASK)  in aspm_l1ss_init()
788 		aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap);  in aspm_l1ss_init()
791 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_aspm_cap_init()  argument
793 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_aspm_cap_init()
800 		link->aspm_enabled = PCIE_LINK_STATE_ASPM_ALL;  in pcie_aspm_cap_init()
801 		link->aspm_disable = PCIE_LINK_STATE_ASPM_ALL;  in pcie_aspm_cap_init()
806 	 * If ASPM not supported, don't mess with the clocks and link,  in pcie_aspm_cap_init()
815 	pcie_aspm_configure_common_clock(link);  in pcie_aspm_cap_init()
820 	 * read-only Link Capabilities may change depending on common clock  in pcie_aspm_cap_init()
841 	 * given link unless components on both sides of the link each  in pcie_aspm_cap_init()
845 		link->aspm_support |= PCIE_LINK_STATE_L0S;  in pcie_aspm_cap_init()
848 		link->aspm_enabled |= PCIE_LINK_STATE_L0S_UP;  in pcie_aspm_cap_init()
850 		link->aspm_enabled |= PCIE_LINK_STATE_L0S_DW;  in pcie_aspm_cap_init()
854 		link->aspm_support |= PCIE_LINK_STATE_L1;  in pcie_aspm_cap_init()
857 		link->aspm_enabled |= PCIE_LINK_STATE_L1;  in pcie_aspm_cap_init()
859 	aspm_l1ss_init(link);  in pcie_aspm_cap_init()
869 	link->aspm_default = link->aspm_enabled;  in pcie_aspm_cap_init()
872 	link->aspm_capable = link->aspm_support;  in pcie_aspm_cap_init()
885 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_l1ss()  argument
888 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_l1ss()
926 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_link()  argument
929 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_link()
933 	state &= (link->aspm_capable & ~link->aspm_disable);  in pcie_config_aspm_link()
942 		state |= (link->aspm_enabled & PCIE_LINK_STATE_L1_SS_PCIPM);  in pcie_config_aspm_link()
945 	/* Nothing to do if the link is already in the requested state */  in pcie_config_aspm_link()
946 	if (link->aspm_enabled == state)  in pcie_config_aspm_link()
976 	if (link->aspm_capable & PCIE_LINK_STATE_L1SS)  in pcie_config_aspm_link()
977 		pcie_config_aspm_l1ss(link, state);  in pcie_config_aspm_link()
983 	link->aspm_enabled = state;  in pcie_config_aspm_link()
986 	pci_save_aspm_l1ss_state(link->downstream);  in pcie_config_aspm_link()
987 	pci_update_aspm_saved_state(link->downstream);  in pcie_config_aspm_link()
992 static void pcie_config_aspm_path(struct pcie_link_state *link)  in pcie_config_aspm_path()  argument
994 	while (link) {  in pcie_config_aspm_path()
995 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_config_aspm_path()
996 		link = link->parent;  in pcie_config_aspm_path()
1000 static void free_link_state(struct pcie_link_state *link)  in free_link_state()  argument
1002 	link->pdev->link_state = NULL;  in free_link_state()
1003 	kfree(link);  in free_link_state()
1043 	struct pcie_link_state *link;  in alloc_pcie_link_state()  local
1045 	link = kzalloc(sizeof(*link), GFP_KERNEL);  in alloc_pcie_link_state()
1046 	if (!link)  in alloc_pcie_link_state()
1049 	INIT_LIST_HEAD(&link->sibling);  in alloc_pcie_link_state()
1050 	link->pdev = pdev;  in alloc_pcie_link_state()
1051 	link->downstream = pci_function_0(pdev->subordinate);  in alloc_pcie_link_state()
1057 	 * a switch may become the root of the link state chain for all  in alloc_pcie_link_state()
1063 		link->root = link;  in alloc_pcie_link_state()
1069 			kfree(link);  in alloc_pcie_link_state()
1073 		link->parent = parent;  in alloc_pcie_link_state()
1074 		link->root = link->parent->root;  in alloc_pcie_link_state()
1077 	list_add(&link->sibling, &link_list);  in alloc_pcie_link_state()
1078 	pdev->link_state = link;  in alloc_pcie_link_state()
1079 	return link;  in alloc_pcie_link_state()
1091  * pcie_aspm_init_link_state: Initiate PCI express link state.
1097 	struct pcie_link_state *link;  in pcie_aspm_init_link_state()  local
1108 	 * end of a Link, so there's nothing to do unless this device is  in pcie_aspm_init_link_state()
1124 	link = alloc_pcie_link_state(pdev);  in pcie_aspm_init_link_state()
1125 	if (!link)  in pcie_aspm_init_link_state()
1132 	pcie_aspm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
1135 	pcie_clkpm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
1139 	 * link policy setting. Enabling ASPM on broken hardware can cripple  in pcie_aspm_init_link_state()
1147 		pcie_config_aspm_path(link);  in pcie_aspm_init_link_state()
1148 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_init_link_state()
1234 	struct pcie_link_state *link;  in pcie_update_aspm_capable()  local
1236 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
1237 		if (link->root != root)  in pcie_update_aspm_capable()
1239 		link->aspm_capable = link->aspm_support;  in pcie_update_aspm_capable()
1241 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
1243 		struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_update_aspm_capable()
1244 		if (link->root != root)  in pcie_update_aspm_capable()
1259 	struct pcie_link_state *link, *root, *parent_link;  in pcie_aspm_exit_link_state()  local
1267 	link = parent->link_state;  in pcie_aspm_exit_link_state()
1268 	root = link->root;  in pcie_aspm_exit_link_state()
1269 	parent_link = link->parent;  in pcie_aspm_exit_link_state()
1272 	 * Free the parent link state, no later than function 0 (i.e.  in pcie_aspm_exit_link_state()
1273 	 * link->downstream) being removed.  in pcie_aspm_exit_link_state()
1275 	 * Do not free the link state any earlier. If function 0 is a  in pcie_aspm_exit_link_state()
1276 	 * switch upstream port, this link state is parent_link to all  in pcie_aspm_exit_link_state()
1279 	if (pdev != link->downstream)  in pcie_aspm_exit_link_state()
1282 	pcie_config_aspm_link(link, 0);  in pcie_aspm_exit_link_state()
1283 	list_del(&link->sibling);  in pcie_aspm_exit_link_state()
1284 	free_link_state(link);  in pcie_aspm_exit_link_state()
1303 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_pm_state_change()  local
1305 	if (aspm_disabled || !link)  in pcie_aspm_pm_state_change()
1314 	pcie_update_aspm_capable(link->root);  in pcie_aspm_pm_state_change()
1315 	pcie_config_aspm_path(link);  in pcie_aspm_pm_state_change()
1323 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_powersave_config_link()  local
1325 	if (aspm_disabled || !link)  in pcie_aspm_powersave_config_link()
1334 	pcie_config_aspm_path(link);  in pcie_aspm_powersave_config_link()
1335 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_powersave_config_link()
1378 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_disable_link_state()  local
1380 	if (!link)  in __pci_disable_link_state()
1398 	link->aspm_disable |= pci_calc_aspm_disable_mask(state);  in __pci_disable_link_state()
1399 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_disable_link_state()
1402 		link->clkpm_disable = 1;  in __pci_disable_link_state()
1403 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_disable_link_state()
1420  * pci_disable_link_state - Disable device's link state, so the link will
1426  * @state: ASPM link state to disable
1436 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_enable_link_state()  local
1438 	if (!link)  in __pci_enable_link_state()
1454 	link->aspm_default = pci_calc_aspm_enable_mask(state);  in __pci_enable_link_state()
1455 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_enable_link_state()
1457 	link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;  in __pci_enable_link_state()
1458 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_enable_link_state()
1467  * pci_enable_link_state - Clear and set the default device link state so that
1468  * the link may be allowed to enter the specified states. Note that if the
1477  * @state: Mask of ASPM link states to enable
1486  * pci_enable_link_state_locked - Clear and set the default device link state
1487  * so that the link may be allowed to enter the specified states. Note that if
1496  * @state: Mask of ASPM link states to enable
1512 	struct pcie_link_state *link;  in pcie_aspm_set_policy()  local
1525 	list_for_each_entry(link, &link_list, sibling) {  in pcie_aspm_set_policy()
1526 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_aspm_set_policy()
1527 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_set_policy()
1560 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in pcie_aspm_enabled()  local
1562 	if (!link)  in pcie_aspm_enabled()
1565 	return link->aspm_enabled;  in pcie_aspm_enabled()
1574 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_show_common()  local
1576 	return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);  in aspm_attr_show_common()
1584 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_store_common()  local
1594 		link->aspm_disable &= ~state;  in aspm_attr_store_common()
1597 			link->aspm_disable &= ~PCIE_LINK_STATE_L1;  in aspm_attr_store_common()
1599 		link->aspm_disable |= state;  in aspm_attr_store_common()
1601 			link->aspm_disable |= PCIE_LINK_STATE_L1SS;  in aspm_attr_store_common()
1604 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in aspm_attr_store_common()
1633 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in ASPM_ATTR()  local
1635 	return sysfs_emit(buf, "%d\n", link->clkpm_enabled);  in ASPM_ATTR()
1643 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in clkpm_store()  local
1652 	link->clkpm_disable = !state_enable;  in clkpm_store()
1653 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in clkpm_store()
1685 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_ctrl_attrs_are_visible()  local
1695 	if (aspm_disabled || !link)  in aspm_ctrl_attrs_are_visible()
1699 		return link->clkpm_capable ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1701 	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1705 	.name = "link",