Lines Matching +full:dma +full:- +full:router
1 // SPDX-License-Identifier: GPL-2.0
3 * Thunderbolt driver - bus logic (NHI independent)
30 * direction. This is 40G - 10% guard band bandwidth.
53 * struct tb_cm - Simple Thunderbolt connection manager
74 return ((void *)tcm - sizeof(struct tb)); in tcm_to_tb()
101 ev->tb = tb; in tb_queue_hotplug()
102 ev->route = route; in tb_queue_hotplug()
103 ev->port = port; in tb_queue_hotplug()
104 ev->unplug = unplug; in tb_queue_hotplug()
105 INIT_DELAYED_WORK(&ev->work, tb_handle_hotplug); in tb_queue_hotplug()
106 queue_delayed_work(tb->wq, &ev->work, 0); in tb_queue_hotplug()
113 struct tb_cm *tcm = tb_priv(sw->tb); in tb_add_dp_resources()
124 * If DP IN on device router exist, position it at the in tb_add_dp_resources()
126 * before DP IN of the host router. This way external GPU(s) in tb_add_dp_resources()
130 list_add(&port->list, &tcm->dp_resources); in tb_add_dp_resources()
132 list_add_tail(&port->list, &tcm->dp_resources); in tb_add_dp_resources()
140 struct tb_cm *tcm = tb_priv(sw->tb); in tb_remove_dp_resources()
146 tb_remove_dp_resources(port->remote->sw); in tb_remove_dp_resources()
149 list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) { in tb_remove_dp_resources()
150 if (port->sw == sw) { in tb_remove_dp_resources()
152 list_del_init(&port->list); in tb_remove_dp_resources()
162 list_for_each_entry(p, &tcm->dp_resources, list) { in tb_discover_dp_resource()
169 list_add_tail(&port->list, &tcm->dp_resources); in tb_discover_dp_resource()
177 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_discover_dp_resources()
179 tb_discover_dp_resource(tb, tunnel->dst_port); in tb_discover_dp_resources()
183 /* Enables CL states up to host router */
186 struct tb_cm *tcm = tb_priv(sw->tb); in tb_enable_clx()
208 * If we are re-enabling then check if there is an active DMA in tb_enable_clx()
211 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_enable_clx()
223 if (ret == -EOPNOTSUPP) in tb_enable_clx()
225 return ret == -EOPNOTSUPP ? 0 : ret; in tb_enable_clx()
229 * tb_disable_clx() - Disable CL states up to host router
230 * @sw: Router to start
232 * Disables CL states from @sw up to the host router. Returns true if
292 * accuracy of first depth child routers (and the host router) in tb_increase_tmu_accuracy()
299 sw = tunnel->tb->root_switch; in tb_increase_tmu_accuracy()
300 device_for_each_child(&sw->dev, NULL, tb_increase_switch_tmu_accuracy); in tb_increase_tmu_accuracy()
317 return device_for_each_child(&tb->dev, NULL, in tb_tmu_hifi_uni_required()
327 * enable the enhanched uni-directional mode. That covers all in tb_enable_tmu()
334 if (ret == -EOPNOTSUPP) { in tb_enable_tmu()
337 * Figure out uni-directional HiFi TMU requirements in tb_enable_tmu()
339 * uni-directional HiFi requirements we can put the TMU in tb_enable_tmu()
342 * Deliberately skip bi-directional HiFi links in tb_enable_tmu()
346 if (tb_tmu_hifi_uni_required(sw->tb)) in tb_enable_tmu()
356 /* If not supported, fallback to bi-directional HiFi */ in tb_enable_tmu()
357 if (ret == -EOPNOTSUPP) in tb_enable_tmu()
382 struct tb *tb = sw->tb; in tb_switch_discover_tunnels()
388 switch (port->config.type) { in tb_switch_discover_tunnels()
407 list_add_tail(&tunnel->list, list); in tb_switch_discover_tunnels()
412 tb_switch_discover_tunnels(port->remote->sw, list, in tb_switch_discover_tunnels()
420 if (tb_switch_is_usb4(port->sw)) in tb_port_configure_xdomain()
427 if (tb_switch_is_usb4(port->sw)) in tb_port_unconfigure_xdomain()
435 struct tb_switch *sw = port->sw; in tb_scan_xdomain()
436 struct tb *tb = sw->tb; in tb_scan_xdomain()
450 xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid, in tb_scan_xdomain()
453 tb_port_at(route, sw)->xdomain = xd; in tb_scan_xdomain()
460 * tb_find_unused_port() - return the first inactive port on @sw
472 if (port->config.type != type) in tb_find_unused_port()
474 if (!port->cap_adap) in tb_find_unused_port()
501 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_find_tunnel()
502 if (tunnel->type == type && in tb_find_tunnel()
503 ((src_port && src_port == tunnel->src_port) || in tb_find_tunnel()
504 (dst_port && dst_port == tunnel->dst_port))) { in tb_find_tunnel()
519 /* Pick the router that is deepest in the topology */ in tb_find_first_usb3_tunnel()
521 sw = dst_port->sw; in tb_find_first_usb3_tunnel()
523 sw = src_port->sw; in tb_find_first_usb3_tunnel()
525 /* Can't be the host router */ in tb_find_first_usb3_tunnel()
526 if (sw == tb->root_switch) in tb_find_first_usb3_tunnel()
529 /* Find the downstream USB4 port that leads to this router */ in tb_find_first_usb3_tunnel()
530 port = tb_port_at(tb_route(sw), tb->root_switch); in tb_find_first_usb3_tunnel()
531 /* Find the corresponding host router USB3 downstream port */ in tb_find_first_usb3_tunnel()
532 usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port); in tb_find_first_usb3_tunnel()
540 * tb_consumed_usb3_pcie_bandwidth() - Consumed USB3/PCIe bandwidth over a single link
589 * tb_consumed_dp_bandwidth() - Consumed DP bandwidth over a single link
624 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_consumed_dp_bandwidth()
642 group = tunnel->src_port->group; in tb_consumed_dp_bandwidth()
643 if (group && group->reserved && !group_reserved[group->index]) in tb_consumed_dp_bandwidth()
644 group_reserved[group->index] = group->reserved; in tb_consumed_dp_bandwidth()
649 * re-calculating estimated bandwidth. in tb_consumed_dp_bandwidth()
651 if (tunnel->src_port == src_port && in tb_consumed_dp_bandwidth()
652 tunnel->dst_port == dst_port) in tb_consumed_dp_bandwidth()
690 * tb_maximum_bandwidth() - Maximum bandwidth over a single link
720 link_speed = port->sw->link_speed; in tb_maximum_bandwidth()
722 * sw->link_width is from upstream perspective so we use in tb_maximum_bandwidth()
723 * the opposite for downstream of the host router. in tb_maximum_bandwidth()
725 if (port->sw->link_width == TB_LINK_WIDTH_ASYM_TX) { in tb_maximum_bandwidth()
728 } else if (port->sw->link_width == TB_LINK_WIDTH_ASYM_RX) { in tb_maximum_bandwidth()
746 up_bw = link_speed * port->sw->link_width * 1000; in tb_maximum_bandwidth()
785 *max_up = up_bw - up_bw / 10; in tb_maximum_bandwidth()
786 *max_down = down_bw - down_bw / 10; in tb_maximum_bandwidth()
793 * tb_available_bandwidth() - Available bandwidth for tunneling
838 max_up -= consumed_up; in tb_available_bandwidth()
839 max_down -= consumed_down; in tb_available_bandwidth()
845 max_up -= consumed_up; in tb_available_bandwidth()
846 max_down -= consumed_down; in tb_available_bandwidth()
888 ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port, in tb_reclaim_usb3_bandwidth()
918 if (!sw->link_usb4) in tb_tunnel_usb3()
958 ret = -ENOMEM; in tb_tunnel_usb3()
965 ret = -EIO; in tb_tunnel_usb3()
969 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_usb3()
993 ret = tb_tunnel_usb3(sw->tb, sw); in tb_create_usb3_tunnels()
1001 ret = tb_create_usb3_tunnels(port->remote->sw); in tb_create_usb3_tunnels()
1010 * tb_configure_asym() - Transition links to asymmetric if needed
1037 /* Pick up router deepest in the hierarchy */ in tb_configure_asym()
1039 sw = dst_port->sw; in tb_configure_asym()
1041 sw = src_port->sw; in tb_configure_asym()
1044 struct tb_port *down = tb_switch_downstream_port(up->sw); in tb_configure_asym()
1056 * (40G - guard band 10%), and the requested is above in tb_configure_asym()
1060 ret = -ENOBUFS; in tb_configure_asym()
1072 ret = -ENOBUFS; in tb_configure_asym()
1082 if (up->sw->link_width == width_up) in tb_configure_asym()
1099 tb_sw_dbg(up->sw, "configuring asymmetric link\n"); in tb_configure_asym()
1105 ret = tb_switch_set_link_width(up->sw, width_up); in tb_configure_asym()
1107 tb_sw_warn(up->sw, "failed to set link width\n"); in tb_configure_asym()
1112 /* Re-enable CL states if they were previosly enabled */ in tb_configure_asym()
1120 * tb_configure_sym() - Transition links to symmetric if possible
1142 /* Pick up router deepest in the hierarchy */ in tb_configure_sym()
1144 sw = dst_port->sw; in tb_configure_sym()
1146 sw = src_port->sw; in tb_configure_sym()
1152 if (up->sw->link_width <= TB_LINK_WIDTH_DUAL) in tb_configure_sym()
1155 if (up->sw->is_unplugged) in tb_configure_sym()
1177 if (up->sw->link_width == TB_LINK_WIDTH_DUAL) in tb_configure_sym()
1184 * However, if the router prefers asymmetric link we in tb_configure_sym()
1188 up->sw->preferred_link_width > TB_LINK_WIDTH_DUAL) { in tb_configure_sym()
1189 tb_sw_dbg(up->sw, "keeping preferred asymmetric link\n"); in tb_configure_sym()
1199 tb_sw_dbg(up->sw, "configuring symmetric link\n"); in tb_configure_sym()
1201 ret = tb_switch_set_link_width(up->sw, TB_LINK_WIDTH_DUAL); in tb_configure_sym()
1203 tb_sw_warn(up->sw, "failed to set link width\n"); in tb_configure_sym()
1208 /* Re-enable CL states if they were previosly enabled */ in tb_configure_sym()
1218 struct tb *tb = sw->tb; in tb_configure_link()
1221 down->remote = up; in tb_configure_link()
1222 up->remote = down; in tb_configure_link()
1223 if (down->dual_link_port && up->dual_link_port) { in tb_configure_link()
1224 down->dual_link_port->remote = up->dual_link_port; in tb_configure_link()
1225 up->dual_link_port->remote = down->dual_link_port; in tb_configure_link()
1232 if (sw->link_width < TB_LINK_WIDTH_DUAL) in tb_configure_link()
1236 * Device router that comes up as symmetric link is in tb_configure_link()
1242 up->sw->link_width == TB_LINK_WIDTH_DUAL) { in tb_configure_link()
1245 host_port = tb_port_at(tb_route(sw), tb->root_switch); in tb_configure_link()
1254 * tb_scan_switch() - scan for and initialize downstream switches
1260 pm_runtime_get_sync(&sw->dev); in tb_scan_switch()
1265 pm_runtime_mark_last_busy(&sw->dev); in tb_scan_switch()
1266 pm_runtime_put_autosuspend(&sw->dev); in tb_scan_switch()
1270 * tb_scan_port() - check for and initialize switches below port
1274 struct tb_cm *tcm = tb_priv(port->sw->tb); in tb_scan_port()
1285 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port, in tb_scan_port()
1290 if (port->config.type != TB_TYPE_PORT) in tb_scan_port()
1292 if (port->dual_link_port && port->link_nr) in tb_scan_port()
1298 if (port->usb4) in tb_scan_port()
1299 pm_runtime_get_sync(&port->usb4->dev); in tb_scan_port()
1303 if (port->remote) { in tb_scan_port()
1310 sw = tb_switch_alloc(port->sw->tb, &port->sw->dev, in tb_scan_port()
1318 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL) in tb_scan_port()
1332 if (port->xdomain) { in tb_scan_port()
1333 tb_xdomain_remove(port->xdomain); in tb_scan_port()
1335 port->xdomain = NULL; in tb_scan_port()
1343 if (!tcm->hotplug_active) { in tb_scan_port()
1344 dev_set_uevent_suppress(&sw->dev, true); in tb_scan_port()
1352 sw->rpm = sw->generation > 1; in tb_scan_port()
1376 * enabled for the upstream port of the router so we do it here. in tb_scan_port()
1389 if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw)) in tb_scan_port()
1396 if (port->usb4) { in tb_scan_port()
1397 pm_runtime_mark_last_busy(&port->usb4->dev); in tb_scan_port()
1398 pm_runtime_put_autosuspend(&port->usb4->dev); in tb_scan_port()
1406 struct tb *tb = group->tb; in tb_recalc_estimated_bandwidth_for_group()
1410 tb_dbg(tb, "re-calculating bandwidth estimation for group %u\n", in tb_recalc_estimated_bandwidth_for_group()
1411 group->index); in tb_recalc_estimated_bandwidth_for_group()
1414 list_for_each_entry(in, &group->ports, group_list) { in tb_recalc_estimated_bandwidth_for_group()
1429 * tunnels under the host router USB4 port, even in tb_recalc_estimated_bandwidth_for_group()
1430 * if they do not begin from the host router, we in tb_recalc_estimated_bandwidth_for_group()
1436 first_tunnel->src_port, first_tunnel->dst_port); in tb_recalc_estimated_bandwidth_for_group()
1444 out = tunnel->dst_port; in tb_recalc_estimated_bandwidth_for_group()
1449 "failed to re-calculate estimated bandwidth\n"); in tb_recalc_estimated_bandwidth_for_group()
1455 * - already allocated bandwidth for the DP tunnel in tb_recalc_estimated_bandwidth_for_group()
1456 * - available bandwidth along the path in tb_recalc_estimated_bandwidth_for_group()
1457 * - bandwidth allocated for USB 3.x but not used. in tb_recalc_estimated_bandwidth_for_group()
1469 "re-calculated estimated bandwidth %u (+ %u reserved) = %u Mb/s\n", in tb_recalc_estimated_bandwidth_for_group()
1470 estimated_bw, group->reserved, in tb_recalc_estimated_bandwidth_for_group()
1471 estimated_bw + group->reserved); in tb_recalc_estimated_bandwidth_for_group()
1474 estimated_bw + group->reserved)) in tb_recalc_estimated_bandwidth_for_group()
1480 tb_reclaim_usb3_bandwidth(tb, first_tunnel->src_port, in tb_recalc_estimated_bandwidth_for_group()
1481 first_tunnel->dst_port); in tb_recalc_estimated_bandwidth_for_group()
1483 tb_dbg(tb, "bandwidth estimation for group %u done\n", group->index); in tb_recalc_estimated_bandwidth_for_group()
1491 tb_dbg(tb, "bandwidth consumption changed, re-calculating estimated bandwidth\n"); in tb_recalc_estimated_bandwidth()
1493 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_recalc_estimated_bandwidth()
1494 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_recalc_estimated_bandwidth()
1496 if (!list_empty(&group->ports)) in tb_recalc_estimated_bandwidth()
1500 tb_dbg(tb, "bandwidth re-calculation done\n"); in tb_recalc_estimated_bandwidth()
1505 if (group->reserved) { in __release_group_bandwidth()
1506 tb_dbg(group->tb, "group %d released total %d Mb/s\n", group->index, in __release_group_bandwidth()
1507 group->reserved); in __release_group_bandwidth()
1508 group->reserved = 0; in __release_group_bandwidth()
1519 if (list_empty(&group->ports)) in __configure_group_sym()
1532 in = list_first_entry(&group->ports, struct tb_port, group_list); in __configure_group_sym()
1533 tunnel = tb_find_tunnel(group->tb, TB_TUNNEL_DP, in, NULL); in __configure_group_sym()
1535 tb_configure_sym(group->tb, in, tunnel->dst_port, true); in __configure_group_sym()
1542 struct tb *tb = group->tb; in tb_bandwidth_group_release_work()
1544 mutex_lock(&tb->lock); in tb_bandwidth_group_release_work()
1548 mutex_unlock(&tb->lock); in tb_bandwidth_group_release_work()
1555 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_init_bandwidth_groups()
1556 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_init_bandwidth_groups()
1558 group->tb = tcm_to_tb(tcm); in tb_init_bandwidth_groups()
1559 group->index = i + 1; in tb_init_bandwidth_groups()
1560 INIT_LIST_HEAD(&group->ports); in tb_init_bandwidth_groups()
1561 INIT_DELAYED_WORK(&group->release_work, in tb_init_bandwidth_groups()
1569 if (!group || WARN_ON(in->group)) in tb_bandwidth_group_attach_port()
1572 in->group = group; in tb_bandwidth_group_attach_port()
1573 list_add_tail(&in->group_list, &group->ports); in tb_bandwidth_group_attach_port()
1575 tb_port_dbg(in, "attached to bandwidth group %d\n", group->index); in tb_bandwidth_group_attach_port()
1582 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_find_free_bandwidth_group()
1583 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_find_free_bandwidth_group()
1585 if (list_empty(&group->ports)) in tb_find_free_bandwidth_group()
1605 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_attach_bandwidth_group()
1609 if (tunnel->src_port->sw == in->sw && in tb_attach_bandwidth_group()
1610 tunnel->dst_port->sw == out->sw) { in tb_attach_bandwidth_group()
1611 group = tunnel->src_port->group; in tb_attach_bandwidth_group()
1636 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_discover_bandwidth_group()
1637 if (tcm->groups[i].index == index) { in tb_discover_bandwidth_group()
1638 tb_bandwidth_group_attach_port(&tcm->groups[i], in); in tb_discover_bandwidth_group()
1649 struct tb_bandwidth_group *group = in->group; in tb_detach_bandwidth_group()
1652 in->group = NULL; in tb_detach_bandwidth_group()
1653 list_del_init(&in->group_list); in tb_detach_bandwidth_group()
1655 tb_port_dbg(in, "detached from bandwidth group %d\n", group->index); in tb_detach_bandwidth_group()
1658 if (list_empty(&group->ports)) { in tb_detach_bandwidth_group()
1659 cancel_delayed_work(&group->release_work); in tb_detach_bandwidth_group()
1670 tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true); in tb_discover_tunnels()
1672 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_discover_tunnels()
1674 struct tb_switch *parent = tunnel->dst_port->sw; in tb_discover_tunnels()
1676 while (parent != tunnel->src_port->sw) { in tb_discover_tunnels()
1677 parent->boot = true; in tb_discover_tunnels()
1681 struct tb_port *in = tunnel->src_port; in tb_discover_tunnels()
1682 struct tb_port *out = tunnel->dst_port; in tb_discover_tunnels()
1685 pm_runtime_get_sync(&in->sw->dev); in tb_discover_tunnels()
1686 pm_runtime_get_sync(&out->sw->dev); in tb_discover_tunnels()
1702 list_del(&tunnel->list); in tb_deactivate_and_free_tunnel()
1704 tb = tunnel->tb; in tb_deactivate_and_free_tunnel()
1705 src_port = tunnel->src_port; in tb_deactivate_and_free_tunnel()
1706 dst_port = tunnel->dst_port; in tb_deactivate_and_free_tunnel()
1708 switch (tunnel->type) { in tb_deactivate_and_free_tunnel()
1715 tb_switch_dealloc_dp_resource(src_port->sw, src_port); in tb_deactivate_and_free_tunnel()
1722 pm_runtime_mark_last_busy(&dst_port->sw->dev); in tb_deactivate_and_free_tunnel()
1723 pm_runtime_put_autosuspend(&dst_port->sw->dev); in tb_deactivate_and_free_tunnel()
1724 pm_runtime_mark_last_busy(&src_port->sw->dev); in tb_deactivate_and_free_tunnel()
1725 pm_runtime_put_autosuspend(&src_port->sw->dev); in tb_deactivate_and_free_tunnel()
1734 * PCIe and DMA tunnels do not consume guaranteed in tb_deactivate_and_free_tunnel()
1744 * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
1752 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_free_invalid_tunnels()
1759 * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
1769 if (port->remote->sw->is_unplugged) { in tb_free_unplugged_children()
1771 tb_remove_dp_resources(port->remote->sw); in tb_free_unplugged_children()
1772 tb_switch_unconfigure_link(port->remote->sw); in tb_free_unplugged_children()
1773 tb_switch_set_link_width(port->remote->sw, in tb_free_unplugged_children()
1775 tb_switch_remove(port->remote->sw); in tb_free_unplugged_children()
1776 port->remote = NULL; in tb_free_unplugged_children()
1777 if (port->dual_link_port) in tb_free_unplugged_children()
1778 port->dual_link_port->remote = NULL; in tb_free_unplugged_children()
1780 tb_free_unplugged_children(port->remote->sw); in tb_free_unplugged_children()
1797 int phy_port = tb_phy_port_from_link(port->port); in tb_find_pcie_down()
1801 * Hard-coded Thunderbolt port to PCIe down port mapping in tb_find_pcie_down()
1814 /* Validate the hard-coding */ in tb_find_pcie_down()
1815 if (WARN_ON(index > sw->config.max_port_number)) in tb_find_pcie_down()
1818 down = &sw->ports[index]; in tb_find_pcie_down()
1839 host_port = tb_route(in->sw) ? in tb_find_dp_out()
1840 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL; in tb_find_dp_out()
1842 list_for_each_entry(port, &tcm->dp_resources, list) { in tb_find_dp_out()
1852 if (in->sw == port->sw) { in tb_find_dp_out()
1853 tb_port_dbg(port, "skipping DP OUT on same router\n"); in tb_find_dp_out()
1861 * the same host router downstream port. in tb_find_dp_out()
1863 if (host_port && tb_route(port->sw)) { in tb_find_dp_out()
1866 p = tb_port_at(tb_route(port->sw), tb->root_switch); in tb_find_dp_out()
1879 struct tb_port *in = tunnel->src_port; in tb_dp_tunnel_active()
1880 struct tb_port *out = tunnel->dst_port; in tb_dp_tunnel_active()
1883 mutex_lock(&tb->lock); in tb_dp_tunnel_active()
1911 * router's 1st children TMU mode to HiFi for in tb_dp_tunnel_active()
1917 struct tb_port *in = tunnel->src_port; in tb_dp_tunnel_active()
1924 * discrete router. in tb_dp_tunnel_active()
1928 * also tear down the tunnel and try to re-use the in tb_dp_tunnel_active()
1937 mutex_unlock(&tb->lock); in tb_dp_tunnel_active()
1956 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_tunnel_one_dp()
1971 pm_runtime_get_sync(&in->sw->dev); in tb_tunnel_one_dp()
1972 pm_runtime_get_sync(&out->sw->dev); in tb_tunnel_one_dp()
1974 if (tb_switch_alloc_dp_resource(in->sw, in)) { in tb_tunnel_one_dp()
2005 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_one_dp()
2008 if (ret && ret != -EINPROGRESS) { in tb_tunnel_one_dp()
2010 list_del(&tunnel->list); in tb_tunnel_one_dp()
2024 tb_switch_dealloc_dp_resource(in->sw, in); in tb_tunnel_one_dp()
2026 pm_runtime_mark_last_busy(&out->sw->dev); in tb_tunnel_one_dp()
2027 pm_runtime_put_autosuspend(&out->sw->dev); in tb_tunnel_one_dp()
2028 pm_runtime_mark_last_busy(&in->sw->dev); in tb_tunnel_one_dp()
2029 pm_runtime_put_autosuspend(&in->sw->dev); in tb_tunnel_one_dp()
2046 tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n"); in tb_tunnel_dp()
2050 list_for_each_entry(port, &tcm->dp_resources, list) { in tb_tunnel_dp()
2075 struct tb_switch *sw = port->sw; in tb_enter_redrive()
2077 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_enter_redrive()
2081 * If we get hot-unplug for the DP IN port of the host router in tb_enter_redrive()
2083 * is a monitor connected directly to the Type-C port and we are in tb_enter_redrive()
2092 port->redrive = true; in tb_enter_redrive()
2093 pm_runtime_get(&sw->dev); in tb_enter_redrive()
2100 struct tb_switch *sw = port->sw; in tb_exit_redrive()
2102 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_exit_redrive()
2109 if (port->redrive && tb_switch_query_dp_resource(sw, port)) { in tb_exit_redrive()
2110 port->redrive = false; in tb_exit_redrive()
2111 pm_runtime_put(&sw->dev); in tb_exit_redrive()
2132 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_switch_exit_redrive()
2139 if (port->redrive) { in tb_switch_exit_redrive()
2140 port->redrive = false; in tb_switch_exit_redrive()
2141 pm_runtime_put(&sw->dev); in tb_switch_exit_redrive()
2168 list_del_init(&port->list); in tb_dp_resource_unavailable()
2186 list_for_each_entry(p, &tcm->dp_resources, list) { in tb_dp_resource_available()
2193 list_add_tail(&port->list, &tcm->dp_resources); in tb_dp_resource_available()
2196 /* Look for suitable DP IN <-> DP OUT pairs now */ in tb_dp_resource_available()
2207 * will be re-established after resume based on plug events. in tb_disconnect_and_release_dp()
2209 list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) { in tb_disconnect_and_release_dp()
2214 while (!list_empty(&tcm->dp_resources)) { in tb_disconnect_and_release_dp()
2217 port = list_first_entry(&tcm->dp_resources, in tb_disconnect_and_release_dp()
2219 list_del_init(&port->list); in tb_disconnect_and_release_dp()
2230 return -ENODEV; in tb_disconnect_pci()
2234 return -ENODEV; in tb_disconnect_pci()
2239 list_del(&tunnel->list); in tb_disconnect_pci()
2265 return -ENOMEM; in tb_tunnel_pci()
2271 return -EIO; in tb_tunnel_pci()
2284 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_pci()
2298 sw = tb_to_switch(xd->dev.parent); in tb_approve_xdomain_paths()
2299 dst_port = tb_port_at(xd->route, sw); in tb_approve_xdomain_paths()
2300 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); in tb_approve_xdomain_paths()
2302 mutex_lock(&tb->lock); in tb_approve_xdomain_paths()
2305 * When tunneling DMA paths the link should not enter CL states in tb_approve_xdomain_paths()
2313 ret = -ENOMEM; in tb_approve_xdomain_paths()
2319 "DMA tunnel activation failed, aborting\n"); in tb_approve_xdomain_paths()
2320 ret = -EIO; in tb_approve_xdomain_paths()
2324 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_approve_xdomain_paths()
2325 mutex_unlock(&tb->lock); in tb_approve_xdomain_paths()
2332 mutex_unlock(&tb->lock); in tb_approve_xdomain_paths()
2346 sw = tb_to_switch(xd->dev.parent); in __tb_disconnect_xdomain_paths()
2347 dst_port = tb_port_at(xd->route, sw); in __tb_disconnect_xdomain_paths()
2348 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); in __tb_disconnect_xdomain_paths()
2350 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in __tb_disconnect_xdomain_paths()
2353 if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port) in __tb_disconnect_xdomain_paths()
2362 * Try to re-enable CL states now, it is OK if this fails in __tb_disconnect_xdomain_paths()
2363 * because we may still have another DMA tunnel active through in __tb_disconnect_xdomain_paths()
2364 * the same host router USB4 downstream port. in __tb_disconnect_xdomain_paths()
2373 if (!xd->is_unplugged) { in tb_disconnect_xdomain_paths()
2374 mutex_lock(&tb->lock); in tb_disconnect_xdomain_paths()
2378 mutex_unlock(&tb->lock); in tb_disconnect_xdomain_paths()
2386 * tb_handle_hotplug() - handle hotplug event
2388 * Executes on tb->wq.
2393 struct tb *tb = ev->tb; in tb_handle_hotplug()
2399 pm_runtime_get_sync(&tb->dev); in tb_handle_hotplug()
2401 mutex_lock(&tb->lock); in tb_handle_hotplug()
2402 if (!tcm->hotplug_active) in tb_handle_hotplug()
2405 sw = tb_switch_find_by_route(tb, ev->route); in tb_handle_hotplug()
2409 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2412 if (ev->port > sw->config.max_port_number) { in tb_handle_hotplug()
2415 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2418 port = &sw->ports[ev->port]; in tb_handle_hotplug()
2421 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2425 pm_runtime_get_sync(&sw->dev); in tb_handle_hotplug()
2427 if (ev->unplug) { in tb_handle_hotplug()
2432 tb_sw_set_unplugged(port->remote->sw); in tb_handle_hotplug()
2434 tb_remove_dp_resources(port->remote->sw); in tb_handle_hotplug()
2435 tb_switch_tmu_disable(port->remote->sw); in tb_handle_hotplug()
2436 tb_switch_unconfigure_link(port->remote->sw); in tb_handle_hotplug()
2437 tb_switch_set_link_width(port->remote->sw, in tb_handle_hotplug()
2439 tb_switch_remove(port->remote->sw); in tb_handle_hotplug()
2440 port->remote = NULL; in tb_handle_hotplug()
2441 if (port->dual_link_port) in tb_handle_hotplug()
2442 port->dual_link_port->remote = NULL; in tb_handle_hotplug()
2446 } else if (port->xdomain) { in tb_handle_hotplug()
2447 struct tb_xdomain *xd = tb_xdomain_get(port->xdomain); in tb_handle_hotplug()
2457 xd->is_unplugged = true; in tb_handle_hotplug()
2459 port->xdomain = NULL; in tb_handle_hotplug()
2460 __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1); in tb_handle_hotplug()
2465 } else if (!port->port) { in tb_handle_hotplug()
2472 } else if (port->remote) { in tb_handle_hotplug()
2474 } else if (!port->port && sw->authorized) { in tb_handle_hotplug()
2481 if (!port->remote) in tb_handle_hotplug()
2488 pm_runtime_mark_last_busy(&sw->dev); in tb_handle_hotplug()
2489 pm_runtime_put_autosuspend(&sw->dev); in tb_handle_hotplug()
2494 mutex_unlock(&tb->lock); in tb_handle_hotplug()
2496 pm_runtime_mark_last_busy(&tb->dev); in tb_handle_hotplug()
2497 pm_runtime_put_autosuspend(&tb->dev); in tb_handle_hotplug()
2509 struct tb *tb = tunnel->tb; in tb_alloc_dp_bandwidth()
2517 in = tunnel->src_port; in tb_alloc_dp_bandwidth()
2518 out = tunnel->dst_port; in tb_alloc_dp_bandwidth()
2571 ret = -ENOBUFS; in tb_alloc_dp_bandwidth()
2576 group = in->group; in tb_alloc_dp_bandwidth()
2580 if (tunnel->bw_mode) { in tb_alloc_dp_bandwidth()
2591 reserved = allocated_down - *requested_down; in tb_alloc_dp_bandwidth()
2593 reserved = allocated_up - *requested_up; in tb_alloc_dp_bandwidth()
2596 group->reserved += reserved; in tb_alloc_dp_bandwidth()
2598 group->index, reserved, group->reserved); in tb_alloc_dp_bandwidth()
2608 mod_delayed_work(system_wq, &group->release_work, in tb_alloc_dp_bandwidth()
2636 available_up, available_down, group->reserved); in tb_alloc_dp_bandwidth()
2639 available_up + group->reserved >= requested_up_corrected) || in tb_alloc_dp_bandwidth()
2641 available_down + group->reserved >= requested_down_corrected)) { in tb_alloc_dp_bandwidth()
2664 released = *requested_down - available_down; in tb_alloc_dp_bandwidth()
2667 released = *requested_up - available_up; in tb_alloc_dp_bandwidth()
2670 group->reserved -= released; in tb_alloc_dp_bandwidth()
2672 group->index, released, group->reserved); in tb_alloc_dp_bandwidth()
2675 ret = -ENOBUFS; in tb_alloc_dp_bandwidth()
2681 if (ret && ret != -ENODEV) { in tb_alloc_dp_bandwidth()
2700 struct tb *tb = ev->tb; in tb_handle_dp_bandwidth_request()
2705 pm_runtime_get_sync(&tb->dev); in tb_handle_dp_bandwidth_request()
2707 mutex_lock(&tb->lock); in tb_handle_dp_bandwidth_request()
2708 if (!tcm->hotplug_active) in tb_handle_dp_bandwidth_request()
2711 sw = tb_switch_find_by_route(tb, ev->route); in tb_handle_dp_bandwidth_request()
2713 tb_warn(tb, "bandwidth request from non-existent router %llx\n", in tb_handle_dp_bandwidth_request()
2714 ev->route); in tb_handle_dp_bandwidth_request()
2718 in = &sw->ports[ev->port]; in tb_handle_dp_bandwidth_request()
2720 tb_port_warn(in, "bandwidth request to non-DP IN adapter\n"); in tb_handle_dp_bandwidth_request()
2724 tb_port_dbg(in, "handling bandwidth allocation request, retry %d\n", ev->retry); in tb_handle_dp_bandwidth_request()
2733 if (tunnel->bw_mode) { in tb_handle_dp_bandwidth_request()
2738 tunnel->bw_mode = false; in tb_handle_dp_bandwidth_request()
2748 if (ret == -ENODATA) { in tb_handle_dp_bandwidth_request()
2768 requested_up = -1; in tb_handle_dp_bandwidth_request()
2772 requested_down = -1; in tb_handle_dp_bandwidth_request()
2777 if (ret == -ENOBUFS) { in tb_handle_dp_bandwidth_request()
2780 } else if (ret == -ENOTCONN) { in tb_handle_dp_bandwidth_request()
2789 if (ev->retry < TB_BW_ALLOC_RETRIES) { in tb_handle_dp_bandwidth_request()
2792 tb_queue_dp_bandwidth_request(tb, ev->route, in tb_handle_dp_bandwidth_request()
2793 ev->port, in tb_handle_dp_bandwidth_request()
2794 ev->retry + 1, in tb_handle_dp_bandwidth_request()
2816 mutex_unlock(&tb->lock); in tb_handle_dp_bandwidth_request()
2818 pm_runtime_mark_last_busy(&tb->dev); in tb_handle_dp_bandwidth_request()
2819 pm_runtime_put_autosuspend(&tb->dev); in tb_handle_dp_bandwidth_request()
2833 ev->tb = tb; in tb_queue_dp_bandwidth_request()
2834 ev->route = route; in tb_queue_dp_bandwidth_request()
2835 ev->port = port; in tb_queue_dp_bandwidth_request()
2836 ev->retry = retry; in tb_queue_dp_bandwidth_request()
2837 INIT_DELAYED_WORK(&ev->work, tb_handle_dp_bandwidth_request); in tb_queue_dp_bandwidth_request()
2838 queue_delayed_work(tb->wq, &ev->work, delay); in tb_queue_dp_bandwidth_request()
2845 switch (error->error) { in tb_handle_notification()
2849 if (tb_cfg_ack_notification(tb->ctl, route, error)) in tb_handle_notification()
2855 if (tb_cfg_ack_notification(tb->ctl, route, error)) in tb_handle_notification()
2858 tb_queue_dp_bandwidth_request(tb, route, error->port, 0, 0); in tb_handle_notification()
2868 * tb_schedule_hotplug_handler() - callback function for the control channel
2876 u64 route = tb_cfg_get_route(&pkg->header); in tb_handle_event()
2889 if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) { in tb_handle_event()
2891 pkg->port); in tb_handle_event()
2894 tb_queue_hotplug(tb, route, pkg->port, pkg->unplug); in tb_handle_event()
2903 cancel_delayed_work(&tcm->remove_work); in tb_stop()
2905 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_stop()
2907 * DMA tunnels require the driver to be functional so we in tb_stop()
2915 tb_switch_remove(tb->root_switch); in tb_stop()
2916 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ in tb_stop()
2925 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) in tb_deinit()
2926 cancel_delayed_work_sync(&tcm->groups[i].release_work); in tb_deinit()
2939 if (sw->boot) in tb_scan_finalize_switch()
2940 sw->authorized = 1; in tb_scan_finalize_switch()
2943 kobject_uevent(&dev->kobj, KOBJ_ADD); in tb_scan_finalize_switch()
2956 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0); in tb_start()
2957 if (IS_ERR(tb->root_switch)) in tb_start()
2958 return PTR_ERR(tb->root_switch); in tb_start()
2966 * implement the necessary router operations. in tb_start()
2968 tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch); in tb_start()
2970 tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch); in tb_start()
2972 ret = tb_switch_configure(tb->root_switch); in tb_start()
2974 tb_switch_put(tb->root_switch); in tb_start()
2979 ret = tb_switch_add(tb->root_switch); in tb_start()
2981 tb_switch_put(tb->root_switch); in tb_start()
2986 * To support highest CLx state, we set host router's TMU to in tb_start()
2989 tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES); in tb_start()
2991 tb_switch_tmu_enable(tb->root_switch); in tb_start()
2999 if (reset && tb_switch_is_usb4(tb->root_switch)) { in tb_start()
3001 if (usb4_switch_version(tb->root_switch) == 1) in tb_start()
3002 tb_switch_reset(tb->root_switch); in tb_start()
3007 tb_scan_switch(tb->root_switch); in tb_start()
3018 tb_create_usb3_tunnels(tb->root_switch); in tb_start()
3020 tb_add_dp_resources(tb->root_switch); in tb_start()
3021 tb_switch_enter_redrive(tb->root_switch); in tb_start()
3023 device_for_each_child(&tb->root_switch->dev, NULL, in tb_start()
3027 tcm->hotplug_active = true; in tb_start()
3037 tb_switch_exit_redrive(tb->root_switch); in tb_suspend_noirq()
3038 tb_switch_suspend(tb->root_switch, false); in tb_suspend_noirq()
3039 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ in tb_suspend_noirq()
3049 /* No need to restore if the router is already unplugged */ in tb_restore_children()
3050 if (sw->is_unplugged) in tb_restore_children()
3054 tb_sw_warn(sw, "failed to re-enable CL states\n"); in tb_restore_children()
3062 if (!tb_port_has_remote(port) && !port->xdomain) in tb_restore_children()
3065 if (port->remote) { in tb_restore_children()
3066 tb_switch_set_link_width(port->remote->sw, in tb_restore_children()
3067 port->remote->sw->link_width); in tb_restore_children()
3068 tb_switch_configure_link(port->remote->sw); in tb_restore_children()
3070 tb_restore_children(port->remote->sw); in tb_restore_children()
3071 } else if (port->xdomain) { in tb_restore_children()
3072 tb_port_configure_xdomain(port, port->xdomain); in tb_restore_children()
3087 * For non-USB4 hosts (Apple systems) remove any PCIe devices in tb_resume_noirq()
3090 if (!tb_switch_is_usb4(tb->root_switch)) in tb_resume_noirq()
3091 tb_switch_reset(tb->root_switch); in tb_resume_noirq()
3093 tb_switch_resume(tb->root_switch, false); in tb_resume_noirq()
3095 tb_free_unplugged_children(tb->root_switch); in tb_resume_noirq()
3096 tb_restore_children(tb->root_switch); in tb_resume_noirq()
3104 tb_switch_discover_tunnels(tb->root_switch, &tunnels, false); in tb_resume_noirq()
3112 /* Re-create our tunnels now */ in tb_resume_noirq()
3113 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_resume_noirq()
3114 /* USB3 requires delay before it can be re-activated */ in tb_resume_noirq()
3122 if (!list_empty(&tcm->tunnel_list)) { in tb_resume_noirq()
3130 tb_switch_enter_redrive(tb->root_switch); in tb_resume_noirq()
3132 tcm->hotplug_active = true; in tb_resume_noirq()
3146 if (port->xdomain && port->xdomain->is_unplugged) { in tb_free_unplugged_xdomains()
3148 tb_xdomain_remove(port->xdomain); in tb_free_unplugged_xdomains()
3150 port->xdomain = NULL; in tb_free_unplugged_xdomains()
3152 } else if (port->remote) { in tb_free_unplugged_xdomains()
3153 ret += tb_free_unplugged_xdomains(port->remote->sw); in tb_free_unplugged_xdomains()
3164 tcm->hotplug_active = false; in tb_freeze_noirq()
3172 tcm->hotplug_active = true; in tb_thaw_noirq()
3183 mutex_lock(&tb->lock); in tb_complete()
3184 if (tb_free_unplugged_xdomains(tb->root_switch)) in tb_complete()
3185 tb_scan_switch(tb->root_switch); in tb_complete()
3186 mutex_unlock(&tb->lock); in tb_complete()
3193 mutex_lock(&tb->lock); in tb_runtime_suspend()
3196 * re-entering redrive mode. in tb_runtime_suspend()
3199 tb_switch_exit_redrive(tb->root_switch); in tb_runtime_suspend()
3200 tb_switch_suspend(tb->root_switch, true); in tb_runtime_suspend()
3201 tcm->hotplug_active = false; in tb_runtime_suspend()
3202 mutex_unlock(&tb->lock); in tb_runtime_suspend()
3212 mutex_lock(&tb->lock); in tb_remove_work()
3213 if (tb->root_switch) { in tb_remove_work()
3214 tb_free_unplugged_children(tb->root_switch); in tb_remove_work()
3215 tb_free_unplugged_xdomains(tb->root_switch); in tb_remove_work()
3217 mutex_unlock(&tb->lock); in tb_remove_work()
3225 mutex_lock(&tb->lock); in tb_runtime_resume()
3226 tb_switch_resume(tb->root_switch, true); in tb_runtime_resume()
3228 tb_restore_children(tb->root_switch); in tb_runtime_resume()
3229 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) in tb_runtime_resume()
3231 tb_switch_enter_redrive(tb->root_switch); in tb_runtime_resume()
3232 tcm->hotplug_active = true; in tb_runtime_resume()
3233 mutex_unlock(&tb->lock); in tb_runtime_resume()
3240 queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50)); in tb_runtime_resume()
3277 switch (nhi->pdev->device) { in tb_apple_add_links()
3287 upstream = pci_upstream_bridge(nhi->pdev); in tb_apple_add_links()
3301 * back to NHI so that PCIe tunnels can be re-established after in tb_apple_add_links()
3305 for_each_pci_bridge(pdev, upstream->subordinate) { in tb_apple_add_links()
3311 !pdev->is_hotplug_bridge) in tb_apple_add_links()
3314 link = device_link_add(&pdev->dev, &nhi->pdev->dev, in tb_apple_add_links()
3318 dev_dbg(&nhi->pdev->dev, "created link from %s\n", in tb_apple_add_links()
3319 dev_name(&pdev->dev)); in tb_apple_add_links()
3322 dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n", in tb_apple_add_links()
3323 dev_name(&pdev->dev)); in tb_apple_add_links()
3340 tb->security_level = TB_SECURITY_USER; in tb_probe()
3342 tb->security_level = TB_SECURITY_NOPCIE; in tb_probe()
3344 tb->cm_ops = &tb_cm_ops; in tb_probe()
3347 INIT_LIST_HEAD(&tcm->tunnel_list); in tb_probe()
3348 INIT_LIST_HEAD(&tcm->dp_resources); in tb_probe()
3349 INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work); in tb_probe()