1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/ethtool_netlink.h> 4 #include <linux/netdevice.h> 5 #include <net/netdev_lock.h> 6 #include <net/netdev_queues.h> 7 #include <net/netdev_rx_queue.h> 8 #include <net/page_pool/memory_provider.h> 9 10 #include "dev.h" 11 #include "page_pool_priv.h" 12 13 void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst, 14 struct netdev_rx_queue *rxq_src) 15 { 16 netdev_assert_locked(rxq_src->dev); 17 netdev_assert_locked(rxq_dst->dev); 18 19 netdev_hold(rxq_src->dev, &rxq_src->lease_tracker, GFP_KERNEL); 20 21 WRITE_ONCE(rxq_src->lease, rxq_dst); 22 WRITE_ONCE(rxq_dst->lease, rxq_src); 23 } 24 25 void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst, 26 struct netdev_rx_queue *rxq_src) 27 { 28 netdev_assert_locked(rxq_dst->dev); 29 netdev_assert_locked(rxq_src->dev); 30 31 netif_rxq_cleanup_unlease(rxq_src, rxq_dst); 32 33 WRITE_ONCE(rxq_src->lease, NULL); 34 WRITE_ONCE(rxq_dst->lease, NULL); 35 36 netdev_put(rxq_src->dev, &rxq_src->lease_tracker); 37 } 38 39 bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx) 40 { 41 if (rxq_idx < dev->real_num_rx_queues) 42 return READ_ONCE(__netif_get_rx_queue(dev, rxq_idx)->lease); 43 return false; 44 } 45 46 /* Virtual devices eligible for leasing have no dev->dev.parent, while 47 * physical devices always have one. Use this to enforce the correct 48 * lease traversal direction. 49 */ 50 static bool netif_lease_dir_ok(const struct net_device *dev, 51 enum netif_lease_dir dir) 52 { 53 if (dir == NETIF_VIRT_TO_PHYS && !dev->dev.parent) 54 return true; 55 if (dir == NETIF_PHYS_TO_VIRT && dev->dev.parent) 56 return true; 57 return false; 58 } 59 60 bool netif_is_queue_leasee(const struct net_device *dev) 61 { 62 return netif_lease_dir_ok(dev, NETIF_VIRT_TO_PHYS); 63 } 64 65 struct netdev_rx_queue * 66 __netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq_idx, 67 enum netif_lease_dir dir) 68 { 69 struct net_device *orig_dev = *dev; 70 struct netdev_rx_queue *rxq = __netif_get_rx_queue(orig_dev, *rxq_idx); 71 72 if (rxq->lease) { 73 if (!netif_lease_dir_ok(orig_dev, dir)) 74 return NULL; 75 rxq = rxq->lease; 76 *rxq_idx = get_netdev_rx_queue_index(rxq); 77 *dev = rxq->dev; 78 } 79 return rxq; 80 } 81 82 /* See also page_pool_is_unreadable() */ 83 bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx) 84 { 85 if (rxq_idx < dev->real_num_rx_queues) 86 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_ops; 87 return false; 88 } 89 EXPORT_SYMBOL(netif_rxq_has_unreadable_mp); 90 91 bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx) 92 { 93 if (rxq_idx < dev->real_num_rx_queues) 94 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_priv; 95 return false; 96 } 97 98 static int netdev_rx_queue_reconfig(struct net_device *dev, 99 unsigned int rxq_idx, 100 struct netdev_queue_config *qcfg_old, 101 struct netdev_queue_config *qcfg_new) 102 { 103 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx); 104 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; 105 void *new_mem, *old_mem; 106 int err; 107 108 if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free || 109 !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start) 110 return -EOPNOTSUPP; 111 112 netdev_assert_locked(dev); 113 114 new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 115 if (!new_mem) 116 return -ENOMEM; 117 118 old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 119 if (!old_mem) { 120 err = -ENOMEM; 121 goto err_free_new_mem; 122 } 123 124 err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx); 125 if (err) 126 goto err_free_old_mem; 127 128 err = page_pool_check_memory_provider(dev, rxq); 129 if (err) 130 goto err_free_new_queue_mem; 131 132 if (netif_running(dev)) { 133 err = qops->ndo_queue_stop(dev, old_mem, rxq_idx); 134 if (err) 135 goto err_free_new_queue_mem; 136 137 err = qops->ndo_queue_start(dev, qcfg_new, new_mem, rxq_idx); 138 if (err) 139 goto err_start_queue; 140 } else { 141 swap(new_mem, old_mem); 142 } 143 144 qops->ndo_queue_mem_free(dev, old_mem); 145 146 kvfree(old_mem); 147 kvfree(new_mem); 148 149 return 0; 150 151 err_start_queue: 152 /* Restarting the queue with old_mem should be successful as we haven't 153 * changed any of the queue configuration, and there is not much we can 154 * do to recover from a failure here. 155 * 156 * WARN if we fail to recover the old rx queue, and at least free 157 * old_mem so we don't also leak that. 158 */ 159 if (qops->ndo_queue_start(dev, qcfg_old, old_mem, rxq_idx)) { 160 WARN(1, 161 "Failed to restart old queue in error path. RX queue %d may be unhealthy.", 162 rxq_idx); 163 qops->ndo_queue_mem_free(dev, old_mem); 164 } 165 166 err_free_new_queue_mem: 167 qops->ndo_queue_mem_free(dev, new_mem); 168 169 err_free_old_mem: 170 kvfree(old_mem); 171 172 err_free_new_mem: 173 kvfree(new_mem); 174 175 return err; 176 } 177 178 int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) 179 { 180 struct netdev_queue_config qcfg; 181 182 netdev_queue_config(dev, rxq_idx, &qcfg); 183 return netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg, &qcfg); 184 } 185 EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL"); 186 187 static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 188 const struct pp_memory_provider_params *p, 189 struct netlink_ext_ack *extack) 190 { 191 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; 192 struct netdev_queue_config qcfg[2]; 193 struct netdev_rx_queue *rxq; 194 int ret; 195 196 if (!qops) 197 return -EOPNOTSUPP; 198 199 if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) { 200 NL_SET_ERR_MSG(extack, "tcp-data-split is disabled"); 201 return -EINVAL; 202 } 203 if (dev->cfg->hds_thresh) { 204 NL_SET_ERR_MSG(extack, "hds-thresh is not zero"); 205 return -EINVAL; 206 } 207 if (dev_xdp_prog_count(dev)) { 208 NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached"); 209 return -EEXIST; 210 } 211 if (p->rx_page_size && !(qops->supported_params & QCFG_RX_PAGE_SIZE)) { 212 NL_SET_ERR_MSG(extack, "device does not support: rx_page_size"); 213 return -EOPNOTSUPP; 214 } 215 216 rxq = __netif_get_rx_queue(dev, rxq_idx); 217 if (rxq->mp_params.mp_ops) { 218 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound"); 219 return -EEXIST; 220 } 221 #ifdef CONFIG_XDP_SOCKETS 222 if (rxq->pool) { 223 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP"); 224 return -EBUSY; 225 } 226 #endif 227 228 netdev_queue_config(dev, rxq_idx, &qcfg[0]); 229 rxq->mp_params = *p; 230 ret = netdev_queue_config_validate(dev, rxq_idx, &qcfg[1], extack); 231 if (ret) 232 goto err_clear_mp; 233 234 ret = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]); 235 if (ret) 236 goto err_clear_mp; 237 238 return 0; 239 240 err_clear_mp: 241 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params)); 242 return ret; 243 } 244 245 int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 246 const struct pp_memory_provider_params *p, 247 struct netlink_ext_ack *extack) 248 { 249 int ret; 250 251 if (!netdev_need_ops_lock(dev)) 252 return -EOPNOTSUPP; 253 254 if (rxq_idx >= dev->real_num_rx_queues) { 255 NL_SET_ERR_MSG(extack, "rx queue index out of range"); 256 return -ERANGE; 257 } 258 rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues); 259 260 if (!netif_rxq_is_leased(dev, rxq_idx)) 261 return __netif_mp_open_rxq(dev, rxq_idx, p, extack); 262 263 if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) { 264 NL_SET_ERR_MSG(extack, "rx queue leased to a virtual netdev"); 265 return -EBUSY; 266 } 267 if (!dev->dev.parent) { 268 NL_SET_ERR_MSG(extack, "rx queue belongs to a virtual netdev"); 269 return -EOPNOTSUPP; 270 } 271 272 netdev_lock(dev); 273 ret = __netif_mp_open_rxq(dev, rxq_idx, p, extack); 274 netdev_unlock(dev); 275 return ret; 276 } 277 278 static void __netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, 279 const struct pp_memory_provider_params *old_p) 280 { 281 struct netdev_queue_config qcfg[2]; 282 struct netdev_rx_queue *rxq; 283 int err; 284 285 rxq = __netif_get_rx_queue(dev, rxq_idx); 286 287 /* Callers holding a netdev ref may get here after we already 288 * went thru shutdown via dev_memory_provider_uninstall(). 289 */ 290 if (dev->reg_state > NETREG_REGISTERED && 291 !rxq->mp_params.mp_ops) 292 return; 293 294 if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops || 295 rxq->mp_params.mp_priv != old_p->mp_priv)) 296 return; 297 298 netdev_queue_config(dev, rxq_idx, &qcfg[0]); 299 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params)); 300 netdev_queue_config(dev, rxq_idx, &qcfg[1]); 301 302 err = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]); 303 WARN_ON(err && err != -ENETDOWN); 304 } 305 306 void netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, 307 const struct pp_memory_provider_params *old_p) 308 { 309 if (WARN_ON_ONCE(rxq_idx >= dev->real_num_rx_queues)) 310 return; 311 if (!netif_rxq_is_leased(dev, rxq_idx)) 312 return __netif_mp_close_rxq(dev, rxq_idx, old_p); 313 314 if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) { 315 WARN_ON_ONCE(1); 316 return; 317 } 318 netdev_lock(dev); 319 __netif_mp_close_rxq(dev, rxq_idx, old_p); 320 netdev_unlock(dev); 321 } 322 323 void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, 324 const struct pp_memory_provider_params *p) 325 { 326 if (p->mp_ops && p->mp_ops->uninstall) 327 p->mp_ops->uninstall(p->mp_priv, rxq); 328 } 329 330 /* Clean up memory provider state when a queue lease is torn down. If 331 * a memory provider was installed on the physical queue via the lease, 332 * close it now. The memory provider is a property of the queue itself, 333 * and it was _guaranteed_ to be installed on the physical queue via 334 * the lease redirection. The extra __netif_mp_close_rxq is needed 335 * since the physical queue can outlive the virtual queue in the lease 336 * case, so it needs to be reconfigured to clear the memory provider. 337 */ 338 void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq, 339 struct netdev_rx_queue *virt_rxq) 340 { 341 struct pp_memory_provider_params *p = &phys_rxq->mp_params; 342 unsigned int rxq_idx = get_netdev_rx_queue_index(phys_rxq); 343 344 if (!p->mp_ops) 345 return; 346 347 __netif_mp_uninstall_rxq(virt_rxq, p); 348 __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, p); 349 } 350