1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/ethtool_netlink.h> 4 #include <linux/netdevice.h> 5 #include <net/netdev_lock.h> 6 #include <net/netdev_queues.h> 7 #include <net/netdev_rx_queue.h> 8 #include <net/page_pool/memory_provider.h> 9 10 #include "dev.h" 11 #include "page_pool_priv.h" 12 13 void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst, 14 struct netdev_rx_queue *rxq_src) 15 { 16 netdev_assert_locked(rxq_src->dev); 17 netdev_assert_locked(rxq_dst->dev); 18 19 netdev_hold(rxq_src->dev, &rxq_src->lease_tracker, GFP_KERNEL); 20 21 WRITE_ONCE(rxq_src->lease, rxq_dst); 22 WRITE_ONCE(rxq_dst->lease, rxq_src); 23 } 24 25 void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst, 26 struct netdev_rx_queue *rxq_src) 27 { 28 netdev_assert_locked(rxq_dst->dev); 29 netdev_assert_locked(rxq_src->dev); 30 31 netif_rxq_cleanup_unlease(rxq_src, rxq_dst); 32 33 WRITE_ONCE(rxq_src->lease, NULL); 34 WRITE_ONCE(rxq_dst->lease, NULL); 35 36 netdev_put(rxq_src->dev, &rxq_src->lease_tracker); 37 } 38 39 bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx) 40 { 41 if (rxq_idx < dev->real_num_rx_queues) 42 return READ_ONCE(__netif_get_rx_queue(dev, rxq_idx)->lease); 43 return false; 44 } 45 46 /* Virtual devices eligible for leasing have no dev->dev.parent, while 47 * physical devices always have one. Use this to enforce the correct 48 * lease traversal direction. 49 */ 50 static bool netif_lease_dir_ok(const struct net_device *dev, 51 enum netif_lease_dir dir) 52 { 53 if (dir == NETIF_VIRT_TO_PHYS && !dev->dev.parent) 54 return true; 55 if (dir == NETIF_PHYS_TO_VIRT && dev->dev.parent) 56 return true; 57 return false; 58 } 59 60 struct netdev_rx_queue * 61 __netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq_idx, 62 enum netif_lease_dir dir) 63 { 64 struct net_device *orig_dev = *dev; 65 struct netdev_rx_queue *rxq = __netif_get_rx_queue(orig_dev, *rxq_idx); 66 67 if (rxq->lease) { 68 if (!netif_lease_dir_ok(orig_dev, dir)) 69 return NULL; 70 rxq = rxq->lease; 71 *rxq_idx = get_netdev_rx_queue_index(rxq); 72 *dev = rxq->dev; 73 } 74 return rxq; 75 } 76 77 struct netdev_rx_queue * 78 netif_get_rx_queue_lease_locked(struct net_device **dev, unsigned int *rxq_idx) 79 { 80 struct net_device *orig_dev = *dev; 81 struct netdev_rx_queue *rxq; 82 83 /* Locking order is always from the virtual to the physical device 84 * see netdev_nl_queue_create_doit(). 85 */ 86 netdev_ops_assert_locked(orig_dev); 87 rxq = __netif_get_rx_queue_lease(dev, rxq_idx, NETIF_VIRT_TO_PHYS); 88 if (rxq && orig_dev != *dev) 89 netdev_lock(*dev); 90 return rxq; 91 } 92 93 void netif_put_rx_queue_lease_locked(struct net_device *orig_dev, 94 struct net_device *dev) 95 { 96 if (orig_dev != dev) 97 netdev_unlock(dev); 98 } 99 100 /* See also page_pool_is_unreadable() */ 101 bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx) 102 { 103 if (rxq_idx < dev->real_num_rx_queues) 104 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_ops; 105 return false; 106 } 107 EXPORT_SYMBOL(netif_rxq_has_unreadable_mp); 108 109 bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx) 110 { 111 if (rxq_idx < dev->real_num_rx_queues) 112 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_priv; 113 return false; 114 } 115 116 static int netdev_rx_queue_reconfig(struct net_device *dev, 117 unsigned int rxq_idx, 118 struct netdev_queue_config *qcfg_old, 119 struct netdev_queue_config *qcfg_new) 120 { 121 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx); 122 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; 123 void *new_mem, *old_mem; 124 int err; 125 126 if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free || 127 !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start) 128 return -EOPNOTSUPP; 129 130 netdev_assert_locked(dev); 131 132 new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 133 if (!new_mem) 134 return -ENOMEM; 135 136 old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 137 if (!old_mem) { 138 err = -ENOMEM; 139 goto err_free_new_mem; 140 } 141 142 err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx); 143 if (err) 144 goto err_free_old_mem; 145 146 err = page_pool_check_memory_provider(dev, rxq); 147 if (err) 148 goto err_free_new_queue_mem; 149 150 if (netif_running(dev)) { 151 err = qops->ndo_queue_stop(dev, old_mem, rxq_idx); 152 if (err) 153 goto err_free_new_queue_mem; 154 155 err = qops->ndo_queue_start(dev, qcfg_new, new_mem, rxq_idx); 156 if (err) 157 goto err_start_queue; 158 } else { 159 swap(new_mem, old_mem); 160 } 161 162 qops->ndo_queue_mem_free(dev, old_mem); 163 164 kvfree(old_mem); 165 kvfree(new_mem); 166 167 return 0; 168 169 err_start_queue: 170 /* Restarting the queue with old_mem should be successful as we haven't 171 * changed any of the queue configuration, and there is not much we can 172 * do to recover from a failure here. 173 * 174 * WARN if we fail to recover the old rx queue, and at least free 175 * old_mem so we don't also leak that. 176 */ 177 if (qops->ndo_queue_start(dev, qcfg_old, old_mem, rxq_idx)) { 178 WARN(1, 179 "Failed to restart old queue in error path. RX queue %d may be unhealthy.", 180 rxq_idx); 181 qops->ndo_queue_mem_free(dev, old_mem); 182 } 183 184 err_free_new_queue_mem: 185 qops->ndo_queue_mem_free(dev, new_mem); 186 187 err_free_old_mem: 188 kvfree(old_mem); 189 190 err_free_new_mem: 191 kvfree(new_mem); 192 193 return err; 194 } 195 196 int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) 197 { 198 struct netdev_queue_config qcfg; 199 200 netdev_queue_config(dev, rxq_idx, &qcfg); 201 return netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg, &qcfg); 202 } 203 EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL"); 204 205 static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 206 const struct pp_memory_provider_params *p, 207 struct netlink_ext_ack *extack) 208 { 209 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; 210 struct netdev_queue_config qcfg[2]; 211 struct netdev_rx_queue *rxq; 212 int ret; 213 214 if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) { 215 NL_SET_ERR_MSG(extack, "tcp-data-split is disabled"); 216 return -EINVAL; 217 } 218 if (dev->cfg->hds_thresh) { 219 NL_SET_ERR_MSG(extack, "hds-thresh is not zero"); 220 return -EINVAL; 221 } 222 if (dev_xdp_prog_count(dev)) { 223 NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached"); 224 return -EEXIST; 225 } 226 if (p->rx_page_size && !(qops->supported_params & QCFG_RX_PAGE_SIZE)) { 227 NL_SET_ERR_MSG(extack, "device does not support: rx_page_size"); 228 return -EOPNOTSUPP; 229 } 230 231 rxq = __netif_get_rx_queue(dev, rxq_idx); 232 if (rxq->mp_params.mp_ops) { 233 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound"); 234 return -EEXIST; 235 } 236 #ifdef CONFIG_XDP_SOCKETS 237 if (rxq->pool) { 238 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP"); 239 return -EBUSY; 240 } 241 #endif 242 243 netdev_queue_config(dev, rxq_idx, &qcfg[0]); 244 rxq->mp_params = *p; 245 ret = netdev_queue_config_validate(dev, rxq_idx, &qcfg[1], extack); 246 if (ret) 247 goto err_clear_mp; 248 249 ret = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]); 250 if (ret) 251 goto err_clear_mp; 252 253 return 0; 254 255 err_clear_mp: 256 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params)); 257 return ret; 258 } 259 260 int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 261 const struct pp_memory_provider_params *p, 262 struct netlink_ext_ack *extack) 263 { 264 struct net_device *orig_dev = dev; 265 int ret; 266 267 if (!netdev_need_ops_lock(dev)) 268 return -EOPNOTSUPP; 269 270 if (rxq_idx >= dev->real_num_rx_queues) { 271 NL_SET_ERR_MSG(extack, "rx queue index out of range"); 272 return -ERANGE; 273 } 274 rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues); 275 276 if (!netif_rxq_is_leased(dev, rxq_idx)) 277 return __netif_mp_open_rxq(dev, rxq_idx, p, extack); 278 279 if (!netif_get_rx_queue_lease_locked(&dev, &rxq_idx)) { 280 NL_SET_ERR_MSG(extack, "rx queue leased to a virtual netdev"); 281 return -EBUSY; 282 } 283 if (!dev->dev.parent) { 284 NL_SET_ERR_MSG(extack, "rx queue belongs to a virtual netdev"); 285 ret = -EOPNOTSUPP; 286 goto out; 287 } 288 289 ret = __netif_mp_open_rxq(dev, rxq_idx, p, extack); 290 out: 291 netif_put_rx_queue_lease_locked(orig_dev, dev); 292 return ret; 293 } 294 295 static void __netif_mp_close_rxq(struct net_device *dev, unsigned int ifq_idx, 296 const struct pp_memory_provider_params *old_p) 297 { 298 struct netdev_queue_config qcfg[2]; 299 struct netdev_rx_queue *rxq; 300 int err; 301 302 rxq = __netif_get_rx_queue(dev, ifq_idx); 303 304 /* Callers holding a netdev ref may get here after we already 305 * went thru shutdown via dev_memory_provider_uninstall(). 306 */ 307 if (dev->reg_state > NETREG_REGISTERED && 308 !rxq->mp_params.mp_ops) 309 return; 310 311 if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops || 312 rxq->mp_params.mp_priv != old_p->mp_priv)) 313 return; 314 315 netdev_queue_config(dev, ifq_idx, &qcfg[0]); 316 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params)); 317 netdev_queue_config(dev, ifq_idx, &qcfg[1]); 318 319 err = netdev_rx_queue_reconfig(dev, ifq_idx, &qcfg[0], &qcfg[1]); 320 WARN_ON(err && err != -ENETDOWN); 321 } 322 323 void netif_mp_close_rxq(struct net_device *dev, unsigned int ifq_idx, 324 const struct pp_memory_provider_params *old_p) 325 { 326 struct net_device *orig_dev = dev; 327 328 if (WARN_ON_ONCE(ifq_idx >= dev->real_num_rx_queues)) 329 return; 330 if (!netif_rxq_is_leased(dev, ifq_idx)) 331 return __netif_mp_close_rxq(dev, ifq_idx, old_p); 332 333 if (WARN_ON_ONCE(!netif_get_rx_queue_lease_locked(&dev, &ifq_idx))) 334 return; 335 336 __netif_mp_close_rxq(dev, ifq_idx, old_p); 337 netif_put_rx_queue_lease_locked(orig_dev, dev); 338 } 339 340 void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, 341 const struct pp_memory_provider_params *p) 342 { 343 if (p->mp_ops && p->mp_ops->uninstall) 344 p->mp_ops->uninstall(p->mp_priv, rxq); 345 } 346 347 /* Clean up memory provider state when a queue lease is torn down. If 348 * a memory provider was installed on the physical queue via the lease, 349 * close it now. The memory provider is a property of the queue itself, 350 * and it was _guaranteed_ to be installed on the physical queue via 351 * the lease redirection. The extra __netif_mp_close_rxq is needed 352 * since the physical queue can outlive the virtual queue in the lease 353 * case, so it needs to be reconfigured to clear the memory provider. 354 */ 355 void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq, 356 struct netdev_rx_queue *virt_rxq) 357 { 358 struct pp_memory_provider_params *p = &phys_rxq->mp_params; 359 unsigned int ifq_idx = get_netdev_rx_queue_index(phys_rxq); 360 361 if (!p->mp_ops) 362 return; 363 364 __netif_mp_uninstall_rxq(virt_rxq, p); 365 __netif_mp_close_rxq(phys_rxq->dev, ifq_idx, p); 366 } 367