1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/ethtool_netlink.h> 4 #include <linux/netdevice.h> 5 #include <net/netdev_lock.h> 6 #include <net/netdev_queues.h> 7 #include <net/netdev_rx_queue.h> 8 #include <net/page_pool/memory_provider.h> 9 10 #include "page_pool_priv.h" 11 12 void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst, 13 struct netdev_rx_queue *rxq_src) 14 { 15 netdev_assert_locked(rxq_src->dev); 16 netdev_assert_locked(rxq_dst->dev); 17 18 netdev_hold(rxq_src->dev, &rxq_src->lease_tracker, GFP_KERNEL); 19 20 WRITE_ONCE(rxq_src->lease, rxq_dst); 21 WRITE_ONCE(rxq_dst->lease, rxq_src); 22 } 23 24 void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst, 25 struct netdev_rx_queue *rxq_src) 26 { 27 netdev_assert_locked(rxq_dst->dev); 28 netdev_assert_locked(rxq_src->dev); 29 30 WRITE_ONCE(rxq_src->lease, NULL); 31 WRITE_ONCE(rxq_dst->lease, NULL); 32 33 netdev_put(rxq_src->dev, &rxq_src->lease_tracker); 34 } 35 36 bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx) 37 { 38 if (rxq_idx < dev->real_num_rx_queues) 39 return READ_ONCE(__netif_get_rx_queue(dev, rxq_idx)->lease); 40 return false; 41 } 42 43 static bool netif_lease_dir_ok(const struct net_device *dev, 44 enum netif_lease_dir dir) 45 { 46 if (dir == NETIF_VIRT_TO_PHYS && !dev->dev.parent) 47 return true; 48 if (dir == NETIF_PHYS_TO_VIRT && dev->dev.parent) 49 return true; 50 return false; 51 } 52 53 struct netdev_rx_queue * 54 __netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq_idx, 55 enum netif_lease_dir dir) 56 { 57 struct net_device *orig_dev = *dev; 58 struct netdev_rx_queue *rxq = __netif_get_rx_queue(orig_dev, *rxq_idx); 59 60 if (rxq->lease) { 61 if (!netif_lease_dir_ok(orig_dev, dir)) 62 return NULL; 63 rxq = rxq->lease; 64 *rxq_idx = get_netdev_rx_queue_index(rxq); 65 *dev = rxq->dev; 66 } 67 return rxq; 68 } 69 70 struct netdev_rx_queue * 71 netif_get_rx_queue_lease_locked(struct net_device **dev, unsigned int *rxq_idx) 72 { 73 struct net_device *orig_dev = *dev; 74 struct netdev_rx_queue *rxq; 75 76 /* Locking order is always from the virtual to the physical device 77 * see netdev_nl_queue_create_doit(). 78 */ 79 netdev_ops_assert_locked(orig_dev); 80 rxq = __netif_get_rx_queue_lease(dev, rxq_idx, NETIF_VIRT_TO_PHYS); 81 if (rxq && orig_dev != *dev) 82 netdev_lock(*dev); 83 return rxq; 84 } 85 86 void netif_put_rx_queue_lease_locked(struct net_device *orig_dev, 87 struct net_device *dev) 88 { 89 if (orig_dev != dev) 90 netdev_unlock(dev); 91 } 92 93 bool netif_rx_queue_lease_get_owner(struct net_device **dev, 94 unsigned int *rxq_idx) 95 { 96 struct net_device *orig_dev = *dev; 97 struct netdev_rx_queue *rxq; 98 99 /* The physical device needs to be locked. If there is indeed a lease, 100 * then the virtual device holds a reference on the physical device 101 * and the lease stays active until the virtual device is torn down. 102 * When queues get {un,}leased both devices are always locked. 103 */ 104 netdev_ops_assert_locked(orig_dev); 105 rxq = __netif_get_rx_queue_lease(dev, rxq_idx, NETIF_PHYS_TO_VIRT); 106 if (rxq && orig_dev != *dev) 107 return true; 108 return false; 109 } 110 111 /* See also page_pool_is_unreadable() */ 112 bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx) 113 { 114 if (rxq_idx < dev->real_num_rx_queues) 115 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_ops; 116 return false; 117 } 118 EXPORT_SYMBOL(netif_rxq_has_unreadable_mp); 119 120 bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx) 121 { 122 if (rxq_idx < dev->real_num_rx_queues) 123 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_priv; 124 return false; 125 } 126 127 int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) 128 { 129 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx); 130 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; 131 void *new_mem, *old_mem; 132 int err; 133 134 if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free || 135 !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start) 136 return -EOPNOTSUPP; 137 138 netdev_assert_locked(dev); 139 140 new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 141 if (!new_mem) 142 return -ENOMEM; 143 144 old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); 145 if (!old_mem) { 146 err = -ENOMEM; 147 goto err_free_new_mem; 148 } 149 150 err = qops->ndo_queue_mem_alloc(dev, new_mem, rxq_idx); 151 if (err) 152 goto err_free_old_mem; 153 154 err = page_pool_check_memory_provider(dev, rxq); 155 if (err) 156 goto err_free_new_queue_mem; 157 158 if (netif_running(dev)) { 159 err = qops->ndo_queue_stop(dev, old_mem, rxq_idx); 160 if (err) 161 goto err_free_new_queue_mem; 162 163 err = qops->ndo_queue_start(dev, new_mem, rxq_idx); 164 if (err) 165 goto err_start_queue; 166 } else { 167 swap(new_mem, old_mem); 168 } 169 170 qops->ndo_queue_mem_free(dev, old_mem); 171 172 kvfree(old_mem); 173 kvfree(new_mem); 174 175 return 0; 176 177 err_start_queue: 178 /* Restarting the queue with old_mem should be successful as we haven't 179 * changed any of the queue configuration, and there is not much we can 180 * do to recover from a failure here. 181 * 182 * WARN if we fail to recover the old rx queue, and at least free 183 * old_mem so we don't also leak that. 184 */ 185 if (qops->ndo_queue_start(dev, old_mem, rxq_idx)) { 186 WARN(1, 187 "Failed to restart old queue in error path. RX queue %d may be unhealthy.", 188 rxq_idx); 189 qops->ndo_queue_mem_free(dev, old_mem); 190 } 191 192 err_free_new_queue_mem: 193 qops->ndo_queue_mem_free(dev, new_mem); 194 195 err_free_old_mem: 196 kvfree(old_mem); 197 198 err_free_new_mem: 199 kvfree(new_mem); 200 201 return err; 202 } 203 EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL"); 204 205 int __net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 206 const struct pp_memory_provider_params *p, 207 struct netlink_ext_ack *extack) 208 { 209 struct net_device *orig_dev = dev; 210 struct netdev_rx_queue *rxq; 211 int ret; 212 213 if (!netdev_need_ops_lock(dev)) 214 return -EOPNOTSUPP; 215 if (rxq_idx >= dev->real_num_rx_queues) { 216 NL_SET_ERR_MSG(extack, "rx queue index out of range"); 217 return -ERANGE; 218 } 219 220 rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues); 221 rxq = netif_get_rx_queue_lease_locked(&dev, &rxq_idx); 222 if (!rxq) { 223 NL_SET_ERR_MSG(extack, "rx queue peered to a virtual netdev"); 224 return -EBUSY; 225 } 226 if (!dev->dev.parent) { 227 NL_SET_ERR_MSG(extack, "rx queue is mapped to a virtual netdev"); 228 ret = -EBUSY; 229 goto out; 230 } 231 if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) { 232 NL_SET_ERR_MSG(extack, "tcp-data-split is disabled"); 233 ret = -EINVAL; 234 goto out; 235 } 236 if (dev->cfg->hds_thresh) { 237 NL_SET_ERR_MSG(extack, "hds-thresh is not zero"); 238 ret = -EINVAL; 239 goto out; 240 } 241 if (dev_xdp_prog_count(dev)) { 242 NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached"); 243 ret = -EEXIST; 244 goto out; 245 } 246 if (rxq->mp_params.mp_ops) { 247 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound"); 248 ret = -EEXIST; 249 goto out; 250 } 251 #ifdef CONFIG_XDP_SOCKETS 252 if (rxq->pool) { 253 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP"); 254 ret = -EBUSY; 255 goto out; 256 } 257 #endif 258 rxq->mp_params = *p; 259 ret = netdev_rx_queue_restart(dev, rxq_idx); 260 if (ret) { 261 rxq->mp_params.mp_ops = NULL; 262 rxq->mp_params.mp_priv = NULL; 263 } 264 out: 265 netif_put_rx_queue_lease_locked(orig_dev, dev); 266 return ret; 267 } 268 269 int net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, 270 struct pp_memory_provider_params *p) 271 { 272 int ret; 273 274 netdev_lock(dev); 275 ret = __net_mp_open_rxq(dev, rxq_idx, p, NULL); 276 netdev_unlock(dev); 277 return ret; 278 } 279 280 void __net_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, 281 const struct pp_memory_provider_params *old_p) 282 { 283 struct net_device *orig_dev = dev; 284 struct netdev_rx_queue *rxq; 285 int err; 286 287 if (WARN_ON_ONCE(rxq_idx >= dev->real_num_rx_queues)) 288 return; 289 290 rxq = netif_get_rx_queue_lease_locked(&dev, &rxq_idx); 291 if (WARN_ON_ONCE(!rxq)) 292 return; 293 294 /* Callers holding a netdev ref may get here after we already 295 * went thru shutdown via dev_memory_provider_uninstall(). 296 */ 297 if (dev->reg_state > NETREG_REGISTERED && 298 !rxq->mp_params.mp_ops) 299 goto out; 300 301 if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops || 302 rxq->mp_params.mp_priv != old_p->mp_priv)) 303 goto out; 304 305 rxq->mp_params.mp_ops = NULL; 306 rxq->mp_params.mp_priv = NULL; 307 err = netdev_rx_queue_restart(dev, rxq_idx); 308 WARN_ON(err && err != -ENETDOWN); 309 out: 310 netif_put_rx_queue_lease_locked(orig_dev, dev); 311 } 312 313 void net_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, 314 struct pp_memory_provider_params *old_p) 315 { 316 netdev_lock(dev); 317 __net_mp_close_rxq(dev, rxq_idx, old_p); 318 netdev_unlock(dev); 319 } 320