1 // SPDX-License-Identifier: GPL-2.0 2 /* XDP user-space packet buffer 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6 #include <linux/init.h> 7 #include <linux/sched/mm.h> 8 #include <linux/sched/signal.h> 9 #include <linux/sched/task.h> 10 #include <linux/uaccess.h> 11 #include <linux/slab.h> 12 #include <linux/bpf.h> 13 #include <linux/mm.h> 14 #include <linux/netdevice.h> 15 #include <linux/rtnetlink.h> 16 #include <linux/idr.h> 17 #include <linux/highmem.h> 18 19 #include "xdp_umem.h" 20 #include "xsk_queue.h" 21 22 #define XDP_UMEM_MIN_CHUNK_SIZE 2048 23 24 static DEFINE_IDA(umem_ida); 25 26 void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 27 { 28 unsigned long flags; 29 30 spin_lock_irqsave(&umem->xsk_list_lock, flags); 31 list_add_rcu(&xs->list, &umem->xsk_list); 32 spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 33 } 34 35 void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 36 { 37 unsigned long flags; 38 39 spin_lock_irqsave(&umem->xsk_list_lock, flags); 40 list_del_rcu(&xs->list); 41 spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 42 } 43 44 /* The umem is stored both in the _rx struct and the _tx struct as we do 45 * not know if the device has more tx queues than rx, or the opposite. 46 * This might also change during run time. 47 */ 48 static int xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, 49 u16 queue_id) 50 { 51 if (queue_id >= max_t(unsigned int, 52 dev->real_num_rx_queues, 53 dev->real_num_tx_queues)) 54 return -EINVAL; 55 56 if (queue_id < dev->real_num_rx_queues) 57 dev->_rx[queue_id].umem = umem; 58 if (queue_id < dev->real_num_tx_queues) 59 dev->_tx[queue_id].umem = umem; 60 61 return 0; 62 } 63 64 struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 65 u16 queue_id) 66 { 67 if (queue_id < dev->real_num_rx_queues) 68 return dev->_rx[queue_id].umem; 69 if (queue_id < dev->real_num_tx_queues) 70 return dev->_tx[queue_id].umem; 71 72 return NULL; 73 } 74 EXPORT_SYMBOL(xdp_get_umem_from_qid); 75 76 static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) 77 { 78 if (queue_id < dev->real_num_rx_queues) 79 dev->_rx[queue_id].umem = NULL; 80 if (queue_id < dev->real_num_tx_queues) 81 dev->_tx[queue_id].umem = NULL; 82 } 83 84 int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 85 u16 queue_id, u16 flags) 86 { 87 bool force_zc, force_copy; 88 struct netdev_bpf bpf; 89 int err = 0; 90 91 ASSERT_RTNL(); 92 93 force_zc = flags & XDP_ZEROCOPY; 94 force_copy = flags & XDP_COPY; 95 96 if (force_zc && force_copy) 97 return -EINVAL; 98 99 if (xdp_get_umem_from_qid(dev, queue_id)) 100 return -EBUSY; 101 102 err = xdp_reg_umem_at_qid(dev, umem, queue_id); 103 if (err) 104 return err; 105 106 umem->dev = dev; 107 umem->queue_id = queue_id; 108 109 if (flags & XDP_USE_NEED_WAKEUP) { 110 umem->flags |= XDP_UMEM_USES_NEED_WAKEUP; 111 /* Tx needs to be explicitly woken up the first time. 112 * Also for supporting drivers that do not implement this 113 * feature. They will always have to call sendto(). 114 */ 115 xsk_set_tx_need_wakeup(umem); 116 } 117 118 dev_hold(dev); 119 120 if (force_copy) 121 /* For copy-mode, we are done. */ 122 return 0; 123 124 if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) { 125 err = -EOPNOTSUPP; 126 goto err_unreg_umem; 127 } 128 129 bpf.command = XDP_SETUP_XSK_UMEM; 130 bpf.xsk.umem = umem; 131 bpf.xsk.queue_id = queue_id; 132 133 err = dev->netdev_ops->ndo_bpf(dev, &bpf); 134 if (err) 135 goto err_unreg_umem; 136 137 umem->zc = true; 138 return 0; 139 140 err_unreg_umem: 141 if (!force_zc) 142 err = 0; /* fallback to copy mode */ 143 if (err) 144 xdp_clear_umem_at_qid(dev, queue_id); 145 return err; 146 } 147 148 void xdp_umem_clear_dev(struct xdp_umem *umem) 149 { 150 struct netdev_bpf bpf; 151 int err; 152 153 ASSERT_RTNL(); 154 155 if (!umem->dev) 156 return; 157 158 if (umem->zc) { 159 bpf.command = XDP_SETUP_XSK_UMEM; 160 bpf.xsk.umem = NULL; 161 bpf.xsk.queue_id = umem->queue_id; 162 163 err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 164 165 if (err) 166 WARN(1, "failed to disable umem!\n"); 167 } 168 169 xdp_clear_umem_at_qid(umem->dev, umem->queue_id); 170 171 dev_put(umem->dev); 172 umem->dev = NULL; 173 umem->zc = false; 174 } 175 176 static void xdp_umem_unmap_pages(struct xdp_umem *umem) 177 { 178 unsigned int i; 179 180 for (i = 0; i < umem->npgs; i++) 181 kunmap(umem->pgs[i]); 182 } 183 184 static void xdp_umem_unpin_pages(struct xdp_umem *umem) 185 { 186 unsigned int i; 187 188 for (i = 0; i < umem->npgs; i++) { 189 struct page *page = umem->pgs[i]; 190 191 set_page_dirty_lock(page); 192 put_page(page); 193 } 194 195 kfree(umem->pgs); 196 umem->pgs = NULL; 197 } 198 199 static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 200 { 201 if (umem->user) { 202 atomic_long_sub(umem->npgs, &umem->user->locked_vm); 203 free_uid(umem->user); 204 } 205 } 206 207 static void xdp_umem_release(struct xdp_umem *umem) 208 { 209 rtnl_lock(); 210 xdp_umem_clear_dev(umem); 211 rtnl_unlock(); 212 213 ida_simple_remove(&umem_ida, umem->id); 214 215 if (umem->fq) { 216 xskq_destroy(umem->fq); 217 umem->fq = NULL; 218 } 219 220 if (umem->cq) { 221 xskq_destroy(umem->cq); 222 umem->cq = NULL; 223 } 224 225 xsk_reuseq_destroy(umem); 226 227 xdp_umem_unmap_pages(umem); 228 xdp_umem_unpin_pages(umem); 229 230 kfree(umem->pages); 231 umem->pages = NULL; 232 233 xdp_umem_unaccount_pages(umem); 234 kfree(umem); 235 } 236 237 static void xdp_umem_release_deferred(struct work_struct *work) 238 { 239 struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 240 241 xdp_umem_release(umem); 242 } 243 244 void xdp_get_umem(struct xdp_umem *umem) 245 { 246 refcount_inc(&umem->users); 247 } 248 249 void xdp_put_umem(struct xdp_umem *umem) 250 { 251 if (!umem) 252 return; 253 254 if (refcount_dec_and_test(&umem->users)) { 255 INIT_WORK(&umem->work, xdp_umem_release_deferred); 256 schedule_work(&umem->work); 257 } 258 } 259 260 static int xdp_umem_pin_pages(struct xdp_umem *umem) 261 { 262 unsigned int gup_flags = FOLL_WRITE; 263 long npgs; 264 int err; 265 266 umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), 267 GFP_KERNEL | __GFP_NOWARN); 268 if (!umem->pgs) 269 return -ENOMEM; 270 271 down_read(¤t->mm->mmap_sem); 272 npgs = get_user_pages(umem->address, umem->npgs, 273 gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL); 274 up_read(¤t->mm->mmap_sem); 275 276 if (npgs != umem->npgs) { 277 if (npgs >= 0) { 278 umem->npgs = npgs; 279 err = -ENOMEM; 280 goto out_pin; 281 } 282 err = npgs; 283 goto out_pgs; 284 } 285 return 0; 286 287 out_pin: 288 xdp_umem_unpin_pages(umem); 289 out_pgs: 290 kfree(umem->pgs); 291 umem->pgs = NULL; 292 return err; 293 } 294 295 static int xdp_umem_account_pages(struct xdp_umem *umem) 296 { 297 unsigned long lock_limit, new_npgs, old_npgs; 298 299 if (capable(CAP_IPC_LOCK)) 300 return 0; 301 302 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 303 umem->user = get_uid(current_user()); 304 305 do { 306 old_npgs = atomic_long_read(&umem->user->locked_vm); 307 new_npgs = old_npgs + umem->npgs; 308 if (new_npgs > lock_limit) { 309 free_uid(umem->user); 310 umem->user = NULL; 311 return -ENOBUFS; 312 } 313 } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 314 new_npgs) != old_npgs); 315 return 0; 316 } 317 318 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 319 { 320 u32 chunk_size = mr->chunk_size, headroom = mr->headroom; 321 unsigned int chunks, chunks_per_page; 322 u64 addr = mr->addr, size = mr->len; 323 int size_chk, err, i; 324 325 if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { 326 /* Strictly speaking we could support this, if: 327 * - huge pages, or* 328 * - using an IOMMU, or 329 * - making sure the memory area is consecutive 330 * but for now, we simply say "computer says no". 331 */ 332 return -EINVAL; 333 } 334 335 if (!is_power_of_2(chunk_size)) 336 return -EINVAL; 337 338 if (!PAGE_ALIGNED(addr)) { 339 /* Memory area has to be page size aligned. For 340 * simplicity, this might change. 341 */ 342 return -EINVAL; 343 } 344 345 if ((addr + size) < addr) 346 return -EINVAL; 347 348 chunks = (unsigned int)div_u64(size, chunk_size); 349 if (chunks == 0) 350 return -EINVAL; 351 352 chunks_per_page = PAGE_SIZE / chunk_size; 353 if (chunks < chunks_per_page || chunks % chunks_per_page) 354 return -EINVAL; 355 356 headroom = ALIGN(headroom, 64); 357 358 size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM; 359 if (size_chk < 0) 360 return -EINVAL; 361 362 umem->address = (unsigned long)addr; 363 umem->chunk_mask = ~((u64)chunk_size - 1); 364 umem->size = size; 365 umem->headroom = headroom; 366 umem->chunk_size_nohr = chunk_size - headroom; 367 umem->npgs = size / PAGE_SIZE; 368 umem->pgs = NULL; 369 umem->user = NULL; 370 INIT_LIST_HEAD(&umem->xsk_list); 371 spin_lock_init(&umem->xsk_list_lock); 372 373 refcount_set(&umem->users, 1); 374 375 err = xdp_umem_account_pages(umem); 376 if (err) 377 return err; 378 379 err = xdp_umem_pin_pages(umem); 380 if (err) 381 goto out_account; 382 383 umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL); 384 if (!umem->pages) { 385 err = -ENOMEM; 386 goto out_account; 387 } 388 389 for (i = 0; i < umem->npgs; i++) 390 umem->pages[i].addr = kmap(umem->pgs[i]); 391 392 return 0; 393 394 out_account: 395 xdp_umem_unaccount_pages(umem); 396 return err; 397 } 398 399 struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 400 { 401 struct xdp_umem *umem; 402 int err; 403 404 umem = kzalloc(sizeof(*umem), GFP_KERNEL); 405 if (!umem) 406 return ERR_PTR(-ENOMEM); 407 408 err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL); 409 if (err < 0) { 410 kfree(umem); 411 return ERR_PTR(err); 412 } 413 umem->id = err; 414 415 err = xdp_umem_reg(umem, mr); 416 if (err) { 417 ida_simple_remove(&umem_ida, umem->id); 418 kfree(umem); 419 return ERR_PTR(err); 420 } 421 422 return umem; 423 } 424 425 bool xdp_umem_validate_queues(struct xdp_umem *umem) 426 { 427 return umem->fq && umem->cq; 428 } 429