1c0c77d8fSBjörn Töpel // SPDX-License-Identifier: GPL-2.0 2c0c77d8fSBjörn Töpel /* XDP user-space packet buffer 3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4c0c77d8fSBjörn Töpel */ 5c0c77d8fSBjörn Töpel 6c0c77d8fSBjörn Töpel #include <linux/init.h> 7c0c77d8fSBjörn Töpel #include <linux/sched/mm.h> 8c0c77d8fSBjörn Töpel #include <linux/sched/signal.h> 9c0c77d8fSBjörn Töpel #include <linux/sched/task.h> 10c0c77d8fSBjörn Töpel #include <linux/uaccess.h> 11c0c77d8fSBjörn Töpel #include <linux/slab.h> 12c0c77d8fSBjörn Töpel #include <linux/bpf.h> 13c0c77d8fSBjörn Töpel #include <linux/mm.h> 1484c6b868SJakub Kicinski #include <linux/netdevice.h> 1584c6b868SJakub Kicinski #include <linux/rtnetlink.h> 1650e74c01SBjörn Töpel #include <linux/idr.h> 17624676e7SIvan Khoronzhuk #include <linux/vmalloc.h> 18c0c77d8fSBjörn Töpel 19c0c77d8fSBjörn Töpel #include "xdp_umem.h" 20e61e62b9SBjörn Töpel #include "xsk_queue.h" 21c0c77d8fSBjörn Töpel 22bbff2f32SBjörn Töpel #define XDP_UMEM_MIN_CHUNK_SIZE 2048 23c0c77d8fSBjörn Töpel 2450e74c01SBjörn Töpel static DEFINE_IDA(umem_ida); 2550e74c01SBjörn Töpel 26ac98d8aaSMagnus Karlsson void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 27ac98d8aaSMagnus Karlsson { 28ac98d8aaSMagnus Karlsson unsigned long flags; 29ac98d8aaSMagnus Karlsson 30ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 31ac98d8aaSMagnus Karlsson list_add_rcu(&xs->list, &umem->xsk_list); 32ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 33ac98d8aaSMagnus Karlsson } 34ac98d8aaSMagnus Karlsson 35ac98d8aaSMagnus Karlsson void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 36ac98d8aaSMagnus Karlsson { 37ac98d8aaSMagnus Karlsson unsigned long flags; 38ac98d8aaSMagnus Karlsson 39ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 40ac98d8aaSMagnus Karlsson list_del_rcu(&xs->list); 41ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 42ac98d8aaSMagnus Karlsson } 43ac98d8aaSMagnus Karlsson 44c9b47cc1SMagnus Karlsson /* The umem is stored both in the _rx struct and the _tx struct as we do 45c9b47cc1SMagnus Karlsson * not know if the device has more tx queues than rx, or the opposite. 46c9b47cc1SMagnus Karlsson * This might also change during run time. 47c9b47cc1SMagnus Karlsson */ 48cc5b5d35SKrzysztof Kazimierczak static int xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, 49c9b47cc1SMagnus Karlsson u16 queue_id) 5084c6b868SJakub Kicinski { 51cc5b5d35SKrzysztof Kazimierczak if (queue_id >= max_t(unsigned int, 52cc5b5d35SKrzysztof Kazimierczak dev->real_num_rx_queues, 53cc5b5d35SKrzysztof Kazimierczak dev->real_num_tx_queues)) 54cc5b5d35SKrzysztof Kazimierczak return -EINVAL; 55cc5b5d35SKrzysztof Kazimierczak 56c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 57c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = umem; 58c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 59c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = umem; 60cc5b5d35SKrzysztof Kazimierczak 61cc5b5d35SKrzysztof Kazimierczak return 0; 62c9b47cc1SMagnus Karlsson } 6384c6b868SJakub Kicinski 641661d346SJakub Kicinski struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 65c9b47cc1SMagnus Karlsson u16 queue_id) 66c9b47cc1SMagnus Karlsson { 67c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 68c9b47cc1SMagnus Karlsson return dev->_rx[queue_id].umem; 69c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 70c9b47cc1SMagnus Karlsson return dev->_tx[queue_id].umem; 7184c6b868SJakub Kicinski 72c9b47cc1SMagnus Karlsson return NULL; 73c9b47cc1SMagnus Karlsson } 745f4f3b2dSJan Sokolowski EXPORT_SYMBOL(xdp_get_umem_from_qid); 7584c6b868SJakub Kicinski 76c9b47cc1SMagnus Karlsson static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) 77c9b47cc1SMagnus Karlsson { 78a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 79c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = NULL; 80a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 81c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = NULL; 8284c6b868SJakub Kicinski } 8384c6b868SJakub Kicinski 84173d3adbSBjörn Töpel int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 85c9b47cc1SMagnus Karlsson u16 queue_id, u16 flags) 86173d3adbSBjörn Töpel { 87173d3adbSBjörn Töpel bool force_zc, force_copy; 88173d3adbSBjörn Töpel struct netdev_bpf bpf; 89c9b47cc1SMagnus Karlsson int err = 0; 90173d3adbSBjörn Töpel 915464c3a0SIlya Maximets ASSERT_RTNL(); 925464c3a0SIlya Maximets 93173d3adbSBjörn Töpel force_zc = flags & XDP_ZEROCOPY; 94173d3adbSBjörn Töpel force_copy = flags & XDP_COPY; 95173d3adbSBjörn Töpel 96173d3adbSBjörn Töpel if (force_zc && force_copy) 97173d3adbSBjörn Töpel return -EINVAL; 98173d3adbSBjörn Töpel 995464c3a0SIlya Maximets if (xdp_get_umem_from_qid(dev, queue_id)) 1005464c3a0SIlya Maximets return -EBUSY; 101c9b47cc1SMagnus Karlsson 102cc5b5d35SKrzysztof Kazimierczak err = xdp_reg_umem_at_qid(dev, umem, queue_id); 103cc5b5d35SKrzysztof Kazimierczak if (err) 1045464c3a0SIlya Maximets return err; 105cc5b5d35SKrzysztof Kazimierczak 106c9b47cc1SMagnus Karlsson umem->dev = dev; 107c9b47cc1SMagnus Karlsson umem->queue_id = queue_id; 108162c820eSIlya Maximets 10977cd0d7bSMagnus Karlsson if (flags & XDP_USE_NEED_WAKEUP) { 11077cd0d7bSMagnus Karlsson umem->flags |= XDP_UMEM_USES_NEED_WAKEUP; 11177cd0d7bSMagnus Karlsson /* Tx needs to be explicitly woken up the first time. 11277cd0d7bSMagnus Karlsson * Also for supporting drivers that do not implement this 11377cd0d7bSMagnus Karlsson * feature. They will always have to call sendto(). 11477cd0d7bSMagnus Karlsson */ 11577cd0d7bSMagnus Karlsson xsk_set_tx_need_wakeup(umem); 11677cd0d7bSMagnus Karlsson } 11777cd0d7bSMagnus Karlsson 118162c820eSIlya Maximets dev_hold(dev); 119162c820eSIlya Maximets 120c9b47cc1SMagnus Karlsson if (force_copy) 121c9b47cc1SMagnus Karlsson /* For copy-mode, we are done. */ 1225464c3a0SIlya Maximets return 0; 123c9b47cc1SMagnus Karlsson 1249116e5e2SMagnus Karlsson if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) { 125c9b47cc1SMagnus Karlsson err = -EOPNOTSUPP; 126c9b47cc1SMagnus Karlsson goto err_unreg_umem; 12784c6b868SJakub Kicinski } 128173d3adbSBjörn Töpel 129173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 130173d3adbSBjörn Töpel bpf.xsk.umem = umem; 131173d3adbSBjörn Töpel bpf.xsk.queue_id = queue_id; 132173d3adbSBjörn Töpel 133173d3adbSBjörn Töpel err = dev->netdev_ops->ndo_bpf(dev, &bpf); 134f734607eSJakub Kicinski if (err) 135c9b47cc1SMagnus Karlsson goto err_unreg_umem; 136173d3adbSBjörn Töpel 137173d3adbSBjörn Töpel umem->zc = true; 138173d3adbSBjörn Töpel return 0; 13984c6b868SJakub Kicinski 140c9b47cc1SMagnus Karlsson err_unreg_umem: 141c9b47cc1SMagnus Karlsson if (!force_zc) 142c9b47cc1SMagnus Karlsson err = 0; /* fallback to copy mode */ 1431e405c1aSBjörn Töpel if (err) 1441e405c1aSBjörn Töpel xdp_clear_umem_at_qid(dev, queue_id); 145c9b47cc1SMagnus Karlsson return err; 146173d3adbSBjörn Töpel } 147173d3adbSBjörn Töpel 148455302d1SIlya Maximets void xdp_umem_clear_dev(struct xdp_umem *umem) 149173d3adbSBjörn Töpel { 150173d3adbSBjörn Töpel struct netdev_bpf bpf; 151173d3adbSBjörn Töpel int err; 152173d3adbSBjörn Töpel 153455302d1SIlya Maximets ASSERT_RTNL(); 154455302d1SIlya Maximets 15501d76b53SIlya Maximets if (!umem->dev) 15601d76b53SIlya Maximets return; 15701d76b53SIlya Maximets 158c9b47cc1SMagnus Karlsson if (umem->zc) { 159173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 160173d3adbSBjörn Töpel bpf.xsk.umem = NULL; 161173d3adbSBjörn Töpel bpf.xsk.queue_id = umem->queue_id; 162173d3adbSBjörn Töpel 163173d3adbSBjörn Töpel err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 164173d3adbSBjörn Töpel 165173d3adbSBjörn Töpel if (err) 166173d3adbSBjörn Töpel WARN(1, "failed to disable umem!\n"); 167c9b47cc1SMagnus Karlsson } 168173d3adbSBjörn Töpel 169c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(umem->dev, umem->queue_id); 170c9b47cc1SMagnus Karlsson 171173d3adbSBjörn Töpel dev_put(umem->dev); 172162c820eSIlya Maximets umem->dev = NULL; 173c9b47cc1SMagnus Karlsson umem->zc = false; 174173d3adbSBjörn Töpel } 175173d3adbSBjörn Töpel 176d9973cecSIvan Khoronzhuk static void xdp_umem_unmap_pages(struct xdp_umem *umem) 177d9973cecSIvan Khoronzhuk { 178d9973cecSIvan Khoronzhuk unsigned int i; 179d9973cecSIvan Khoronzhuk 180d9973cecSIvan Khoronzhuk for (i = 0; i < umem->npgs; i++) 181624676e7SIvan Khoronzhuk if (PageHighMem(umem->pgs[i])) 182624676e7SIvan Khoronzhuk vunmap(umem->pages[i].addr); 183624676e7SIvan Khoronzhuk } 184624676e7SIvan Khoronzhuk 185624676e7SIvan Khoronzhuk static int xdp_umem_map_pages(struct xdp_umem *umem) 186624676e7SIvan Khoronzhuk { 187624676e7SIvan Khoronzhuk unsigned int i; 188624676e7SIvan Khoronzhuk void *addr; 189624676e7SIvan Khoronzhuk 190624676e7SIvan Khoronzhuk for (i = 0; i < umem->npgs; i++) { 191624676e7SIvan Khoronzhuk if (PageHighMem(umem->pgs[i])) 192624676e7SIvan Khoronzhuk addr = vmap(&umem->pgs[i], 1, VM_MAP, PAGE_KERNEL); 193624676e7SIvan Khoronzhuk else 194624676e7SIvan Khoronzhuk addr = page_address(umem->pgs[i]); 195624676e7SIvan Khoronzhuk 196624676e7SIvan Khoronzhuk if (!addr) { 197624676e7SIvan Khoronzhuk xdp_umem_unmap_pages(umem); 198624676e7SIvan Khoronzhuk return -ENOMEM; 199624676e7SIvan Khoronzhuk } 200624676e7SIvan Khoronzhuk 201624676e7SIvan Khoronzhuk umem->pages[i].addr = addr; 202624676e7SIvan Khoronzhuk } 203624676e7SIvan Khoronzhuk 204624676e7SIvan Khoronzhuk return 0; 205d9973cecSIvan Khoronzhuk } 206d9973cecSIvan Khoronzhuk 207c0c77d8fSBjörn Töpel static void xdp_umem_unpin_pages(struct xdp_umem *umem) 208c0c77d8fSBjörn Töpel { 209*1edc9769SJohn Hubbard put_user_pages_dirty_lock(umem->pgs, umem->npgs, true); 210c0c77d8fSBjörn Töpel 211c0c77d8fSBjörn Töpel kfree(umem->pgs); 212c0c77d8fSBjörn Töpel umem->pgs = NULL; 213c0c77d8fSBjörn Töpel } 214c0c77d8fSBjörn Töpel 215c0c77d8fSBjörn Töpel static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 216c0c77d8fSBjörn Töpel { 217c09290c5SDaniel Borkmann if (umem->user) { 218c0c77d8fSBjörn Töpel atomic_long_sub(umem->npgs, &umem->user->locked_vm); 219c0c77d8fSBjörn Töpel free_uid(umem->user); 220c0c77d8fSBjörn Töpel } 221c09290c5SDaniel Borkmann } 222c0c77d8fSBjörn Töpel 223c0c77d8fSBjörn Töpel static void xdp_umem_release(struct xdp_umem *umem) 224c0c77d8fSBjörn Töpel { 225455302d1SIlya Maximets rtnl_lock(); 226173d3adbSBjörn Töpel xdp_umem_clear_dev(umem); 227455302d1SIlya Maximets rtnl_unlock(); 228173d3adbSBjörn Töpel 22950e74c01SBjörn Töpel ida_simple_remove(&umem_ida, umem->id); 23050e74c01SBjörn Töpel 231423f3832SMagnus Karlsson if (umem->fq) { 232423f3832SMagnus Karlsson xskq_destroy(umem->fq); 233423f3832SMagnus Karlsson umem->fq = NULL; 234423f3832SMagnus Karlsson } 235423f3832SMagnus Karlsson 236fe230832SMagnus Karlsson if (umem->cq) { 237fe230832SMagnus Karlsson xskq_destroy(umem->cq); 238fe230832SMagnus Karlsson umem->cq = NULL; 239fe230832SMagnus Karlsson } 240fe230832SMagnus Karlsson 241f5bd9138SJakub Kicinski xsk_reuseq_destroy(umem); 242f5bd9138SJakub Kicinski 243d9973cecSIvan Khoronzhuk xdp_umem_unmap_pages(umem); 244c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 245c0c77d8fSBjörn Töpel 2468aef7340SBjörn Töpel kfree(umem->pages); 2478aef7340SBjörn Töpel umem->pages = NULL; 2488aef7340SBjörn Töpel 249c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 250c0c77d8fSBjörn Töpel kfree(umem); 251c0c77d8fSBjörn Töpel } 252c0c77d8fSBjörn Töpel 253c0c77d8fSBjörn Töpel static void xdp_umem_release_deferred(struct work_struct *work) 254c0c77d8fSBjörn Töpel { 255c0c77d8fSBjörn Töpel struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 256c0c77d8fSBjörn Töpel 257c0c77d8fSBjörn Töpel xdp_umem_release(umem); 258c0c77d8fSBjörn Töpel } 259c0c77d8fSBjörn Töpel 260c0c77d8fSBjörn Töpel void xdp_get_umem(struct xdp_umem *umem) 261c0c77d8fSBjörn Töpel { 262d3b42f14SBjörn Töpel refcount_inc(&umem->users); 263c0c77d8fSBjörn Töpel } 264c0c77d8fSBjörn Töpel 265c0c77d8fSBjörn Töpel void xdp_put_umem(struct xdp_umem *umem) 266c0c77d8fSBjörn Töpel { 267c0c77d8fSBjörn Töpel if (!umem) 268c0c77d8fSBjörn Töpel return; 269c0c77d8fSBjörn Töpel 270d3b42f14SBjörn Töpel if (refcount_dec_and_test(&umem->users)) { 271c0c77d8fSBjörn Töpel INIT_WORK(&umem->work, xdp_umem_release_deferred); 272c0c77d8fSBjörn Töpel schedule_work(&umem->work); 273c0c77d8fSBjörn Töpel } 274c0c77d8fSBjörn Töpel } 275c0c77d8fSBjörn Töpel 276c0c77d8fSBjörn Töpel static int xdp_umem_pin_pages(struct xdp_umem *umem) 277c0c77d8fSBjörn Töpel { 278c0c77d8fSBjörn Töpel unsigned int gup_flags = FOLL_WRITE; 279c0c77d8fSBjörn Töpel long npgs; 280c0c77d8fSBjörn Töpel int err; 281c0c77d8fSBjörn Töpel 282a343993cSBjörn Töpel umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), 283a343993cSBjörn Töpel GFP_KERNEL | __GFP_NOWARN); 284c0c77d8fSBjörn Töpel if (!umem->pgs) 285c0c77d8fSBjörn Töpel return -ENOMEM; 286c0c77d8fSBjörn Töpel 287e451eb51SDavidlohr Bueso down_read(¤t->mm->mmap_sem); 288932f4a63SIra Weiny npgs = get_user_pages(umem->address, umem->npgs, 289932f4a63SIra Weiny gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL); 290e451eb51SDavidlohr Bueso up_read(¤t->mm->mmap_sem); 291c0c77d8fSBjörn Töpel 292c0c77d8fSBjörn Töpel if (npgs != umem->npgs) { 293c0c77d8fSBjörn Töpel if (npgs >= 0) { 294c0c77d8fSBjörn Töpel umem->npgs = npgs; 295c0c77d8fSBjörn Töpel err = -ENOMEM; 296c0c77d8fSBjörn Töpel goto out_pin; 297c0c77d8fSBjörn Töpel } 298c0c77d8fSBjörn Töpel err = npgs; 299c0c77d8fSBjörn Töpel goto out_pgs; 300c0c77d8fSBjörn Töpel } 301c0c77d8fSBjörn Töpel return 0; 302c0c77d8fSBjörn Töpel 303c0c77d8fSBjörn Töpel out_pin: 304c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 305c0c77d8fSBjörn Töpel out_pgs: 306c0c77d8fSBjörn Töpel kfree(umem->pgs); 307c0c77d8fSBjörn Töpel umem->pgs = NULL; 308c0c77d8fSBjörn Töpel return err; 309c0c77d8fSBjörn Töpel } 310c0c77d8fSBjörn Töpel 311c0c77d8fSBjörn Töpel static int xdp_umem_account_pages(struct xdp_umem *umem) 312c0c77d8fSBjörn Töpel { 313c0c77d8fSBjörn Töpel unsigned long lock_limit, new_npgs, old_npgs; 314c0c77d8fSBjörn Töpel 315c0c77d8fSBjörn Töpel if (capable(CAP_IPC_LOCK)) 316c0c77d8fSBjörn Töpel return 0; 317c0c77d8fSBjörn Töpel 318c0c77d8fSBjörn Töpel lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 319c0c77d8fSBjörn Töpel umem->user = get_uid(current_user()); 320c0c77d8fSBjörn Töpel 321c0c77d8fSBjörn Töpel do { 322c0c77d8fSBjörn Töpel old_npgs = atomic_long_read(&umem->user->locked_vm); 323c0c77d8fSBjörn Töpel new_npgs = old_npgs + umem->npgs; 324c0c77d8fSBjörn Töpel if (new_npgs > lock_limit) { 325c0c77d8fSBjörn Töpel free_uid(umem->user); 326c0c77d8fSBjörn Töpel umem->user = NULL; 327c0c77d8fSBjörn Töpel return -ENOBUFS; 328c0c77d8fSBjörn Töpel } 329c0c77d8fSBjörn Töpel } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 330c0c77d8fSBjörn Töpel new_npgs) != old_npgs); 331c0c77d8fSBjörn Töpel return 0; 332c0c77d8fSBjörn Töpel } 333c0c77d8fSBjörn Töpel 334a49049eaSBjörn Töpel static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 335c0c77d8fSBjörn Töpel { 336c05cd364SKevin Laatz bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; 337bbff2f32SBjörn Töpel u32 chunk_size = mr->chunk_size, headroom = mr->headroom; 338bbff2f32SBjörn Töpel unsigned int chunks, chunks_per_page; 339c0c77d8fSBjörn Töpel u64 addr = mr->addr, size = mr->len; 340624676e7SIvan Khoronzhuk int size_chk, err; 341c0c77d8fSBjörn Töpel 342bbff2f32SBjörn Töpel if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { 343c0c77d8fSBjörn Töpel /* Strictly speaking we could support this, if: 344c0c77d8fSBjörn Töpel * - huge pages, or* 345c0c77d8fSBjörn Töpel * - using an IOMMU, or 346c0c77d8fSBjörn Töpel * - making sure the memory area is consecutive 347c0c77d8fSBjörn Töpel * but for now, we simply say "computer says no". 348c0c77d8fSBjörn Töpel */ 349c0c77d8fSBjörn Töpel return -EINVAL; 350c0c77d8fSBjörn Töpel } 351c0c77d8fSBjörn Töpel 352c05cd364SKevin Laatz if (mr->flags & ~(XDP_UMEM_UNALIGNED_CHUNK_FLAG | 353c05cd364SKevin Laatz XDP_UMEM_USES_NEED_WAKEUP)) 354c05cd364SKevin Laatz return -EINVAL; 355c05cd364SKevin Laatz 356c05cd364SKevin Laatz if (!unaligned_chunks && !is_power_of_2(chunk_size)) 357c0c77d8fSBjörn Töpel return -EINVAL; 358c0c77d8fSBjörn Töpel 359c0c77d8fSBjörn Töpel if (!PAGE_ALIGNED(addr)) { 360c0c77d8fSBjörn Töpel /* Memory area has to be page size aligned. For 361c0c77d8fSBjörn Töpel * simplicity, this might change. 362c0c77d8fSBjörn Töpel */ 363c0c77d8fSBjörn Töpel return -EINVAL; 364c0c77d8fSBjörn Töpel } 365c0c77d8fSBjörn Töpel 366c0c77d8fSBjörn Töpel if ((addr + size) < addr) 367c0c77d8fSBjörn Töpel return -EINVAL; 368c0c77d8fSBjörn Töpel 369bbff2f32SBjörn Töpel chunks = (unsigned int)div_u64(size, chunk_size); 370bbff2f32SBjörn Töpel if (chunks == 0) 371c0c77d8fSBjörn Töpel return -EINVAL; 372c0c77d8fSBjörn Töpel 373c05cd364SKevin Laatz if (!unaligned_chunks) { 374bbff2f32SBjörn Töpel chunks_per_page = PAGE_SIZE / chunk_size; 375bbff2f32SBjörn Töpel if (chunks < chunks_per_page || chunks % chunks_per_page) 376c0c77d8fSBjörn Töpel return -EINVAL; 377c05cd364SKevin Laatz } 378c0c77d8fSBjörn Töpel 379bbff2f32SBjörn Töpel headroom = ALIGN(headroom, 64); 380c0c77d8fSBjörn Töpel 381bbff2f32SBjörn Töpel size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM; 382c0c77d8fSBjörn Töpel if (size_chk < 0) 383c0c77d8fSBjörn Töpel return -EINVAL; 384c0c77d8fSBjörn Töpel 385c0c77d8fSBjörn Töpel umem->address = (unsigned long)addr; 386c05cd364SKevin Laatz umem->chunk_mask = unaligned_chunks ? XSK_UNALIGNED_BUF_ADDR_MASK 387c05cd364SKevin Laatz : ~((u64)chunk_size - 1); 38893ee30f3SMagnus Karlsson umem->size = size; 389bbff2f32SBjörn Töpel umem->headroom = headroom; 390bbff2f32SBjörn Töpel umem->chunk_size_nohr = chunk_size - headroom; 391c0c77d8fSBjörn Töpel umem->npgs = size / PAGE_SIZE; 392c0c77d8fSBjörn Töpel umem->pgs = NULL; 393c0c77d8fSBjörn Töpel umem->user = NULL; 394c05cd364SKevin Laatz umem->flags = mr->flags; 395ac98d8aaSMagnus Karlsson INIT_LIST_HEAD(&umem->xsk_list); 396ac98d8aaSMagnus Karlsson spin_lock_init(&umem->xsk_list_lock); 397c0c77d8fSBjörn Töpel 398d3b42f14SBjörn Töpel refcount_set(&umem->users, 1); 399c0c77d8fSBjörn Töpel 400c0c77d8fSBjörn Töpel err = xdp_umem_account_pages(umem); 401c0c77d8fSBjörn Töpel if (err) 402044175a0SBjörn Töpel return err; 403c0c77d8fSBjörn Töpel 404c0c77d8fSBjörn Töpel err = xdp_umem_pin_pages(umem); 405c0c77d8fSBjörn Töpel if (err) 406c0c77d8fSBjörn Töpel goto out_account; 4078aef7340SBjörn Töpel 4088aef7340SBjörn Töpel umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL); 4098aef7340SBjörn Töpel if (!umem->pages) { 4108aef7340SBjörn Töpel err = -ENOMEM; 411fb89c394SIvan Khoronzhuk goto out_pin; 4128aef7340SBjörn Töpel } 4138aef7340SBjörn Töpel 414624676e7SIvan Khoronzhuk err = xdp_umem_map_pages(umem); 415624676e7SIvan Khoronzhuk if (!err) 416c0c77d8fSBjörn Töpel return 0; 417c0c77d8fSBjörn Töpel 418624676e7SIvan Khoronzhuk kfree(umem->pages); 419624676e7SIvan Khoronzhuk 420fb89c394SIvan Khoronzhuk out_pin: 421fb89c394SIvan Khoronzhuk xdp_umem_unpin_pages(umem); 422c0c77d8fSBjörn Töpel out_account: 423c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 424c0c77d8fSBjörn Töpel return err; 425c0c77d8fSBjörn Töpel } 426965a9909SMagnus Karlsson 427a49049eaSBjörn Töpel struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 428a49049eaSBjörn Töpel { 429a49049eaSBjörn Töpel struct xdp_umem *umem; 430a49049eaSBjörn Töpel int err; 431a49049eaSBjörn Töpel 432a49049eaSBjörn Töpel umem = kzalloc(sizeof(*umem), GFP_KERNEL); 433a49049eaSBjörn Töpel if (!umem) 434a49049eaSBjörn Töpel return ERR_PTR(-ENOMEM); 435a49049eaSBjörn Töpel 43650e74c01SBjörn Töpel err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL); 43750e74c01SBjörn Töpel if (err < 0) { 43850e74c01SBjörn Töpel kfree(umem); 43950e74c01SBjörn Töpel return ERR_PTR(err); 44050e74c01SBjörn Töpel } 44150e74c01SBjörn Töpel umem->id = err; 44250e74c01SBjörn Töpel 443a49049eaSBjörn Töpel err = xdp_umem_reg(umem, mr); 444a49049eaSBjörn Töpel if (err) { 44550e74c01SBjörn Töpel ida_simple_remove(&umem_ida, umem->id); 446a49049eaSBjörn Töpel kfree(umem); 447a49049eaSBjörn Töpel return ERR_PTR(err); 448a49049eaSBjörn Töpel } 449a49049eaSBjörn Töpel 450a49049eaSBjörn Töpel return umem; 451a49049eaSBjörn Töpel } 452a49049eaSBjörn Töpel 453965a9909SMagnus Karlsson bool xdp_umem_validate_queues(struct xdp_umem *umem) 454965a9909SMagnus Karlsson { 455da60cf00SBjörn Töpel return umem->fq && umem->cq; 456965a9909SMagnus Karlsson } 457