1c0c77d8fSBjörn Töpel // SPDX-License-Identifier: GPL-2.0 2c0c77d8fSBjörn Töpel /* XDP user-space packet buffer 3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4c0c77d8fSBjörn Töpel */ 5c0c77d8fSBjörn Töpel 6c0c77d8fSBjörn Töpel #include <linux/init.h> 7c0c77d8fSBjörn Töpel #include <linux/sched/mm.h> 8c0c77d8fSBjörn Töpel #include <linux/sched/signal.h> 9c0c77d8fSBjörn Töpel #include <linux/sched/task.h> 10c0c77d8fSBjörn Töpel #include <linux/uaccess.h> 11c0c77d8fSBjörn Töpel #include <linux/slab.h> 12c0c77d8fSBjörn Töpel #include <linux/bpf.h> 13c0c77d8fSBjörn Töpel #include <linux/mm.h> 1484c6b868SJakub Kicinski #include <linux/netdevice.h> 1584c6b868SJakub Kicinski #include <linux/rtnetlink.h> 1650e74c01SBjörn Töpel #include <linux/idr.h> 17624676e7SIvan Khoronzhuk #include <linux/vmalloc.h> 18c0c77d8fSBjörn Töpel 19c0c77d8fSBjörn Töpel #include "xdp_umem.h" 20e61e62b9SBjörn Töpel #include "xsk_queue.h" 21c0c77d8fSBjörn Töpel 22bbff2f32SBjörn Töpel #define XDP_UMEM_MIN_CHUNK_SIZE 2048 23c0c77d8fSBjörn Töpel 2450e74c01SBjörn Töpel static DEFINE_IDA(umem_ida); 2550e74c01SBjörn Töpel 26ac98d8aaSMagnus Karlsson void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 27ac98d8aaSMagnus Karlsson { 28ac98d8aaSMagnus Karlsson unsigned long flags; 29ac98d8aaSMagnus Karlsson 302afd23f7SMagnus Karlsson if (!xs->tx) 312afd23f7SMagnus Karlsson return; 322afd23f7SMagnus Karlsson 33e4e5aefcSMagnus Karlsson spin_lock_irqsave(&umem->xsk_tx_list_lock, flags); 34e4e5aefcSMagnus Karlsson list_add_rcu(&xs->list, &umem->xsk_tx_list); 35e4e5aefcSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags); 36ac98d8aaSMagnus Karlsson } 37ac98d8aaSMagnus Karlsson 38ac98d8aaSMagnus Karlsson void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 39ac98d8aaSMagnus Karlsson { 40ac98d8aaSMagnus Karlsson unsigned long flags; 41ac98d8aaSMagnus Karlsson 422afd23f7SMagnus Karlsson if (!xs->tx) 432afd23f7SMagnus Karlsson return; 442afd23f7SMagnus Karlsson 45e4e5aefcSMagnus Karlsson spin_lock_irqsave(&umem->xsk_tx_list_lock, flags); 46ac98d8aaSMagnus Karlsson list_del_rcu(&xs->list); 47e4e5aefcSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags); 48ac98d8aaSMagnus Karlsson } 49ac98d8aaSMagnus Karlsson 50c9b47cc1SMagnus Karlsson /* The umem is stored both in the _rx struct and the _tx struct as we do 51c9b47cc1SMagnus Karlsson * not know if the device has more tx queues than rx, or the opposite. 52c9b47cc1SMagnus Karlsson * This might also change during run time. 53c9b47cc1SMagnus Karlsson */ 54cc5b5d35SKrzysztof Kazimierczak static int xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, 55c9b47cc1SMagnus Karlsson u16 queue_id) 5684c6b868SJakub Kicinski { 57cc5b5d35SKrzysztof Kazimierczak if (queue_id >= max_t(unsigned int, 58cc5b5d35SKrzysztof Kazimierczak dev->real_num_rx_queues, 59cc5b5d35SKrzysztof Kazimierczak dev->real_num_tx_queues)) 60cc5b5d35SKrzysztof Kazimierczak return -EINVAL; 61cc5b5d35SKrzysztof Kazimierczak 62c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 63c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = umem; 64c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 65c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = umem; 66cc5b5d35SKrzysztof Kazimierczak 67cc5b5d35SKrzysztof Kazimierczak return 0; 68c9b47cc1SMagnus Karlsson } 6984c6b868SJakub Kicinski 701661d346SJakub Kicinski struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 71c9b47cc1SMagnus Karlsson u16 queue_id) 72c9b47cc1SMagnus Karlsson { 73c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 74c9b47cc1SMagnus Karlsson return dev->_rx[queue_id].umem; 75c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 76c9b47cc1SMagnus Karlsson return dev->_tx[queue_id].umem; 7784c6b868SJakub Kicinski 78c9b47cc1SMagnus Karlsson return NULL; 79c9b47cc1SMagnus Karlsson } 805f4f3b2dSJan Sokolowski EXPORT_SYMBOL(xdp_get_umem_from_qid); 8184c6b868SJakub Kicinski 82c9b47cc1SMagnus Karlsson static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) 83c9b47cc1SMagnus Karlsson { 84a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 85c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = NULL; 86a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 87c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = NULL; 8884c6b868SJakub Kicinski } 8984c6b868SJakub Kicinski 90173d3adbSBjörn Töpel int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 91c9b47cc1SMagnus Karlsson u16 queue_id, u16 flags) 92173d3adbSBjörn Töpel { 93173d3adbSBjörn Töpel bool force_zc, force_copy; 94173d3adbSBjörn Töpel struct netdev_bpf bpf; 95c9b47cc1SMagnus Karlsson int err = 0; 96173d3adbSBjörn Töpel 975464c3a0SIlya Maximets ASSERT_RTNL(); 985464c3a0SIlya Maximets 99173d3adbSBjörn Töpel force_zc = flags & XDP_ZEROCOPY; 100173d3adbSBjörn Töpel force_copy = flags & XDP_COPY; 101173d3adbSBjörn Töpel 102173d3adbSBjörn Töpel if (force_zc && force_copy) 103173d3adbSBjörn Töpel return -EINVAL; 104173d3adbSBjörn Töpel 1055464c3a0SIlya Maximets if (xdp_get_umem_from_qid(dev, queue_id)) 1065464c3a0SIlya Maximets return -EBUSY; 107c9b47cc1SMagnus Karlsson 108cc5b5d35SKrzysztof Kazimierczak err = xdp_reg_umem_at_qid(dev, umem, queue_id); 109cc5b5d35SKrzysztof Kazimierczak if (err) 1105464c3a0SIlya Maximets return err; 111cc5b5d35SKrzysztof Kazimierczak 112c9b47cc1SMagnus Karlsson umem->dev = dev; 113c9b47cc1SMagnus Karlsson umem->queue_id = queue_id; 114162c820eSIlya Maximets 11577cd0d7bSMagnus Karlsson if (flags & XDP_USE_NEED_WAKEUP) { 11677cd0d7bSMagnus Karlsson umem->flags |= XDP_UMEM_USES_NEED_WAKEUP; 11777cd0d7bSMagnus Karlsson /* Tx needs to be explicitly woken up the first time. 11877cd0d7bSMagnus Karlsson * Also for supporting drivers that do not implement this 11977cd0d7bSMagnus Karlsson * feature. They will always have to call sendto(). 12077cd0d7bSMagnus Karlsson */ 12177cd0d7bSMagnus Karlsson xsk_set_tx_need_wakeup(umem); 12277cd0d7bSMagnus Karlsson } 12377cd0d7bSMagnus Karlsson 124162c820eSIlya Maximets dev_hold(dev); 125162c820eSIlya Maximets 126c9b47cc1SMagnus Karlsson if (force_copy) 127c9b47cc1SMagnus Karlsson /* For copy-mode, we are done. */ 1285464c3a0SIlya Maximets return 0; 129c9b47cc1SMagnus Karlsson 1309116e5e2SMagnus Karlsson if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) { 131c9b47cc1SMagnus Karlsson err = -EOPNOTSUPP; 132c9b47cc1SMagnus Karlsson goto err_unreg_umem; 13384c6b868SJakub Kicinski } 134173d3adbSBjörn Töpel 135173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 136173d3adbSBjörn Töpel bpf.xsk.umem = umem; 137173d3adbSBjörn Töpel bpf.xsk.queue_id = queue_id; 138173d3adbSBjörn Töpel 139173d3adbSBjörn Töpel err = dev->netdev_ops->ndo_bpf(dev, &bpf); 140f734607eSJakub Kicinski if (err) 141c9b47cc1SMagnus Karlsson goto err_unreg_umem; 142173d3adbSBjörn Töpel 143173d3adbSBjörn Töpel umem->zc = true; 144173d3adbSBjörn Töpel return 0; 14584c6b868SJakub Kicinski 146c9b47cc1SMagnus Karlsson err_unreg_umem: 147c9b47cc1SMagnus Karlsson if (!force_zc) 148c9b47cc1SMagnus Karlsson err = 0; /* fallback to copy mode */ 1491e405c1aSBjörn Töpel if (err) 1501e405c1aSBjörn Töpel xdp_clear_umem_at_qid(dev, queue_id); 151c9b47cc1SMagnus Karlsson return err; 152173d3adbSBjörn Töpel } 153173d3adbSBjörn Töpel 154455302d1SIlya Maximets void xdp_umem_clear_dev(struct xdp_umem *umem) 155173d3adbSBjörn Töpel { 156173d3adbSBjörn Töpel struct netdev_bpf bpf; 157173d3adbSBjörn Töpel int err; 158173d3adbSBjörn Töpel 159455302d1SIlya Maximets ASSERT_RTNL(); 160455302d1SIlya Maximets 16101d76b53SIlya Maximets if (!umem->dev) 16201d76b53SIlya Maximets return; 16301d76b53SIlya Maximets 164c9b47cc1SMagnus Karlsson if (umem->zc) { 165173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 166173d3adbSBjörn Töpel bpf.xsk.umem = NULL; 167173d3adbSBjörn Töpel bpf.xsk.queue_id = umem->queue_id; 168173d3adbSBjörn Töpel 169173d3adbSBjörn Töpel err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 170173d3adbSBjörn Töpel 171173d3adbSBjörn Töpel if (err) 172173d3adbSBjörn Töpel WARN(1, "failed to disable umem!\n"); 173c9b47cc1SMagnus Karlsson } 174173d3adbSBjörn Töpel 175c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(umem->dev, umem->queue_id); 176c9b47cc1SMagnus Karlsson 177173d3adbSBjörn Töpel dev_put(umem->dev); 178162c820eSIlya Maximets umem->dev = NULL; 179c9b47cc1SMagnus Karlsson umem->zc = false; 180173d3adbSBjörn Töpel } 181173d3adbSBjörn Töpel 182c0c77d8fSBjörn Töpel static void xdp_umem_unpin_pages(struct xdp_umem *umem) 183c0c77d8fSBjörn Töpel { 184f1f6a7ddSJohn Hubbard unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); 185c0c77d8fSBjörn Töpel 186c0c77d8fSBjörn Töpel kfree(umem->pgs); 187c0c77d8fSBjörn Töpel umem->pgs = NULL; 188c0c77d8fSBjörn Töpel } 189c0c77d8fSBjörn Töpel 190c0c77d8fSBjörn Töpel static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 191c0c77d8fSBjörn Töpel { 192c09290c5SDaniel Borkmann if (umem->user) { 193c0c77d8fSBjörn Töpel atomic_long_sub(umem->npgs, &umem->user->locked_vm); 194c0c77d8fSBjörn Töpel free_uid(umem->user); 195c0c77d8fSBjörn Töpel } 196c09290c5SDaniel Borkmann } 197c0c77d8fSBjörn Töpel 198c0c77d8fSBjörn Töpel static void xdp_umem_release(struct xdp_umem *umem) 199c0c77d8fSBjörn Töpel { 200455302d1SIlya Maximets rtnl_lock(); 201173d3adbSBjörn Töpel xdp_umem_clear_dev(umem); 202455302d1SIlya Maximets rtnl_unlock(); 203173d3adbSBjörn Töpel 20450e74c01SBjörn Töpel ida_simple_remove(&umem_ida, umem->id); 20550e74c01SBjörn Töpel 206423f3832SMagnus Karlsson if (umem->fq) { 207423f3832SMagnus Karlsson xskq_destroy(umem->fq); 208423f3832SMagnus Karlsson umem->fq = NULL; 209423f3832SMagnus Karlsson } 210423f3832SMagnus Karlsson 211fe230832SMagnus Karlsson if (umem->cq) { 212fe230832SMagnus Karlsson xskq_destroy(umem->cq); 213fe230832SMagnus Karlsson umem->cq = NULL; 214fe230832SMagnus Karlsson } 215fe230832SMagnus Karlsson 2162b43470aSBjörn Töpel xp_destroy(umem->pool); 217c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 218c0c77d8fSBjörn Töpel 219c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 220c0c77d8fSBjörn Töpel kfree(umem); 221c0c77d8fSBjörn Töpel } 222c0c77d8fSBjörn Töpel 223c0c77d8fSBjörn Töpel static void xdp_umem_release_deferred(struct work_struct *work) 224c0c77d8fSBjörn Töpel { 225c0c77d8fSBjörn Töpel struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 226c0c77d8fSBjörn Töpel 227c0c77d8fSBjörn Töpel xdp_umem_release(umem); 228c0c77d8fSBjörn Töpel } 229c0c77d8fSBjörn Töpel 230c0c77d8fSBjörn Töpel void xdp_get_umem(struct xdp_umem *umem) 231c0c77d8fSBjörn Töpel { 232d3b42f14SBjörn Töpel refcount_inc(&umem->users); 233c0c77d8fSBjörn Töpel } 234c0c77d8fSBjörn Töpel 235c0c77d8fSBjörn Töpel void xdp_put_umem(struct xdp_umem *umem) 236c0c77d8fSBjörn Töpel { 237c0c77d8fSBjörn Töpel if (!umem) 238c0c77d8fSBjörn Töpel return; 239c0c77d8fSBjörn Töpel 240d3b42f14SBjörn Töpel if (refcount_dec_and_test(&umem->users)) { 241c0c77d8fSBjörn Töpel INIT_WORK(&umem->work, xdp_umem_release_deferred); 242c0c77d8fSBjörn Töpel schedule_work(&umem->work); 243c0c77d8fSBjörn Töpel } 244c0c77d8fSBjörn Töpel } 245c0c77d8fSBjörn Töpel 24607bf2d97SMagnus Karlsson static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address) 247c0c77d8fSBjörn Töpel { 248c0c77d8fSBjörn Töpel unsigned int gup_flags = FOLL_WRITE; 249c0c77d8fSBjörn Töpel long npgs; 250c0c77d8fSBjörn Töpel int err; 251c0c77d8fSBjörn Töpel 252a343993cSBjörn Töpel umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), 253a343993cSBjörn Töpel GFP_KERNEL | __GFP_NOWARN); 254c0c77d8fSBjörn Töpel if (!umem->pgs) 255c0c77d8fSBjörn Töpel return -ENOMEM; 256c0c77d8fSBjörn Töpel 257*d8ed45c5SMichel Lespinasse mmap_read_lock(current->mm); 25807bf2d97SMagnus Karlsson npgs = pin_user_pages(address, umem->npgs, 259932f4a63SIra Weiny gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL); 260*d8ed45c5SMichel Lespinasse mmap_read_unlock(current->mm); 261c0c77d8fSBjörn Töpel 262c0c77d8fSBjörn Töpel if (npgs != umem->npgs) { 263c0c77d8fSBjörn Töpel if (npgs >= 0) { 264c0c77d8fSBjörn Töpel umem->npgs = npgs; 265c0c77d8fSBjörn Töpel err = -ENOMEM; 266c0c77d8fSBjörn Töpel goto out_pin; 267c0c77d8fSBjörn Töpel } 268c0c77d8fSBjörn Töpel err = npgs; 269c0c77d8fSBjörn Töpel goto out_pgs; 270c0c77d8fSBjörn Töpel } 271c0c77d8fSBjörn Töpel return 0; 272c0c77d8fSBjörn Töpel 273c0c77d8fSBjörn Töpel out_pin: 274c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 275c0c77d8fSBjörn Töpel out_pgs: 276c0c77d8fSBjörn Töpel kfree(umem->pgs); 277c0c77d8fSBjörn Töpel umem->pgs = NULL; 278c0c77d8fSBjörn Töpel return err; 279c0c77d8fSBjörn Töpel } 280c0c77d8fSBjörn Töpel 281c0c77d8fSBjörn Töpel static int xdp_umem_account_pages(struct xdp_umem *umem) 282c0c77d8fSBjörn Töpel { 283c0c77d8fSBjörn Töpel unsigned long lock_limit, new_npgs, old_npgs; 284c0c77d8fSBjörn Töpel 285c0c77d8fSBjörn Töpel if (capable(CAP_IPC_LOCK)) 286c0c77d8fSBjörn Töpel return 0; 287c0c77d8fSBjörn Töpel 288c0c77d8fSBjörn Töpel lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 289c0c77d8fSBjörn Töpel umem->user = get_uid(current_user()); 290c0c77d8fSBjörn Töpel 291c0c77d8fSBjörn Töpel do { 292c0c77d8fSBjörn Töpel old_npgs = atomic_long_read(&umem->user->locked_vm); 293c0c77d8fSBjörn Töpel new_npgs = old_npgs + umem->npgs; 294c0c77d8fSBjörn Töpel if (new_npgs > lock_limit) { 295c0c77d8fSBjörn Töpel free_uid(umem->user); 296c0c77d8fSBjörn Töpel umem->user = NULL; 297c0c77d8fSBjörn Töpel return -ENOBUFS; 298c0c77d8fSBjörn Töpel } 299c0c77d8fSBjörn Töpel } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 300c0c77d8fSBjörn Töpel new_npgs) != old_npgs); 301c0c77d8fSBjörn Töpel return 0; 302c0c77d8fSBjörn Töpel } 303c0c77d8fSBjörn Töpel 304a49049eaSBjörn Töpel static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 305c0c77d8fSBjörn Töpel { 306c05cd364SKevin Laatz bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; 307bbff2f32SBjörn Töpel u32 chunk_size = mr->chunk_size, headroom = mr->headroom; 308b16a87d0SBjörn Töpel u64 npgs, addr = mr->addr, size = mr->len; 309bbff2f32SBjörn Töpel unsigned int chunks, chunks_per_page; 31099e3a236SMagnus Karlsson int err; 311c0c77d8fSBjörn Töpel 312bbff2f32SBjörn Töpel if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { 313c0c77d8fSBjörn Töpel /* Strictly speaking we could support this, if: 314c0c77d8fSBjörn Töpel * - huge pages, or* 315c0c77d8fSBjörn Töpel * - using an IOMMU, or 316c0c77d8fSBjörn Töpel * - making sure the memory area is consecutive 317c0c77d8fSBjörn Töpel * but for now, we simply say "computer says no". 318c0c77d8fSBjörn Töpel */ 319c0c77d8fSBjörn Töpel return -EINVAL; 320c0c77d8fSBjörn Töpel } 321c0c77d8fSBjörn Töpel 322c05cd364SKevin Laatz if (mr->flags & ~(XDP_UMEM_UNALIGNED_CHUNK_FLAG | 323c05cd364SKevin Laatz XDP_UMEM_USES_NEED_WAKEUP)) 324c05cd364SKevin Laatz return -EINVAL; 325c05cd364SKevin Laatz 326c05cd364SKevin Laatz if (!unaligned_chunks && !is_power_of_2(chunk_size)) 327c0c77d8fSBjörn Töpel return -EINVAL; 328c0c77d8fSBjörn Töpel 329c0c77d8fSBjörn Töpel if (!PAGE_ALIGNED(addr)) { 330c0c77d8fSBjörn Töpel /* Memory area has to be page size aligned. For 331c0c77d8fSBjörn Töpel * simplicity, this might change. 332c0c77d8fSBjörn Töpel */ 333c0c77d8fSBjörn Töpel return -EINVAL; 334c0c77d8fSBjörn Töpel } 335c0c77d8fSBjörn Töpel 336c0c77d8fSBjörn Töpel if ((addr + size) < addr) 337c0c77d8fSBjörn Töpel return -EINVAL; 338c0c77d8fSBjörn Töpel 3397d877c35SPavel Machek npgs = size >> PAGE_SHIFT; 340b16a87d0SBjörn Töpel if (npgs > U32_MAX) 341b16a87d0SBjörn Töpel return -EINVAL; 342b16a87d0SBjörn Töpel 343bbff2f32SBjörn Töpel chunks = (unsigned int)div_u64(size, chunk_size); 344bbff2f32SBjörn Töpel if (chunks == 0) 345c0c77d8fSBjörn Töpel return -EINVAL; 346c0c77d8fSBjörn Töpel 347c05cd364SKevin Laatz if (!unaligned_chunks) { 348bbff2f32SBjörn Töpel chunks_per_page = PAGE_SIZE / chunk_size; 349bbff2f32SBjörn Töpel if (chunks < chunks_per_page || chunks % chunks_per_page) 350c0c77d8fSBjörn Töpel return -EINVAL; 351c05cd364SKevin Laatz } 352c0c77d8fSBjörn Töpel 35399e3a236SMagnus Karlsson if (headroom >= chunk_size - XDP_PACKET_HEADROOM) 354c0c77d8fSBjörn Töpel return -EINVAL; 355c0c77d8fSBjörn Töpel 35693ee30f3SMagnus Karlsson umem->size = size; 357bbff2f32SBjörn Töpel umem->headroom = headroom; 3582b43470aSBjörn Töpel umem->chunk_size = chunk_size; 359b16a87d0SBjörn Töpel umem->npgs = (u32)npgs; 360c0c77d8fSBjörn Töpel umem->pgs = NULL; 361c0c77d8fSBjörn Töpel umem->user = NULL; 362c05cd364SKevin Laatz umem->flags = mr->flags; 363e4e5aefcSMagnus Karlsson INIT_LIST_HEAD(&umem->xsk_tx_list); 364e4e5aefcSMagnus Karlsson spin_lock_init(&umem->xsk_tx_list_lock); 365c0c77d8fSBjörn Töpel 366d3b42f14SBjörn Töpel refcount_set(&umem->users, 1); 367c0c77d8fSBjörn Töpel 368c0c77d8fSBjörn Töpel err = xdp_umem_account_pages(umem); 369c0c77d8fSBjörn Töpel if (err) 370044175a0SBjörn Töpel return err; 371c0c77d8fSBjörn Töpel 37207bf2d97SMagnus Karlsson err = xdp_umem_pin_pages(umem, (unsigned long)addr); 373c0c77d8fSBjörn Töpel if (err) 374c0c77d8fSBjörn Töpel goto out_account; 3758aef7340SBjörn Töpel 3762b43470aSBjörn Töpel umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size, 3772b43470aSBjörn Töpel headroom, size, unaligned_chunks); 3782b43470aSBjörn Töpel if (!umem->pool) { 3792b43470aSBjörn Töpel err = -ENOMEM; 3800807892eSBjörn Töpel goto out_pin; 3812b43470aSBjörn Töpel } 382c0c77d8fSBjörn Töpel return 0; 383c0c77d8fSBjörn Töpel 384fb89c394SIvan Khoronzhuk out_pin: 385fb89c394SIvan Khoronzhuk xdp_umem_unpin_pages(umem); 386c0c77d8fSBjörn Töpel out_account: 387c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 388c0c77d8fSBjörn Töpel return err; 389c0c77d8fSBjörn Töpel } 390965a9909SMagnus Karlsson 391a49049eaSBjörn Töpel struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 392a49049eaSBjörn Töpel { 393a49049eaSBjörn Töpel struct xdp_umem *umem; 394a49049eaSBjörn Töpel int err; 395a49049eaSBjörn Töpel 396a49049eaSBjörn Töpel umem = kzalloc(sizeof(*umem), GFP_KERNEL); 397a49049eaSBjörn Töpel if (!umem) 398a49049eaSBjörn Töpel return ERR_PTR(-ENOMEM); 399a49049eaSBjörn Töpel 40050e74c01SBjörn Töpel err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL); 40150e74c01SBjörn Töpel if (err < 0) { 40250e74c01SBjörn Töpel kfree(umem); 40350e74c01SBjörn Töpel return ERR_PTR(err); 40450e74c01SBjörn Töpel } 40550e74c01SBjörn Töpel umem->id = err; 40650e74c01SBjörn Töpel 407a49049eaSBjörn Töpel err = xdp_umem_reg(umem, mr); 408a49049eaSBjörn Töpel if (err) { 40950e74c01SBjörn Töpel ida_simple_remove(&umem_ida, umem->id); 410a49049eaSBjörn Töpel kfree(umem); 411a49049eaSBjörn Töpel return ERR_PTR(err); 412a49049eaSBjörn Töpel } 413a49049eaSBjörn Töpel 414a49049eaSBjörn Töpel return umem; 415a49049eaSBjörn Töpel } 416a49049eaSBjörn Töpel 417965a9909SMagnus Karlsson bool xdp_umem_validate_queues(struct xdp_umem *umem) 418965a9909SMagnus Karlsson { 419da60cf00SBjörn Töpel return umem->fq && umem->cq; 420965a9909SMagnus Karlsson } 421