1.. SPDX-License-Identifier: GPL-2.0 2 3============= 4Page Pool API 5============= 6 7.. kernel-doc:: include/net/page_pool/helpers.h 8 :doc: page_pool allocator 9 10Architecture overview 11===================== 12 13.. code-block:: none 14 15 +------------------+ 16 | Driver | 17 +------------------+ 18 ^ 19 | 20 | 21 | 22 v 23 +--------------------------------------------+ 24 | request memory | 25 +--------------------------------------------+ 26 ^ ^ 27 | | 28 | Pool empty | Pool has entries 29 | | 30 v v 31 +-----------------------+ +------------------------+ 32 | alloc (and map) pages | | get page from cache | 33 +-----------------------+ +------------------------+ 34 ^ ^ 35 | | 36 | cache available | No entries, refill 37 | | from ptr-ring 38 | | 39 v v 40 +-----------------+ +------------------+ 41 | Fast cache | | ptr-ring cache | 42 +-----------------+ +------------------+ 43 44API interface 45============= 46The number of pools created **must** match the number of hardware queues 47unless hardware restrictions make that impossible. This would otherwise beat the 48purpose of page pool, which is allocate pages fast from cache without locking. 49This lockless guarantee naturally comes from running under a NAPI softirq. 50The protection doesn't strictly have to be NAPI, any guarantee that allocating 51a page will cause no race conditions is enough. 52 53.. kernel-doc:: net/core/page_pool.c 54 :identifiers: page_pool_create 55 56.. kernel-doc:: include/net/page_pool/types.h 57 :identifiers: struct page_pool_params 58 59.. kernel-doc:: include/net/page_pool/helpers.h 60 :identifiers: page_pool_put_page page_pool_put_full_page 61 page_pool_recycle_direct page_pool_free_va 62 page_pool_dev_alloc_pages page_pool_dev_alloc_frag 63 page_pool_dev_alloc page_pool_dev_alloc_va 64 page_pool_get_dma_addr page_pool_get_dma_dir 65 66.. kernel-doc:: net/core/page_pool.c 67 :identifiers: page_pool_put_page_bulk page_pool_get_stats 68 69DMA sync 70-------- 71Driver is always responsible for syncing the pages for the CPU. 72Drivers may choose to take care of syncing for the device as well 73or set the ``PP_FLAG_DMA_SYNC_DEV`` flag to request that pages 74allocated from the page pool are already synced for the device. 75 76If ``PP_FLAG_DMA_SYNC_DEV`` is set, the driver must inform the core what portion 77of the buffer has to be synced. This allows the core to avoid syncing the entire 78page when the drivers knows that the device only accessed a portion of the page. 79 80Most drivers will reserve headroom in front of the frame. This part 81of the buffer is not touched by the device, so to avoid syncing 82it drivers can set the ``offset`` field in struct page_pool_params 83appropriately. 84 85For pages recycled on the XDP xmit and skb paths the page pool will 86use the ``max_len`` member of struct page_pool_params to decide how 87much of the page needs to be synced (starting at ``offset``). 88When directly freeing pages in the driver (page_pool_put_page()) 89the ``dma_sync_size`` argument specifies how much of the buffer needs 90to be synced. 91 92If in doubt set ``offset`` to 0, ``max_len`` to ``PAGE_SIZE`` and 93pass -1 as ``dma_sync_size``. That combination of arguments is always 94correct. 95 96Note that the syncing parameters are for the entire page. 97This is important to remember when using fragments (``PP_FLAG_PAGE_FRAG``), 98where allocated buffers may be smaller than a full page. 99Unless the driver author really understands page pool internals 100it's recommended to always use ``offset = 0``, ``max_len = PAGE_SIZE`` 101with fragmented page pools. 102 103Stats API and structures 104------------------------ 105If the kernel is configured with ``CONFIG_PAGE_POOL_STATS=y``, the API 106page_pool_get_stats() and structures described below are available. 107It takes a pointer to a ``struct page_pool`` and a pointer to a struct 108page_pool_stats allocated by the caller. 109 110The API will fill in the provided struct page_pool_stats with 111statistics about the page_pool. 112 113.. kernel-doc:: include/net/page_pool/types.h 114 :identifiers: struct page_pool_recycle_stats 115 struct page_pool_alloc_stats 116 struct page_pool_stats 117 118Coding examples 119=============== 120 121Registration 122------------ 123 124.. code-block:: c 125 126 /* Page pool registration */ 127 struct page_pool_params pp_params = { 0 }; 128 struct xdp_rxq_info xdp_rxq; 129 int err; 130 131 pp_params.order = 0; 132 /* internal DMA mapping in page_pool */ 133 pp_params.flags = PP_FLAG_DMA_MAP; 134 pp_params.pool_size = DESC_NUM; 135 pp_params.nid = NUMA_NO_NODE; 136 pp_params.dev = priv->dev; 137 pp_params.napi = napi; /* only if locking is tied to NAPI */ 138 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 139 page_pool = page_pool_create(&pp_params); 140 141 err = xdp_rxq_info_reg(&xdp_rxq, ndev, 0); 142 if (err) 143 goto err_out; 144 145 err = xdp_rxq_info_reg_mem_model(&xdp_rxq, MEM_TYPE_PAGE_POOL, page_pool); 146 if (err) 147 goto err_out; 148 149NAPI poller 150----------- 151 152 153.. code-block:: c 154 155 /* NAPI Rx poller */ 156 enum dma_data_direction dma_dir; 157 158 dma_dir = page_pool_get_dma_dir(dring->page_pool); 159 while (done < budget) { 160 if (some error) 161 page_pool_recycle_direct(page_pool, page); 162 if (packet_is_xdp) { 163 if XDP_DROP: 164 page_pool_recycle_direct(page_pool, page); 165 } else (packet_is_skb) { 166 skb_mark_for_recycle(skb); 167 new_page = page_pool_dev_alloc_pages(page_pool); 168 } 169 } 170 171Stats 172----- 173 174.. code-block:: c 175 176 #ifdef CONFIG_PAGE_POOL_STATS 177 /* retrieve stats */ 178 struct page_pool_stats stats = { 0 }; 179 if (page_pool_get_stats(page_pool, &stats)) { 180 /* perhaps the driver reports statistics with ethool */ 181 ethtool_print_allocation_stats(&stats.alloc_stats); 182 ethtool_print_recycle_stats(&stats.recycle_stats); 183 } 184 #endif 185 186Driver unload 187------------- 188 189.. code-block:: c 190 191 /* Driver unload */ 192 page_pool_put_full_page(page_pool, page, false); 193 xdp_rxq_info_unreg(&xdp_rxq); 194