1.. SPDX-License-Identifier: GPL-2.0 2 3============= 4Page Pool API 5============= 6 7.. kernel-doc:: include/net/page_pool/helpers.h 8 :doc: page_pool allocator 9 10Architecture overview 11===================== 12 13.. code-block:: none 14 15 +------------------+ 16 | Driver | 17 +------------------+ 18 ^ 19 | 20 | 21 | 22 v 23 +--------------------------------------------+ 24 | request memory | 25 +--------------------------------------------+ 26 ^ ^ 27 | | 28 | Pool empty | Pool has entries 29 | | 30 v v 31 +-----------------------+ +------------------------+ 32 | alloc (and map) pages | | get page from cache | 33 +-----------------------+ +------------------------+ 34 ^ ^ 35 | | 36 | cache available | No entries, refill 37 | | from ptr-ring 38 | | 39 v v 40 +-----------------+ +------------------+ 41 | Fast cache | | ptr-ring cache | 42 +-----------------+ +------------------+ 43 44API interface 45============= 46The number of pools created **must** match the number of hardware queues 47unless hardware restrictions make that impossible. This would otherwise beat the 48purpose of page pool, which is allocate pages fast from cache without locking. 49This lockless guarantee naturally comes from running under a NAPI softirq. 50The protection doesn't strictly have to be NAPI, any guarantee that allocating 51a page will cause no race conditions is enough. 52 53.. kernel-doc:: net/core/page_pool.c 54 :identifiers: page_pool_create 55 56.. kernel-doc:: include/net/page_pool/types.h 57 :identifiers: struct page_pool_params 58 59.. kernel-doc:: include/net/page_pool/helpers.h 60 :identifiers: page_pool_put_page page_pool_put_full_page 61 page_pool_recycle_direct page_pool_dev_alloc_pages 62 page_pool_get_dma_addr page_pool_get_dma_dir 63 64.. kernel-doc:: net/core/page_pool.c 65 :identifiers: page_pool_put_page_bulk page_pool_get_stats 66 67DMA sync 68-------- 69Driver is always responsible for syncing the pages for the CPU. 70Drivers may choose to take care of syncing for the device as well 71or set the ``PP_FLAG_DMA_SYNC_DEV`` flag to request that pages 72allocated from the page pool are already synced for the device. 73 74If ``PP_FLAG_DMA_SYNC_DEV`` is set, the driver must inform the core what portion 75of the buffer has to be synced. This allows the core to avoid syncing the entire 76page when the drivers knows that the device only accessed a portion of the page. 77 78Most drivers will reserve headroom in front of the frame. This part 79of the buffer is not touched by the device, so to avoid syncing 80it drivers can set the ``offset`` field in struct page_pool_params 81appropriately. 82 83For pages recycled on the XDP xmit and skb paths the page pool will 84use the ``max_len`` member of struct page_pool_params to decide how 85much of the page needs to be synced (starting at ``offset``). 86When directly freeing pages in the driver (page_pool_put_page()) 87the ``dma_sync_size`` argument specifies how much of the buffer needs 88to be synced. 89 90If in doubt set ``offset`` to 0, ``max_len`` to ``PAGE_SIZE`` and 91pass -1 as ``dma_sync_size``. That combination of arguments is always 92correct. 93 94Note that the syncing parameters are for the entire page. 95This is important to remember when using fragments (``PP_FLAG_PAGE_FRAG``), 96where allocated buffers may be smaller than a full page. 97Unless the driver author really understands page pool internals 98it's recommended to always use ``offset = 0``, ``max_len = PAGE_SIZE`` 99with fragmented page pools. 100 101Stats API and structures 102------------------------ 103If the kernel is configured with ``CONFIG_PAGE_POOL_STATS=y``, the API 104page_pool_get_stats() and structures described below are available. 105It takes a pointer to a ``struct page_pool`` and a pointer to a struct 106page_pool_stats allocated by the caller. 107 108The API will fill in the provided struct page_pool_stats with 109statistics about the page_pool. 110 111.. kernel-doc:: include/net/page_pool/types.h 112 :identifiers: struct page_pool_recycle_stats 113 struct page_pool_alloc_stats 114 struct page_pool_stats 115 116Coding examples 117=============== 118 119Registration 120------------ 121 122.. code-block:: c 123 124 /* Page pool registration */ 125 struct page_pool_params pp_params = { 0 }; 126 struct xdp_rxq_info xdp_rxq; 127 int err; 128 129 pp_params.order = 0; 130 /* internal DMA mapping in page_pool */ 131 pp_params.flags = PP_FLAG_DMA_MAP; 132 pp_params.pool_size = DESC_NUM; 133 pp_params.nid = NUMA_NO_NODE; 134 pp_params.dev = priv->dev; 135 pp_params.napi = napi; /* only if locking is tied to NAPI */ 136 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 137 page_pool = page_pool_create(&pp_params); 138 139 err = xdp_rxq_info_reg(&xdp_rxq, ndev, 0); 140 if (err) 141 goto err_out; 142 143 err = xdp_rxq_info_reg_mem_model(&xdp_rxq, MEM_TYPE_PAGE_POOL, page_pool); 144 if (err) 145 goto err_out; 146 147NAPI poller 148----------- 149 150 151.. code-block:: c 152 153 /* NAPI Rx poller */ 154 enum dma_data_direction dma_dir; 155 156 dma_dir = page_pool_get_dma_dir(dring->page_pool); 157 while (done < budget) { 158 if (some error) 159 page_pool_recycle_direct(page_pool, page); 160 if (packet_is_xdp) { 161 if XDP_DROP: 162 page_pool_recycle_direct(page_pool, page); 163 } else (packet_is_skb) { 164 skb_mark_for_recycle(skb); 165 new_page = page_pool_dev_alloc_pages(page_pool); 166 } 167 } 168 169Stats 170----- 171 172.. code-block:: c 173 174 #ifdef CONFIG_PAGE_POOL_STATS 175 /* retrieve stats */ 176 struct page_pool_stats stats = { 0 }; 177 if (page_pool_get_stats(page_pool, &stats)) { 178 /* perhaps the driver reports statistics with ethool */ 179 ethtool_print_allocation_stats(&stats.alloc_stats); 180 ethtool_print_recycle_stats(&stats.recycle_stats); 181 } 182 #endif 183 184Driver unload 185------------- 186 187.. code-block:: c 188 189 /* Driver unload */ 190 page_pool_put_full_page(page_pool, page, false); 191 xdp_rxq_info_unreg(&xdp_rxq); 192