1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2024 Intel Corporation */ 3 4 #include <net/libeth/rx.h> 5 6 /* Rx buffer management */ 7 8 /** 9 * libeth_rx_hw_len_mtu - get the actual buffer size to be passed to HW 10 * @pp: &page_pool_params of the netdev to calculate the size for 11 * @max_len: maximum buffer size for a single descriptor 12 * 13 * Return: HW-writeable length per one buffer to pass it to the HW accounting: 14 * MTU the @dev has, HW required alignment, minimum and maximum allowed values, 15 * and system's page size. 16 */ 17 static u32 libeth_rx_hw_len_mtu(const struct page_pool_params *pp, u32 max_len) 18 { 19 u32 len; 20 21 len = READ_ONCE(pp->netdev->mtu) + LIBETH_RX_LL_LEN; 22 len = ALIGN(len, LIBETH_RX_BUF_STRIDE); 23 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE), 24 pp->max_len); 25 26 return len; 27 } 28 29 /** 30 * libeth_rx_hw_len_truesize - get the short buffer size to be passed to HW 31 * @pp: &page_pool_params of the netdev to calculate the size for 32 * @max_len: maximum buffer size for a single descriptor 33 * @truesize: desired truesize for the buffers 34 * 35 * Return: HW-writeable length per one buffer to pass it to the HW ignoring the 36 * MTU and closest to the passed truesize. Can be used for "short" buffer 37 * queues to fragment pages more efficiently. 38 */ 39 static u32 libeth_rx_hw_len_truesize(const struct page_pool_params *pp, 40 u32 max_len, u32 truesize) 41 { 42 u32 min, len; 43 44 min = SKB_HEAD_ALIGN(pp->offset + LIBETH_RX_BUF_STRIDE); 45 truesize = clamp(roundup_pow_of_two(truesize), roundup_pow_of_two(min), 46 PAGE_SIZE << LIBETH_RX_PAGE_ORDER); 47 48 len = SKB_WITH_OVERHEAD(truesize - pp->offset); 49 len = ALIGN_DOWN(len, LIBETH_RX_BUF_STRIDE) ? : LIBETH_RX_BUF_STRIDE; 50 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE), 51 pp->max_len); 52 53 return len; 54 } 55 56 /** 57 * libeth_rx_page_pool_params - calculate params with the stack overhead 58 * @fq: buffer queue to calculate the size for 59 * @pp: &page_pool_params of the netdev 60 * 61 * Set the PP params to will all needed stack overhead (headroom, tailroom) and 62 * both the HW buffer length and the truesize for all types of buffers. For 63 * "short" buffers, truesize never exceeds the "wanted" one; for the rest, 64 * it can be up to the page size. 65 * 66 * Return: true on success, false on invalid input params. 67 */ 68 static bool libeth_rx_page_pool_params(struct libeth_fq *fq, 69 struct page_pool_params *pp) 70 { 71 pp->offset = LIBETH_SKB_HEADROOM; 72 /* HW-writeable / syncable length per one page */ 73 pp->max_len = LIBETH_RX_PAGE_LEN(pp->offset); 74 75 /* HW-writeable length per buffer */ 76 switch (fq->type) { 77 case LIBETH_FQE_MTU: 78 fq->buf_len = libeth_rx_hw_len_mtu(pp, fq->buf_len); 79 break; 80 case LIBETH_FQE_SHORT: 81 fq->buf_len = libeth_rx_hw_len_truesize(pp, fq->buf_len, 82 fq->truesize); 83 break; 84 case LIBETH_FQE_HDR: 85 fq->buf_len = ALIGN(LIBETH_MAX_HEAD, LIBETH_RX_BUF_STRIDE); 86 break; 87 default: 88 return false; 89 } 90 91 /* Buffer size to allocate */ 92 fq->truesize = roundup_pow_of_two(SKB_HEAD_ALIGN(pp->offset + 93 fq->buf_len)); 94 95 return true; 96 } 97 98 /** 99 * libeth_rx_page_pool_params_zc - calculate params without the stack overhead 100 * @fq: buffer queue to calculate the size for 101 * @pp: &page_pool_params of the netdev 102 * 103 * Set the PP params to exclude the stack overhead and both the buffer length 104 * and the truesize, which are equal for the data buffers. Note that this 105 * requires separate header buffers to be always active and account the 106 * overhead. 107 * With the MTU == ``PAGE_SIZE``, this allows the kernel to enable the zerocopy 108 * mode. 109 * 110 * Return: true on success, false on invalid input params. 111 */ 112 static bool libeth_rx_page_pool_params_zc(struct libeth_fq *fq, 113 struct page_pool_params *pp) 114 { 115 u32 mtu, max; 116 117 pp->offset = 0; 118 pp->max_len = PAGE_SIZE << LIBETH_RX_PAGE_ORDER; 119 120 switch (fq->type) { 121 case LIBETH_FQE_MTU: 122 mtu = READ_ONCE(pp->netdev->mtu); 123 break; 124 case LIBETH_FQE_SHORT: 125 mtu = fq->truesize; 126 break; 127 default: 128 return false; 129 } 130 131 mtu = roundup_pow_of_two(mtu); 132 max = min(rounddown_pow_of_two(fq->buf_len ? : U32_MAX), 133 pp->max_len); 134 135 fq->buf_len = clamp(mtu, LIBETH_RX_BUF_STRIDE, max); 136 fq->truesize = fq->buf_len; 137 138 return true; 139 } 140 141 /** 142 * libeth_rx_fq_create - create a PP with the default libeth settings 143 * @fq: buffer queue struct to fill 144 * @napi: &napi_struct covering this PP (no usage outside its poll loops) 145 * 146 * Return: %0 on success, -%errno on failure. 147 */ 148 int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi) 149 { 150 struct page_pool_params pp = { 151 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 152 .order = LIBETH_RX_PAGE_ORDER, 153 .pool_size = fq->count, 154 .nid = fq->nid, 155 .dev = napi->dev->dev.parent, 156 .netdev = napi->dev, 157 .napi = napi, 158 .dma_dir = DMA_FROM_DEVICE, 159 }; 160 struct libeth_fqe *fqes; 161 struct page_pool *pool; 162 bool ret; 163 164 if (!fq->hsplit) 165 ret = libeth_rx_page_pool_params(fq, &pp); 166 else 167 ret = libeth_rx_page_pool_params_zc(fq, &pp); 168 if (!ret) 169 return -EINVAL; 170 171 pool = page_pool_create(&pp); 172 if (IS_ERR(pool)) 173 return PTR_ERR(pool); 174 175 fqes = kvcalloc_node(fq->count, sizeof(*fqes), GFP_KERNEL, fq->nid); 176 if (!fqes) 177 goto err_buf; 178 179 fq->fqes = fqes; 180 fq->pp = pool; 181 182 return 0; 183 184 err_buf: 185 page_pool_destroy(pool); 186 187 return -ENOMEM; 188 } 189 EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, LIBETH); 190 191 /** 192 * libeth_rx_fq_destroy - destroy a &page_pool created by libeth 193 * @fq: buffer queue to process 194 */ 195 void libeth_rx_fq_destroy(struct libeth_fq *fq) 196 { 197 kvfree(fq->fqes); 198 page_pool_destroy(fq->pp); 199 } 200 EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, LIBETH); 201 202 /** 203 * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context 204 * @page: page to recycle 205 * 206 * To be used on exceptions or rare cases not requiring fast inline recycling. 207 */ 208 void libeth_rx_recycle_slow(struct page *page) 209 { 210 page_pool_recycle_direct(page->pp, page); 211 } 212 EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, LIBETH); 213 214 /* Converting abstract packet type numbers into a software structure with 215 * the packet parameters to do O(1) lookup on Rx. 216 */ 217 218 static const u16 libeth_rx_pt_xdp_oip[] = { 219 [LIBETH_RX_PT_OUTER_L2] = XDP_RSS_TYPE_NONE, 220 [LIBETH_RX_PT_OUTER_IPV4] = XDP_RSS_L3_IPV4, 221 [LIBETH_RX_PT_OUTER_IPV6] = XDP_RSS_L3_IPV6, 222 }; 223 224 static const u16 libeth_rx_pt_xdp_iprot[] = { 225 [LIBETH_RX_PT_INNER_NONE] = XDP_RSS_TYPE_NONE, 226 [LIBETH_RX_PT_INNER_UDP] = XDP_RSS_L4_UDP, 227 [LIBETH_RX_PT_INNER_TCP] = XDP_RSS_L4_TCP, 228 [LIBETH_RX_PT_INNER_SCTP] = XDP_RSS_L4_SCTP, 229 [LIBETH_RX_PT_INNER_ICMP] = XDP_RSS_L4_ICMP, 230 [LIBETH_RX_PT_INNER_TIMESYNC] = XDP_RSS_TYPE_NONE, 231 }; 232 233 static const u16 libeth_rx_pt_xdp_pl[] = { 234 [LIBETH_RX_PT_PAYLOAD_NONE] = XDP_RSS_TYPE_NONE, 235 [LIBETH_RX_PT_PAYLOAD_L2] = XDP_RSS_TYPE_NONE, 236 [LIBETH_RX_PT_PAYLOAD_L3] = XDP_RSS_TYPE_NONE, 237 [LIBETH_RX_PT_PAYLOAD_L4] = XDP_RSS_L4, 238 }; 239 240 /** 241 * libeth_rx_pt_gen_hash_type - generate an XDP RSS hash type for a PT 242 * @pt: PT structure to evaluate 243 * 244 * Generates ```hash_type``` field with XDP RSS type values from the parsed 245 * packet parameters if they're obtained dynamically at runtime. 246 */ 247 void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt) 248 { 249 pt->hash_type = 0; 250 pt->hash_type |= libeth_rx_pt_xdp_oip[pt->outer_ip]; 251 pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot]; 252 pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer]; 253 } 254 EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, LIBETH); 255 256 /* Module */ 257 258 MODULE_DESCRIPTION("Common Ethernet library"); 259 MODULE_LICENSE("GPL"); 260