1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google, Inc. 5 */ 6 7 #include <net/xdp_sock_drv.h> 8 #include "gve.h" 9 #include "gve_utils.h" 10 11 int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs) 12 { 13 return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias; 14 } 15 16 struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx) 17 { 18 struct gve_rx_buf_state_dqo *buf_state; 19 s16 buffer_id; 20 21 buffer_id = rx->dqo.free_buf_states; 22 if (unlikely(buffer_id == -1)) 23 return NULL; 24 25 buf_state = &rx->dqo.buf_states[buffer_id]; 26 27 /* Remove buf_state from free list */ 28 rx->dqo.free_buf_states = buf_state->next; 29 30 /* Point buf_state to itself to mark it as allocated */ 31 buf_state->next = buffer_id; 32 33 /* Clear the buffer pointers */ 34 buf_state->page_info.page = NULL; 35 buf_state->xsk_buff = NULL; 36 37 return buf_state; 38 } 39 40 bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, 41 struct gve_rx_buf_state_dqo *buf_state) 42 { 43 s16 buffer_id = buf_state - rx->dqo.buf_states; 44 45 return buf_state->next == buffer_id; 46 } 47 48 void gve_free_buf_state(struct gve_rx_ring *rx, 49 struct gve_rx_buf_state_dqo *buf_state) 50 { 51 s16 buffer_id = buf_state - rx->dqo.buf_states; 52 53 buf_state->next = rx->dqo.free_buf_states; 54 rx->dqo.free_buf_states = buffer_id; 55 } 56 57 struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx, 58 struct gve_index_list *list) 59 { 60 struct gve_rx_buf_state_dqo *buf_state; 61 s16 buffer_id; 62 63 buffer_id = list->head; 64 if (unlikely(buffer_id == -1)) 65 return NULL; 66 67 buf_state = &rx->dqo.buf_states[buffer_id]; 68 69 /* Remove buf_state from list */ 70 list->head = buf_state->next; 71 if (buf_state->next == -1) 72 list->tail = -1; 73 74 /* Point buf_state to itself to mark it as allocated */ 75 buf_state->next = buffer_id; 76 77 return buf_state; 78 } 79 80 void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list, 81 struct gve_rx_buf_state_dqo *buf_state) 82 { 83 s16 buffer_id = buf_state - rx->dqo.buf_states; 84 85 buf_state->next = -1; 86 87 if (list->head == -1) { 88 list->head = buffer_id; 89 list->tail = buffer_id; 90 } else { 91 int tail = list->tail; 92 93 rx->dqo.buf_states[tail].next = buffer_id; 94 list->tail = buffer_id; 95 } 96 } 97 98 struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx) 99 { 100 struct gve_rx_buf_state_dqo *buf_state; 101 int i; 102 103 /* Recycled buf states are immediately usable. */ 104 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states); 105 if (likely(buf_state)) 106 return buf_state; 107 108 if (unlikely(rx->dqo.used_buf_states.head == -1)) 109 return NULL; 110 111 /* Used buf states are only usable when ref count reaches 0, which means 112 * no SKBs refer to them. 113 * 114 * Search a limited number before giving up. 115 */ 116 for (i = 0; i < 5; i++) { 117 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); 118 if (gve_buf_ref_cnt(buf_state) == 0) { 119 rx->dqo.used_buf_states_cnt--; 120 return buf_state; 121 } 122 123 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 124 } 125 126 return NULL; 127 } 128 129 int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, 130 struct gve_rx_buf_state_dqo *buf_state) 131 { 132 struct gve_priv *priv = rx->gve; 133 u32 idx; 134 135 idx = rx->dqo.next_qpl_page_idx; 136 if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) { 137 net_err_ratelimited("%s: Out of QPL pages\n", 138 priv->dev->name); 139 return -ENOMEM; 140 } 141 buf_state->page_info.page = rx->dqo.qpl->pages[idx]; 142 buf_state->addr = rx->dqo.qpl->page_buses[idx]; 143 rx->dqo.next_qpl_page_idx++; 144 buf_state->page_info.page_offset = 0; 145 buf_state->page_info.page_address = 146 page_address(buf_state->page_info.page); 147 buf_state->page_info.buf_size = rx->packet_buffer_truesize; 148 buf_state->page_info.pad = rx->rx_headroom; 149 buf_state->last_single_ref_offset = 0; 150 151 /* The page already has 1 ref. */ 152 page_ref_add(buf_state->page_info.page, INT_MAX - 1); 153 buf_state->page_info.pagecnt_bias = INT_MAX; 154 155 return 0; 156 } 157 158 void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state) 159 { 160 if (!buf_state->page_info.page) 161 return; 162 163 page_ref_sub(buf_state->page_info.page, 164 buf_state->page_info.pagecnt_bias - 1); 165 buf_state->page_info.page = NULL; 166 } 167 168 void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx, 169 struct gve_rx_buf_state_dqo *buf_state) 170 { 171 const u16 data_buffer_size = rx->packet_buffer_truesize; 172 int pagecount; 173 174 /* Can't reuse if we only fit one buffer per page */ 175 if (data_buffer_size * 2 > PAGE_SIZE) 176 goto mark_used; 177 178 pagecount = gve_buf_ref_cnt(buf_state); 179 180 /* Record the offset when we have a single remaining reference. 181 * 182 * When this happens, we know all of the other offsets of the page are 183 * usable. 184 */ 185 if (pagecount == 1) { 186 buf_state->last_single_ref_offset = 187 buf_state->page_info.page_offset; 188 } 189 190 /* Use the next buffer sized chunk in the page. */ 191 buf_state->page_info.page_offset += data_buffer_size; 192 buf_state->page_info.page_offset &= (PAGE_SIZE - 1); 193 194 /* If we wrap around to the same offset without ever dropping to 1 195 * reference, then we don't know if this offset was ever freed. 196 */ 197 if (buf_state->page_info.page_offset == 198 buf_state->last_single_ref_offset) { 199 goto mark_used; 200 } 201 202 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); 203 return; 204 205 mark_used: 206 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 207 rx->dqo.used_buf_states_cnt++; 208 } 209 210 void gve_free_to_page_pool(struct gve_rx_ring *rx, 211 struct gve_rx_buf_state_dqo *buf_state, 212 bool allow_direct) 213 { 214 netmem_ref netmem = buf_state->page_info.netmem; 215 216 if (!netmem) 217 return; 218 219 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, allow_direct); 220 buf_state->page_info.netmem = 0; 221 } 222 223 static int gve_alloc_from_page_pool(struct gve_rx_ring *rx, 224 struct gve_rx_buf_state_dqo *buf_state) 225 { 226 netmem_ref netmem; 227 228 buf_state->page_info.buf_size = rx->packet_buffer_truesize; 229 netmem = page_pool_alloc_netmem(rx->dqo.page_pool, 230 &buf_state->page_info.page_offset, 231 &buf_state->page_info.buf_size, 232 GFP_ATOMIC); 233 234 if (!netmem) 235 return -ENOMEM; 236 237 buf_state->page_info.netmem = netmem; 238 buf_state->page_info.page_address = netmem_address(netmem); 239 buf_state->addr = page_pool_get_dma_addr_netmem(netmem); 240 buf_state->page_info.pad = rx->dqo.page_pool->p.offset; 241 242 return 0; 243 } 244 245 struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, 246 struct gve_rx_ring *rx, 247 bool xdp) 248 { 249 u32 ntfy_id = gve_rx_idx_to_ntfy(priv, rx->q_num); 250 struct page_pool_params pp = { 251 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 252 .order = 0, 253 .pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt, 254 .nid = priv->numa_node, 255 .dev = &priv->pdev->dev, 256 .netdev = priv->dev, 257 .napi = &priv->ntfy_blocks[ntfy_id].napi, 258 .max_len = PAGE_SIZE, 259 .dma_dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 260 .offset = xdp ? XDP_PACKET_HEADROOM : 0, 261 }; 262 263 if (priv->header_split_enabled) { 264 pp.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 265 pp.queue_idx = rx->q_num; 266 } 267 268 return page_pool_create(&pp); 269 } 270 271 void gve_free_buffer(struct gve_rx_ring *rx, 272 struct gve_rx_buf_state_dqo *buf_state) 273 { 274 if (rx->dqo.page_pool) { 275 gve_free_to_page_pool(rx, buf_state, true); 276 gve_free_buf_state(rx, buf_state); 277 } else { 278 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, 279 buf_state); 280 } 281 } 282 283 void gve_reuse_buffer(struct gve_rx_ring *rx, 284 struct gve_rx_buf_state_dqo *buf_state) 285 { 286 if (rx->dqo.page_pool) { 287 buf_state->page_info.netmem = 0; 288 gve_free_buf_state(rx, buf_state); 289 } else { 290 gve_dec_pagecnt_bias(&buf_state->page_info); 291 gve_try_recycle_buf(rx->gve, rx, buf_state); 292 } 293 } 294 295 int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc) 296 { 297 struct gve_rx_buf_state_dqo *buf_state; 298 299 if (rx->xsk_pool) { 300 buf_state = gve_alloc_buf_state(rx); 301 if (unlikely(!buf_state)) 302 return -ENOMEM; 303 304 buf_state->xsk_buff = xsk_buff_alloc(rx->xsk_pool); 305 if (unlikely(!buf_state->xsk_buff)) { 306 xsk_set_rx_need_wakeup(rx->xsk_pool); 307 gve_free_buf_state(rx, buf_state); 308 return -ENOMEM; 309 } 310 /* Allocated xsk buffer. Clear wakeup in case it was set. */ 311 xsk_clear_rx_need_wakeup(rx->xsk_pool); 312 desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); 313 desc->buf_addr = 314 cpu_to_le64(xsk_buff_xdp_get_dma(buf_state->xsk_buff)); 315 return 0; 316 } else if (rx->dqo.page_pool) { 317 buf_state = gve_alloc_buf_state(rx); 318 if (WARN_ON_ONCE(!buf_state)) 319 return -ENOMEM; 320 321 if (gve_alloc_from_page_pool(rx, buf_state)) 322 goto free_buf_state; 323 } else { 324 buf_state = gve_get_recycled_buf_state(rx); 325 if (unlikely(!buf_state)) { 326 buf_state = gve_alloc_buf_state(rx); 327 if (unlikely(!buf_state)) 328 return -ENOMEM; 329 330 if (unlikely(gve_alloc_qpl_page_dqo(rx, buf_state))) 331 goto free_buf_state; 332 } 333 } 334 desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); 335 desc->buf_addr = cpu_to_le64(buf_state->addr + 336 buf_state->page_info.page_offset + 337 buf_state->page_info.pad); 338 339 return 0; 340 341 free_buf_state: 342 gve_free_buf_state(rx, buf_state); 343 return -ENOMEM; 344 } 345