1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google, Inc. 5 */ 6 7 #include <net/xdp_sock_drv.h> 8 #include "gve.h" 9 #include "gve_utils.h" 10 11 int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs) 12 { 13 return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias; 14 } 15 16 struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx) 17 { 18 struct gve_rx_buf_state_dqo *buf_state; 19 s16 buffer_id; 20 21 buffer_id = rx->dqo.free_buf_states; 22 if (unlikely(buffer_id == -1)) 23 return NULL; 24 25 buf_state = &rx->dqo.buf_states[buffer_id]; 26 27 /* Remove buf_state from free list */ 28 rx->dqo.free_buf_states = buf_state->next; 29 30 /* Point buf_state to itself to mark it as allocated */ 31 buf_state->next = buffer_id; 32 33 /* Clear the buffer pointers */ 34 buf_state->page_info.page = NULL; 35 buf_state->xsk_buff = NULL; 36 37 return buf_state; 38 } 39 40 bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, 41 struct gve_rx_buf_state_dqo *buf_state) 42 { 43 s16 buffer_id = buf_state - rx->dqo.buf_states; 44 45 return buf_state->next == buffer_id; 46 } 47 48 void gve_free_buf_state(struct gve_rx_ring *rx, 49 struct gve_rx_buf_state_dqo *buf_state) 50 { 51 s16 buffer_id = buf_state - rx->dqo.buf_states; 52 53 buf_state->next = rx->dqo.free_buf_states; 54 rx->dqo.free_buf_states = buffer_id; 55 } 56 57 struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx, 58 struct gve_index_list *list) 59 { 60 struct gve_rx_buf_state_dqo *buf_state; 61 s16 buffer_id; 62 63 buffer_id = list->head; 64 if (unlikely(buffer_id == -1)) 65 return NULL; 66 67 buf_state = &rx->dqo.buf_states[buffer_id]; 68 69 /* Remove buf_state from list */ 70 list->head = buf_state->next; 71 if (buf_state->next == -1) 72 list->tail = -1; 73 74 /* Point buf_state to itself to mark it as allocated */ 75 buf_state->next = buffer_id; 76 77 return buf_state; 78 } 79 80 void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list, 81 struct gve_rx_buf_state_dqo *buf_state) 82 { 83 s16 buffer_id = buf_state - rx->dqo.buf_states; 84 85 buf_state->next = -1; 86 87 if (list->head == -1) { 88 list->head = buffer_id; 89 list->tail = buffer_id; 90 } else { 91 int tail = list->tail; 92 93 rx->dqo.buf_states[tail].next = buffer_id; 94 list->tail = buffer_id; 95 } 96 } 97 98 struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx) 99 { 100 struct gve_rx_buf_state_dqo *buf_state; 101 int i; 102 103 /* Recycled buf states are immediately usable. */ 104 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states); 105 if (likely(buf_state)) 106 return buf_state; 107 108 if (unlikely(rx->dqo.used_buf_states.head == -1)) 109 return NULL; 110 111 /* Used buf states are only usable when ref count reaches 0, which means 112 * no SKBs refer to them. 113 * 114 * Search a limited number before giving up. 115 */ 116 for (i = 0; i < 5; i++) { 117 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); 118 if (gve_buf_ref_cnt(buf_state) == 0) { 119 rx->dqo.used_buf_states_cnt--; 120 return buf_state; 121 } 122 123 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 124 } 125 126 return NULL; 127 } 128 129 int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, 130 struct gve_rx_buf_state_dqo *buf_state) 131 { 132 struct gve_priv *priv = rx->gve; 133 u32 idx; 134 135 idx = rx->dqo.next_qpl_page_idx; 136 if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) { 137 net_err_ratelimited("%s: Out of QPL pages\n", 138 priv->dev->name); 139 return -ENOMEM; 140 } 141 buf_state->page_info.page = rx->dqo.qpl->pages[idx]; 142 buf_state->addr = rx->dqo.qpl->page_buses[idx]; 143 rx->dqo.next_qpl_page_idx++; 144 buf_state->page_info.page_offset = 0; 145 buf_state->page_info.page_address = 146 page_address(buf_state->page_info.page); 147 buf_state->page_info.buf_size = rx->packet_buffer_truesize; 148 buf_state->page_info.pad = rx->rx_headroom; 149 buf_state->last_single_ref_offset = 0; 150 151 /* The page already has 1 ref. */ 152 page_ref_add(buf_state->page_info.page, INT_MAX - 1); 153 buf_state->page_info.pagecnt_bias = INT_MAX; 154 155 return 0; 156 } 157 158 void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state) 159 { 160 if (!buf_state->page_info.page) 161 return; 162 163 page_ref_sub(buf_state->page_info.page, 164 buf_state->page_info.pagecnt_bias - 1); 165 buf_state->page_info.page = NULL; 166 } 167 168 void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx, 169 struct gve_rx_buf_state_dqo *buf_state) 170 { 171 const u16 data_buffer_size = rx->packet_buffer_truesize; 172 int pagecount; 173 174 /* Can't reuse if we only fit one buffer per page */ 175 if (data_buffer_size * 2 > PAGE_SIZE) 176 goto mark_used; 177 178 pagecount = gve_buf_ref_cnt(buf_state); 179 180 /* Record the offset when we have a single remaining reference. 181 * 182 * When this happens, we know all of the other offsets of the page are 183 * usable. 184 */ 185 if (pagecount == 1) { 186 buf_state->last_single_ref_offset = 187 buf_state->page_info.page_offset; 188 } 189 190 /* Use the next buffer sized chunk in the page. */ 191 buf_state->page_info.page_offset += data_buffer_size; 192 buf_state->page_info.page_offset &= (PAGE_SIZE - 1); 193 194 /* If we wrap around to the same offset without ever dropping to 1 195 * reference, then we don't know if this offset was ever freed. 196 */ 197 if (buf_state->page_info.page_offset == 198 buf_state->last_single_ref_offset) { 199 goto mark_used; 200 } 201 202 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); 203 return; 204 205 mark_used: 206 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 207 rx->dqo.used_buf_states_cnt++; 208 } 209 210 void gve_free_to_page_pool(struct gve_rx_ring *rx, 211 struct gve_rx_buf_state_dqo *buf_state, 212 bool allow_direct) 213 { 214 netmem_ref netmem = buf_state->page_info.netmem; 215 216 if (!netmem) 217 return; 218 219 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, allow_direct); 220 buf_state->page_info.netmem = 0; 221 } 222 223 static int gve_alloc_from_page_pool(struct gve_rx_ring *rx, 224 struct gve_rx_buf_state_dqo *buf_state) 225 { 226 netmem_ref netmem; 227 228 buf_state->page_info.buf_size = rx->packet_buffer_truesize; 229 netmem = page_pool_alloc_netmem(rx->dqo.page_pool, 230 &buf_state->page_info.page_offset, 231 &buf_state->page_info.buf_size, 232 GFP_ATOMIC); 233 234 if (!netmem) 235 return -ENOMEM; 236 237 buf_state->page_info.netmem = netmem; 238 buf_state->page_info.page_address = netmem_address(netmem); 239 buf_state->addr = page_pool_get_dma_addr_netmem(netmem); 240 buf_state->page_info.pad = rx->dqo.page_pool->p.offset; 241 242 return 0; 243 } 244 245 struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, 246 struct gve_rx_ring *rx, 247 bool xdp) 248 { 249 u32 ntfy_id = gve_rx_idx_to_ntfy(priv, rx->q_num); 250 struct page_pool_params pp = { 251 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 252 .order = 0, 253 .pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt, 254 .nid = priv->numa_node, 255 .dev = &priv->pdev->dev, 256 .netdev = priv->dev, 257 .napi = &priv->ntfy_blocks[ntfy_id].napi, 258 .max_len = PAGE_SIZE, 259 .dma_dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 260 .offset = xdp ? XDP_PACKET_HEADROOM : 0, 261 }; 262 263 return page_pool_create(&pp); 264 } 265 266 void gve_free_buffer(struct gve_rx_ring *rx, 267 struct gve_rx_buf_state_dqo *buf_state) 268 { 269 if (rx->dqo.page_pool) { 270 gve_free_to_page_pool(rx, buf_state, true); 271 gve_free_buf_state(rx, buf_state); 272 } else { 273 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, 274 buf_state); 275 } 276 } 277 278 void gve_reuse_buffer(struct gve_rx_ring *rx, 279 struct gve_rx_buf_state_dqo *buf_state) 280 { 281 if (rx->dqo.page_pool) { 282 buf_state->page_info.netmem = 0; 283 gve_free_buf_state(rx, buf_state); 284 } else { 285 gve_dec_pagecnt_bias(&buf_state->page_info); 286 gve_try_recycle_buf(rx->gve, rx, buf_state); 287 } 288 } 289 290 int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc) 291 { 292 struct gve_rx_buf_state_dqo *buf_state; 293 294 if (rx->xsk_pool) { 295 buf_state = gve_alloc_buf_state(rx); 296 if (unlikely(!buf_state)) 297 return -ENOMEM; 298 299 buf_state->xsk_buff = xsk_buff_alloc(rx->xsk_pool); 300 if (unlikely(!buf_state->xsk_buff)) { 301 xsk_set_rx_need_wakeup(rx->xsk_pool); 302 gve_free_buf_state(rx, buf_state); 303 return -ENOMEM; 304 } 305 /* Allocated xsk buffer. Clear wakeup in case it was set. */ 306 xsk_clear_rx_need_wakeup(rx->xsk_pool); 307 desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); 308 desc->buf_addr = 309 cpu_to_le64(xsk_buff_xdp_get_dma(buf_state->xsk_buff)); 310 return 0; 311 } else if (rx->dqo.page_pool) { 312 buf_state = gve_alloc_buf_state(rx); 313 if (WARN_ON_ONCE(!buf_state)) 314 return -ENOMEM; 315 316 if (gve_alloc_from_page_pool(rx, buf_state)) 317 goto free_buf_state; 318 } else { 319 buf_state = gve_get_recycled_buf_state(rx); 320 if (unlikely(!buf_state)) { 321 buf_state = gve_alloc_buf_state(rx); 322 if (unlikely(!buf_state)) 323 return -ENOMEM; 324 325 if (unlikely(gve_alloc_qpl_page_dqo(rx, buf_state))) 326 goto free_buf_state; 327 } 328 } 329 desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); 330 desc->buf_addr = cpu_to_le64(buf_state->addr + 331 buf_state->page_info.page_offset + 332 buf_state->page_info.pad); 333 334 return 0; 335 336 free_buf_state: 337 gve_free_buf_state(rx, buf_state); 338 return -ENOMEM; 339 } 340