xref: /freebsd/sys/dev/gve/gve.h (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 2023 Google LLC
5  *
6  * Redistribution and use in source and binary forms, with or without modification,
7  * are permitted provided that the following conditions are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright notice, this
10  *    list of conditions and the following disclaimer.
11  *
12  * 2. Redistributions in binary form must reproduce the above copyright notice,
13  *    this list of conditions and the following disclaimer in the documentation
14  *    and/or other materials provided with the distribution.
15  *
16  * 3. Neither the name of the copyright holder nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software without
18  *    specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
24  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
25  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
27  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31 #ifndef _GVE_FBSD_H
32 #define _GVE_FBSD_H
33 
34 #include "gve_desc.h"
35 #include "gve_plat.h"
36 #include "gve_register.h"
37 
38 #ifndef PCI_VENDOR_ID_GOOGLE
39 #define PCI_VENDOR_ID_GOOGLE	0x1ae0
40 #endif
41 
42 #define PCI_DEV_ID_GVNIC	0x0042
43 #define GVE_REGISTER_BAR	0
44 #define GVE_DOORBELL_BAR	2
45 
46 /* Driver can alloc up to 2 segments for the header and 2 for the payload. */
47 #define GVE_TX_MAX_DESCS	4
48 #define GVE_TX_BUFRING_ENTRIES	4096
49 
50 #define ADMINQ_SIZE PAGE_SIZE
51 
52 #define GVE_DEFAULT_RX_BUFFER_SIZE 2048
53 /* Each RX bounce buffer page can fit two packet buffers. */
54 #define GVE_DEFAULT_RX_BUFFER_OFFSET (PAGE_SIZE / 2)
55 
56 /*
57  * Number of descriptors per queue page list.
58  * Page count AKA QPL size can be derived by dividing the number of elements in
59  * a page by the number of descriptors available.
60  */
61 #define GVE_QPL_DIVISOR	16
62 
63 static MALLOC_DEFINE(M_GVE, "gve", "gve allocations");
64 
65 struct gve_dma_handle {
66 	bus_addr_t	bus_addr;
67 	void		*cpu_addr;
68 	bus_dma_tag_t	tag;
69 	bus_dmamap_t	map;
70 };
71 
72 union gve_tx_desc {
73 	struct gve_tx_pkt_desc pkt; /* first desc for a packet */
74 	struct gve_tx_mtd_desc mtd; /* optional metadata descriptor */
75 	struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
76 };
77 
78 /* Tracks the memory in the fifo occupied by a segment of a packet */
79 struct gve_tx_iovec {
80 	uint32_t iov_offset; /* offset into this segment */
81 	uint32_t iov_len; /* length */
82 	uint32_t iov_padding; /* padding associated with this segment */
83 };
84 
85 /* Tracks allowed and current queue settings */
86 struct gve_queue_config {
87 	uint16_t max_queues;
88 	uint16_t num_queues; /* current */
89 };
90 
91 struct gve_irq_db {
92 	__be32 index;
93 } __aligned(CACHE_LINE_SIZE);
94 
95 /*
96  * GVE_QUEUE_FORMAT_UNSPECIFIED must be zero since 0 is the default value
97  * when the entire configure_device_resources command is zeroed out and the
98  * queue_format is not specified.
99  */
100 enum gve_queue_format {
101 	GVE_QUEUE_FORMAT_UNSPECIFIED	= 0x0,
102 	GVE_GQI_RDA_FORMAT		= 0x1,
103 	GVE_GQI_QPL_FORMAT		= 0x2,
104 	GVE_DQO_RDA_FORMAT		= 0x3,
105 };
106 
107 enum gve_state_flags_bit {
108 	GVE_STATE_FLAG_ADMINQ_OK,
109 	GVE_STATE_FLAG_RESOURCES_OK,
110 	GVE_STATE_FLAG_QPLREG_OK,
111 	GVE_STATE_FLAG_RX_RINGS_OK,
112 	GVE_STATE_FLAG_TX_RINGS_OK,
113 	GVE_STATE_FLAG_QUEUES_UP,
114 	GVE_STATE_FLAG_LINK_UP,
115 	GVE_STATE_FLAG_DO_RESET,
116 	GVE_STATE_FLAG_IN_RESET,
117 	GVE_NUM_STATE_FLAGS /* Not part of the enum space */
118 };
119 
120 BITSET_DEFINE(gve_state_flags, GVE_NUM_STATE_FLAGS);
121 
122 #define GVE_DEVICE_STATUS_RESET (0x1 << 1)
123 #define GVE_DEVICE_STATUS_LINK_STATUS (0x1 << 2)
124 
125 #define GVE_RING_LOCK(ring)	mtx_lock(&(ring)->ring_mtx)
126 #define GVE_RING_TRYLOCK(ring)	mtx_trylock(&(ring)->ring_mtx)
127 #define GVE_RING_UNLOCK(ring)	mtx_unlock(&(ring)->ring_mtx)
128 #define GVE_RING_ASSERT(ring)	mtx_assert(&(ring)->ring_mtx, MA_OWNED)
129 
130 #define GVE_IFACE_LOCK_INIT(lock)     sx_init(&lock, "gve interface lock")
131 #define GVE_IFACE_LOCK_DESTROY(lock)  sx_destroy(&lock)
132 #define GVE_IFACE_LOCK_LOCK(lock)     sx_xlock(&lock)
133 #define GVE_IFACE_LOCK_UNLOCK(lock)   sx_unlock(&lock)
134 #define GVE_IFACE_LOCK_ASSERT(lock)   sx_assert(&lock, SA_XLOCKED)
135 
136 struct gve_queue_page_list {
137 	uint32_t id;
138 	uint32_t num_dmas;
139 	uint32_t num_pages;
140 	vm_offset_t kva;
141 	vm_page_t *pages;
142 	struct gve_dma_handle *dmas;
143 };
144 
145 struct gve_irq {
146 	struct resource *res;
147 	void *cookie;
148 };
149 
150 struct gve_rx_slot_page_info {
151 	void *page_address;
152 	vm_page_t page;
153 	uint32_t page_offset;
154 	uint16_t pad;
155 };
156 
157 /*
158  * A single received packet split across multiple buffers may be
159  * reconstructed using the information in this structure.
160  */
161 struct gve_rx_ctx {
162 	/* head and tail of mbuf chain for the current packet */
163 	struct mbuf *mbuf_head;
164 	struct mbuf *mbuf_tail;
165 	uint32_t total_size;
166 	uint8_t frag_cnt;
167 	bool is_tcp;
168 	bool drop_pkt;
169 };
170 
171 struct gve_ring_com {
172 	struct gve_priv *priv;
173 	uint32_t id;
174 
175 	/*
176 	 * BAR2 offset for this ring's doorbell and the
177 	 * counter-array offset for this ring's counter.
178 	 * Acquired from the device individually for each
179 	 * queue in the queue_create adminq command.
180 	 */
181 	struct gve_queue_resources *q_resources;
182 	struct gve_dma_handle q_resources_mem;
183 
184 	/* Byte offset into BAR2 where this ring's 4-byte irq doorbell lies. */
185 	uint32_t irq_db_offset;
186 	/* Byte offset into BAR2 where this ring's 4-byte doorbell lies. */
187 	uint32_t db_offset;
188 	/*
189 	 * Index, not byte-offset, into the counter array where this ring's
190 	 * 4-byte counter lies.
191 	 */
192 	uint32_t counter_idx;
193 
194 	/*
195 	 * The index of the MSIX vector that was assigned to
196 	 * this ring in `gve_alloc_irqs`.
197 	 *
198 	 * It is passed to the device in the queue_create adminq
199 	 * command.
200 	 *
201 	 * Additionally, this also serves as the index into
202 	 * `priv->irq_db_indices` where this ring's irq doorbell's
203 	 * BAR2 offset, `irq_db_idx`, can be found.
204 	 */
205 	int ntfy_id;
206 
207 	/*
208 	 * The fixed bounce buffer for this ring.
209 	 * Once allocated, has to be offered to the device
210 	 * over the register-page-list adminq command.
211 	 */
212 	struct gve_queue_page_list *qpl;
213 
214 	struct task cleanup_task;
215 	struct taskqueue *cleanup_tq;
216 } __aligned(CACHE_LINE_SIZE);
217 
218 struct gve_rxq_stats {
219 	counter_u64_t rbytes;
220 	counter_u64_t rpackets;
221 	counter_u64_t rx_dropped_pkt;
222 	counter_u64_t rx_copybreak_cnt;
223 	counter_u64_t rx_frag_flip_cnt;
224 	counter_u64_t rx_frag_copy_cnt;
225 	counter_u64_t rx_dropped_pkt_desc_err;
226 	counter_u64_t rx_dropped_pkt_mbuf_alloc_fail;
227 };
228 
229 #define NUM_RX_STATS (sizeof(struct gve_rxq_stats) / sizeof(counter_u64_t))
230 
231 /* power-of-2 sized receive ring */
232 struct gve_rx_ring {
233 	struct gve_ring_com com;
234 	struct gve_dma_handle desc_ring_mem;
235 	struct gve_dma_handle data_ring_mem;
236 
237 	/* accessed in the receive hot path */
238 	struct {
239 		struct gve_rx_desc *desc_ring;
240 		union gve_rx_data_slot *data_ring;
241 		struct gve_rx_slot_page_info *page_info;
242 
243 		struct gve_rx_ctx ctx;
244 		struct lro_ctrl lro;
245 		uint8_t seq_no; /* helps traverse the descriptor ring */
246 		uint32_t cnt; /* free-running total number of completed packets */
247 		uint32_t fill_cnt; /* free-running total number of descs and buffs posted */
248 		uint32_t mask; /* masks the cnt and fill_cnt to the size of the ring */
249 		struct gve_rxq_stats stats;
250 	} __aligned(CACHE_LINE_SIZE);
251 
252 } __aligned(CACHE_LINE_SIZE);
253 
254 /*
255  * A contiguous representation of the pages composing the Tx bounce buffer.
256  * The xmit taskqueue and the completion taskqueue both simultaneously use it.
257  * Both operate on `available`: the xmit tq lowers it and the completion tq
258  * raises it. `head` is the last location written at and so only the xmit tq
259  * uses it.
260  */
261 struct gve_tx_fifo {
262 	vm_offset_t base; /* address of base of FIFO */
263 	uint32_t size; /* total size */
264 	volatile int available; /* how much space is still available */
265 	uint32_t head; /* offset to write at */
266 };
267 
268 struct gve_tx_buffer_state {
269 	struct mbuf *mbuf;
270 	struct gve_tx_iovec iov[GVE_TX_MAX_DESCS];
271 };
272 
273 struct gve_txq_stats {
274 	counter_u64_t tbytes;
275 	counter_u64_t tpackets;
276 	counter_u64_t tso_packet_cnt;
277 	counter_u64_t tx_dropped_pkt;
278 	counter_u64_t tx_dropped_pkt_nospace_device;
279 	counter_u64_t tx_dropped_pkt_nospace_bufring;
280 	counter_u64_t tx_dropped_pkt_vlan;
281 };
282 
283 #define NUM_TX_STATS (sizeof(struct gve_txq_stats) / sizeof(counter_u64_t))
284 
285 /* power-of-2 sized transmit ring */
286 struct gve_tx_ring {
287 	struct gve_ring_com com;
288 	struct gve_dma_handle desc_ring_mem;
289 
290 	struct task xmit_task;
291 	struct taskqueue *xmit_tq;
292 
293 	/* accessed in the transmit hot path */
294 	struct {
295 		union gve_tx_desc *desc_ring;
296 		struct gve_tx_buffer_state *info;
297 		struct buf_ring *br;
298 
299 		struct gve_tx_fifo fifo;
300 		struct mtx ring_mtx;
301 
302 		uint32_t req; /* free-running total number of packets written to the nic */
303 		uint32_t done; /* free-running total number of completed packets */
304 		uint32_t mask; /* masks the req and done to the size of the ring */
305 		struct gve_txq_stats stats;
306 	} __aligned(CACHE_LINE_SIZE);
307 
308 } __aligned(CACHE_LINE_SIZE);
309 
310 struct gve_priv {
311 	if_t ifp;
312 	device_t dev;
313 	struct ifmedia media;
314 
315 	uint8_t mac[ETHER_ADDR_LEN];
316 
317 	struct gve_dma_handle aq_mem;
318 
319 	struct resource *reg_bar; /* BAR0 */
320 	struct resource *db_bar; /* BAR2 */
321 	struct resource *msix_table;
322 
323 	uint32_t mgmt_msix_idx;
324 	uint32_t rx_copybreak;
325 
326 	uint16_t num_event_counters;
327 	uint16_t default_num_queues;
328 	uint16_t tx_desc_cnt;
329 	uint16_t rx_desc_cnt;
330 	uint16_t rx_pages_per_qpl;
331 	uint64_t max_registered_pages;
332 	uint64_t num_registered_pages;
333 	uint32_t supported_features;
334 	uint16_t max_mtu;
335 
336 	struct gve_dma_handle counter_array_mem;
337 	__be32 *counters;
338 	struct gve_dma_handle irqs_db_mem;
339 	struct gve_irq_db *irq_db_indices;
340 
341 	enum gve_queue_format queue_format;
342 	struct gve_queue_page_list *qpls;
343 	struct gve_queue_config tx_cfg;
344 	struct gve_queue_config rx_cfg;
345 	uint32_t num_queues;
346 
347 	struct gve_irq *irq_tbl;
348 	struct gve_tx_ring *tx;
349 	struct gve_rx_ring *rx;
350 
351 	/*
352 	 * Admin queue - see gve_adminq.h
353 	 * Since AQ cmds do not run in steady state, 32 bit counters suffice
354 	 */
355 	struct gve_adminq_command *adminq;
356 	vm_paddr_t adminq_bus_addr;
357 	uint32_t adminq_mask; /* masks prod_cnt to adminq size */
358 	uint32_t adminq_prod_cnt; /* free-running count of AQ cmds executed */
359 	uint32_t adminq_cmd_fail; /* free-running count of AQ cmds failed */
360 	uint32_t adminq_timeouts; /* free-running count of AQ cmds timeouts */
361 	/* free-running count of each distinct AQ cmd executed */
362 	uint32_t adminq_describe_device_cnt;
363 	uint32_t adminq_cfg_device_resources_cnt;
364 	uint32_t adminq_register_page_list_cnt;
365 	uint32_t adminq_unregister_page_list_cnt;
366 	uint32_t adminq_create_tx_queue_cnt;
367 	uint32_t adminq_create_rx_queue_cnt;
368 	uint32_t adminq_destroy_tx_queue_cnt;
369 	uint32_t adminq_destroy_rx_queue_cnt;
370 	uint32_t adminq_dcfg_device_resources_cnt;
371 	uint32_t adminq_set_driver_parameter_cnt;
372 	uint32_t adminq_verify_driver_compatibility_cnt;
373 
374 	uint32_t interface_up_cnt;
375 	uint32_t interface_down_cnt;
376 	uint32_t reset_cnt;
377 
378 	struct task service_task;
379 	struct taskqueue *service_tq;
380 
381 	struct gve_state_flags state_flags;
382 	struct sx gve_iface_lock;
383 };
384 
385 static inline bool
386 gve_get_state_flag(struct gve_priv *priv, int pos)
387 {
388 	return (BIT_ISSET(GVE_NUM_STATE_FLAGS, pos, &priv->state_flags));
389 }
390 
391 static inline void
392 gve_set_state_flag(struct gve_priv *priv, int pos)
393 {
394 	BIT_SET_ATOMIC(GVE_NUM_STATE_FLAGS, pos, &priv->state_flags);
395 }
396 
397 static inline void
398 gve_clear_state_flag(struct gve_priv *priv, int pos)
399 {
400 	BIT_CLR_ATOMIC(GVE_NUM_STATE_FLAGS, pos, &priv->state_flags);
401 }
402 
403 /* Defined in gve_main.c */
404 void gve_schedule_reset(struct gve_priv *priv);
405 
406 /* Register access functions defined in gve_utils.c */
407 uint32_t gve_reg_bar_read_4(struct gve_priv *priv, bus_size_t offset);
408 void gve_reg_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);
409 void gve_db_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);
410 
411 /* QPL (Queue Page List) functions defined in gve_qpl.c */
412 int gve_alloc_qpls(struct gve_priv *priv);
413 void gve_free_qpls(struct gve_priv *priv);
414 int gve_register_qpls(struct gve_priv *priv);
415 int gve_unregister_qpls(struct gve_priv *priv);
416 
417 /* TX functions defined in gve_tx.c */
418 int gve_alloc_tx_rings(struct gve_priv *priv);
419 void gve_free_tx_rings(struct gve_priv *priv);
420 int gve_create_tx_rings(struct gve_priv *priv);
421 int gve_destroy_tx_rings(struct gve_priv *priv);
422 int gve_tx_intr(void *arg);
423 int gve_xmit_ifp(if_t ifp, struct mbuf *mbuf);
424 void gve_qflush(if_t ifp);
425 void gve_xmit_tq(void *arg, int pending);
426 void gve_tx_cleanup_tq(void *arg, int pending);
427 
428 /* RX functions defined in gve_rx.c */
429 int gve_alloc_rx_rings(struct gve_priv *priv);
430 void gve_free_rx_rings(struct gve_priv *priv);
431 int gve_create_rx_rings(struct gve_priv *priv);
432 int gve_destroy_rx_rings(struct gve_priv *priv);
433 int gve_rx_intr(void *arg);
434 void gve_rx_cleanup_tq(void *arg, int pending);
435 
436 /* DMA functions defined in gve_utils.c */
437 int gve_dma_alloc_coherent(struct gve_priv *priv, int size, int align,
438     struct gve_dma_handle *dma);
439 void gve_dma_free_coherent(struct gve_dma_handle *dma);
440 int gve_dmamap_create(struct gve_priv *priv, int size, int align,
441     struct gve_dma_handle *dma);
442 void gve_dmamap_destroy(struct gve_dma_handle *dma);
443 
444 /* IRQ functions defined in gve_utils.c */
445 void gve_free_irqs(struct gve_priv *priv);
446 int gve_alloc_irqs(struct gve_priv *priv);
447 void gve_unmask_all_queue_irqs(struct gve_priv *priv);
448 void gve_mask_all_queue_irqs(struct gve_priv *priv);
449 
450 /* Systcl functions defined in gve_sysctl.c*/
451 void gve_setup_sysctl(struct gve_priv *priv);
452 void gve_accum_stats(struct gve_priv *priv, uint64_t *rpackets,
453     uint64_t *rbytes, uint64_t *rx_dropped_pkt, uint64_t *tpackets,
454     uint64_t *tbytes, uint64_t *tx_dropped_pkt);
455 
456 /* Stats functions defined in gve_utils.c */
457 void gve_alloc_counters(counter_u64_t *stat, int num_stats);
458 void gve_free_counters(counter_u64_t *stat, int num_stats);
459 
460 #endif /* _GVE_FBSD_H_ */
461