1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT)
4  * Copyright (C) 2015-2024 Google LLC
10 #include <linux/dma-mapping.h>
35 /* 1 for management, 1 for rx, 1 for tx */
38 /* Numbers of gve tx/rx stats in stats report. */
45 /* Numbers of NIC tx/rx stats in stats report. */
51 #define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1))
73 	(GVE_ADMINQ_BUFFER_SIZE / sizeof(((struct gve_adminq_queried_flow_rule *)0)->location))
91 /* 2K buffers for DQO-QPL */
97  * allocs and uses a non-qpl page on the receive path of DQO QPL to free
112 /* The page info for a single slot in the RX data queue */
125 	u16 pad; /* adjustment for rx padding */
129 /* A list of pages registered with the device during setup and used by a queue
144 	struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
150 /* RX buffer queue for posting buffers to HW.
151  * Each RX (completion) queue has a corresponding buffer queue.
161 /* RX completion queue to receive packets from HW. */
167 	 * post more buffers than the queue size to avoid HW overrunning the
168 	 * queue.
206 	/* Linked list index to next element in the list, or -1 if none */
210 /* `head` and `tail` are indices into an array, or -1 if empty. */
236 /* Contains datapath state used to represent an RX queue. */
241 	u16 packet_buffer_truesize;	/* Total size of RX buffer */
267 			 * buf_states, or -1 if empty.
272 			 * buf_states, or -1 if empty.
284 			 * buf_states, or -1 if empty.
291 			/* qpl assigned to this queue */
294 			/* index into queue page list */
300 			/* Address info of the buffers for header-split */
307 	u64 rbytes; /* free-running bytes received */
308 	u64 rx_hsplit_bytes; /* free-running header bytes received */
309 	u64 rpackets; /* free-running packets received */
310 	u32 cnt; /* free-running total number of completed packets */
311 	u32 fill_cnt; /* free-running total number of descs and buffs posted */
313 	u64 rx_hsplit_pkt; /* free-running packets with headers split */
314 	u64 rx_copybreak_pkt; /* free-running count of copybreak packets */
315 	u64 rx_copied_pkt; /* free-running total number of copied packets */
316 	u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */
317 	u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */
318 	u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */
319 	/* free-running count of unsplit packets due to header buffer overflow or hdr_len is 0 */
321 	u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */
322 	u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */
323 	u64 rx_frag_copy_cnt; /* free-running count of rx segments copied */
324 	u64 rx_frag_alloc_cnt; /* free-running count of rx page allocations */
329 	u32 q_num; /* queue index */
332 	dma_addr_t q_resources_bus; /* dma address for the queue resources */
345 	struct gve_tx_pkt_desc pkt; /* first desc for a packet */  member
362 		struct sk_buff *skb; /* skb for this pkt */
366 		u16 size; /* size of xmitted xdp pkt */
370 		struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
378 /* A TX buffer - each queue has one */
389 	struct gve_tx_pkt_desc_dqo pkt;  member
402 	 * re-injection completion.
442 	/* Linked list index to next element in the list, or -1 if none */
445 	/* Linked list index to prev element in the list, or -1 if none.
460 	 * freed if the corresponding re-injection completion is not received
466 /* Contains datapath state used to represent a TX queue. */
468 	/* Cacheline 0 -- Accessed & dirtied during transmit */
483 			 * pending_packets, or -1 if empty.
509 				* tx_qpl_buf_next, or -1 if empty.
531 	/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
549 			 * pending_packets, or -1 if empty.
578 				 * tx_qpl_buf_next, or -1 if empty.
594 	u64 pkt_done; /* free-running - total packets completed */
595 	u64 bytes_done; /* free-running - total bytes completed */
596 	u64 dropped_pkt; /* free-running - total packets dropped */
599 	/* Cacheline 2 -- Read-mostly fields */
623 				/* qpl assigned to this queue */
640 	u32 mask; /* masks req and done down to queue size */
643 	/* Slow-path fields */
644 	u32 q_num ____cacheline_aligned; /* queue idx */
645 	u32 stop_queue; /* count of queue stops */
646 	u32 wake_queue; /* count of queue wakes */
647 	u32 queue_timeout; /* count of queue timeouts */
649 	u32 last_kick_msec; /* Last time the queue was kicked */
651 	dma_addr_t q_resources_bus; /* dma address of the queue resources */
669 	struct gve_rx_ring *rx; /* rx rings on this block */  member
673 /* Tracks allowed and current rx queue settings */
680 /* Tracks allowed and current tx queue settings */
719 /* Parameters for allocating resources for rx queues */
733 	struct gve_rx_ring *rx;  member
795 	struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */  member
823 	unsigned long *xsk_pools; /* bitmap of RX queues with XSK pools */
824 	u32 num_ntfy_blks; /* split between TX and RX so must be even */
835 	/* Admin queue - see gve_adminq.h*/
841 	u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
842 	u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */
843 	u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */
844 	/* free-running count of per AQ cmd executed */
871 	u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */
903 	u16 header_buf_size; /* device configured, header-split supported if non-zero */
945 	return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_get_do_reset()
950 	set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_set_do_reset()
955 	clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_clear_do_reset()
961 			&priv->service_task_flags);  in gve_get_reset_in_progress()
966 	set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_set_reset_in_progress()
971 	clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_reset_in_progress()
977 			&priv->service_task_flags);  in gve_get_probe_in_progress()
982 	set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_set_probe_in_progress()
987 	clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_probe_in_progress()
993 			&priv->service_task_flags);  in gve_get_do_report_stats()
998 	set_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_set_do_report_stats()
1003 	clear_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_clear_do_report_stats()
1008 	return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_get_admin_queue_ok()
1013 	set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_set_admin_queue_ok()
1018 	clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_clear_admin_queue_ok()
1023 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_get_device_resources_ok()
1028 	set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_set_device_resources_ok()
1033 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_clear_device_resources_ok()
1038 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_get_device_rings_ok()
1043 	set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_set_device_rings_ok()
1048 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_clear_device_rings_ok()
1053 	return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_get_napi_enabled()
1058 	set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_set_napi_enabled()
1063 	clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_clear_napi_enabled()
1068 	return test_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_get_report_stats()
1073 	clear_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_clear_report_stats()
1081 	return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)];  in gve_irq_doorbell()
1091 /* Returns the index into ntfy_blocks of the given rx ring's block
1095 	return (priv->num_ntfy_blks / 2) + queue_idx;  in gve_rx_idx_to_ntfy()
1100 	return priv->queue_format == GVE_GQI_QPL_FORMAT ||  in gve_is_qpl()
1101 		priv->queue_format == GVE_DQO_QPL_FORMAT;  in gve_is_qpl()
1104 /* Returns the number of tx queue page lists */
1110 	return tx_cfg->num_queues + tx_cfg->num_xdp_queues;  in gve_num_tx_qpls()
1113 /* Returns the number of rx queue page lists */
1119 	return rx_cfg->num_queues;  in gve_num_rx_qpls()
1129 	return priv->tx_cfg.max_queues + rx_qid;  in gve_rx_qpl_id()
1135 	return tx_cfg->max_queues + rx_qid;  in gve_get_rx_qpl_id()
1151 	 * out-of-order completions. Set it to two times of ring size.  in gve_get_rx_pages_per_qpl_dqo()
1156 /* Returns the correct dma direction for tx and rx qpls */
1160 	if (id < gve_rx_start_qpl_id(&priv->tx_cfg))  in gve_qpl_dma_dir()
1168 	return priv->queue_format == GVE_GQI_RDA_FORMAT ||  in gve_is_gqi()
1169 		priv->queue_format == GVE_GQI_QPL_FORMAT;  in gve_is_gqi()
1174 	return priv->tx_cfg.num_queues + priv->tx_cfg.num_xdp_queues;  in gve_num_tx_queues()
1179 	return priv->tx_cfg.num_queues + queue_id;  in gve_xdp_tx_queue_id()
1189 	switch (priv->queue_format) {  in gve_supports_xdp_xmit()
1236 /* rx handling */
1237 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
1239 bool gve_rx_work_pending(struct gve_rx_ring *rx);
1242 			  struct gve_rx_ring *rx,
1244 void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
1255 /* rx buffer handling */
1259 struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx);
1260 bool gve_buf_state_is_allocated(struct gve_rx_ring *rx,
1262 void gve_free_buf_state(struct gve_rx_ring *rx,
1264 struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
1266 void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
1268 struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx);
1269 void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
1271 void gve_free_to_page_pool(struct gve_rx_ring *rx,
1274 int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx,
1277 void gve_reuse_buffer(struct gve_rx_ring *rx,
1279 void gve_free_buffer(struct gve_rx_ring *rx,
1281 int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc);
1283 					  struct gve_rx_ring *rx,
1315 	return -EOPNOTSUPP;  in gve_clock_nic_ts_read()