xref: /freebsd/sys/contrib/ena-com/ena_eth_com.h (revision 5ab1c5846ff41be24b1f6beb0317bf8258cd4409)
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * * Redistributions of source code must retain the above copyright
12  * notice, this list of conditions and the following disclaimer.
13  * * Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  * * Neither the name of copyright holder nor the names of its
18  * contributors may be used to endorse or promote products derived
19  * from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #ifndef ENA_ETH_COM_H_
35 #define ENA_ETH_COM_H_
36 
37 #if defined(__cplusplus)
38 extern "C" {
39 #endif
40 #include "ena_com.h"
41 
42 /* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */
43 #define ENA_COMP_HEAD_THRESH 4
44 
45 struct ena_com_tx_ctx {
46 	struct ena_com_tx_meta ena_meta;
47 	struct ena_com_buf *ena_bufs;
48 	/* For LLQ, header buffer - pushed to the device mem space */
49 	void *push_header;
50 
51 	enum ena_eth_io_l3_proto_index l3_proto;
52 	enum ena_eth_io_l4_proto_index l4_proto;
53 	u16 num_bufs;
54 	u16 req_id;
55 	/* For regular queue, indicate the size of the header
56 	 * For LLQ, indicate the size of the pushed buffer
57 	 */
58 	u16 header_len;
59 
60 	u8 meta_valid;
61 	u8 tso_enable;
62 	u8 l3_csum_enable;
63 	u8 l4_csum_enable;
64 	u8 l4_csum_partial;
65 	u8 df; /* Don't fragment */
66 };
67 
68 struct ena_com_rx_ctx {
69 	struct ena_com_rx_buf_info *ena_bufs;
70 	enum ena_eth_io_l3_proto_index l3_proto;
71 	enum ena_eth_io_l4_proto_index l4_proto;
72 	bool l3_csum_err;
73 	bool l4_csum_err;
74 	u8 l4_csum_checked;
75 	/* fragmented packet */
76 	bool frag;
77 	u32 hash;
78 	u16 descs;
79 	int max_bufs;
80 };
81 
82 int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
83 		       struct ena_com_tx_ctx *ena_tx_ctx,
84 		       int *nb_hw_desc);
85 
86 int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
87 		   struct ena_com_io_sq *io_sq,
88 		   struct ena_com_rx_ctx *ena_rx_ctx);
89 
90 int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
91 			       struct ena_com_buf *ena_buf,
92 			       u16 req_id);
93 
94 bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
95 
96 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
97 				       struct ena_eth_io_intr_reg *intr_reg)
98 {
99 	ENA_REG_WRITE32(io_cq->bus, intr_reg->intr_control, io_cq->unmask_reg);
100 }
101 
102 static inline int ena_com_free_desc(struct ena_com_io_sq *io_sq)
103 {
104 	u16 tail, next_to_comp, cnt;
105 
106 	next_to_comp = io_sq->next_to_comp;
107 	tail = io_sq->tail;
108 	cnt = tail - next_to_comp;
109 
110 	return io_sq->q_depth - 1 - cnt;
111 }
112 
113 /* Check if the submission queue has enough space to hold required_buffers */
114 static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
115 						u16 required_buffers)
116 {
117 	int temp;
118 
119 	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
120 		return ena_com_free_desc(io_sq) >= required_buffers;
121 
122 	/* This calculation doesn't need to be 100% accurate. So to reduce
123 	 * the calculation overhead just Subtract 2 lines from the free descs
124 	 * (one for the header line and one to compensate the devision
125 	 * down calculation.
126 	 */
127 	temp = required_buffers / io_sq->llq_info.descs_per_entry + 2;
128 
129 	return ena_com_free_desc(io_sq) > temp;
130 }
131 
132 static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
133 					     struct ena_com_tx_ctx *ena_tx_ctx)
134 {
135 	if (!ena_tx_ctx->meta_valid)
136 		return false;
137 
138 	return !!memcmp(&io_sq->cached_tx_meta,
139 			&ena_tx_ctx->ena_meta,
140 			sizeof(struct ena_com_tx_meta));
141 }
142 
143 static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq)
144 {
145 	return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) &&
146 	       io_sq->llq_info.max_entries_in_tx_burst > 0;
147 }
148 
149 static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
150 					      struct ena_com_tx_ctx *ena_tx_ctx)
151 {
152 	struct ena_com_llq_info *llq_info;
153 	int descs_after_first_entry;
154 	int num_entries_needed = 1;
155 	u16 num_descs;
156 
157 	if (!is_llq_max_tx_burst_exists(io_sq))
158 		return false;
159 
160 	llq_info = &io_sq->llq_info;
161 	num_descs = ena_tx_ctx->num_bufs;
162 
163 	if (unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx)))
164 		++num_descs;
165 
166 	if (num_descs > llq_info->descs_num_before_header) {
167 		descs_after_first_entry = num_descs - llq_info->descs_num_before_header;
168 		num_entries_needed += DIV_ROUND_UP(descs_after_first_entry,
169 						   llq_info->descs_per_entry);
170 	}
171 
172 	ena_trc_dbg("queue: %d num_descs: %d num_entries_needed: %d\n",
173 		    io_sq->qid, num_descs, num_entries_needed);
174 
175 	return num_entries_needed > io_sq->entries_in_tx_burst_left;
176 }
177 
178 static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
179 {
180 	u16 tail = io_sq->tail;
181 	u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst;
182 
183 	ena_trc_dbg("write submission queue doorbell for queue: %d tail: %d\n",
184 		    io_sq->qid, tail);
185 
186 	ENA_REG_WRITE32(io_sq->bus, tail, io_sq->db_addr);
187 
188 	if (is_llq_max_tx_burst_exists(io_sq)) {
189 		ena_trc_dbg("reset available entries in tx burst for queue %d to %d\n",
190 			     io_sq->qid, max_entries_in_tx_burst);
191 		io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst;
192 	}
193 
194 	return 0;
195 }
196 
197 static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
198 {
199 	u16 unreported_comp, head;
200 	bool need_update;
201 
202 	head = io_cq->head;
203 	unreported_comp = head - io_cq->last_head_update;
204 	need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH);
205 
206 	if (io_cq->cq_head_db_reg && need_update) {
207 		ena_trc_dbg("Write completion queue doorbell for queue %d: head: %d\n",
208 			    io_cq->qid, head);
209 		ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg);
210 		io_cq->last_head_update = head;
211 	}
212 
213 	return 0;
214 }
215 
216 static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
217 					    u8 numa_node)
218 {
219 	struct ena_eth_io_numa_node_cfg_reg numa_cfg;
220 
221 	if (!io_cq->numa_node_cfg_reg)
222 		return;
223 
224 	numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK)
225 		| ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
226 
227 	ENA_REG_WRITE32(io_cq->bus, numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
228 }
229 
230 static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
231 {
232 	io_sq->next_to_comp += elem;
233 }
234 
235 static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
236 {
237 	io_cq->head++;
238 
239 	/* Switch phase bit in case of wrap around */
240 	if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
241 		io_cq->phase ^= 1;
242 }
243 
244 static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,
245 					     u16 *req_id)
246 {
247 	u8 expected_phase, cdesc_phase;
248 	struct ena_eth_io_tx_cdesc *cdesc;
249 	u16 masked_head;
250 
251 	masked_head = io_cq->head & (io_cq->q_depth - 1);
252 	expected_phase = io_cq->phase;
253 
254 	cdesc = (struct ena_eth_io_tx_cdesc *)
255 		((uintptr_t)io_cq->cdesc_addr.virt_addr +
256 		(masked_head * io_cq->cdesc_entry_size_in_bytes));
257 
258 	/* When the current completion descriptor phase isn't the same as the
259 	 * expected, it mean that the device still didn't update
260 	 * this completion.
261 	 */
262 	cdesc_phase = READ_ONCE16(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
263 	if (cdesc_phase != expected_phase)
264 		return ENA_COM_TRY_AGAIN;
265 
266 	dma_rmb();
267 
268 	*req_id = READ_ONCE16(cdesc->req_id);
269 	if (unlikely(*req_id >= io_cq->q_depth)) {
270 		ena_trc_err("Invalid req id %d\n", cdesc->req_id);
271 		return ENA_COM_INVAL;
272 	}
273 
274 	ena_com_cq_inc_head(io_cq);
275 
276 	return 0;
277 }
278 
279 #if defined(__cplusplus)
280 }
281 #endif
282 #endif /* ENA_ETH_COM_H_ */
283