1 /*-
2 * Copyright (c) 2012-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/cdefs.h>
32 #include "efx.h"
33 #include "efx_impl.h"
34
35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
36
37 static __checkReturn efx_rc_t
efx_mcdi_init_rxq(__in efx_nic_t * enp,__in uint32_t ndescs,__in uint32_t target_evq,__in uint32_t label,__in uint32_t instance,__in efsys_mem_t * esmp,__in boolean_t disable_scatter,__in boolean_t want_inner_classes,__in uint32_t ps_bufsize,__in uint32_t es_bufs_per_desc,__in uint32_t es_max_dma_len,__in uint32_t es_buf_stride,__in uint32_t hol_block_timeout)38 efx_mcdi_init_rxq(
39 __in efx_nic_t *enp,
40 __in uint32_t ndescs,
41 __in uint32_t target_evq,
42 __in uint32_t label,
43 __in uint32_t instance,
44 __in efsys_mem_t *esmp,
45 __in boolean_t disable_scatter,
46 __in boolean_t want_inner_classes,
47 __in uint32_t ps_bufsize,
48 __in uint32_t es_bufs_per_desc,
49 __in uint32_t es_max_dma_len,
50 __in uint32_t es_buf_stride,
51 __in uint32_t hol_block_timeout)
52 {
53 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
54 efx_mcdi_req_t req;
55 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V3_IN_LEN,
56 MC_CMD_INIT_RXQ_V3_OUT_LEN);
57 int npages = EFX_RXQ_NBUFS(ndescs);
58 int i;
59 efx_qword_t *dma_addr;
60 uint64_t addr;
61 efx_rc_t rc;
62 uint32_t dma_mode;
63 boolean_t want_outer_classes;
64
65 EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
66
67 if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) {
68 rc = EINVAL;
69 goto fail1;
70 }
71
72 if (ps_bufsize > 0)
73 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
74 else if (es_bufs_per_desc > 0)
75 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
76 else
77 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
78
79 if (encp->enc_tunnel_encapsulations_supported != 0 &&
80 !want_inner_classes) {
81 /*
82 * WANT_OUTER_CLASSES can only be specified on hardware which
83 * supports tunnel encapsulation offloads, even though it is
84 * effectively the behaviour the hardware gives.
85 *
86 * Also, on hardware which does support such offloads, older
87 * firmware rejects the flag if the offloads are not supported
88 * by the current firmware variant, which means this may fail if
89 * the capabilities are not updated when the firmware variant
90 * changes. This is not an issue on newer firmware, as it was
91 * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
92 * specified on all firmware variants.
93 */
94 want_outer_classes = B_TRUE;
95 } else {
96 want_outer_classes = B_FALSE;
97 }
98
99 req.emr_cmd = MC_CMD_INIT_RXQ;
100 req.emr_in_buf = payload;
101 req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
102 req.emr_out_buf = payload;
103 req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
104
105 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
106 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
107 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
108 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
109 MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
110 INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
111 INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
112 INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
113 INIT_RXQ_EXT_IN_CRC_MODE, 0,
114 INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
115 INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
116 INIT_RXQ_EXT_IN_DMA_MODE,
117 dma_mode,
118 INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
119 INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
120 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
121 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
122
123 if (es_bufs_per_desc > 0) {
124 MCDI_IN_SET_DWORD(req,
125 INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
126 es_bufs_per_desc);
127 MCDI_IN_SET_DWORD(req,
128 INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
129 MCDI_IN_SET_DWORD(req,
130 INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
131 MCDI_IN_SET_DWORD(req,
132 INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
133 hol_block_timeout);
134 }
135
136 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
137 addr = EFSYS_MEM_ADDR(esmp);
138
139 for (i = 0; i < npages; i++) {
140 EFX_POPULATE_QWORD_2(*dma_addr,
141 EFX_DWORD_1, (uint32_t)(addr >> 32),
142 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
143
144 dma_addr++;
145 addr += EFX_BUF_SIZE;
146 }
147
148 efx_mcdi_execute(enp, &req);
149
150 if (req.emr_rc != 0) {
151 rc = req.emr_rc;
152 goto fail2;
153 }
154
155 return (0);
156
157 fail2:
158 EFSYS_PROBE(fail2);
159 fail1:
160 EFSYS_PROBE1(fail1, efx_rc_t, rc);
161
162 return (rc);
163 }
164
165 static __checkReturn efx_rc_t
efx_mcdi_fini_rxq(__in efx_nic_t * enp,__in uint32_t instance)166 efx_mcdi_fini_rxq(
167 __in efx_nic_t *enp,
168 __in uint32_t instance)
169 {
170 efx_mcdi_req_t req;
171 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
172 MC_CMD_FINI_RXQ_OUT_LEN);
173 efx_rc_t rc;
174
175 req.emr_cmd = MC_CMD_FINI_RXQ;
176 req.emr_in_buf = payload;
177 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
178 req.emr_out_buf = payload;
179 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
180
181 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
182
183 efx_mcdi_execute_quiet(enp, &req);
184
185 if (req.emr_rc != 0) {
186 rc = req.emr_rc;
187 goto fail1;
188 }
189
190 return (0);
191
192 fail1:
193 /*
194 * EALREADY is not an error, but indicates that the MC has rebooted and
195 * that the RXQ has already been destroyed.
196 */
197 if (rc != EALREADY)
198 EFSYS_PROBE1(fail1, efx_rc_t, rc);
199
200 return (rc);
201 }
202
203 #if EFSYS_OPT_RX_SCALE
204 static __checkReturn efx_rc_t
efx_mcdi_rss_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__out uint32_t * rss_contextp)205 efx_mcdi_rss_context_alloc(
206 __in efx_nic_t *enp,
207 __in efx_rx_scale_context_type_t type,
208 __in uint32_t num_queues,
209 __out uint32_t *rss_contextp)
210 {
211 efx_mcdi_req_t req;
212 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
213 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
214 uint32_t rss_context;
215 uint32_t context_type;
216 efx_rc_t rc;
217
218 if (num_queues > EFX_MAXRSS) {
219 rc = EINVAL;
220 goto fail1;
221 }
222
223 switch (type) {
224 case EFX_RX_SCALE_EXCLUSIVE:
225 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
226 break;
227 case EFX_RX_SCALE_SHARED:
228 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
229 break;
230 default:
231 rc = EINVAL;
232 goto fail2;
233 }
234
235 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
236 req.emr_in_buf = payload;
237 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
238 req.emr_out_buf = payload;
239 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
240
241 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
242 EVB_PORT_ID_ASSIGNED);
243 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
244
245 /*
246 * For exclusive contexts, NUM_QUEUES is only used to validate
247 * indirection table offsets.
248 * For shared contexts, the provided context will spread traffic over
249 * NUM_QUEUES many queues.
250 */
251 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
252
253 efx_mcdi_execute(enp, &req);
254
255 if (req.emr_rc != 0) {
256 rc = req.emr_rc;
257 goto fail3;
258 }
259
260 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
261 rc = EMSGSIZE;
262 goto fail4;
263 }
264
265 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
266 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
267 rc = ENOENT;
268 goto fail5;
269 }
270
271 *rss_contextp = rss_context;
272
273 return (0);
274
275 fail5:
276 EFSYS_PROBE(fail5);
277 fail4:
278 EFSYS_PROBE(fail4);
279 fail3:
280 EFSYS_PROBE(fail3);
281 fail2:
282 EFSYS_PROBE(fail2);
283 fail1:
284 EFSYS_PROBE1(fail1, efx_rc_t, rc);
285
286 return (rc);
287 }
288 #endif /* EFSYS_OPT_RX_SCALE */
289
290 #if EFSYS_OPT_RX_SCALE
291 static efx_rc_t
efx_mcdi_rss_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)292 efx_mcdi_rss_context_free(
293 __in efx_nic_t *enp,
294 __in uint32_t rss_context)
295 {
296 efx_mcdi_req_t req;
297 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
298 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
299 efx_rc_t rc;
300
301 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
302 rc = EINVAL;
303 goto fail1;
304 }
305
306 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
307 req.emr_in_buf = payload;
308 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
309 req.emr_out_buf = payload;
310 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
311
312 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
313
314 efx_mcdi_execute_quiet(enp, &req);
315
316 if (req.emr_rc != 0) {
317 rc = req.emr_rc;
318 goto fail2;
319 }
320
321 return (0);
322
323 fail2:
324 EFSYS_PROBE(fail2);
325 fail1:
326 EFSYS_PROBE1(fail1, efx_rc_t, rc);
327
328 return (rc);
329 }
330 #endif /* EFSYS_OPT_RX_SCALE */
331
332 #if EFSYS_OPT_RX_SCALE
333 static efx_rc_t
efx_mcdi_rss_context_set_flags(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_type_t type)334 efx_mcdi_rss_context_set_flags(
335 __in efx_nic_t *enp,
336 __in uint32_t rss_context,
337 __in efx_rx_hash_type_t type)
338 {
339 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
340 efx_mcdi_req_t req;
341 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
342 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
343 efx_rc_t rc;
344
345 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
346 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
347 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
348 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
349 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
350 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
351 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
352 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
353 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
354 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
355 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
356 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
357 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
358 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
359 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
360 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
361
362 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
363 rc = EINVAL;
364 goto fail1;
365 }
366
367 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
368 req.emr_in_buf = payload;
369 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
370 req.emr_out_buf = payload;
371 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
372
373 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
374 rss_context);
375
376 /*
377 * If the firmware lacks support for additional modes, RSS_MODE
378 * fields must contain zeros, otherwise the operation will fail.
379 */
380 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
381 type &= EFX_RX_HASH_LEGACY_MASK;
382
383 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
384 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
385 (type & EFX_RX_HASH_IPV4) ? 1 : 0,
386 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
387 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
388 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
389 (type & EFX_RX_HASH_IPV6) ? 1 : 0,
390 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
391 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
392 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
393 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
394 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
395 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
396 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
397 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
398 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
399 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
400 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
401 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
402 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
403 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
404 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
405 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
406 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
407 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
408
409 efx_mcdi_execute(enp, &req);
410
411 if (req.emr_rc != 0) {
412 rc = req.emr_rc;
413 goto fail2;
414 }
415
416 return (0);
417
418 fail2:
419 EFSYS_PROBE(fail2);
420 fail1:
421 EFSYS_PROBE1(fail1, efx_rc_t, rc);
422
423 return (rc);
424 }
425 #endif /* EFSYS_OPT_RX_SCALE */
426
427 #if EFSYS_OPT_RX_SCALE
428 static efx_rc_t
efx_mcdi_rss_context_set_key(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)429 efx_mcdi_rss_context_set_key(
430 __in efx_nic_t *enp,
431 __in uint32_t rss_context,
432 __in_ecount(n) uint8_t *key,
433 __in size_t n)
434 {
435 efx_mcdi_req_t req;
436 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
437 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
438 efx_rc_t rc;
439
440 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
441 rc = EINVAL;
442 goto fail1;
443 }
444
445 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
446 req.emr_in_buf = payload;
447 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
448 req.emr_out_buf = payload;
449 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
450
451 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
452 rss_context);
453
454 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
455 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
456 rc = EINVAL;
457 goto fail2;
458 }
459
460 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
461 key, n);
462
463 efx_mcdi_execute(enp, &req);
464
465 if (req.emr_rc != 0) {
466 rc = req.emr_rc;
467 goto fail3;
468 }
469
470 return (0);
471
472 fail3:
473 EFSYS_PROBE(fail3);
474 fail2:
475 EFSYS_PROBE(fail2);
476 fail1:
477 EFSYS_PROBE1(fail1, efx_rc_t, rc);
478
479 return (rc);
480 }
481 #endif /* EFSYS_OPT_RX_SCALE */
482
483 #if EFSYS_OPT_RX_SCALE
484 static efx_rc_t
efx_mcdi_rss_context_set_table(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)unsigned int * table,__in size_t n)485 efx_mcdi_rss_context_set_table(
486 __in efx_nic_t *enp,
487 __in uint32_t rss_context,
488 __in_ecount(n) unsigned int *table,
489 __in size_t n)
490 {
491 efx_mcdi_req_t req;
492 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
493 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
494 uint8_t *req_table;
495 int i, rc;
496
497 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
498 rc = EINVAL;
499 goto fail1;
500 }
501
502 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
503 req.emr_in_buf = payload;
504 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
505 req.emr_out_buf = payload;
506 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
507
508 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
509 rss_context);
510
511 req_table =
512 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
513
514 for (i = 0;
515 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
516 i++) {
517 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
518 }
519
520 efx_mcdi_execute(enp, &req);
521
522 if (req.emr_rc != 0) {
523 rc = req.emr_rc;
524 goto fail2;
525 }
526
527 return (0);
528
529 fail2:
530 EFSYS_PROBE(fail2);
531 fail1:
532 EFSYS_PROBE1(fail1, efx_rc_t, rc);
533
534 return (rc);
535 }
536 #endif /* EFSYS_OPT_RX_SCALE */
537
538 __checkReturn efx_rc_t
ef10_rx_init(__in efx_nic_t * enp)539 ef10_rx_init(
540 __in efx_nic_t *enp)
541 {
542 #if EFSYS_OPT_RX_SCALE
543
544 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
545 &enp->en_rss_context) == 0) {
546 /*
547 * Allocated an exclusive RSS context, which allows both the
548 * indirection table and key to be modified.
549 */
550 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
551 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
552 } else {
553 /*
554 * Failed to allocate an exclusive RSS context. Continue
555 * operation without support for RSS. The pseudo-header in
556 * received packets will not contain a Toeplitz hash value.
557 */
558 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
559 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
560 }
561
562 #endif /* EFSYS_OPT_RX_SCALE */
563
564 return (0);
565 }
566
567 #if EFSYS_OPT_RX_SCATTER
568 __checkReturn efx_rc_t
ef10_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)569 ef10_rx_scatter_enable(
570 __in efx_nic_t *enp,
571 __in unsigned int buf_size)
572 {
573 _NOTE(ARGUNUSED(enp, buf_size))
574 return (0);
575 }
576 #endif /* EFSYS_OPT_RX_SCATTER */
577
578 #if EFSYS_OPT_RX_SCALE
579 __checkReturn efx_rc_t
ef10_rx_scale_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__out uint32_t * rss_contextp)580 ef10_rx_scale_context_alloc(
581 __in efx_nic_t *enp,
582 __in efx_rx_scale_context_type_t type,
583 __in uint32_t num_queues,
584 __out uint32_t *rss_contextp)
585 {
586 efx_rc_t rc;
587
588 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
589 if (rc != 0)
590 goto fail1;
591
592 return (0);
593
594 fail1:
595 EFSYS_PROBE1(fail1, efx_rc_t, rc);
596 return (rc);
597 }
598 #endif /* EFSYS_OPT_RX_SCALE */
599
600 #if EFSYS_OPT_RX_SCALE
601 __checkReturn efx_rc_t
ef10_rx_scale_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)602 ef10_rx_scale_context_free(
603 __in efx_nic_t *enp,
604 __in uint32_t rss_context)
605 {
606 efx_rc_t rc;
607
608 rc = efx_mcdi_rss_context_free(enp, rss_context);
609 if (rc != 0)
610 goto fail1;
611
612 return (0);
613
614 fail1:
615 EFSYS_PROBE1(fail1, efx_rc_t, rc);
616 return (rc);
617 }
618 #endif /* EFSYS_OPT_RX_SCALE */
619
620 #if EFSYS_OPT_RX_SCALE
621 __checkReturn efx_rc_t
ef10_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)622 ef10_rx_scale_mode_set(
623 __in efx_nic_t *enp,
624 __in uint32_t rss_context,
625 __in efx_rx_hash_alg_t alg,
626 __in efx_rx_hash_type_t type,
627 __in boolean_t insert)
628 {
629 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
630 efx_rc_t rc;
631
632 EFSYS_ASSERT3U(insert, ==, B_TRUE);
633
634 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
635 insert == B_FALSE) {
636 rc = EINVAL;
637 goto fail1;
638 }
639
640 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
641 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
642 rc = ENOTSUP;
643 goto fail2;
644 }
645 rss_context = enp->en_rss_context;
646 }
647
648 if ((rc = efx_mcdi_rss_context_set_flags(enp,
649 rss_context, type)) != 0)
650 goto fail3;
651
652 return (0);
653
654 fail3:
655 EFSYS_PROBE(fail3);
656 fail2:
657 EFSYS_PROBE(fail2);
658 fail1:
659 EFSYS_PROBE1(fail1, efx_rc_t, rc);
660
661 return (rc);
662 }
663 #endif /* EFSYS_OPT_RX_SCALE */
664
665 #if EFSYS_OPT_RX_SCALE
666 __checkReturn efx_rc_t
ef10_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)667 ef10_rx_scale_key_set(
668 __in efx_nic_t *enp,
669 __in uint32_t rss_context,
670 __in_ecount(n) uint8_t *key,
671 __in size_t n)
672 {
673 efx_rc_t rc;
674
675 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
676 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
677
678 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
679 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
680 rc = ENOTSUP;
681 goto fail1;
682 }
683 rss_context = enp->en_rss_context;
684 }
685
686 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
687 goto fail2;
688
689 return (0);
690
691 fail2:
692 EFSYS_PROBE(fail2);
693 fail1:
694 EFSYS_PROBE1(fail1, efx_rc_t, rc);
695
696 return (rc);
697 }
698 #endif /* EFSYS_OPT_RX_SCALE */
699
700 #if EFSYS_OPT_RX_SCALE
701 __checkReturn efx_rc_t
ef10_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)unsigned int * table,__in size_t n)702 ef10_rx_scale_tbl_set(
703 __in efx_nic_t *enp,
704 __in uint32_t rss_context,
705 __in_ecount(n) unsigned int *table,
706 __in size_t n)
707 {
708 efx_rc_t rc;
709
710 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
711 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
712 rc = ENOTSUP;
713 goto fail1;
714 }
715 rss_context = enp->en_rss_context;
716 }
717
718 if ((rc = efx_mcdi_rss_context_set_table(enp,
719 rss_context, table, n)) != 0)
720 goto fail2;
721
722 return (0);
723
724 fail2:
725 EFSYS_PROBE(fail2);
726 fail1:
727 EFSYS_PROBE1(fail1, efx_rc_t, rc);
728
729 return (rc);
730 }
731 #endif /* EFSYS_OPT_RX_SCALE */
732
733 /*
734 * EF10 RX pseudo-header
735 * ---------------------
736 *
737 * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
738 *
739 * +00: Toeplitz hash value.
740 * (32bit little-endian)
741 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
742 * (16bit big-endian)
743 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
744 * (16bit big-endian)
745 * +08: Packet Length. Zero if the RX datapath was in cut-through mode.
746 * (16bit little-endian)
747 * +10: MAC timestamp. Zero if timestamping is not enabled.
748 * (32bit little-endian)
749 *
750 * See "The RX Pseudo-header" in SF-109306-TC.
751 */
752
753 __checkReturn efx_rc_t
ef10_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)754 ef10_rx_prefix_pktlen(
755 __in efx_nic_t *enp,
756 __in uint8_t *buffer,
757 __out uint16_t *lengthp)
758 {
759 _NOTE(ARGUNUSED(enp))
760
761 /*
762 * The RX pseudo-header contains the packet length, excluding the
763 * pseudo-header. If the hardware receive datapath was operating in
764 * cut-through mode then the length in the RX pseudo-header will be
765 * zero, and the packet length must be obtained from the DMA length
766 * reported in the RX event.
767 */
768 *lengthp = buffer[8] | (buffer[9] << 8);
769 return (0);
770 }
771
772 #if EFSYS_OPT_RX_SCALE
773 __checkReturn uint32_t
ef10_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)774 ef10_rx_prefix_hash(
775 __in efx_nic_t *enp,
776 __in efx_rx_hash_alg_t func,
777 __in uint8_t *buffer)
778 {
779 _NOTE(ARGUNUSED(enp))
780
781 switch (func) {
782 case EFX_RX_HASHALG_PACKED_STREAM:
783 case EFX_RX_HASHALG_TOEPLITZ:
784 return (buffer[0] |
785 (buffer[1] << 8) |
786 (buffer[2] << 16) |
787 (buffer[3] << 24));
788
789 default:
790 EFSYS_ASSERT(0);
791 return (0);
792 }
793 }
794 #endif /* EFSYS_OPT_RX_SCALE */
795
796 #if EFSYS_OPT_RX_PACKED_STREAM
797 /*
798 * Fake length for RXQ descriptors in packed stream mode
799 * to make hardware happy
800 */
801 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
802 #endif
803
804 void
ef10_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)805 ef10_rx_qpost(
806 __in efx_rxq_t *erp,
807 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
808 __in size_t size,
809 __in unsigned int ndescs,
810 __in unsigned int completed,
811 __in unsigned int added)
812 {
813 efx_qword_t qword;
814 unsigned int i;
815 unsigned int offset;
816 unsigned int id;
817
818 _NOTE(ARGUNUSED(completed))
819
820 #if EFSYS_OPT_RX_PACKED_STREAM
821 /*
822 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
823 * and equal to 0 after applying mask. Hardware does not like it.
824 */
825 if (erp->er_ev_qstate->eers_rx_packed_stream)
826 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
827 #endif
828
829 /* The client driver must not overfill the queue */
830 EFSYS_ASSERT3U(added - completed + ndescs, <=,
831 EFX_RXQ_LIMIT(erp->er_mask + 1));
832
833 id = added & (erp->er_mask);
834 for (i = 0; i < ndescs; i++) {
835 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
836 unsigned int, id, efsys_dma_addr_t, addrp[i],
837 size_t, size);
838
839 EFX_POPULATE_QWORD_3(qword,
840 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
841 ESF_DZ_RX_KER_BUF_ADDR_DW0,
842 (uint32_t)(addrp[i] & 0xffffffff),
843 ESF_DZ_RX_KER_BUF_ADDR_DW1,
844 (uint32_t)(addrp[i] >> 32));
845
846 offset = id * sizeof (efx_qword_t);
847 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
848
849 id = (id + 1) & (erp->er_mask);
850 }
851 }
852
853 void
ef10_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)854 ef10_rx_qpush(
855 __in efx_rxq_t *erp,
856 __in unsigned int added,
857 __inout unsigned int *pushedp)
858 {
859 efx_nic_t *enp = erp->er_enp;
860 unsigned int pushed = *pushedp;
861 uint32_t wptr;
862 efx_dword_t dword;
863
864 /* Hardware has alignment restriction for WPTR */
865 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN);
866 if (pushed == wptr)
867 return;
868
869 *pushedp = wptr;
870
871 /* Push the populated descriptors out */
872 wptr &= erp->er_mask;
873
874 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
875
876 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
877 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
878 wptr, pushed & erp->er_mask);
879 EFSYS_PIO_WRITE_BARRIER();
880 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
881 erp->er_index, &dword, B_FALSE);
882 }
883
884 #if EFSYS_OPT_RX_PACKED_STREAM
885
886 void
ef10_rx_qpush_ps_credits(__in efx_rxq_t * erp)887 ef10_rx_qpush_ps_credits(
888 __in efx_rxq_t *erp)
889 {
890 efx_nic_t *enp = erp->er_enp;
891 efx_dword_t dword;
892 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
893 uint32_t credits;
894
895 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
896
897 if (rxq_state->eers_rx_packed_stream_credits == 0)
898 return;
899
900 /*
901 * It is a bug if we think that FW has utilized more
902 * credits than it is allowed to have (maximum). However,
903 * make sure that we do not credit more than maximum anyway.
904 */
905 credits = MIN(rxq_state->eers_rx_packed_stream_credits,
906 EFX_RX_PACKED_STREAM_MAX_CREDITS);
907 EFX_POPULATE_DWORD_3(dword,
908 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
909 ERF_DZ_RX_DESC_MAGIC_CMD,
910 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
911 ERF_DZ_RX_DESC_MAGIC_DATA, credits);
912 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
913 erp->er_index, &dword, B_FALSE);
914
915 rxq_state->eers_rx_packed_stream_credits = 0;
916 }
917
918 /*
919 * In accordance with SF-112241-TC the received data has the following layout:
920 * - 8 byte pseudo-header which consist of:
921 * - 4 byte little-endian timestamp
922 * - 2 byte little-endian captured length in bytes
923 * - 2 byte little-endian original packet length in bytes
924 * - captured packet bytes
925 * - optional padding to align to 64 bytes boundary
926 * - 64 bytes scratch space for the host software
927 */
928 __checkReturn uint8_t *
ef10_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)929 ef10_rx_qps_packet_info(
930 __in efx_rxq_t *erp,
931 __in uint8_t *buffer,
932 __in uint32_t buffer_length,
933 __in uint32_t current_offset,
934 __out uint16_t *lengthp,
935 __out uint32_t *next_offsetp,
936 __out uint32_t *timestamp)
937 {
938 uint16_t buf_len;
939 uint8_t *pkt_start;
940 efx_qword_t *qwordp;
941 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
942
943 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
944
945 buffer += current_offset;
946 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
947
948 qwordp = (efx_qword_t *)buffer;
949 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
950 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
951 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
952
953 buf_len = EFX_P2ROUNDUP(uint16_t,
954 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
955 EFX_RX_PACKED_STREAM_ALIGNMENT);
956 *next_offsetp =
957 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
958
959 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
960 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
961
962 if ((*next_offsetp ^ current_offset) &
963 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
964 rxq_state->eers_rx_packed_stream_credits++;
965
966 return (pkt_start);
967 }
968
969 #endif
970
971 __checkReturn efx_rc_t
ef10_rx_qflush(__in efx_rxq_t * erp)972 ef10_rx_qflush(
973 __in efx_rxq_t *erp)
974 {
975 efx_nic_t *enp = erp->er_enp;
976 efx_rc_t rc;
977
978 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
979 goto fail1;
980
981 return (0);
982
983 fail1:
984 /*
985 * EALREADY is not an error, but indicates that the MC has rebooted and
986 * that the RXQ has already been destroyed. Callers need to know that
987 * the RXQ flush has completed to avoid waiting until timeout for a
988 * flush done event that will not be delivered.
989 */
990 if (rc != EALREADY)
991 EFSYS_PROBE1(fail1, efx_rc_t, rc);
992
993 return (rc);
994 }
995
996 void
ef10_rx_qenable(__in efx_rxq_t * erp)997 ef10_rx_qenable(
998 __in efx_rxq_t *erp)
999 {
1000 /* FIXME */
1001 _NOTE(ARGUNUSED(erp))
1002 /* FIXME */
1003 }
1004
1005 __checkReturn efx_rc_t
ef10_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__in efx_rxq_t * erp)1006 ef10_rx_qcreate(
1007 __in efx_nic_t *enp,
1008 __in unsigned int index,
1009 __in unsigned int label,
1010 __in efx_rxq_type_t type,
1011 __in_opt const efx_rxq_type_data_t *type_data,
1012 __in efsys_mem_t *esmp,
1013 __in size_t ndescs,
1014 __in uint32_t id,
1015 __in unsigned int flags,
1016 __in efx_evq_t *eep,
1017 __in efx_rxq_t *erp)
1018 {
1019 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1020 efx_rc_t rc;
1021 boolean_t disable_scatter;
1022 boolean_t want_inner_classes;
1023 unsigned int ps_buf_size;
1024 uint32_t es_bufs_per_desc = 0;
1025 uint32_t es_max_dma_len = 0;
1026 uint32_t es_buf_stride = 0;
1027 uint32_t hol_block_timeout = 0;
1028
1029 _NOTE(ARGUNUSED(id, erp, type_data))
1030
1031 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1032 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1033 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1034
1035 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1036 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1037
1038 if (!ISP2(ndescs) ||
1039 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1040 rc = EINVAL;
1041 goto fail1;
1042 }
1043 if (index >= encp->enc_rxq_limit) {
1044 rc = EINVAL;
1045 goto fail2;
1046 }
1047
1048 switch (type) {
1049 case EFX_RXQ_TYPE_DEFAULT:
1050 ps_buf_size = 0;
1051 break;
1052 #if EFSYS_OPT_RX_PACKED_STREAM
1053 case EFX_RXQ_TYPE_PACKED_STREAM:
1054 if (type_data == NULL) {
1055 rc = EINVAL;
1056 goto fail3;
1057 }
1058 switch (type_data->ertd_packed_stream.eps_buf_size) {
1059 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1060 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1061 break;
1062 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1063 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1064 break;
1065 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1066 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1067 break;
1068 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1069 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1070 break;
1071 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1072 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1073 break;
1074 default:
1075 rc = ENOTSUP;
1076 goto fail4;
1077 }
1078 break;
1079 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1080 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1081 case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1082 if (type_data == NULL) {
1083 rc = EINVAL;
1084 goto fail5;
1085 }
1086 ps_buf_size = 0;
1087 es_bufs_per_desc =
1088 type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1089 es_max_dma_len =
1090 type_data->ertd_es_super_buffer.eessb_max_dma_len;
1091 es_buf_stride =
1092 type_data->ertd_es_super_buffer.eessb_buf_stride;
1093 hol_block_timeout =
1094 type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1095 break;
1096 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1097 default:
1098 rc = ENOTSUP;
1099 goto fail6;
1100 }
1101
1102 #if EFSYS_OPT_RX_PACKED_STREAM
1103 if (ps_buf_size != 0) {
1104 /* Check if datapath firmware supports packed stream mode */
1105 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1106 rc = ENOTSUP;
1107 goto fail7;
1108 }
1109 /* Check if packed stream allows configurable buffer sizes */
1110 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1111 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1112 rc = ENOTSUP;
1113 goto fail8;
1114 }
1115 }
1116 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1117 EFSYS_ASSERT(ps_buf_size == 0);
1118 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1119
1120 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1121 if (es_bufs_per_desc > 0) {
1122 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1123 rc = ENOTSUP;
1124 goto fail9;
1125 }
1126 if (!EFX_IS_P2ALIGNED(uint32_t, es_max_dma_len,
1127 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1128 rc = EINVAL;
1129 goto fail10;
1130 }
1131 if (!EFX_IS_P2ALIGNED(uint32_t, es_buf_stride,
1132 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1133 rc = EINVAL;
1134 goto fail11;
1135 }
1136 }
1137 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1138 EFSYS_ASSERT(es_bufs_per_desc == 0);
1139 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1140
1141 /* Scatter can only be disabled if the firmware supports doing so */
1142 if (flags & EFX_RXQ_FLAG_SCATTER)
1143 disable_scatter = B_FALSE;
1144 else
1145 disable_scatter = encp->enc_rx_disable_scatter_supported;
1146
1147 if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1148 want_inner_classes = B_TRUE;
1149 else
1150 want_inner_classes = B_FALSE;
1151
1152 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1153 esmp, disable_scatter, want_inner_classes,
1154 ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1155 es_buf_stride, hol_block_timeout)) != 0)
1156 goto fail12;
1157
1158 erp->er_eep = eep;
1159 erp->er_label = label;
1160
1161 ef10_ev_rxlabel_init(eep, erp, label, type);
1162
1163 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1164
1165 return (0);
1166
1167 fail12:
1168 EFSYS_PROBE(fail12);
1169 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1170 fail11:
1171 EFSYS_PROBE(fail11);
1172 fail10:
1173 EFSYS_PROBE(fail10);
1174 fail9:
1175 EFSYS_PROBE(fail9);
1176 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1177 #if EFSYS_OPT_RX_PACKED_STREAM
1178 fail8:
1179 EFSYS_PROBE(fail8);
1180 fail7:
1181 EFSYS_PROBE(fail7);
1182 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1183 fail6:
1184 EFSYS_PROBE(fail6);
1185 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1186 fail5:
1187 EFSYS_PROBE(fail5);
1188 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1189 #if EFSYS_OPT_RX_PACKED_STREAM
1190 fail4:
1191 EFSYS_PROBE(fail4);
1192 fail3:
1193 EFSYS_PROBE(fail3);
1194 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1195 fail2:
1196 EFSYS_PROBE(fail2);
1197 fail1:
1198 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1199
1200 return (rc);
1201 }
1202
1203 void
ef10_rx_qdestroy(__in efx_rxq_t * erp)1204 ef10_rx_qdestroy(
1205 __in efx_rxq_t *erp)
1206 {
1207 efx_nic_t *enp = erp->er_enp;
1208 efx_evq_t *eep = erp->er_eep;
1209 unsigned int label = erp->er_label;
1210
1211 ef10_ev_rxlabel_fini(eep, label);
1212
1213 EFSYS_ASSERT(enp->en_rx_qcount != 0);
1214 --enp->en_rx_qcount;
1215
1216 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1217 }
1218
1219 void
ef10_rx_fini(__in efx_nic_t * enp)1220 ef10_rx_fini(
1221 __in efx_nic_t *enp)
1222 {
1223 #if EFSYS_OPT_RX_SCALE
1224 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1225 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1226 enp->en_rss_context = 0;
1227 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1228 #else
1229 _NOTE(ARGUNUSED(enp))
1230 #endif /* EFSYS_OPT_RX_SCALE */
1231 }
1232
1233 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
1234