xref: /illumos-gate/usr/src/uts/common/io/sfxge/common/efx_rx.c (revision 49ef7e0638c8b771d8a136eae78b1c0f99acc8e0)
1 /*
2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30 
31 #include "efx.h"
32 #include "efx_impl.h"
33 
34 
35 #if EFSYS_OPT_SIENA
36 
37 static	__checkReturn	efx_rc_t
38 siena_rx_init(
39 	__in		efx_nic_t *enp);
40 
41 static			void
42 siena_rx_fini(
43 	__in		efx_nic_t *enp);
44 
45 #if EFSYS_OPT_RX_SCATTER
46 static	__checkReturn	efx_rc_t
47 siena_rx_scatter_enable(
48 	__in		efx_nic_t *enp,
49 	__in		unsigned int buf_size);
50 #endif /* EFSYS_OPT_RX_SCATTER */
51 
52 #if EFSYS_OPT_RX_SCALE
53 static	__checkReturn	efx_rc_t
54 siena_rx_scale_mode_set(
55 	__in		efx_nic_t *enp,
56 	__in		efx_rx_hash_alg_t alg,
57 	__in		efx_rx_hash_type_t type,
58 	__in		boolean_t insert);
59 
60 static	__checkReturn	efx_rc_t
61 siena_rx_scale_key_set(
62 	__in		efx_nic_t *enp,
63 	__in_ecount(n)	uint8_t *key,
64 	__in		size_t n);
65 
66 static	__checkReturn	efx_rc_t
67 siena_rx_scale_tbl_set(
68 	__in		efx_nic_t *enp,
69 	__in_ecount(n)	unsigned int *table,
70 	__in		size_t n);
71 
72 static	__checkReturn	uint32_t
73 siena_rx_prefix_hash(
74 	__in		efx_nic_t *enp,
75 	__in		efx_rx_hash_alg_t func,
76 	__in		uint8_t *buffer);
77 
78 #endif /* EFSYS_OPT_RX_SCALE */
79 
80 static	__checkReturn	efx_rc_t
81 siena_rx_prefix_pktlen(
82 	__in		efx_nic_t *enp,
83 	__in		uint8_t *buffer,
84 	__out		uint16_t *lengthp);
85 
86 static			void
87 siena_rx_qpost(
88 	__in		efx_rxq_t *erp,
89 	__in_ecount(n)	efsys_dma_addr_t *addrp,
90 	__in		size_t size,
91 	__in		unsigned int n,
92 	__in		unsigned int completed,
93 	__in		unsigned int added);
94 
95 static			void
96 siena_rx_qpush(
97 	__in		efx_rxq_t *erp,
98 	__in		unsigned int added,
99 	__inout		unsigned int *pushedp);
100 
101 static	__checkReturn	efx_rc_t
102 siena_rx_qflush(
103 	__in		efx_rxq_t *erp);
104 
105 static			void
106 siena_rx_qenable(
107 	__in		efx_rxq_t *erp);
108 
109 static	__checkReturn	efx_rc_t
110 siena_rx_qcreate(
111 	__in		efx_nic_t *enp,
112 	__in		unsigned int index,
113 	__in		unsigned int label,
114 	__in		efx_rxq_type_t type,
115 	__in		efsys_mem_t *esmp,
116 	__in		size_t n,
117 	__in		uint32_t id,
118 	__in		efx_evq_t *eep,
119 	__in		efx_rxq_t *erp);
120 
121 static			void
122 siena_rx_qdestroy(
123 	__in		efx_rxq_t *erp);
124 
125 #endif /* EFSYS_OPT_SIENA */
126 
127 
128 #if EFSYS_OPT_SIENA
129 static const efx_rx_ops_t __efx_rx_siena_ops = {
130 	siena_rx_init,				/* erxo_init */
131 	siena_rx_fini,				/* erxo_fini */
132 #if EFSYS_OPT_RX_SCATTER
133 	siena_rx_scatter_enable,		/* erxo_scatter_enable */
134 #endif
135 #if EFSYS_OPT_RX_SCALE
136 	siena_rx_scale_mode_set,		/* erxo_scale_mode_set */
137 	siena_rx_scale_key_set,			/* erxo_scale_key_set */
138 	siena_rx_scale_tbl_set,			/* erxo_scale_tbl_set */
139 	siena_rx_prefix_hash,			/* erxo_prefix_hash */
140 #endif
141 	siena_rx_prefix_pktlen,			/* erxo_prefix_pktlen */
142 	siena_rx_qpost,				/* erxo_qpost */
143 	siena_rx_qpush,				/* erxo_qpush */
144 	siena_rx_qflush,			/* erxo_qflush */
145 	siena_rx_qenable,			/* erxo_qenable */
146 	siena_rx_qcreate,			/* erxo_qcreate */
147 	siena_rx_qdestroy,			/* erxo_qdestroy */
148 };
149 #endif	/* EFSYS_OPT_SIENA */
150 
151 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
152 static const efx_rx_ops_t __efx_rx_ef10_ops = {
153 	ef10_rx_init,				/* erxo_init */
154 	ef10_rx_fini,				/* erxo_fini */
155 #if EFSYS_OPT_RX_SCATTER
156 	ef10_rx_scatter_enable,			/* erxo_scatter_enable */
157 #endif
158 #if EFSYS_OPT_RX_SCALE
159 	ef10_rx_scale_mode_set,			/* erxo_scale_mode_set */
160 	ef10_rx_scale_key_set,			/* erxo_scale_key_set */
161 	ef10_rx_scale_tbl_set,			/* erxo_scale_tbl_set */
162 	ef10_rx_prefix_hash,			/* erxo_prefix_hash */
163 #endif
164 	ef10_rx_prefix_pktlen,			/* erxo_prefix_pktlen */
165 	ef10_rx_qpost,				/* erxo_qpost */
166 	ef10_rx_qpush,				/* erxo_qpush */
167 	ef10_rx_qflush,				/* erxo_qflush */
168 	ef10_rx_qenable,			/* erxo_qenable */
169 	ef10_rx_qcreate,			/* erxo_qcreate */
170 	ef10_rx_qdestroy,			/* erxo_qdestroy */
171 };
172 #endif	/* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
173 
174 
175 	__checkReturn	efx_rc_t
efx_rx_init(__inout efx_nic_t * enp)176 efx_rx_init(
177 	__inout		efx_nic_t *enp)
178 {
179 	const efx_rx_ops_t *erxop;
180 	efx_rc_t rc;
181 
182 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
183 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
184 
185 	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
186 		rc = EINVAL;
187 		goto fail1;
188 	}
189 
190 	if (enp->en_mod_flags & EFX_MOD_RX) {
191 		rc = EINVAL;
192 		goto fail2;
193 	}
194 
195 	switch (enp->en_family) {
196 #if EFSYS_OPT_SIENA
197 	case EFX_FAMILY_SIENA:
198 		erxop = &__efx_rx_siena_ops;
199 		break;
200 #endif /* EFSYS_OPT_SIENA */
201 
202 #if EFSYS_OPT_HUNTINGTON
203 	case EFX_FAMILY_HUNTINGTON:
204 		erxop = &__efx_rx_ef10_ops;
205 		break;
206 #endif /* EFSYS_OPT_HUNTINGTON */
207 
208 #if EFSYS_OPT_MEDFORD
209 	case EFX_FAMILY_MEDFORD:
210 		erxop = &__efx_rx_ef10_ops;
211 		break;
212 #endif /* EFSYS_OPT_MEDFORD */
213 
214 	default:
215 		EFSYS_ASSERT(0);
216 		rc = ENOTSUP;
217 		goto fail3;
218 	}
219 
220 	if ((rc = erxop->erxo_init(enp)) != 0)
221 		goto fail4;
222 
223 	enp->en_erxop = erxop;
224 	enp->en_mod_flags |= EFX_MOD_RX;
225 	return (0);
226 
227 fail4:
228 	EFSYS_PROBE(fail4);
229 fail3:
230 	EFSYS_PROBE(fail3);
231 fail2:
232 	EFSYS_PROBE(fail2);
233 fail1:
234 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
235 
236 	enp->en_erxop = NULL;
237 	enp->en_mod_flags &= ~EFX_MOD_RX;
238 	return (rc);
239 }
240 
241 			void
efx_rx_fini(__in efx_nic_t * enp)242 efx_rx_fini(
243 	__in		efx_nic_t *enp)
244 {
245 	const efx_rx_ops_t *erxop = enp->en_erxop;
246 
247 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
248 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
249 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
250 	EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
251 
252 	erxop->erxo_fini(enp);
253 
254 	enp->en_erxop = NULL;
255 	enp->en_mod_flags &= ~EFX_MOD_RX;
256 }
257 
258 #if EFSYS_OPT_RX_SCATTER
259 	__checkReturn	efx_rc_t
efx_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)260 efx_rx_scatter_enable(
261 	__in		efx_nic_t *enp,
262 	__in		unsigned int buf_size)
263 {
264 	const efx_rx_ops_t *erxop = enp->en_erxop;
265 	efx_rc_t rc;
266 
267 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
268 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
269 
270 	if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
271 		goto fail1;
272 
273 	return (0);
274 
275 fail1:
276 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
277 	return (rc);
278 }
279 #endif	/* EFSYS_OPT_RX_SCATTER */
280 
281 #if EFSYS_OPT_RX_SCALE
282 	__checkReturn	efx_rc_t
efx_rx_hash_support_get(__in efx_nic_t * enp,__out efx_rx_hash_support_t * supportp)283 efx_rx_hash_support_get(
284 	__in		efx_nic_t *enp,
285 	__out		efx_rx_hash_support_t *supportp)
286 {
287 	efx_rc_t rc;
288 
289 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
291 
292 	if (supportp == NULL) {
293 		rc = EINVAL;
294 		goto fail1;
295 	}
296 
297 	/* Report if resources are available to insert RX hash value */
298 	*supportp = enp->en_hash_support;
299 
300 	return (0);
301 
302 fail1:
303 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
304 
305 	return (rc);
306 }
307 
308 	__checkReturn	efx_rc_t
efx_rx_scale_support_get(__in efx_nic_t * enp,__out efx_rx_scale_support_t * supportp)309 efx_rx_scale_support_get(
310 	__in		efx_nic_t *enp,
311 	__out		efx_rx_scale_support_t *supportp)
312 {
313 	efx_rc_t rc;
314 
315 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
316 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
317 
318 	if (supportp == NULL) {
319 		rc = EINVAL;
320 		goto fail1;
321 	}
322 
323 	/* Report if resources are available to support RSS */
324 	*supportp = enp->en_rss_support;
325 
326 	return (0);
327 
328 fail1:
329 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
330 
331 	return (rc);
332 }
333 
334 	__checkReturn	efx_rc_t
efx_rx_scale_mode_set(__in efx_nic_t * enp,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)335 efx_rx_scale_mode_set(
336 	__in		efx_nic_t *enp,
337 	__in		efx_rx_hash_alg_t alg,
338 	__in		efx_rx_hash_type_t type,
339 	__in		boolean_t insert)
340 {
341 	const efx_rx_ops_t *erxop = enp->en_erxop;
342 	efx_rc_t rc;
343 
344 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
345 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
346 
347 	if (erxop->erxo_scale_mode_set != NULL) {
348 		if ((rc = erxop->erxo_scale_mode_set(enp, alg,
349 			    type, insert)) != 0)
350 			goto fail1;
351 	}
352 
353 	return (0);
354 
355 fail1:
356 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
357 	return (rc);
358 }
359 #endif	/* EFSYS_OPT_RX_SCALE */
360 
361 #if EFSYS_OPT_RX_SCALE
362 	__checkReturn	efx_rc_t
efx_rx_scale_key_set(__in efx_nic_t * enp,__in_ecount (n)uint8_t * key,__in size_t n)363 efx_rx_scale_key_set(
364 	__in		efx_nic_t *enp,
365 	__in_ecount(n)	uint8_t *key,
366 	__in		size_t n)
367 {
368 	const efx_rx_ops_t *erxop = enp->en_erxop;
369 	efx_rc_t rc;
370 
371 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
372 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
373 
374 	if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
375 		goto fail1;
376 
377 	return (0);
378 
379 fail1:
380 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
381 
382 	return (rc);
383 }
384 #endif	/* EFSYS_OPT_RX_SCALE */
385 
386 #if EFSYS_OPT_RX_SCALE
387 	__checkReturn	efx_rc_t
efx_rx_scale_tbl_set(__in efx_nic_t * enp,__in_ecount (n)unsigned int * table,__in size_t n)388 efx_rx_scale_tbl_set(
389 	__in		efx_nic_t *enp,
390 	__in_ecount(n)	unsigned int *table,
391 	__in		size_t n)
392 {
393 	const efx_rx_ops_t *erxop = enp->en_erxop;
394 	efx_rc_t rc;
395 
396 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
397 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
398 
399 	if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
400 		goto fail1;
401 
402 	return (0);
403 
404 fail1:
405 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
406 
407 	return (rc);
408 }
409 #endif	/* EFSYS_OPT_RX_SCALE */
410 
411 			void
efx_rx_qpost(__in efx_rxq_t * erp,__in_ecount (n)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int n,__in unsigned int completed,__in unsigned int added)412 efx_rx_qpost(
413 	__in		efx_rxq_t *erp,
414 	__in_ecount(n)	efsys_dma_addr_t *addrp,
415 	__in		size_t size,
416 	__in		unsigned int n,
417 	__in		unsigned int completed,
418 	__in		unsigned int added)
419 {
420 	efx_nic_t *enp = erp->er_enp;
421 	const efx_rx_ops_t *erxop = enp->en_erxop;
422 
423 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
424 
425 	erxop->erxo_qpost(erp, addrp, size, n, completed, added);
426 }
427 
428 			void
efx_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)429 efx_rx_qpush(
430 	__in		efx_rxq_t *erp,
431 	__in		unsigned int added,
432 	__inout		unsigned int *pushedp)
433 {
434 	efx_nic_t *enp = erp->er_enp;
435 	const efx_rx_ops_t *erxop = enp->en_erxop;
436 
437 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
438 
439 	erxop->erxo_qpush(erp, added, pushedp);
440 }
441 
442 	__checkReturn	efx_rc_t
efx_rx_qflush(__in efx_rxq_t * erp)443 efx_rx_qflush(
444 	__in		efx_rxq_t *erp)
445 {
446 	efx_nic_t *enp = erp->er_enp;
447 	const efx_rx_ops_t *erxop = enp->en_erxop;
448 	efx_rc_t rc;
449 
450 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
451 
452 	if ((rc = erxop->erxo_qflush(erp)) != 0)
453 		goto fail1;
454 
455 	return (0);
456 
457 fail1:
458 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
459 
460 	return (rc);
461 }
462 
463 			void
efx_rx_qenable(__in efx_rxq_t * erp)464 efx_rx_qenable(
465 	__in		efx_rxq_t *erp)
466 {
467 	efx_nic_t *enp = erp->er_enp;
468 	const efx_rx_ops_t *erxop = enp->en_erxop;
469 
470 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
471 
472 	erxop->erxo_qenable(erp);
473 }
474 
475 	__checkReturn	efx_rc_t
efx_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)476 efx_rx_qcreate(
477 	__in		efx_nic_t *enp,
478 	__in		unsigned int index,
479 	__in		unsigned int label,
480 	__in		efx_rxq_type_t type,
481 	__in		efsys_mem_t *esmp,
482 	__in		size_t n,
483 	__in		uint32_t id,
484 	__in		efx_evq_t *eep,
485 	__deref_out	efx_rxq_t **erpp)
486 {
487 	const efx_rx_ops_t *erxop = enp->en_erxop;
488 	efx_rxq_t *erp;
489 	efx_rc_t rc;
490 
491 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
492 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
493 
494 	/* Allocate an RXQ object */
495 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
496 
497 	if (erp == NULL) {
498 		rc = ENOMEM;
499 		goto fail1;
500 	}
501 
502 	erp->er_magic = EFX_RXQ_MAGIC;
503 	erp->er_enp = enp;
504 	erp->er_index = index;
505 	erp->er_mask = n - 1;
506 	erp->er_esmp = esmp;
507 
508 	if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
509 	    eep, erp)) != 0)
510 		goto fail2;
511 
512 	enp->en_rx_qcount++;
513 	*erpp = erp;
514 
515 	return (0);
516 
517 fail2:
518 	EFSYS_PROBE(fail2);
519 
520 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
521 fail1:
522 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
523 
524 	return (rc);
525 }
526 
527 			void
efx_rx_qdestroy(__in efx_rxq_t * erp)528 efx_rx_qdestroy(
529 	__in		efx_rxq_t *erp)
530 {
531 	efx_nic_t *enp = erp->er_enp;
532 	const efx_rx_ops_t *erxop = enp->en_erxop;
533 
534 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
535 
536 	erxop->erxo_qdestroy(erp);
537 }
538 
539 	__checkReturn	efx_rc_t
efx_psuedo_hdr_pkt_length_get(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)540 efx_psuedo_hdr_pkt_length_get(
541 	__in		efx_nic_t *enp,
542 	__in		uint8_t *buffer,
543 	__out		uint16_t *lengthp)
544 {
545 	const efx_rx_ops_t *erxop = enp->en_erxop;
546 
547 	return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
548 }
549 
550 #if EFSYS_OPT_RX_SCALE
551 	__checkReturn	uint32_t
efx_psuedo_hdr_hash_get(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)552 efx_psuedo_hdr_hash_get(
553 	__in		efx_nic_t *enp,
554 	__in		efx_rx_hash_alg_t func,
555 	__in		uint8_t *buffer)
556 {
557 	const efx_rx_ops_t *erxop = enp->en_erxop;
558 
559 	EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
560 	return (erxop->erxo_prefix_hash(enp, func, buffer));
561 }
562 #endif	/* EFSYS_OPT_RX_SCALE */
563 
564 #if EFSYS_OPT_SIENA
565 
566 static	__checkReturn	efx_rc_t
siena_rx_init(__in efx_nic_t * enp)567 siena_rx_init(
568 	__in		efx_nic_t *enp)
569 {
570 	efx_oword_t oword;
571 	unsigned int index;
572 
573 	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
574 
575 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
576 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
577 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
578 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
579 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
580 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
581 	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
582 
583 	/* Zero the RSS table */
584 	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
585 	    index++) {
586 		EFX_ZERO_OWORD(oword);
587 		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
588 				    index, &oword, B_TRUE);
589 	}
590 
591 #if EFSYS_OPT_RX_SCALE
592 	/* The RSS key and indirection table are writable. */
593 	enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
594 
595 	/* Hardware can insert RX hash with/without RSS */
596 	enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
597 #endif	/* EFSYS_OPT_RX_SCALE */
598 
599 	return (0);
600 }
601 
602 #if EFSYS_OPT_RX_SCATTER
603 static	__checkReturn	efx_rc_t
siena_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)604 siena_rx_scatter_enable(
605 	__in		efx_nic_t *enp,
606 	__in		unsigned int buf_size)
607 {
608 	unsigned int nbuf32;
609 	efx_oword_t oword;
610 	efx_rc_t rc;
611 
612 	nbuf32 = buf_size / 32;
613 	if ((nbuf32 == 0) ||
614 	    (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
615 	    ((buf_size % 32) != 0)) {
616 		rc = EINVAL;
617 		goto fail1;
618 	}
619 
620 	if (enp->en_rx_qcount > 0) {
621 		rc = EBUSY;
622 		goto fail2;
623 	}
624 
625 	/* Set scatter buffer size */
626 	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
627 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
628 	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
629 
630 	/* Enable scatter for packets not matching a filter */
631 	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
632 	EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
633 	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
634 
635 	return (0);
636 
637 fail2:
638 	EFSYS_PROBE(fail2);
639 fail1:
640 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
641 
642 	return (rc);
643 }
644 #endif	/* EFSYS_OPT_RX_SCATTER */
645 
646 
647 #define	EFX_RX_LFSR_HASH(_enp, _insert)					\
648 	do {								\
649 		efx_oword_t oword;					\
650 									\
651 		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
652 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);	\
653 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);	\
654 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);	\
655 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
656 		    (_insert) ? 1 : 0);					\
657 		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
658 									\
659 		if ((_enp)->en_family == EFX_FAMILY_SIENA) {		\
660 			EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
661 			    &oword);					\
662 			EFX_SET_OWORD_FIELD(oword,			\
663 			    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);	\
664 			EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
665 			    &oword);					\
666 		}							\
667 									\
668 		_NOTE(CONSTANTCONDITION)				\
669 	} while (B_FALSE)
670 
671 #define	EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)		\
672 	do {								\
673 		efx_oword_t oword;					\
674 									\
675 		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG,	&oword);	\
676 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);	\
677 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,		\
678 		    (_ip) ? 1 : 0);					\
679 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,		\
680 		    (_tcp) ? 0 : 1);					\
681 		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
682 		    (_insert) ? 1 : 0);					\
683 		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
684 									\
685 		_NOTE(CONSTANTCONDITION)				\
686 	} while (B_FALSE)
687 
688 #define	EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)			\
689 	do {								\
690 		efx_oword_t oword;					\
691 									\
692 		EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
693 		EFX_SET_OWORD_FIELD(oword,				\
694 		    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);		\
695 		EFX_SET_OWORD_FIELD(oword,				\
696 		    FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0);	\
697 		EFX_SET_OWORD_FIELD(oword,				\
698 		    FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);	\
699 		EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
700 									\
701 		(_rc) = 0;						\
702 									\
703 		_NOTE(CONSTANTCONDITION)				\
704 	} while (B_FALSE)
705 
706 
707 #if EFSYS_OPT_RX_SCALE
708 
709 static	__checkReturn	efx_rc_t
siena_rx_scale_mode_set(__in efx_nic_t * enp,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)710 siena_rx_scale_mode_set(
711 	__in		efx_nic_t *enp,
712 	__in		efx_rx_hash_alg_t alg,
713 	__in		efx_rx_hash_type_t type,
714 	__in		boolean_t insert)
715 {
716 	efx_rc_t rc;
717 
718 	switch (alg) {
719 	case EFX_RX_HASHALG_LFSR:
720 		EFX_RX_LFSR_HASH(enp, insert);
721 		break;
722 
723 	case EFX_RX_HASHALG_TOEPLITZ:
724 		EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
725 		    type & (1 << EFX_RX_HASH_IPV4),
726 		    type & (1 << EFX_RX_HASH_TCPIPV4));
727 
728 		EFX_RX_TOEPLITZ_IPV6_HASH(enp,
729 		    type & (1 << EFX_RX_HASH_IPV6),
730 		    type & (1 << EFX_RX_HASH_TCPIPV6),
731 		    rc);
732 		if (rc != 0)
733 			goto fail1;
734 
735 		break;
736 
737 	default:
738 		rc = EINVAL;
739 		goto fail2;
740 	}
741 
742 	return (0);
743 
744 fail2:
745 	EFSYS_PROBE(fail2);
746 fail1:
747 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
748 
749 	EFX_RX_LFSR_HASH(enp, B_FALSE);
750 
751 	return (rc);
752 }
753 #endif
754 
755 #if EFSYS_OPT_RX_SCALE
756 static	__checkReturn	efx_rc_t
siena_rx_scale_key_set(__in efx_nic_t * enp,__in_ecount (n)uint8_t * key,__in size_t n)757 siena_rx_scale_key_set(
758 	__in		efx_nic_t *enp,
759 	__in_ecount(n)	uint8_t *key,
760 	__in		size_t n)
761 {
762 	efx_oword_t oword;
763 	unsigned int byte;
764 	unsigned int offset;
765 	efx_rc_t rc;
766 
767 	byte = 0;
768 
769 	/* Write Toeplitz IPv4 hash key */
770 	EFX_ZERO_OWORD(oword);
771 	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
772 	    offset > 0 && byte < n;
773 	    --offset)
774 		oword.eo_u8[offset - 1] = key[byte++];
775 
776 	EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
777 
778 	byte = 0;
779 
780 	/* Verify Toeplitz IPv4 hash key */
781 	EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
782 	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
783 	    offset > 0 && byte < n;
784 	    --offset) {
785 		if (oword.eo_u8[offset - 1] != key[byte++]) {
786 			rc = EFAULT;
787 			goto fail1;
788 		}
789 	}
790 
791 	if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
792 		goto done;
793 
794 	byte = 0;
795 
796 	/* Write Toeplitz IPv6 hash key 3 */
797 	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
798 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
799 	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
800 	    offset > 0 && byte < n;
801 	    --offset)
802 		oword.eo_u8[offset - 1] = key[byte++];
803 
804 	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
805 
806 	/* Write Toeplitz IPv6 hash key 2 */
807 	EFX_ZERO_OWORD(oword);
808 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
809 	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
810 	    offset > 0 && byte < n;
811 	    --offset)
812 		oword.eo_u8[offset - 1] = key[byte++];
813 
814 	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
815 
816 	/* Write Toeplitz IPv6 hash key 1 */
817 	EFX_ZERO_OWORD(oword);
818 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
819 	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
820 	    offset > 0 && byte < n;
821 	    --offset)
822 		oword.eo_u8[offset - 1] = key[byte++];
823 
824 	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
825 
826 	byte = 0;
827 
828 	/* Verify Toeplitz IPv6 hash key 3 */
829 	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
830 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
831 	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
832 	    offset > 0 && byte < n;
833 	    --offset) {
834 		if (oword.eo_u8[offset - 1] != key[byte++]) {
835 			rc = EFAULT;
836 			goto fail2;
837 		}
838 	}
839 
840 	/* Verify Toeplitz IPv6 hash key 2 */
841 	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
842 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
843 	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
844 	    offset > 0 && byte < n;
845 	    --offset) {
846 		if (oword.eo_u8[offset - 1] != key[byte++]) {
847 			rc = EFAULT;
848 			goto fail3;
849 		}
850 	}
851 
852 	/* Verify Toeplitz IPv6 hash key 1 */
853 	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
854 	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
855 	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
856 	    offset > 0 && byte < n;
857 	    --offset) {
858 		if (oword.eo_u8[offset - 1] != key[byte++]) {
859 			rc = EFAULT;
860 			goto fail4;
861 		}
862 	}
863 
864 done:
865 	return (0);
866 
867 fail4:
868 	EFSYS_PROBE(fail4);
869 fail3:
870 	EFSYS_PROBE(fail3);
871 fail2:
872 	EFSYS_PROBE(fail2);
873 fail1:
874 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
875 
876 	return (rc);
877 }
878 #endif
879 
880 #if EFSYS_OPT_RX_SCALE
881 static	__checkReturn	efx_rc_t
siena_rx_scale_tbl_set(__in efx_nic_t * enp,__in_ecount (n)unsigned int * table,__in size_t n)882 siena_rx_scale_tbl_set(
883 	__in		efx_nic_t *enp,
884 	__in_ecount(n)	unsigned int *table,
885 	__in		size_t n)
886 {
887 	efx_oword_t oword;
888 	int index;
889 	efx_rc_t rc;
890 
891 	EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
892 	EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
893 
894 	if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
895 		rc = EINVAL;
896 		goto fail1;
897 	}
898 
899 	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
900 		uint32_t byte;
901 
902 		/* Calculate the entry to place in the table */
903 		byte = (n > 0) ? (uint32_t)table[index % n] : 0;
904 
905 		EFSYS_PROBE2(table, int, index, uint32_t, byte);
906 
907 		EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
908 
909 		/* Write the table */
910 		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
911 				    index, &oword, B_TRUE);
912 	}
913 
914 	for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
915 		uint32_t byte;
916 
917 		/* Determine if we're starting a new batch */
918 		byte = (n > 0) ? (uint32_t)table[index % n] : 0;
919 
920 		/* Read the table */
921 		EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
922 				    index, &oword, B_TRUE);
923 
924 		/* Verify the entry */
925 		if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
926 			rc = EFAULT;
927 			goto fail2;
928 		}
929 	}
930 
931 	return (0);
932 
933 fail2:
934 	EFSYS_PROBE(fail2);
935 fail1:
936 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
937 
938 	return (rc);
939 }
940 #endif
941 
942 /*
943  * Falcon/Siena psuedo-header
944  * --------------------------
945  *
946  * Receive packets are prefixed by an optional 16 byte pseudo-header.
947  * The psuedo-header is a byte array of one of the forms:
948  *
949  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
950  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
951  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
952  *
953  * where:
954  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
955  *   LL.LL         LFSR hash     (16-bit big-endian)
956  */
957 
958 #if EFSYS_OPT_RX_SCALE
959 static	__checkReturn	uint32_t
siena_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)960 siena_rx_prefix_hash(
961 	__in		efx_nic_t *enp,
962 	__in		efx_rx_hash_alg_t func,
963 	__in		uint8_t *buffer)
964 {
965 	_NOTE(ARGUNUSED(enp));
966 
967 	switch (func) {
968 	case EFX_RX_HASHALG_TOEPLITZ:
969 		return ((buffer[12] << 24) |
970 		    (buffer[13] << 16) |
971 		    (buffer[14] <<  8) |
972 		    buffer[15]);
973 
974 	case EFX_RX_HASHALG_LFSR:
975 		return ((buffer[14] << 8) | buffer[15]);
976 
977 	default:
978 		EFSYS_ASSERT(0);
979 		return (0);
980 	}
981 }
982 #endif /* EFSYS_OPT_RX_SCALE */
983 
984 static	__checkReturn	efx_rc_t
siena_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)985 siena_rx_prefix_pktlen(
986 	__in		efx_nic_t *enp,
987 	__in		uint8_t *buffer,
988 	__out		uint16_t *lengthp)
989 {
990 	_NOTE(ARGUNUSED(enp));
991 	_NOTE(ARGUNUSED(buffer));
992 	_NOTE(ARGUNUSED(lengthp));
993 	/* Not supported by Falcon/Siena hardware */
994 	EFSYS_ASSERT(0);
995 	return (ENOTSUP);
996 }
997 
998 
999 static			void
siena_rx_qpost(__in efx_rxq_t * erp,__in_ecount (n)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int n,__in unsigned int completed,__in unsigned int added)1000 siena_rx_qpost(
1001 	__in		efx_rxq_t *erp,
1002 	__in_ecount(n)	efsys_dma_addr_t *addrp,
1003 	__in		size_t size,
1004 	__in		unsigned int n,
1005 	__in		unsigned int completed,
1006 	__in		unsigned int added)
1007 {
1008 	efx_qword_t qword;
1009 	unsigned int i;
1010 	unsigned int offset;
1011 	unsigned int id;
1012 
1013 	/* The client driver must not overfill the queue */
1014 	EFSYS_ASSERT3U(added - completed + n, <=,
1015 	    EFX_RXQ_LIMIT(erp->er_mask + 1));
1016 
1017 	id = added & (erp->er_mask);
1018 	for (i = 0; i < n; i++) {
1019 		EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1020 		    unsigned int, id, efsys_dma_addr_t, addrp[i],
1021 		    size_t, size);
1022 
1023 		EFX_POPULATE_QWORD_3(qword,
1024 		    FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1025 		    FSF_AZ_RX_KER_BUF_ADDR_DW0,
1026 		    (uint32_t)(addrp[i] & 0xffffffff),
1027 		    FSF_AZ_RX_KER_BUF_ADDR_DW1,
1028 		    (uint32_t)(addrp[i] >> 32));
1029 
1030 		offset = id * sizeof (efx_qword_t);
1031 		EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1032 
1033 		id = (id + 1) & (erp->er_mask);
1034 	}
1035 }
1036 
1037 static			void
siena_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)1038 siena_rx_qpush(
1039 	__in	efx_rxq_t *erp,
1040 	__in	unsigned int added,
1041 	__inout	unsigned int *pushedp)
1042 {
1043 	efx_nic_t *enp = erp->er_enp;
1044 	unsigned int pushed = *pushedp;
1045 	uint32_t wptr;
1046 	efx_oword_t oword;
1047 	efx_dword_t dword;
1048 
1049 	/* All descriptors are pushed */
1050 	*pushedp = added;
1051 
1052 	/* Push the populated descriptors out */
1053 	wptr = added & erp->er_mask;
1054 
1055 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1056 
1057 	/* Only write the third DWORD */
1058 	EFX_POPULATE_DWORD_1(dword,
1059 	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1060 
1061 	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1062 	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1063 	    wptr, pushed & erp->er_mask);
1064 	EFSYS_PIO_WRITE_BARRIER();
1065 	EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1066 			    erp->er_index, &dword, B_FALSE);
1067 }
1068 
1069 static	__checkReturn	efx_rc_t
siena_rx_qflush(__in efx_rxq_t * erp)1070 siena_rx_qflush(
1071 	__in	efx_rxq_t *erp)
1072 {
1073 	efx_nic_t *enp = erp->er_enp;
1074 	efx_oword_t oword;
1075 	uint32_t label;
1076 
1077 	label = erp->er_index;
1078 
1079 	/* Flush the queue */
1080 	EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1081 	    FRF_AZ_RX_FLUSH_DESCQ, label);
1082 	EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1083 
1084 	return (0);
1085 }
1086 
1087 static		void
siena_rx_qenable(__in efx_rxq_t * erp)1088 siena_rx_qenable(
1089 	__in	efx_rxq_t *erp)
1090 {
1091 	efx_nic_t *enp = erp->er_enp;
1092 	efx_oword_t oword;
1093 
1094 	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1095 
1096 	EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1097 			    erp->er_index, &oword, B_TRUE);
1098 
1099 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1100 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1101 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1102 
1103 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1104 			    erp->er_index, &oword, B_TRUE);
1105 }
1106 
1107 static	__checkReturn	efx_rc_t
siena_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in efx_evq_t * eep,__in efx_rxq_t * erp)1108 siena_rx_qcreate(
1109 	__in		efx_nic_t *enp,
1110 	__in		unsigned int index,
1111 	__in		unsigned int label,
1112 	__in		efx_rxq_type_t type,
1113 	__in		efsys_mem_t *esmp,
1114 	__in		size_t n,
1115 	__in		uint32_t id,
1116 	__in		efx_evq_t *eep,
1117 	__in		efx_rxq_t *erp)
1118 {
1119 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1120 	efx_oword_t oword;
1121 	uint32_t size;
1122 	boolean_t jumbo;
1123 	efx_rc_t rc;
1124 
1125 	_NOTE(ARGUNUSED(esmp));
1126 
1127 	EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1128 	    (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1129 	EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1130 	EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1131 
1132 	EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1133 	EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1134 
1135 	if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1136 		rc = EINVAL;
1137 		goto fail1;
1138 	}
1139 	if (index >= encp->enc_rxq_limit) {
1140 		rc = EINVAL;
1141 		goto fail2;
1142 	}
1143 	for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1144 	    size++)
1145 		if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1146 			break;
1147 	if (id + (1 << size) >= encp->enc_buftbl_limit) {
1148 		rc = EINVAL;
1149 		goto fail3;
1150 	}
1151 
1152 	switch (type) {
1153 	case EFX_RXQ_TYPE_DEFAULT:
1154 		jumbo = B_FALSE;
1155 		break;
1156 
1157 #if EFSYS_OPT_RX_SCATTER
1158 	case EFX_RXQ_TYPE_SCATTER:
1159 		if (enp->en_family < EFX_FAMILY_SIENA) {
1160 			rc = EINVAL;
1161 			goto fail4;
1162 		}
1163 		jumbo = B_TRUE;
1164 		break;
1165 #endif	/* EFSYS_OPT_RX_SCATTER */
1166 
1167 	default:
1168 		rc = EINVAL;
1169 		goto fail4;
1170 	}
1171 
1172 	/* Set up the new descriptor queue */
1173 	EFX_POPULATE_OWORD_7(oword,
1174 	    FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1175 	    FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1176 	    FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1177 	    FRF_AZ_RX_DESCQ_LABEL, label,
1178 	    FRF_AZ_RX_DESCQ_SIZE, size,
1179 	    FRF_AZ_RX_DESCQ_TYPE, 0,
1180 	    FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1181 
1182 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1183 			    erp->er_index, &oword, B_TRUE);
1184 
1185 	return (0);
1186 
1187 fail4:
1188 	EFSYS_PROBE(fail4);
1189 fail3:
1190 	EFSYS_PROBE(fail3);
1191 fail2:
1192 	EFSYS_PROBE(fail2);
1193 fail1:
1194 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1195 
1196 	return (rc);
1197 }
1198 
1199 static		void
siena_rx_qdestroy(__in efx_rxq_t * erp)1200 siena_rx_qdestroy(
1201 	__in	efx_rxq_t *erp)
1202 {
1203 	efx_nic_t *enp = erp->er_enp;
1204 	efx_oword_t oword;
1205 
1206 	EFSYS_ASSERT(enp->en_rx_qcount != 0);
1207 	--enp->en_rx_qcount;
1208 
1209 	/* Purge descriptor queue */
1210 	EFX_ZERO_OWORD(oword);
1211 
1212 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1213 			    erp->er_index, &oword, B_TRUE);
1214 
1215 	/* Free the RXQ object */
1216 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1217 }
1218 
1219 static		void
siena_rx_fini(__in efx_nic_t * enp)1220 siena_rx_fini(
1221 	__in	efx_nic_t *enp)
1222 {
1223 	_NOTE(ARGUNUSED(enp))
1224 }
1225 
1226 #endif /* EFSYS_OPT_SIENA */
1227