1e948693eSPhilip Paeps /*- 2718cf2ccSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3718cf2ccSPedro F. Giffuni * 4929c7febSAndrew Rybchenko * Copyright (c) 2007-2016 Solarflare Communications Inc. 53c838a9fSAndrew Rybchenko * All rights reserved. 6e948693eSPhilip Paeps * 7e948693eSPhilip Paeps * Redistribution and use in source and binary forms, with or without 83c838a9fSAndrew Rybchenko * modification, are permitted provided that the following conditions are met: 9e948693eSPhilip Paeps * 103c838a9fSAndrew Rybchenko * 1. Redistributions of source code must retain the above copyright notice, 113c838a9fSAndrew Rybchenko * this list of conditions and the following disclaimer. 123c838a9fSAndrew Rybchenko * 2. Redistributions in binary form must reproduce the above copyright notice, 133c838a9fSAndrew Rybchenko * this list of conditions and the following disclaimer in the documentation 143c838a9fSAndrew Rybchenko * and/or other materials provided with the distribution. 153c838a9fSAndrew Rybchenko * 163c838a9fSAndrew Rybchenko * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 173c838a9fSAndrew Rybchenko * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 183c838a9fSAndrew Rybchenko * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 193c838a9fSAndrew Rybchenko * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 203c838a9fSAndrew Rybchenko * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 213c838a9fSAndrew Rybchenko * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 223c838a9fSAndrew Rybchenko * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 233c838a9fSAndrew Rybchenko * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 243c838a9fSAndrew Rybchenko * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 253c838a9fSAndrew Rybchenko * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 263c838a9fSAndrew Rybchenko * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 273c838a9fSAndrew Rybchenko * 283c838a9fSAndrew Rybchenko * The views and conclusions contained in the software and documentation are 293c838a9fSAndrew Rybchenko * those of the authors and should not be interpreted as representing official 303c838a9fSAndrew Rybchenko * policies, either expressed or implied, of the FreeBSD Project. 31e948693eSPhilip Paeps */ 32e948693eSPhilip Paeps 335dee87d7SPhilip Paeps #include <sys/cdefs.h> 345dee87d7SPhilip Paeps __FBSDID("$FreeBSD$"); 355dee87d7SPhilip Paeps 36e948693eSPhilip Paeps #include "efx.h" 37e948693eSPhilip Paeps #include "efx_impl.h" 38e948693eSPhilip Paeps 393c838a9fSAndrew Rybchenko 40e75412c9SAndrew Rybchenko #if EFSYS_OPT_SIENA 413c838a9fSAndrew Rybchenko 42460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 43d2df9a4eSAndrew Rybchenko siena_rx_init( 443c838a9fSAndrew Rybchenko __in efx_nic_t *enp); 453c838a9fSAndrew Rybchenko 463c838a9fSAndrew Rybchenko static void 47d2df9a4eSAndrew Rybchenko siena_rx_fini( 483c838a9fSAndrew Rybchenko __in efx_nic_t *enp); 493c838a9fSAndrew Rybchenko 503c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER 51460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 52d2df9a4eSAndrew Rybchenko siena_rx_scatter_enable( 533c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 543c838a9fSAndrew Rybchenko __in unsigned int buf_size); 553c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */ 563c838a9fSAndrew Rybchenko 573c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 58460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 59d2df9a4eSAndrew Rybchenko siena_rx_scale_mode_set( 603c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 61*82af879cSAndrew Rybchenko __in uint32_t rss_context, 623c838a9fSAndrew Rybchenko __in efx_rx_hash_alg_t alg, 633c838a9fSAndrew Rybchenko __in efx_rx_hash_type_t type, 643c838a9fSAndrew Rybchenko __in boolean_t insert); 653c838a9fSAndrew Rybchenko 66460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 67d2df9a4eSAndrew Rybchenko siena_rx_scale_key_set( 683c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 69*82af879cSAndrew Rybchenko __in uint32_t rss_context, 703c838a9fSAndrew Rybchenko __in_ecount(n) uint8_t *key, 713c838a9fSAndrew Rybchenko __in size_t n); 723c838a9fSAndrew Rybchenko 73460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 74d2df9a4eSAndrew Rybchenko siena_rx_scale_tbl_set( 753c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 76*82af879cSAndrew Rybchenko __in uint32_t rss_context, 773c838a9fSAndrew Rybchenko __in_ecount(n) unsigned int *table, 783c838a9fSAndrew Rybchenko __in size_t n); 793c838a9fSAndrew Rybchenko 800badfd72SAndrew Rybchenko static __checkReturn uint32_t 81d2df9a4eSAndrew Rybchenko siena_rx_prefix_hash( 820badfd72SAndrew Rybchenko __in efx_nic_t *enp, 830badfd72SAndrew Rybchenko __in efx_rx_hash_alg_t func, 840badfd72SAndrew Rybchenko __in uint8_t *buffer); 850badfd72SAndrew Rybchenko 860a91bc3fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 870a91bc3fSAndrew Rybchenko 880badfd72SAndrew Rybchenko static __checkReturn efx_rc_t 89d2df9a4eSAndrew Rybchenko siena_rx_prefix_pktlen( 900badfd72SAndrew Rybchenko __in efx_nic_t *enp, 910badfd72SAndrew Rybchenko __in uint8_t *buffer, 920badfd72SAndrew Rybchenko __out uint16_t *lengthp); 930badfd72SAndrew Rybchenko 943c838a9fSAndrew Rybchenko static void 95d2df9a4eSAndrew Rybchenko siena_rx_qpost( 963c838a9fSAndrew Rybchenko __in efx_rxq_t *erp, 973c838a9fSAndrew Rybchenko __in_ecount(n) efsys_dma_addr_t *addrp, 983c838a9fSAndrew Rybchenko __in size_t size, 993c838a9fSAndrew Rybchenko __in unsigned int n, 1003c838a9fSAndrew Rybchenko __in unsigned int completed, 1013c838a9fSAndrew Rybchenko __in unsigned int added); 1023c838a9fSAndrew Rybchenko 1033c838a9fSAndrew Rybchenko static void 104d2df9a4eSAndrew Rybchenko siena_rx_qpush( 1053c838a9fSAndrew Rybchenko __in efx_rxq_t *erp, 1063c838a9fSAndrew Rybchenko __in unsigned int added, 1073c838a9fSAndrew Rybchenko __inout unsigned int *pushedp); 1083c838a9fSAndrew Rybchenko 1098e0c4827SAndrew Rybchenko #if EFSYS_OPT_RX_PACKED_STREAM 1108e0c4827SAndrew Rybchenko static void 1118e0c4827SAndrew Rybchenko siena_rx_qpush_ps_credits( 1128e0c4827SAndrew Rybchenko __in efx_rxq_t *erp); 1138e0c4827SAndrew Rybchenko 1148e0c4827SAndrew Rybchenko static __checkReturn uint8_t * 1158e0c4827SAndrew Rybchenko siena_rx_qps_packet_info( 1168e0c4827SAndrew Rybchenko __in efx_rxq_t *erp, 1178e0c4827SAndrew Rybchenko __in uint8_t *buffer, 1188e0c4827SAndrew Rybchenko __in uint32_t buffer_length, 1198e0c4827SAndrew Rybchenko __in uint32_t current_offset, 1208e0c4827SAndrew Rybchenko __out uint16_t *lengthp, 1218e0c4827SAndrew Rybchenko __out uint32_t *next_offsetp, 1228e0c4827SAndrew Rybchenko __out uint32_t *timestamp); 1238e0c4827SAndrew Rybchenko #endif 1248e0c4827SAndrew Rybchenko 125460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 126d2df9a4eSAndrew Rybchenko siena_rx_qflush( 1273c838a9fSAndrew Rybchenko __in efx_rxq_t *erp); 1283c838a9fSAndrew Rybchenko 1293c838a9fSAndrew Rybchenko static void 130d2df9a4eSAndrew Rybchenko siena_rx_qenable( 1313c838a9fSAndrew Rybchenko __in efx_rxq_t *erp); 1323c838a9fSAndrew Rybchenko 133460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 134d2df9a4eSAndrew Rybchenko siena_rx_qcreate( 1353c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 1363c838a9fSAndrew Rybchenko __in unsigned int index, 1373c838a9fSAndrew Rybchenko __in unsigned int label, 1383c838a9fSAndrew Rybchenko __in efx_rxq_type_t type, 1393c838a9fSAndrew Rybchenko __in efsys_mem_t *esmp, 1403c838a9fSAndrew Rybchenko __in size_t n, 1413c838a9fSAndrew Rybchenko __in uint32_t id, 1423c838a9fSAndrew Rybchenko __in efx_evq_t *eep, 1433c838a9fSAndrew Rybchenko __in efx_rxq_t *erp); 1443c838a9fSAndrew Rybchenko 1453c838a9fSAndrew Rybchenko static void 146d2df9a4eSAndrew Rybchenko siena_rx_qdestroy( 1473c838a9fSAndrew Rybchenko __in efx_rxq_t *erp); 1483c838a9fSAndrew Rybchenko 149e75412c9SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */ 1503c838a9fSAndrew Rybchenko 1513c838a9fSAndrew Rybchenko 1523c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA 153ec831f7fSAndrew Rybchenko static const efx_rx_ops_t __efx_rx_siena_ops = { 154d2df9a4eSAndrew Rybchenko siena_rx_init, /* erxo_init */ 155d2df9a4eSAndrew Rybchenko siena_rx_fini, /* erxo_fini */ 1563c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER 157d2df9a4eSAndrew Rybchenko siena_rx_scatter_enable, /* erxo_scatter_enable */ 1583c838a9fSAndrew Rybchenko #endif 1593c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 160e6d55a0bSAndrew Rybchenko NULL, /* erxo_scale_context_alloc */ 161e6d55a0bSAndrew Rybchenko NULL, /* erxo_scale_context_free */ 162d2df9a4eSAndrew Rybchenko siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 163d2df9a4eSAndrew Rybchenko siena_rx_scale_key_set, /* erxo_scale_key_set */ 164d2df9a4eSAndrew Rybchenko siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165d2df9a4eSAndrew Rybchenko siena_rx_prefix_hash, /* erxo_prefix_hash */ 1663c838a9fSAndrew Rybchenko #endif 167d2df9a4eSAndrew Rybchenko siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168d2df9a4eSAndrew Rybchenko siena_rx_qpost, /* erxo_qpost */ 169d2df9a4eSAndrew Rybchenko siena_rx_qpush, /* erxo_qpush */ 1708e0c4827SAndrew Rybchenko #if EFSYS_OPT_RX_PACKED_STREAM 1718e0c4827SAndrew Rybchenko siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 1728e0c4827SAndrew Rybchenko siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 1738e0c4827SAndrew Rybchenko #endif 174d2df9a4eSAndrew Rybchenko siena_rx_qflush, /* erxo_qflush */ 175d2df9a4eSAndrew Rybchenko siena_rx_qenable, /* erxo_qenable */ 176d2df9a4eSAndrew Rybchenko siena_rx_qcreate, /* erxo_qcreate */ 177d2df9a4eSAndrew Rybchenko siena_rx_qdestroy, /* erxo_qdestroy */ 1783c838a9fSAndrew Rybchenko }; 1793c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */ 1803c838a9fSAndrew Rybchenko 181b627be98SAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 182ec831f7fSAndrew Rybchenko static const efx_rx_ops_t __efx_rx_ef10_ops = { 183b627be98SAndrew Rybchenko ef10_rx_init, /* erxo_init */ 184b627be98SAndrew Rybchenko ef10_rx_fini, /* erxo_fini */ 1853c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER 186b627be98SAndrew Rybchenko ef10_rx_scatter_enable, /* erxo_scatter_enable */ 1873c838a9fSAndrew Rybchenko #endif 1883c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 189e6d55a0bSAndrew Rybchenko ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 190e6d55a0bSAndrew Rybchenko ef10_rx_scale_context_free, /* erxo_scale_context_free */ 191b627be98SAndrew Rybchenko ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 192b627be98SAndrew Rybchenko ef10_rx_scale_key_set, /* erxo_scale_key_set */ 193b627be98SAndrew Rybchenko ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 1940badfd72SAndrew Rybchenko ef10_rx_prefix_hash, /* erxo_prefix_hash */ 1953c838a9fSAndrew Rybchenko #endif 1960badfd72SAndrew Rybchenko ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 197b627be98SAndrew Rybchenko ef10_rx_qpost, /* erxo_qpost */ 198b627be98SAndrew Rybchenko ef10_rx_qpush, /* erxo_qpush */ 1998e0c4827SAndrew Rybchenko #if EFSYS_OPT_RX_PACKED_STREAM 2008e0c4827SAndrew Rybchenko ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 2018e0c4827SAndrew Rybchenko ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 2028e0c4827SAndrew Rybchenko #endif 203b627be98SAndrew Rybchenko ef10_rx_qflush, /* erxo_qflush */ 204b627be98SAndrew Rybchenko ef10_rx_qenable, /* erxo_qenable */ 205b627be98SAndrew Rybchenko ef10_rx_qcreate, /* erxo_qcreate */ 206b627be98SAndrew Rybchenko ef10_rx_qdestroy, /* erxo_qdestroy */ 2073c838a9fSAndrew Rybchenko }; 208b627be98SAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 2093c838a9fSAndrew Rybchenko 2103c838a9fSAndrew Rybchenko 211460cb568SAndrew Rybchenko __checkReturn efx_rc_t 212e948693eSPhilip Paeps efx_rx_init( 2133c838a9fSAndrew Rybchenko __inout efx_nic_t *enp) 214e948693eSPhilip Paeps { 215ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop; 216460cb568SAndrew Rybchenko efx_rc_t rc; 217e948693eSPhilip Paeps 218e948693eSPhilip Paeps EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 219e948693eSPhilip Paeps EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 220e948693eSPhilip Paeps 221e948693eSPhilip Paeps if (!(enp->en_mod_flags & EFX_MOD_EV)) { 222e948693eSPhilip Paeps rc = EINVAL; 223e948693eSPhilip Paeps goto fail1; 224e948693eSPhilip Paeps } 225e948693eSPhilip Paeps 226e948693eSPhilip Paeps if (enp->en_mod_flags & EFX_MOD_RX) { 227e948693eSPhilip Paeps rc = EINVAL; 228e948693eSPhilip Paeps goto fail2; 229e948693eSPhilip Paeps } 230e948693eSPhilip Paeps 2313c838a9fSAndrew Rybchenko switch (enp->en_family) { 2323c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA 2333c838a9fSAndrew Rybchenko case EFX_FAMILY_SIENA: 234ec831f7fSAndrew Rybchenko erxop = &__efx_rx_siena_ops; 2353c838a9fSAndrew Rybchenko break; 2363c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */ 2373c838a9fSAndrew Rybchenko 2383c838a9fSAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON 2393c838a9fSAndrew Rybchenko case EFX_FAMILY_HUNTINGTON: 240ec831f7fSAndrew Rybchenko erxop = &__efx_rx_ef10_ops; 2413c838a9fSAndrew Rybchenko break; 2423c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */ 2433c838a9fSAndrew Rybchenko 244b627be98SAndrew Rybchenko #if EFSYS_OPT_MEDFORD 245b627be98SAndrew Rybchenko case EFX_FAMILY_MEDFORD: 246ec831f7fSAndrew Rybchenko erxop = &__efx_rx_ef10_ops; 247b627be98SAndrew Rybchenko break; 248b627be98SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD */ 249b627be98SAndrew Rybchenko 2503c838a9fSAndrew Rybchenko default: 2513c838a9fSAndrew Rybchenko EFSYS_ASSERT(0); 2523c838a9fSAndrew Rybchenko rc = ENOTSUP; 2533c838a9fSAndrew Rybchenko goto fail3; 2543c838a9fSAndrew Rybchenko } 2553c838a9fSAndrew Rybchenko 2563c838a9fSAndrew Rybchenko if ((rc = erxop->erxo_init(enp)) != 0) 2573c838a9fSAndrew Rybchenko goto fail4; 2583c838a9fSAndrew Rybchenko 2593c838a9fSAndrew Rybchenko enp->en_erxop = erxop; 2603c838a9fSAndrew Rybchenko enp->en_mod_flags |= EFX_MOD_RX; 2613c838a9fSAndrew Rybchenko return (0); 2623c838a9fSAndrew Rybchenko 2633c838a9fSAndrew Rybchenko fail4: 2643c838a9fSAndrew Rybchenko EFSYS_PROBE(fail4); 2653c838a9fSAndrew Rybchenko fail3: 2663c838a9fSAndrew Rybchenko EFSYS_PROBE(fail3); 2673c838a9fSAndrew Rybchenko fail2: 2683c838a9fSAndrew Rybchenko EFSYS_PROBE(fail2); 2693c838a9fSAndrew Rybchenko fail1: 270460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 2713c838a9fSAndrew Rybchenko 2723c838a9fSAndrew Rybchenko enp->en_erxop = NULL; 2733c838a9fSAndrew Rybchenko enp->en_mod_flags &= ~EFX_MOD_RX; 2743c838a9fSAndrew Rybchenko return (rc); 2753c838a9fSAndrew Rybchenko } 2763c838a9fSAndrew Rybchenko 2773c838a9fSAndrew Rybchenko void 2783c838a9fSAndrew Rybchenko efx_rx_fini( 2793c838a9fSAndrew Rybchenko __in efx_nic_t *enp) 2803c838a9fSAndrew Rybchenko { 281ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 2823c838a9fSAndrew Rybchenko 2833c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 2843c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 2853c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 2863c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 2873c838a9fSAndrew Rybchenko 2883c838a9fSAndrew Rybchenko erxop->erxo_fini(enp); 2893c838a9fSAndrew Rybchenko 2903c838a9fSAndrew Rybchenko enp->en_erxop = NULL; 2913c838a9fSAndrew Rybchenko enp->en_mod_flags &= ~EFX_MOD_RX; 2923c838a9fSAndrew Rybchenko } 2933c838a9fSAndrew Rybchenko 2943c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER 295460cb568SAndrew Rybchenko __checkReturn efx_rc_t 2963c838a9fSAndrew Rybchenko efx_rx_scatter_enable( 2973c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 2983c838a9fSAndrew Rybchenko __in unsigned int buf_size) 2993c838a9fSAndrew Rybchenko { 300ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 301460cb568SAndrew Rybchenko efx_rc_t rc; 3023c838a9fSAndrew Rybchenko 3033c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 3043c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 3053c838a9fSAndrew Rybchenko 3063c838a9fSAndrew Rybchenko if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 3073c838a9fSAndrew Rybchenko goto fail1; 3083c838a9fSAndrew Rybchenko 3093c838a9fSAndrew Rybchenko return (0); 3103c838a9fSAndrew Rybchenko 3113c838a9fSAndrew Rybchenko fail1: 312460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 3133c838a9fSAndrew Rybchenko return (rc); 3143c838a9fSAndrew Rybchenko } 3153c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */ 3163c838a9fSAndrew Rybchenko 3173c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 318460cb568SAndrew Rybchenko __checkReturn efx_rc_t 31939023729SAndrew Rybchenko efx_rx_hash_default_support_get( 3203c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 3213c838a9fSAndrew Rybchenko __out efx_rx_hash_support_t *supportp) 3223c838a9fSAndrew Rybchenko { 323460cb568SAndrew Rybchenko efx_rc_t rc; 3243c838a9fSAndrew Rybchenko 3253c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 3263c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 3273c838a9fSAndrew Rybchenko 3283c838a9fSAndrew Rybchenko if (supportp == NULL) { 3293c838a9fSAndrew Rybchenko rc = EINVAL; 3303c838a9fSAndrew Rybchenko goto fail1; 3313c838a9fSAndrew Rybchenko } 3323c838a9fSAndrew Rybchenko 33339023729SAndrew Rybchenko /* 33439023729SAndrew Rybchenko * Report the hashing support the client gets by default if it 33539023729SAndrew Rybchenko * does not allocate an RSS context itself. 33639023729SAndrew Rybchenko */ 3373c838a9fSAndrew Rybchenko *supportp = enp->en_hash_support; 3383c838a9fSAndrew Rybchenko 3393c838a9fSAndrew Rybchenko return (0); 3403c838a9fSAndrew Rybchenko 3413c838a9fSAndrew Rybchenko fail1: 342460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 3433c838a9fSAndrew Rybchenko 3443c838a9fSAndrew Rybchenko return (rc); 3453c838a9fSAndrew Rybchenko } 3463c838a9fSAndrew Rybchenko 347460cb568SAndrew Rybchenko __checkReturn efx_rc_t 34839023729SAndrew Rybchenko efx_rx_scale_default_support_get( 3493c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 35039023729SAndrew Rybchenko __out efx_rx_scale_context_type_t *typep) 3513c838a9fSAndrew Rybchenko { 352460cb568SAndrew Rybchenko efx_rc_t rc; 3533c838a9fSAndrew Rybchenko 3543c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 3553c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 3563c838a9fSAndrew Rybchenko 35739023729SAndrew Rybchenko if (typep == NULL) { 3583c838a9fSAndrew Rybchenko rc = EINVAL; 3593c838a9fSAndrew Rybchenko goto fail1; 3603c838a9fSAndrew Rybchenko } 3613c838a9fSAndrew Rybchenko 36239023729SAndrew Rybchenko /* 36339023729SAndrew Rybchenko * Report the RSS support the client gets by default if it 36439023729SAndrew Rybchenko * does not allocate an RSS context itself. 36539023729SAndrew Rybchenko */ 36639023729SAndrew Rybchenko *typep = enp->en_rss_context_type; 3673c838a9fSAndrew Rybchenko 3683c838a9fSAndrew Rybchenko return (0); 3693c838a9fSAndrew Rybchenko 3703c838a9fSAndrew Rybchenko fail1: 371460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 3723c838a9fSAndrew Rybchenko 3733c838a9fSAndrew Rybchenko return (rc); 3743c838a9fSAndrew Rybchenko } 375e6d55a0bSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 3763c838a9fSAndrew Rybchenko 377e6d55a0bSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 378e6d55a0bSAndrew Rybchenko __checkReturn efx_rc_t 379e6d55a0bSAndrew Rybchenko efx_rx_scale_context_alloc( 380e6d55a0bSAndrew Rybchenko __in efx_nic_t *enp, 381e6d55a0bSAndrew Rybchenko __in efx_rx_scale_context_type_t type, 382e6d55a0bSAndrew Rybchenko __in uint32_t num_queues, 383e6d55a0bSAndrew Rybchenko __out uint32_t *rss_contextp) 384e6d55a0bSAndrew Rybchenko { 385e6d55a0bSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 386e6d55a0bSAndrew Rybchenko efx_rc_t rc; 387e6d55a0bSAndrew Rybchenko 388e6d55a0bSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 389e6d55a0bSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 390e6d55a0bSAndrew Rybchenko 391e6d55a0bSAndrew Rybchenko if (erxop->erxo_scale_context_alloc == NULL) { 392e6d55a0bSAndrew Rybchenko rc = ENOTSUP; 393e6d55a0bSAndrew Rybchenko goto fail1; 394e6d55a0bSAndrew Rybchenko } 395e6d55a0bSAndrew Rybchenko if ((rc = erxop->erxo_scale_context_alloc(enp, type, 396e6d55a0bSAndrew Rybchenko num_queues, rss_contextp)) != 0) { 397e6d55a0bSAndrew Rybchenko goto fail2; 398e6d55a0bSAndrew Rybchenko } 399e6d55a0bSAndrew Rybchenko 400e6d55a0bSAndrew Rybchenko return (0); 401e6d55a0bSAndrew Rybchenko 402e6d55a0bSAndrew Rybchenko fail2: 403e6d55a0bSAndrew Rybchenko EFSYS_PROBE(fail2); 404e6d55a0bSAndrew Rybchenko fail1: 405e6d55a0bSAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 406e6d55a0bSAndrew Rybchenko return (rc); 407e6d55a0bSAndrew Rybchenko } 408e6d55a0bSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 409e6d55a0bSAndrew Rybchenko 410e6d55a0bSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 411e6d55a0bSAndrew Rybchenko __checkReturn efx_rc_t 412e6d55a0bSAndrew Rybchenko efx_rx_scale_context_free( 413e6d55a0bSAndrew Rybchenko __in efx_nic_t *enp, 414e6d55a0bSAndrew Rybchenko __in uint32_t rss_context) 415e6d55a0bSAndrew Rybchenko { 416e6d55a0bSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 417e6d55a0bSAndrew Rybchenko efx_rc_t rc; 418e6d55a0bSAndrew Rybchenko 419e6d55a0bSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 420e6d55a0bSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 421e6d55a0bSAndrew Rybchenko 422e6d55a0bSAndrew Rybchenko if (erxop->erxo_scale_context_free == NULL) { 423e6d55a0bSAndrew Rybchenko rc = ENOTSUP; 424e6d55a0bSAndrew Rybchenko goto fail1; 425e6d55a0bSAndrew Rybchenko } 426e6d55a0bSAndrew Rybchenko if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 427e6d55a0bSAndrew Rybchenko goto fail2; 428e6d55a0bSAndrew Rybchenko 429e6d55a0bSAndrew Rybchenko return (0); 430e6d55a0bSAndrew Rybchenko 431e6d55a0bSAndrew Rybchenko fail2: 432e6d55a0bSAndrew Rybchenko EFSYS_PROBE(fail2); 433e6d55a0bSAndrew Rybchenko fail1: 434e6d55a0bSAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 435e6d55a0bSAndrew Rybchenko return (rc); 436e6d55a0bSAndrew Rybchenko } 437e6d55a0bSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 438e6d55a0bSAndrew Rybchenko 439e6d55a0bSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 440460cb568SAndrew Rybchenko __checkReturn efx_rc_t 4413c838a9fSAndrew Rybchenko efx_rx_scale_mode_set( 4423c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 443*82af879cSAndrew Rybchenko __in uint32_t rss_context, 4443c838a9fSAndrew Rybchenko __in efx_rx_hash_alg_t alg, 4453c838a9fSAndrew Rybchenko __in efx_rx_hash_type_t type, 4463c838a9fSAndrew Rybchenko __in boolean_t insert) 4473c838a9fSAndrew Rybchenko { 448ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 449460cb568SAndrew Rybchenko efx_rc_t rc; 4503c838a9fSAndrew Rybchenko 4513c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 4523c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 4533c838a9fSAndrew Rybchenko 4543c838a9fSAndrew Rybchenko if (erxop->erxo_scale_mode_set != NULL) { 455*82af879cSAndrew Rybchenko if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 4563c838a9fSAndrew Rybchenko type, insert)) != 0) 4573c838a9fSAndrew Rybchenko goto fail1; 4583c838a9fSAndrew Rybchenko } 4593c838a9fSAndrew Rybchenko 4603c838a9fSAndrew Rybchenko return (0); 4613c838a9fSAndrew Rybchenko 4623c838a9fSAndrew Rybchenko fail1: 463460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 4643c838a9fSAndrew Rybchenko return (rc); 4653c838a9fSAndrew Rybchenko } 4663c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 4673c838a9fSAndrew Rybchenko 4683c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 469460cb568SAndrew Rybchenko __checkReturn efx_rc_t 4703c838a9fSAndrew Rybchenko efx_rx_scale_key_set( 4713c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 472*82af879cSAndrew Rybchenko __in uint32_t rss_context, 4733c838a9fSAndrew Rybchenko __in_ecount(n) uint8_t *key, 4743c838a9fSAndrew Rybchenko __in size_t n) 4753c838a9fSAndrew Rybchenko { 476ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 477460cb568SAndrew Rybchenko efx_rc_t rc; 4783c838a9fSAndrew Rybchenko 4793c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 4803c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 4813c838a9fSAndrew Rybchenko 482*82af879cSAndrew Rybchenko if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 4833c838a9fSAndrew Rybchenko goto fail1; 4843c838a9fSAndrew Rybchenko 4853c838a9fSAndrew Rybchenko return (0); 4863c838a9fSAndrew Rybchenko 4873c838a9fSAndrew Rybchenko fail1: 488460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 4893c838a9fSAndrew Rybchenko 4903c838a9fSAndrew Rybchenko return (rc); 4913c838a9fSAndrew Rybchenko } 4923c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 4933c838a9fSAndrew Rybchenko 4943c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 495460cb568SAndrew Rybchenko __checkReturn efx_rc_t 4963c838a9fSAndrew Rybchenko efx_rx_scale_tbl_set( 4973c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 498*82af879cSAndrew Rybchenko __in uint32_t rss_context, 4993c838a9fSAndrew Rybchenko __in_ecount(n) unsigned int *table, 5003c838a9fSAndrew Rybchenko __in size_t n) 5013c838a9fSAndrew Rybchenko { 502ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 503460cb568SAndrew Rybchenko efx_rc_t rc; 5043c838a9fSAndrew Rybchenko 5053c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 5063c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 5073c838a9fSAndrew Rybchenko 508*82af879cSAndrew Rybchenko if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 5093c838a9fSAndrew Rybchenko goto fail1; 5103c838a9fSAndrew Rybchenko 5113c838a9fSAndrew Rybchenko return (0); 5123c838a9fSAndrew Rybchenko 5133c838a9fSAndrew Rybchenko fail1: 514460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 5153c838a9fSAndrew Rybchenko 5163c838a9fSAndrew Rybchenko return (rc); 5173c838a9fSAndrew Rybchenko } 5183c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 5193c838a9fSAndrew Rybchenko 5203c838a9fSAndrew Rybchenko void 5213c838a9fSAndrew Rybchenko efx_rx_qpost( 5223c838a9fSAndrew Rybchenko __in efx_rxq_t *erp, 5233c838a9fSAndrew Rybchenko __in_ecount(n) efsys_dma_addr_t *addrp, 5243c838a9fSAndrew Rybchenko __in size_t size, 5253c838a9fSAndrew Rybchenko __in unsigned int n, 5263c838a9fSAndrew Rybchenko __in unsigned int completed, 5273c838a9fSAndrew Rybchenko __in unsigned int added) 5283c838a9fSAndrew Rybchenko { 5293c838a9fSAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 530ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 5313c838a9fSAndrew Rybchenko 5323c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 5333c838a9fSAndrew Rybchenko 5343c838a9fSAndrew Rybchenko erxop->erxo_qpost(erp, addrp, size, n, completed, added); 5353c838a9fSAndrew Rybchenko } 5363c838a9fSAndrew Rybchenko 5378e0c4827SAndrew Rybchenko #if EFSYS_OPT_RX_PACKED_STREAM 5388e0c4827SAndrew Rybchenko 5398e0c4827SAndrew Rybchenko void 5408e0c4827SAndrew Rybchenko efx_rx_qpush_ps_credits( 5418e0c4827SAndrew Rybchenko __in efx_rxq_t *erp) 5428e0c4827SAndrew Rybchenko { 5438e0c4827SAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 5448e0c4827SAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 5458e0c4827SAndrew Rybchenko 5468e0c4827SAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 5478e0c4827SAndrew Rybchenko 5488e0c4827SAndrew Rybchenko erxop->erxo_qpush_ps_credits(erp); 5498e0c4827SAndrew Rybchenko } 5508e0c4827SAndrew Rybchenko 5518e0c4827SAndrew Rybchenko __checkReturn uint8_t * 5528e0c4827SAndrew Rybchenko efx_rx_qps_packet_info( 5538e0c4827SAndrew Rybchenko __in efx_rxq_t *erp, 5548e0c4827SAndrew Rybchenko __in uint8_t *buffer, 5558e0c4827SAndrew Rybchenko __in uint32_t buffer_length, 5568e0c4827SAndrew Rybchenko __in uint32_t current_offset, 5578e0c4827SAndrew Rybchenko __out uint16_t *lengthp, 5588e0c4827SAndrew Rybchenko __out uint32_t *next_offsetp, 5598e0c4827SAndrew Rybchenko __out uint32_t *timestamp) 5608e0c4827SAndrew Rybchenko { 5618e0c4827SAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 5628e0c4827SAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 5638e0c4827SAndrew Rybchenko 5648e0c4827SAndrew Rybchenko return (erxop->erxo_qps_packet_info(erp, buffer, 5658e0c4827SAndrew Rybchenko buffer_length, current_offset, lengthp, 5668e0c4827SAndrew Rybchenko next_offsetp, timestamp)); 5678e0c4827SAndrew Rybchenko } 5688e0c4827SAndrew Rybchenko 5698e0c4827SAndrew Rybchenko #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 5708e0c4827SAndrew Rybchenko 5713c838a9fSAndrew Rybchenko void 5723c838a9fSAndrew Rybchenko efx_rx_qpush( 5733c838a9fSAndrew Rybchenko __in efx_rxq_t *erp, 5743c838a9fSAndrew Rybchenko __in unsigned int added, 5753c838a9fSAndrew Rybchenko __inout unsigned int *pushedp) 5763c838a9fSAndrew Rybchenko { 5773c838a9fSAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 578ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 5793c838a9fSAndrew Rybchenko 5803c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 5813c838a9fSAndrew Rybchenko 5823c838a9fSAndrew Rybchenko erxop->erxo_qpush(erp, added, pushedp); 5833c838a9fSAndrew Rybchenko } 5843c838a9fSAndrew Rybchenko 585460cb568SAndrew Rybchenko __checkReturn efx_rc_t 5863c838a9fSAndrew Rybchenko efx_rx_qflush( 5873c838a9fSAndrew Rybchenko __in efx_rxq_t *erp) 5883c838a9fSAndrew Rybchenko { 5893c838a9fSAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 590ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 591460cb568SAndrew Rybchenko efx_rc_t rc; 5923c838a9fSAndrew Rybchenko 5933c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 5943c838a9fSAndrew Rybchenko 5953c838a9fSAndrew Rybchenko if ((rc = erxop->erxo_qflush(erp)) != 0) 5963c838a9fSAndrew Rybchenko goto fail1; 5973c838a9fSAndrew Rybchenko 5983c838a9fSAndrew Rybchenko return (0); 5993c838a9fSAndrew Rybchenko 6003c838a9fSAndrew Rybchenko fail1: 601460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 6023c838a9fSAndrew Rybchenko 6033c838a9fSAndrew Rybchenko return (rc); 6043c838a9fSAndrew Rybchenko } 6053c838a9fSAndrew Rybchenko 6063c838a9fSAndrew Rybchenko void 6073c838a9fSAndrew Rybchenko efx_rx_qenable( 6083c838a9fSAndrew Rybchenko __in efx_rxq_t *erp) 6093c838a9fSAndrew Rybchenko { 6103c838a9fSAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 611ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 6123c838a9fSAndrew Rybchenko 6133c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 6143c838a9fSAndrew Rybchenko 6153c838a9fSAndrew Rybchenko erxop->erxo_qenable(erp); 6163c838a9fSAndrew Rybchenko } 6173c838a9fSAndrew Rybchenko 618460cb568SAndrew Rybchenko __checkReturn efx_rc_t 6193c838a9fSAndrew Rybchenko efx_rx_qcreate( 6203c838a9fSAndrew Rybchenko __in efx_nic_t *enp, 6213c838a9fSAndrew Rybchenko __in unsigned int index, 6223c838a9fSAndrew Rybchenko __in unsigned int label, 6233c838a9fSAndrew Rybchenko __in efx_rxq_type_t type, 6243c838a9fSAndrew Rybchenko __in efsys_mem_t *esmp, 6253c838a9fSAndrew Rybchenko __in size_t n, 6263c838a9fSAndrew Rybchenko __in uint32_t id, 6273c838a9fSAndrew Rybchenko __in efx_evq_t *eep, 6283c838a9fSAndrew Rybchenko __deref_out efx_rxq_t **erpp) 6293c838a9fSAndrew Rybchenko { 630ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 6313c838a9fSAndrew Rybchenko efx_rxq_t *erp; 632460cb568SAndrew Rybchenko efx_rc_t rc; 6333c838a9fSAndrew Rybchenko 6343c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 6353c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 6363c838a9fSAndrew Rybchenko 6373c838a9fSAndrew Rybchenko /* Allocate an RXQ object */ 6383c838a9fSAndrew Rybchenko EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 6393c838a9fSAndrew Rybchenko 6403c838a9fSAndrew Rybchenko if (erp == NULL) { 6413c838a9fSAndrew Rybchenko rc = ENOMEM; 6423c838a9fSAndrew Rybchenko goto fail1; 6433c838a9fSAndrew Rybchenko } 6443c838a9fSAndrew Rybchenko 6453c838a9fSAndrew Rybchenko erp->er_magic = EFX_RXQ_MAGIC; 6463c838a9fSAndrew Rybchenko erp->er_enp = enp; 6473c838a9fSAndrew Rybchenko erp->er_index = index; 6483c838a9fSAndrew Rybchenko erp->er_mask = n - 1; 6493c838a9fSAndrew Rybchenko erp->er_esmp = esmp; 6503c838a9fSAndrew Rybchenko 6513c838a9fSAndrew Rybchenko if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, 6523c838a9fSAndrew Rybchenko eep, erp)) != 0) 6533c838a9fSAndrew Rybchenko goto fail2; 6543c838a9fSAndrew Rybchenko 6553c838a9fSAndrew Rybchenko enp->en_rx_qcount++; 6563c838a9fSAndrew Rybchenko *erpp = erp; 6573c838a9fSAndrew Rybchenko 6583c838a9fSAndrew Rybchenko return (0); 6593c838a9fSAndrew Rybchenko 6603c838a9fSAndrew Rybchenko fail2: 6613c838a9fSAndrew Rybchenko EFSYS_PROBE(fail2); 6623c838a9fSAndrew Rybchenko 6633c838a9fSAndrew Rybchenko EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 6643c838a9fSAndrew Rybchenko fail1: 665460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 6663c838a9fSAndrew Rybchenko 6673c838a9fSAndrew Rybchenko return (rc); 6683c838a9fSAndrew Rybchenko } 6693c838a9fSAndrew Rybchenko 6703c838a9fSAndrew Rybchenko void 6713c838a9fSAndrew Rybchenko efx_rx_qdestroy( 6723c838a9fSAndrew Rybchenko __in efx_rxq_t *erp) 6733c838a9fSAndrew Rybchenko { 6743c838a9fSAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 675ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 6763c838a9fSAndrew Rybchenko 6773c838a9fSAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 6783c838a9fSAndrew Rybchenko 6793c838a9fSAndrew Rybchenko erxop->erxo_qdestroy(erp); 6803c838a9fSAndrew Rybchenko } 6813c838a9fSAndrew Rybchenko 682460cb568SAndrew Rybchenko __checkReturn efx_rc_t 68354c1459cSAndrew Rybchenko efx_pseudo_hdr_pkt_length_get( 684dae57086SAndrew Rybchenko __in efx_rxq_t *erp, 6853c838a9fSAndrew Rybchenko __in uint8_t *buffer, 6860badfd72SAndrew Rybchenko __out uint16_t *lengthp) 6873c838a9fSAndrew Rybchenko { 688dae57086SAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 689ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 6903c838a9fSAndrew Rybchenko 691dae57086SAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 692dae57086SAndrew Rybchenko 6930badfd72SAndrew Rybchenko return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 6943c838a9fSAndrew Rybchenko } 6953c838a9fSAndrew Rybchenko 6963c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 6970badfd72SAndrew Rybchenko __checkReturn uint32_t 69854c1459cSAndrew Rybchenko efx_pseudo_hdr_hash_get( 699dae57086SAndrew Rybchenko __in efx_rxq_t *erp, 7003c838a9fSAndrew Rybchenko __in efx_rx_hash_alg_t func, 7013c838a9fSAndrew Rybchenko __in uint8_t *buffer) 7023c838a9fSAndrew Rybchenko { 703dae57086SAndrew Rybchenko efx_nic_t *enp = erp->er_enp; 704ec831f7fSAndrew Rybchenko const efx_rx_ops_t *erxop = enp->en_erxop; 7053c838a9fSAndrew Rybchenko 706dae57086SAndrew Rybchenko EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 707dae57086SAndrew Rybchenko 7080badfd72SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 7090badfd72SAndrew Rybchenko return (erxop->erxo_prefix_hash(enp, func, buffer)); 7100badfd72SAndrew Rybchenko } 7113c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 7123c838a9fSAndrew Rybchenko 713e75412c9SAndrew Rybchenko #if EFSYS_OPT_SIENA 7143c838a9fSAndrew Rybchenko 715460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 716d2df9a4eSAndrew Rybchenko siena_rx_init( 7173c838a9fSAndrew Rybchenko __in efx_nic_t *enp) 7183c838a9fSAndrew Rybchenko { 7193c838a9fSAndrew Rybchenko efx_oword_t oword; 7203c838a9fSAndrew Rybchenko unsigned int index; 7213c838a9fSAndrew Rybchenko 722e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 723e948693eSPhilip Paeps 724e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 725e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 726e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 727e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 728e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 729e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 730e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 731e948693eSPhilip Paeps 732e948693eSPhilip Paeps /* Zero the RSS table */ 733e948693eSPhilip Paeps for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 734e948693eSPhilip Paeps index++) { 735e948693eSPhilip Paeps EFX_ZERO_OWORD(oword); 736e948693eSPhilip Paeps EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 7373c838a9fSAndrew Rybchenko index, &oword, B_TRUE); 738e948693eSPhilip Paeps } 739e948693eSPhilip Paeps 7403c838a9fSAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 7413c838a9fSAndrew Rybchenko /* The RSS key and indirection table are writable. */ 74239023729SAndrew Rybchenko enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 7433c838a9fSAndrew Rybchenko 7443c838a9fSAndrew Rybchenko /* Hardware can insert RX hash with/without RSS */ 7453c838a9fSAndrew Rybchenko enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 7463c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 7473c838a9fSAndrew Rybchenko 748e948693eSPhilip Paeps return (0); 749e948693eSPhilip Paeps } 750e948693eSPhilip Paeps 751e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCATTER 752460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 753d2df9a4eSAndrew Rybchenko siena_rx_scatter_enable( 754e948693eSPhilip Paeps __in efx_nic_t *enp, 755e948693eSPhilip Paeps __in unsigned int buf_size) 756e948693eSPhilip Paeps { 757e948693eSPhilip Paeps unsigned int nbuf32; 758e948693eSPhilip Paeps efx_oword_t oword; 759460cb568SAndrew Rybchenko efx_rc_t rc; 760e948693eSPhilip Paeps 761e948693eSPhilip Paeps nbuf32 = buf_size / 32; 762e948693eSPhilip Paeps if ((nbuf32 == 0) || 763e948693eSPhilip Paeps (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 764e948693eSPhilip Paeps ((buf_size % 32) != 0)) { 765e948693eSPhilip Paeps rc = EINVAL; 766e948693eSPhilip Paeps goto fail1; 767e948693eSPhilip Paeps } 768e948693eSPhilip Paeps 769e948693eSPhilip Paeps if (enp->en_rx_qcount > 0) { 770e948693eSPhilip Paeps rc = EBUSY; 771e948693eSPhilip Paeps goto fail2; 772e948693eSPhilip Paeps } 773e948693eSPhilip Paeps 774e948693eSPhilip Paeps /* Set scatter buffer size */ 775e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 776e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 777e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 778e948693eSPhilip Paeps 779e948693eSPhilip Paeps /* Enable scatter for packets not matching a filter */ 780e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 781e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 782e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 783e948693eSPhilip Paeps 784e948693eSPhilip Paeps return (0); 785e948693eSPhilip Paeps 786e948693eSPhilip Paeps fail2: 787e948693eSPhilip Paeps EFSYS_PROBE(fail2); 788e948693eSPhilip Paeps fail1: 789460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 790e948693eSPhilip Paeps 791e948693eSPhilip Paeps return (rc); 792e948693eSPhilip Paeps } 793e948693eSPhilip Paeps #endif /* EFSYS_OPT_RX_SCATTER */ 794e948693eSPhilip Paeps 795e948693eSPhilip Paeps 796e948693eSPhilip Paeps #define EFX_RX_LFSR_HASH(_enp, _insert) \ 797e948693eSPhilip Paeps do { \ 798e948693eSPhilip Paeps efx_oword_t oword; \ 799e948693eSPhilip Paeps \ 800e948693eSPhilip Paeps EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 801e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 802e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 803e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 804e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 805e948693eSPhilip Paeps (_insert) ? 1 : 0); \ 806e948693eSPhilip Paeps EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 807e948693eSPhilip Paeps \ 808e948693eSPhilip Paeps if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 809e948693eSPhilip Paeps EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 810e948693eSPhilip Paeps &oword); \ 811e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, \ 812e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 813e948693eSPhilip Paeps EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 814e948693eSPhilip Paeps &oword); \ 815e948693eSPhilip Paeps } \ 816e948693eSPhilip Paeps \ 817e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 818e948693eSPhilip Paeps } while (B_FALSE) 819e948693eSPhilip Paeps 820e948693eSPhilip Paeps #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 821e948693eSPhilip Paeps do { \ 822e948693eSPhilip Paeps efx_oword_t oword; \ 823e948693eSPhilip Paeps \ 824e948693eSPhilip Paeps EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 825e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 826e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 827e948693eSPhilip Paeps (_ip) ? 1 : 0); \ 828e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 829e948693eSPhilip Paeps (_tcp) ? 0 : 1); \ 830e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 831e948693eSPhilip Paeps (_insert) ? 1 : 0); \ 832e948693eSPhilip Paeps EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 833e948693eSPhilip Paeps \ 834e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 835e948693eSPhilip Paeps } while (B_FALSE) 836e948693eSPhilip Paeps 837e948693eSPhilip Paeps #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 838e948693eSPhilip Paeps do { \ 839e948693eSPhilip Paeps efx_oword_t oword; \ 840e948693eSPhilip Paeps \ 841e948693eSPhilip Paeps EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 842e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, \ 843e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 844e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, \ 845e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 846e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, \ 847e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 848e948693eSPhilip Paeps EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 849e948693eSPhilip Paeps \ 850e948693eSPhilip Paeps (_rc) = 0; \ 851e948693eSPhilip Paeps \ 852e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 853e948693eSPhilip Paeps } while (B_FALSE) 854e948693eSPhilip Paeps 855e948693eSPhilip Paeps 856e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE 8573c838a9fSAndrew Rybchenko 858460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 859d2df9a4eSAndrew Rybchenko siena_rx_scale_mode_set( 860e948693eSPhilip Paeps __in efx_nic_t *enp, 861*82af879cSAndrew Rybchenko __in uint32_t rss_context, 862e948693eSPhilip Paeps __in efx_rx_hash_alg_t alg, 863e948693eSPhilip Paeps __in efx_rx_hash_type_t type, 864e948693eSPhilip Paeps __in boolean_t insert) 865e948693eSPhilip Paeps { 866460cb568SAndrew Rybchenko efx_rc_t rc; 867e948693eSPhilip Paeps 868*82af879cSAndrew Rybchenko if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 869*82af879cSAndrew Rybchenko rc = EINVAL; 870*82af879cSAndrew Rybchenko goto fail1; 871*82af879cSAndrew Rybchenko } 872*82af879cSAndrew Rybchenko 873e948693eSPhilip Paeps switch (alg) { 874e948693eSPhilip Paeps case EFX_RX_HASHALG_LFSR: 875e948693eSPhilip Paeps EFX_RX_LFSR_HASH(enp, insert); 876e948693eSPhilip Paeps break; 877e948693eSPhilip Paeps 878e948693eSPhilip Paeps case EFX_RX_HASHALG_TOEPLITZ: 879e948693eSPhilip Paeps EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 88019734dbbSAndrew Rybchenko type & EFX_RX_HASH_IPV4, 88119734dbbSAndrew Rybchenko type & EFX_RX_HASH_TCPIPV4); 882e948693eSPhilip Paeps 883e948693eSPhilip Paeps EFX_RX_TOEPLITZ_IPV6_HASH(enp, 88419734dbbSAndrew Rybchenko type & EFX_RX_HASH_IPV6, 88519734dbbSAndrew Rybchenko type & EFX_RX_HASH_TCPIPV6, 886e948693eSPhilip Paeps rc); 887e948693eSPhilip Paeps if (rc != 0) 888*82af879cSAndrew Rybchenko goto fail2; 889e948693eSPhilip Paeps 890e948693eSPhilip Paeps break; 891e948693eSPhilip Paeps 892e948693eSPhilip Paeps default: 893e948693eSPhilip Paeps rc = EINVAL; 894*82af879cSAndrew Rybchenko goto fail3; 895e948693eSPhilip Paeps } 896e948693eSPhilip Paeps 897e948693eSPhilip Paeps return (0); 898e948693eSPhilip Paeps 899*82af879cSAndrew Rybchenko fail3: 900*82af879cSAndrew Rybchenko EFSYS_PROBE(fail3); 901e948693eSPhilip Paeps fail2: 902e948693eSPhilip Paeps EFSYS_PROBE(fail2); 903e948693eSPhilip Paeps fail1: 904460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 905e948693eSPhilip Paeps 906e948693eSPhilip Paeps EFX_RX_LFSR_HASH(enp, B_FALSE); 907e948693eSPhilip Paeps 908e948693eSPhilip Paeps return (rc); 909e948693eSPhilip Paeps } 910e948693eSPhilip Paeps #endif 911e948693eSPhilip Paeps 912e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE 913460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 914d2df9a4eSAndrew Rybchenko siena_rx_scale_key_set( 915e948693eSPhilip Paeps __in efx_nic_t *enp, 916*82af879cSAndrew Rybchenko __in uint32_t rss_context, 917e948693eSPhilip Paeps __in_ecount(n) uint8_t *key, 918e948693eSPhilip Paeps __in size_t n) 919e948693eSPhilip Paeps { 920e948693eSPhilip Paeps efx_oword_t oword; 921e948693eSPhilip Paeps unsigned int byte; 922e948693eSPhilip Paeps unsigned int offset; 923460cb568SAndrew Rybchenko efx_rc_t rc; 924e948693eSPhilip Paeps 925*82af879cSAndrew Rybchenko if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 926*82af879cSAndrew Rybchenko rc = EINVAL; 927*82af879cSAndrew Rybchenko goto fail1; 928*82af879cSAndrew Rybchenko } 929*82af879cSAndrew Rybchenko 930e948693eSPhilip Paeps byte = 0; 931e948693eSPhilip Paeps 9323c838a9fSAndrew Rybchenko /* Write Toeplitz IPv4 hash key */ 933e948693eSPhilip Paeps EFX_ZERO_OWORD(oword); 934e948693eSPhilip Paeps for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 935e948693eSPhilip Paeps offset > 0 && byte < n; 936e948693eSPhilip Paeps --offset) 937e948693eSPhilip Paeps oword.eo_u8[offset - 1] = key[byte++]; 938e948693eSPhilip Paeps 939e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 940e948693eSPhilip Paeps 941e948693eSPhilip Paeps byte = 0; 942e948693eSPhilip Paeps 9433c838a9fSAndrew Rybchenko /* Verify Toeplitz IPv4 hash key */ 944e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 945e948693eSPhilip Paeps for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 946e948693eSPhilip Paeps offset > 0 && byte < n; 947e948693eSPhilip Paeps --offset) { 948e948693eSPhilip Paeps if (oword.eo_u8[offset - 1] != key[byte++]) { 949e948693eSPhilip Paeps rc = EFAULT; 950*82af879cSAndrew Rybchenko goto fail2; 951e948693eSPhilip Paeps } 952e948693eSPhilip Paeps } 953e948693eSPhilip Paeps 9543c838a9fSAndrew Rybchenko if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 9553c838a9fSAndrew Rybchenko goto done; 956e948693eSPhilip Paeps 957e948693eSPhilip Paeps byte = 0; 958e948693eSPhilip Paeps 9593c838a9fSAndrew Rybchenko /* Write Toeplitz IPv6 hash key 3 */ 960e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 961e948693eSPhilip Paeps for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 962e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 963e948693eSPhilip Paeps offset > 0 && byte < n; 964e948693eSPhilip Paeps --offset) 965e948693eSPhilip Paeps oword.eo_u8[offset - 1] = key[byte++]; 966e948693eSPhilip Paeps 967e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 968e948693eSPhilip Paeps 9693c838a9fSAndrew Rybchenko /* Write Toeplitz IPv6 hash key 2 */ 970e948693eSPhilip Paeps EFX_ZERO_OWORD(oword); 971e948693eSPhilip Paeps for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 972e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 973e948693eSPhilip Paeps offset > 0 && byte < n; 974e948693eSPhilip Paeps --offset) 975e948693eSPhilip Paeps oword.eo_u8[offset - 1] = key[byte++]; 976e948693eSPhilip Paeps 977e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 978e948693eSPhilip Paeps 9793c838a9fSAndrew Rybchenko /* Write Toeplitz IPv6 hash key 1 */ 980e948693eSPhilip Paeps EFX_ZERO_OWORD(oword); 981e948693eSPhilip Paeps for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 982e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 983e948693eSPhilip Paeps offset > 0 && byte < n; 984e948693eSPhilip Paeps --offset) 985e948693eSPhilip Paeps oword.eo_u8[offset - 1] = key[byte++]; 986e948693eSPhilip Paeps 987e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 988e948693eSPhilip Paeps 989e948693eSPhilip Paeps byte = 0; 990e948693eSPhilip Paeps 9913c838a9fSAndrew Rybchenko /* Verify Toeplitz IPv6 hash key 3 */ 992e948693eSPhilip Paeps EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 993e948693eSPhilip Paeps for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 994e948693eSPhilip Paeps FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 995e948693eSPhilip Paeps offset > 0 && byte < n; 996e948693eSPhilip Paeps --offset) { 997e948693eSPhilip Paeps if (oword.eo_u8[offset - 1] != key[byte++]) { 998e948693eSPhilip Paeps rc = EFAULT; 999*82af879cSAndrew Rybchenko goto fail3; 1000e948693eSPhilip Paeps } 1001e948693eSPhilip Paeps } 1002e948693eSPhilip Paeps 10033c838a9fSAndrew Rybchenko /* Verify Toeplitz IPv6 hash key 2 */ 10043c838a9fSAndrew Rybchenko EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 10053c838a9fSAndrew Rybchenko for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 10063c838a9fSAndrew Rybchenko FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1007e948693eSPhilip Paeps offset > 0 && byte < n; 1008e948693eSPhilip Paeps --offset) { 1009e948693eSPhilip Paeps if (oword.eo_u8[offset - 1] != key[byte++]) { 1010e948693eSPhilip Paeps rc = EFAULT; 1011*82af879cSAndrew Rybchenko goto fail4; 1012e948693eSPhilip Paeps } 1013e948693eSPhilip Paeps } 1014e948693eSPhilip Paeps 10153c838a9fSAndrew Rybchenko /* Verify Toeplitz IPv6 hash key 1 */ 10163c838a9fSAndrew Rybchenko EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 10173c838a9fSAndrew Rybchenko for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 10183c838a9fSAndrew Rybchenko FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 10193c838a9fSAndrew Rybchenko offset > 0 && byte < n; 10203c838a9fSAndrew Rybchenko --offset) { 10213c838a9fSAndrew Rybchenko if (oword.eo_u8[offset - 1] != key[byte++]) { 10223c838a9fSAndrew Rybchenko rc = EFAULT; 1023*82af879cSAndrew Rybchenko goto fail5; 10243c838a9fSAndrew Rybchenko } 10253c838a9fSAndrew Rybchenko } 10263c838a9fSAndrew Rybchenko 10273c838a9fSAndrew Rybchenko done: 1028e948693eSPhilip Paeps return (0); 1029e948693eSPhilip Paeps 1030*82af879cSAndrew Rybchenko fail5: 1031*82af879cSAndrew Rybchenko EFSYS_PROBE(fail5); 10323c838a9fSAndrew Rybchenko fail4: 10333c838a9fSAndrew Rybchenko EFSYS_PROBE(fail4); 1034e948693eSPhilip Paeps fail3: 1035e948693eSPhilip Paeps EFSYS_PROBE(fail3); 1036e948693eSPhilip Paeps fail2: 1037e948693eSPhilip Paeps EFSYS_PROBE(fail2); 1038e948693eSPhilip Paeps fail1: 1039460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 1040e948693eSPhilip Paeps 1041e948693eSPhilip Paeps return (rc); 1042e948693eSPhilip Paeps } 1043e948693eSPhilip Paeps #endif 1044e948693eSPhilip Paeps 1045e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE 1046460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 1047d2df9a4eSAndrew Rybchenko siena_rx_scale_tbl_set( 1048e948693eSPhilip Paeps __in efx_nic_t *enp, 1049*82af879cSAndrew Rybchenko __in uint32_t rss_context, 1050e948693eSPhilip Paeps __in_ecount(n) unsigned int *table, 1051e948693eSPhilip Paeps __in size_t n) 1052e948693eSPhilip Paeps { 1053e948693eSPhilip Paeps efx_oword_t oword; 1054e948693eSPhilip Paeps int index; 1055460cb568SAndrew Rybchenko efx_rc_t rc; 1056e948693eSPhilip Paeps 1057e948693eSPhilip Paeps EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1058e948693eSPhilip Paeps EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1059e948693eSPhilip Paeps 1060*82af879cSAndrew Rybchenko if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1061e948693eSPhilip Paeps rc = EINVAL; 1062e948693eSPhilip Paeps goto fail1; 1063e948693eSPhilip Paeps } 1064e948693eSPhilip Paeps 1065*82af879cSAndrew Rybchenko if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1066*82af879cSAndrew Rybchenko rc = EINVAL; 1067*82af879cSAndrew Rybchenko goto fail2; 1068*82af879cSAndrew Rybchenko } 1069*82af879cSAndrew Rybchenko 1070e948693eSPhilip Paeps for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1071e948693eSPhilip Paeps uint32_t byte; 1072e948693eSPhilip Paeps 1073e948693eSPhilip Paeps /* Calculate the entry to place in the table */ 10743c838a9fSAndrew Rybchenko byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1075e948693eSPhilip Paeps 1076e948693eSPhilip Paeps EFSYS_PROBE2(table, int, index, uint32_t, byte); 1077e948693eSPhilip Paeps 1078e948693eSPhilip Paeps EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1079e948693eSPhilip Paeps 1080e948693eSPhilip Paeps /* Write the table */ 1081e948693eSPhilip Paeps EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 10823c838a9fSAndrew Rybchenko index, &oword, B_TRUE); 1083e948693eSPhilip Paeps } 1084e948693eSPhilip Paeps 1085e948693eSPhilip Paeps for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1086e948693eSPhilip Paeps uint32_t byte; 1087e948693eSPhilip Paeps 1088e948693eSPhilip Paeps /* Determine if we're starting a new batch */ 10893c838a9fSAndrew Rybchenko byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1090e948693eSPhilip Paeps 1091e948693eSPhilip Paeps /* Read the table */ 1092e948693eSPhilip Paeps EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 10933c838a9fSAndrew Rybchenko index, &oword, B_TRUE); 1094e948693eSPhilip Paeps 1095e948693eSPhilip Paeps /* Verify the entry */ 1096e948693eSPhilip Paeps if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1097e948693eSPhilip Paeps rc = EFAULT; 1098*82af879cSAndrew Rybchenko goto fail3; 1099e948693eSPhilip Paeps } 1100e948693eSPhilip Paeps } 1101e948693eSPhilip Paeps 1102e948693eSPhilip Paeps return (0); 1103e948693eSPhilip Paeps 1104*82af879cSAndrew Rybchenko fail3: 1105*82af879cSAndrew Rybchenko EFSYS_PROBE(fail3); 1106e948693eSPhilip Paeps fail2: 1107e948693eSPhilip Paeps EFSYS_PROBE(fail2); 1108e948693eSPhilip Paeps fail1: 1109460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 1110e948693eSPhilip Paeps 1111e948693eSPhilip Paeps return (rc); 1112e948693eSPhilip Paeps } 1113e948693eSPhilip Paeps #endif 1114e948693eSPhilip Paeps 11150badfd72SAndrew Rybchenko /* 111654c1459cSAndrew Rybchenko * Falcon/Siena pseudo-header 11170badfd72SAndrew Rybchenko * -------------------------- 11180badfd72SAndrew Rybchenko * 11190badfd72SAndrew Rybchenko * Receive packets are prefixed by an optional 16 byte pseudo-header. 112054c1459cSAndrew Rybchenko * The pseudo-header is a byte array of one of the forms: 11210badfd72SAndrew Rybchenko * 11220badfd72SAndrew Rybchenko * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 11230badfd72SAndrew Rybchenko * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 11240badfd72SAndrew Rybchenko * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 11250badfd72SAndrew Rybchenko * 11260badfd72SAndrew Rybchenko * where: 11270badfd72SAndrew Rybchenko * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 11280badfd72SAndrew Rybchenko * LL.LL LFSR hash (16-bit big-endian) 11290badfd72SAndrew Rybchenko */ 11300badfd72SAndrew Rybchenko 11310badfd72SAndrew Rybchenko #if EFSYS_OPT_RX_SCALE 11320badfd72SAndrew Rybchenko static __checkReturn uint32_t 1133d2df9a4eSAndrew Rybchenko siena_rx_prefix_hash( 11340badfd72SAndrew Rybchenko __in efx_nic_t *enp, 11350badfd72SAndrew Rybchenko __in efx_rx_hash_alg_t func, 11360badfd72SAndrew Rybchenko __in uint8_t *buffer) 11370badfd72SAndrew Rybchenko { 1138a92a2133SAndrew Rybchenko _NOTE(ARGUNUSED(enp)) 1139a92a2133SAndrew Rybchenko 11400badfd72SAndrew Rybchenko switch (func) { 11410badfd72SAndrew Rybchenko case EFX_RX_HASHALG_TOEPLITZ: 11420badfd72SAndrew Rybchenko return ((buffer[12] << 24) | 11430badfd72SAndrew Rybchenko (buffer[13] << 16) | 11440badfd72SAndrew Rybchenko (buffer[14] << 8) | 11450badfd72SAndrew Rybchenko buffer[15]); 11460badfd72SAndrew Rybchenko 11470badfd72SAndrew Rybchenko case EFX_RX_HASHALG_LFSR: 11480badfd72SAndrew Rybchenko return ((buffer[14] << 8) | buffer[15]); 11490badfd72SAndrew Rybchenko 11500badfd72SAndrew Rybchenko default: 11510badfd72SAndrew Rybchenko EFSYS_ASSERT(0); 11520badfd72SAndrew Rybchenko return (0); 11530badfd72SAndrew Rybchenko } 11540badfd72SAndrew Rybchenko } 11550badfd72SAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCALE */ 11560badfd72SAndrew Rybchenko 11570badfd72SAndrew Rybchenko static __checkReturn efx_rc_t 1158d2df9a4eSAndrew Rybchenko siena_rx_prefix_pktlen( 11590badfd72SAndrew Rybchenko __in efx_nic_t *enp, 11600badfd72SAndrew Rybchenko __in uint8_t *buffer, 11610badfd72SAndrew Rybchenko __out uint16_t *lengthp) 11620badfd72SAndrew Rybchenko { 1163a92a2133SAndrew Rybchenko _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1164a92a2133SAndrew Rybchenko 11650badfd72SAndrew Rybchenko /* Not supported by Falcon/Siena hardware */ 11660badfd72SAndrew Rybchenko EFSYS_ASSERT(0); 11670badfd72SAndrew Rybchenko return (ENOTSUP); 11680badfd72SAndrew Rybchenko } 11690badfd72SAndrew Rybchenko 11700badfd72SAndrew Rybchenko 11713c838a9fSAndrew Rybchenko static void 1172d2df9a4eSAndrew Rybchenko siena_rx_qpost( 1173e948693eSPhilip Paeps __in efx_rxq_t *erp, 1174e948693eSPhilip Paeps __in_ecount(n) efsys_dma_addr_t *addrp, 1175e948693eSPhilip Paeps __in size_t size, 1176e948693eSPhilip Paeps __in unsigned int n, 1177e948693eSPhilip Paeps __in unsigned int completed, 1178e948693eSPhilip Paeps __in unsigned int added) 1179e948693eSPhilip Paeps { 1180e948693eSPhilip Paeps efx_qword_t qword; 1181e948693eSPhilip Paeps unsigned int i; 1182e948693eSPhilip Paeps unsigned int offset; 1183e948693eSPhilip Paeps unsigned int id; 1184e948693eSPhilip Paeps 1185e948693eSPhilip Paeps /* The client driver must not overfill the queue */ 1186e948693eSPhilip Paeps EFSYS_ASSERT3U(added - completed + n, <=, 1187e948693eSPhilip Paeps EFX_RXQ_LIMIT(erp->er_mask + 1)); 1188e948693eSPhilip Paeps 1189e948693eSPhilip Paeps id = added & (erp->er_mask); 1190e948693eSPhilip Paeps for (i = 0; i < n; i++) { 1191e948693eSPhilip Paeps EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1192e948693eSPhilip Paeps unsigned int, id, efsys_dma_addr_t, addrp[i], 1193e948693eSPhilip Paeps size_t, size); 1194e948693eSPhilip Paeps 1195e948693eSPhilip Paeps EFX_POPULATE_QWORD_3(qword, 1196e948693eSPhilip Paeps FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1197e948693eSPhilip Paeps FSF_AZ_RX_KER_BUF_ADDR_DW0, 1198e948693eSPhilip Paeps (uint32_t)(addrp[i] & 0xffffffff), 1199e948693eSPhilip Paeps FSF_AZ_RX_KER_BUF_ADDR_DW1, 1200e948693eSPhilip Paeps (uint32_t)(addrp[i] >> 32)); 1201e948693eSPhilip Paeps 1202e948693eSPhilip Paeps offset = id * sizeof (efx_qword_t); 1203e948693eSPhilip Paeps EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1204e948693eSPhilip Paeps 1205e948693eSPhilip Paeps id = (id + 1) & (erp->er_mask); 1206e948693eSPhilip Paeps } 1207e948693eSPhilip Paeps } 1208e948693eSPhilip Paeps 12093c838a9fSAndrew Rybchenko static void 1210d2df9a4eSAndrew Rybchenko siena_rx_qpush( 1211e948693eSPhilip Paeps __in efx_rxq_t *erp, 12123c838a9fSAndrew Rybchenko __in unsigned int added, 12133c838a9fSAndrew Rybchenko __inout unsigned int *pushedp) 1214e948693eSPhilip Paeps { 1215e948693eSPhilip Paeps efx_nic_t *enp = erp->er_enp; 12163c838a9fSAndrew Rybchenko unsigned int pushed = *pushedp; 1217e948693eSPhilip Paeps uint32_t wptr; 1218e948693eSPhilip Paeps efx_oword_t oword; 1219e948693eSPhilip Paeps efx_dword_t dword; 1220e948693eSPhilip Paeps 12213c838a9fSAndrew Rybchenko /* All descriptors are pushed */ 12223c838a9fSAndrew Rybchenko *pushedp = added; 1223e948693eSPhilip Paeps 1224e948693eSPhilip Paeps /* Push the populated descriptors out */ 1225e948693eSPhilip Paeps wptr = added & erp->er_mask; 1226e948693eSPhilip Paeps 1227e948693eSPhilip Paeps EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1228e948693eSPhilip Paeps 1229e948693eSPhilip Paeps /* Only write the third DWORD */ 1230e948693eSPhilip Paeps EFX_POPULATE_DWORD_1(dword, 1231e948693eSPhilip Paeps EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 12323c838a9fSAndrew Rybchenko 12333c838a9fSAndrew Rybchenko /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 12343c838a9fSAndrew Rybchenko EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 12353c838a9fSAndrew Rybchenko wptr, pushed & erp->er_mask); 12363c838a9fSAndrew Rybchenko EFSYS_PIO_WRITE_BARRIER(); 1237e948693eSPhilip Paeps EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1238e948693eSPhilip Paeps erp->er_index, &dword, B_FALSE); 1239e948693eSPhilip Paeps } 1240e948693eSPhilip Paeps 12418e0c4827SAndrew Rybchenko #if EFSYS_OPT_RX_PACKED_STREAM 12428e0c4827SAndrew Rybchenko static void 12438e0c4827SAndrew Rybchenko siena_rx_qpush_ps_credits( 12448e0c4827SAndrew Rybchenko __in efx_rxq_t *erp) 12458e0c4827SAndrew Rybchenko { 12468e0c4827SAndrew Rybchenko /* Not supported by Siena hardware */ 12478e0c4827SAndrew Rybchenko EFSYS_ASSERT(0); 12488e0c4827SAndrew Rybchenko } 12498e0c4827SAndrew Rybchenko 12508e0c4827SAndrew Rybchenko static uint8_t * 12518e0c4827SAndrew Rybchenko siena_rx_qps_packet_info( 12528e0c4827SAndrew Rybchenko __in efx_rxq_t *erp, 12538e0c4827SAndrew Rybchenko __in uint8_t *buffer, 12548e0c4827SAndrew Rybchenko __in uint32_t buffer_length, 12558e0c4827SAndrew Rybchenko __in uint32_t current_offset, 12568e0c4827SAndrew Rybchenko __out uint16_t *lengthp, 12578e0c4827SAndrew Rybchenko __out uint32_t *next_offsetp, 12588e0c4827SAndrew Rybchenko __out uint32_t *timestamp) 12598e0c4827SAndrew Rybchenko { 12608e0c4827SAndrew Rybchenko /* Not supported by Siena hardware */ 12618e0c4827SAndrew Rybchenko EFSYS_ASSERT(0); 12628e0c4827SAndrew Rybchenko 12638e0c4827SAndrew Rybchenko return (NULL); 12648e0c4827SAndrew Rybchenko } 12658e0c4827SAndrew Rybchenko #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 12668e0c4827SAndrew Rybchenko 1267460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 1268d2df9a4eSAndrew Rybchenko siena_rx_qflush( 1269e948693eSPhilip Paeps __in efx_rxq_t *erp) 1270e948693eSPhilip Paeps { 1271e948693eSPhilip Paeps efx_nic_t *enp = erp->er_enp; 1272e948693eSPhilip Paeps efx_oword_t oword; 1273e948693eSPhilip Paeps uint32_t label; 1274e948693eSPhilip Paeps 1275e948693eSPhilip Paeps label = erp->er_index; 1276e948693eSPhilip Paeps 1277e948693eSPhilip Paeps /* Flush the queue */ 1278e948693eSPhilip Paeps EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1279e948693eSPhilip Paeps FRF_AZ_RX_FLUSH_DESCQ, label); 1280e948693eSPhilip Paeps EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 12813c838a9fSAndrew Rybchenko 12823c838a9fSAndrew Rybchenko return (0); 1283e948693eSPhilip Paeps } 1284e948693eSPhilip Paeps 12853c838a9fSAndrew Rybchenko static void 1286d2df9a4eSAndrew Rybchenko siena_rx_qenable( 1287e948693eSPhilip Paeps __in efx_rxq_t *erp) 1288e948693eSPhilip Paeps { 1289e948693eSPhilip Paeps efx_nic_t *enp = erp->er_enp; 1290e948693eSPhilip Paeps efx_oword_t oword; 1291e948693eSPhilip Paeps 1292e948693eSPhilip Paeps EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1293e948693eSPhilip Paeps 1294e948693eSPhilip Paeps EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 12953c838a9fSAndrew Rybchenko erp->er_index, &oword, B_TRUE); 1296e948693eSPhilip Paeps 1297e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1298e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1299e948693eSPhilip Paeps EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1300e948693eSPhilip Paeps 1301e948693eSPhilip Paeps EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 13023c838a9fSAndrew Rybchenko erp->er_index, &oword, B_TRUE); 1303e948693eSPhilip Paeps } 1304e948693eSPhilip Paeps 1305460cb568SAndrew Rybchenko static __checkReturn efx_rc_t 1306d2df9a4eSAndrew Rybchenko siena_rx_qcreate( 1307e948693eSPhilip Paeps __in efx_nic_t *enp, 1308e948693eSPhilip Paeps __in unsigned int index, 1309e948693eSPhilip Paeps __in unsigned int label, 1310e948693eSPhilip Paeps __in efx_rxq_type_t type, 1311e948693eSPhilip Paeps __in efsys_mem_t *esmp, 1312e948693eSPhilip Paeps __in size_t n, 1313e948693eSPhilip Paeps __in uint32_t id, 1314e948693eSPhilip Paeps __in efx_evq_t *eep, 13153c838a9fSAndrew Rybchenko __in efx_rxq_t *erp) 1316e948693eSPhilip Paeps { 1317e948693eSPhilip Paeps efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1318e948693eSPhilip Paeps efx_oword_t oword; 1319e948693eSPhilip Paeps uint32_t size; 1320e948693eSPhilip Paeps boolean_t jumbo; 1321460cb568SAndrew Rybchenko efx_rc_t rc; 1322e948693eSPhilip Paeps 1323a92a2133SAndrew Rybchenko _NOTE(ARGUNUSED(esmp)) 1324a92a2133SAndrew Rybchenko 13250ff23789SAndrew Rybchenko EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 13260ff23789SAndrew Rybchenko (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1327e948693eSPhilip Paeps EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1328e948693eSPhilip Paeps EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1329e948693eSPhilip Paeps 13303c838a9fSAndrew Rybchenko EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 13313c838a9fSAndrew Rybchenko EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 13323c838a9fSAndrew Rybchenko 13333c838a9fSAndrew Rybchenko if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 1334e948693eSPhilip Paeps rc = EINVAL; 1335e948693eSPhilip Paeps goto fail1; 1336e948693eSPhilip Paeps } 1337e948693eSPhilip Paeps if (index >= encp->enc_rxq_limit) { 1338e948693eSPhilip Paeps rc = EINVAL; 1339e948693eSPhilip Paeps goto fail2; 1340e948693eSPhilip Paeps } 1341e948693eSPhilip Paeps for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1342e948693eSPhilip Paeps size++) 1343e948693eSPhilip Paeps if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 1344e948693eSPhilip Paeps break; 1345e948693eSPhilip Paeps if (id + (1 << size) >= encp->enc_buftbl_limit) { 1346e948693eSPhilip Paeps rc = EINVAL; 1347e948693eSPhilip Paeps goto fail3; 1348e948693eSPhilip Paeps } 1349e948693eSPhilip Paeps 1350e948693eSPhilip Paeps switch (type) { 1351e948693eSPhilip Paeps case EFX_RXQ_TYPE_DEFAULT: 1352e948693eSPhilip Paeps jumbo = B_FALSE; 1353e948693eSPhilip Paeps break; 1354e948693eSPhilip Paeps 1355e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCATTER 1356e948693eSPhilip Paeps case EFX_RXQ_TYPE_SCATTER: 1357e948693eSPhilip Paeps if (enp->en_family < EFX_FAMILY_SIENA) { 1358e948693eSPhilip Paeps rc = EINVAL; 1359e948693eSPhilip Paeps goto fail4; 1360e948693eSPhilip Paeps } 1361e948693eSPhilip Paeps jumbo = B_TRUE; 1362e948693eSPhilip Paeps break; 1363e948693eSPhilip Paeps #endif /* EFSYS_OPT_RX_SCATTER */ 1364e948693eSPhilip Paeps 1365e948693eSPhilip Paeps default: 1366e948693eSPhilip Paeps rc = EINVAL; 1367e948693eSPhilip Paeps goto fail4; 1368e948693eSPhilip Paeps } 1369e948693eSPhilip Paeps 1370e948693eSPhilip Paeps /* Set up the new descriptor queue */ 13711759d5d5SAndrew Rybchenko EFX_POPULATE_OWORD_7(oword, 1372e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1373e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1374e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1375e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_LABEL, label, 1376e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_SIZE, size, 1377e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_TYPE, 0, 1378e948693eSPhilip Paeps FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1379e948693eSPhilip Paeps 1380e948693eSPhilip Paeps EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 13813c838a9fSAndrew Rybchenko erp->er_index, &oword, B_TRUE); 1382e948693eSPhilip Paeps 1383e948693eSPhilip Paeps return (0); 1384e948693eSPhilip Paeps 1385e948693eSPhilip Paeps fail4: 1386e948693eSPhilip Paeps EFSYS_PROBE(fail4); 1387e948693eSPhilip Paeps fail3: 1388e948693eSPhilip Paeps EFSYS_PROBE(fail3); 1389e948693eSPhilip Paeps fail2: 1390e948693eSPhilip Paeps EFSYS_PROBE(fail2); 1391e948693eSPhilip Paeps fail1: 1392460cb568SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc); 1393e948693eSPhilip Paeps 1394e948693eSPhilip Paeps return (rc); 1395e948693eSPhilip Paeps } 1396e948693eSPhilip Paeps 13973c838a9fSAndrew Rybchenko static void 1398d2df9a4eSAndrew Rybchenko siena_rx_qdestroy( 1399e948693eSPhilip Paeps __in efx_rxq_t *erp) 1400e948693eSPhilip Paeps { 1401e948693eSPhilip Paeps efx_nic_t *enp = erp->er_enp; 1402e948693eSPhilip Paeps efx_oword_t oword; 1403e948693eSPhilip Paeps 1404e948693eSPhilip Paeps EFSYS_ASSERT(enp->en_rx_qcount != 0); 1405e948693eSPhilip Paeps --enp->en_rx_qcount; 1406e948693eSPhilip Paeps 1407e948693eSPhilip Paeps /* Purge descriptor queue */ 1408e948693eSPhilip Paeps EFX_ZERO_OWORD(oword); 1409e948693eSPhilip Paeps 1410e948693eSPhilip Paeps EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 14113c838a9fSAndrew Rybchenko erp->er_index, &oword, B_TRUE); 1412e948693eSPhilip Paeps 1413e948693eSPhilip Paeps /* Free the RXQ object */ 1414e948693eSPhilip Paeps EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1415e948693eSPhilip Paeps } 1416e948693eSPhilip Paeps 14173c838a9fSAndrew Rybchenko static void 1418d2df9a4eSAndrew Rybchenko siena_rx_fini( 1419e948693eSPhilip Paeps __in efx_nic_t *enp) 1420e948693eSPhilip Paeps { 14213c838a9fSAndrew Rybchenko _NOTE(ARGUNUSED(enp)) 1422e948693eSPhilip Paeps } 14233c838a9fSAndrew Rybchenko 1424e75412c9SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */ 1425