1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IB_ADAPTERS_TAVOR_CQ_H 28 #define _SYS_IB_ADAPTERS_TAVOR_CQ_H 29 30 /* 31 * tavor_cq.h 32 * Contains all of the prototypes, #defines, and structures necessary 33 * for the Completion Queue Processing routines. 34 * Specifically it contains the various completion types, flags, 35 * structures used for managing Tavor completion queues, and prototypes 36 * for many of the functions consumed by other parts of the Tavor driver 37 * (including those routines directly exposed through the IBTF CI 38 * interface). 39 */ 40 41 #include <sys/types.h> 42 #include <sys/conf.h> 43 #include <sys/ddi.h> 44 #include <sys/sunddi.h> 45 46 #include <sys/ib/adapters/tavor/tavor_misc.h> 47 48 #ifdef __cplusplus 49 extern "C" { 50 #endif 51 52 /* 53 * The following defines specify the default number of Completion Queues (CQ) 54 * their maximum size. Settings exist for the supported DDR DIMM sizes of 55 * 128MB and 256MB. If a DIMM greater than 256 is found, then the 256MB 56 * profile is used. See tavor_cfg.c for more discussion on config profiles. 57 * 58 * For manual configuration (not using config profiles), these values are 59 * controllable through the "tavor_log_max_cq_sz" and "tavor_log_num_cq" 60 * configuration variables, respectively. To override config profile settings 61 * the 'tavor_alt_config_enable' configuration variable must first be set. 62 * 63 * Note: We also have a define for the minimum size of a CQ. CQs allocated 64 * with size 0, 1, 2, or 3 will always get back a CQ of size 4. This is the 65 * smallest size that Tavor hardware and software can correctly handle. 66 */ 67 #define TAVOR_NUM_CQ_SHIFT_128 0x10 68 #define TAVOR_NUM_CQ_SHIFT_256 0x11 69 #define TAVOR_CQ_SZ_SHIFT 0x10 70 #define TAVOR_CQ_SZ (1 << TAVOR_CQ_SZ_SHIFT) 71 #define TAVOR_CQ_MIN_SIZE 0x3 72 73 /* 74 * Minimal configuration values. 75 */ 76 #define TAVOR_NUM_CQ_SHIFT_MIN 0xC 77 #define TAVOR_CQ_SZ_SHIFT_MIN 0xC 78 79 /* 80 * The following macro determines whether the contents of CQ memory (CQEs) 81 * need to be sync'd (with ddi_dma_sync()). This decision is based on whether 82 * the CQ memory is in DDR memory (no sync) or system memory (sync required). 83 * Note: It doesn't make much sense to put CQEs in DDR memory (since they are 84 * primarily written by HW and read by the CPU), but the driver does support 85 * that possibility. And it also supports the possibility that if a CQ in 86 * system memory is mapped DDI_DMA_CONSISTENT, it can be configured to not be 87 * sync'd because of the "sync override" parameter in the config profile. 88 */ 89 #define TAVOR_CQ_IS_SYNC_REQ(state, cqinfo) \ 90 ((((((state)->ts_cfg_profile->cp_streaming_consistent) && \ 91 ((state)->ts_cfg_profile->cp_consistent_syncoverride))) || \ 92 ((cqinfo).qa_location == TAVOR_QUEUE_LOCATION_INDDR)) \ 93 ? 0 : 1) 94 95 /* 96 * The following defines specify the size of the individual Completion Queue 97 * Context (CQC) entries 98 */ 99 #define TAVOR_CQC_SIZE_SHIFT 0x6 100 #define TAVOR_CQC_SIZE (1 << TAVOR_CQC_SIZE_SHIFT) 101 102 /* 103 * These are the defines for the Tavor CQ completion statuses. They are 104 * specified by the Tavor register specification. 105 */ 106 #define TAVOR_CQE_SUCCESS 0x0 107 #define TAVOR_CQE_LOC_LEN_ERR 0x1 108 #define TAVOR_CQE_LOC_OP_ERR 0x2 109 #define TAVOR_CQE_LOC_EEC_ERR 0x3 /* unsupported: RD */ 110 #define TAVOR_CQE_LOC_PROT_ERR 0x4 111 #define TAVOR_CQE_WR_FLUSHED_ERR 0x5 112 #define TAVOR_CQE_MW_BIND_ERR 0x6 113 #define TAVOR_CQE_BAD_RESPONSE_ERR 0x10 114 #define TAVOR_CQE_LOCAL_ACCESS_ERR 0x11 115 #define TAVOR_CQE_REM_INV_REQ_ERR 0x12 116 #define TAVOR_CQE_REM_ACC_ERR 0x13 117 #define TAVOR_CQE_REM_OP_ERR 0x14 118 #define TAVOR_CQE_TRANS_TO_ERR 0x15 119 #define TAVOR_CQE_RNRNAK_TO_ERR 0x16 120 #define TAVOR_CQE_LOCAL_RDD_VIO_ERR 0x20 /* unsupported: RD */ 121 #define TAVOR_CQE_REM_INV_RD_REQ_ERR 0x21 /* unsupported: RD */ 122 #define TAVOR_CQE_EEC_REM_ABORTED_ERR 0x22 /* unsupported: RD */ 123 #define TAVOR_CQE_INV_EEC_NUM_ERR 0x23 /* unsupported: RD */ 124 #define TAVOR_CQE_INV_EEC_STATE_ERR 0x24 /* unsupported: RD */ 125 126 /* 127 * These are the defines for the Tavor CQ entry types. They are also 128 * specified by the Tavor register specification. They indicate what type 129 * of work request is completing (for successful completions). Note: The 130 * "SND" or "RCV" in each define is used to indicate whether the completion 131 * work request was from the Send work queue or the Receive work queue on 132 * the associated QP. 133 */ 134 #define TAVOR_CQE_SND_RDMAWR 0x8 135 #define TAVOR_CQE_SND_RDMAWR_IMM 0x9 136 #define TAVOR_CQE_SND_SEND 0xA 137 #define TAVOR_CQE_SND_SEND_IMM 0xB 138 #define TAVOR_CQE_SND_RDMARD 0x10 139 #define TAVOR_CQE_SND_ATOMIC_CS 0x11 140 #define TAVOR_CQE_SND_ATOMIC_FA 0x12 141 #define TAVOR_CQE_SND_BIND_MW 0x18 142 #define TAVOR_CQE_RCV_RECV_IMM 0x3 143 #define TAVOR_CQE_RCV_RECV_IMM2 0x5 144 #define TAVOR_CQE_RCV_RECV 0x2 145 #define TAVOR_CQE_RCV_RECV2 0x4 146 #define TAVOR_CQE_RCV_RDMAWR_IMM 0x9 147 #define TAVOR_CQE_RCV_RDMAWR_IMM2 0xB 148 149 /* Define for maximum CQ number mask (CQ number is 24 bits) */ 150 #define TAVOR_CQ_MAXNUMBER_MSK 0xFFFFFF 151 152 /* 153 * This define and the following macro are used to find an event queue for a 154 * new CQ based on its completion queue number. Note: This is a rather 155 * simple method that we use today. We simply choose from one of the first 156 * 32 EQs based on the 5 least significant bits of the CQ number. 157 */ 158 #define TAVOR_CQ_TO_EQ_MASK 0x1F 159 #define TAVOR_CQ_EQNUM_GET(cqnum) ((cqnum) & TAVOR_CQ_TO_EQ_MASK) 160 161 /* 162 * The following macro is even simpler than the above one. This is used to 163 * find an event queue for CQ errors for a new CQ. In theory we could do this 164 * based on the CQ's number (as we do above). Today, however, all CQ error 165 * events go to one specific EQ (i.e. EQ #32). 166 */ 167 #define TAVOR_CQ_ERREQNUM_GET(cqnum) 0x20 168 169 /* 170 * The following defines are used for Tavor CQ error handling. Note: For 171 * CQEs which correspond to error events, the Tavor device requires some 172 * special handling by software. These defines are used to identify and 173 * extract the necessary information from each error CQE, including status 174 * code (above), doorbell count, and whether a error completion is for a 175 * send or receive work request. 176 */ 177 #define TAVOR_CQE_ERR_STATUS_SHIFT 24 178 #define TAVOR_CQE_ERR_STATUS_MASK 0xFF 179 #define TAVOR_CQE_ERR_DBDCNT_MASK 0xFFFF 180 #define TAVOR_CQE_SEND_ERR_OPCODE 0xFF 181 #define TAVOR_CQE_RECV_ERR_OPCODE 0xFE 182 #define TAVOR_CQ_SYNC_AND_DB 0 183 #define TAVOR_CQ_RECYCLE_ENTRY 1 184 185 /* Defines for tracking whether a CQ is being used with special QP or not */ 186 #define TAVOR_CQ_IS_NORMAL 0 187 #define TAVOR_CQ_IS_SPECIAL 1 188 189 /* 190 * The tavor_sw_cq_s structure is also referred to using the "tavor_cqhdl_t" 191 * typedef (see tavor_typedef.h). It encodes all the information necessary 192 * to track the various resources needed to allocate, initialize, poll, resize, 193 * and (later) free a completion queue (CQ). 194 * 195 * Specifically, it has a consumer index and a lock to ensure single threaded 196 * access to it. It has pointers to the various resources allocated for the 197 * completion queue, i.e. a CQC resource and the memory for the completion 198 * queue itself. It has flags to indicate whether the CQ requires 199 * ddi_dma_sync() ("cq_sync"). It also has a reference count and the number(s) 200 * of the EQs to which it is associated (for success and for errors). 201 * 202 * Additionally, it has a pointer to the associated MR handle (for the mapped 203 * queue memory) and a void pointer that holds the argument that should be 204 * passed back to the IBTF when events are generated on the CQ. 205 * 206 * We also have the always necessary backpointer to the resource for the 207 * CQ handle structure itself. But we also have pointers to the "Work Request 208 * ID" processing lists (both the lock and the regular list, as well as the 209 * head and tail for the "reapable" list). See tavor_wrid.c for more details. 210 */ 211 struct tavor_sw_cq_s { 212 kmutex_t cq_lock; 213 uint32_t cq_consindx; 214 uint32_t cq_cqnum; 215 tavor_hw_cqe_t *cq_buf; 216 tavor_mrhdl_t cq_mrhdl; 217 uint32_t cq_bufsz; 218 uint_t cq_sync; 219 uint_t cq_refcnt; 220 uint32_t cq_eqnum; 221 uint32_t cq_erreqnum; 222 uint_t cq_is_special; 223 uint_t cq_is_umap; 224 uint32_t cq_uarpg; 225 devmap_cookie_t cq_umap_dhp; 226 tavor_rsrc_t *cq_cqcrsrcp; 227 tavor_rsrc_t *cq_rsrcp; 228 229 void *cq_hdlrarg; 230 231 /* For Work Request ID processing */ 232 kmutex_t cq_wrid_wqhdr_lock; 233 avl_tree_t cq_wrid_wqhdr_avl_tree; 234 tavor_wrid_list_hdr_t *cq_wrid_reap_head; 235 tavor_wrid_list_hdr_t *cq_wrid_reap_tail; 236 237 struct tavor_qalloc_info_s cq_cqinfo; 238 }; 239 _NOTE(READ_ONLY_DATA(tavor_sw_cq_s::cq_cqnum 240 tavor_sw_cq_s::cq_eqnum 241 tavor_sw_cq_s::cq_erreqnum 242 tavor_sw_cq_s::cq_cqcrsrcp 243 tavor_sw_cq_s::cq_rsrcp 244 tavor_sw_cq_s::cq_hdlrarg 245 tavor_sw_cq_s::cq_is_umap 246 tavor_sw_cq_s::cq_uarpg)) 247 _NOTE(DATA_READABLE_WITHOUT_LOCK(tavor_sw_cq_s::cq_bufsz 248 tavor_sw_cq_s::cq_cqinfo)) 249 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_lock, 250 tavor_sw_cq_s::cq_consindx 251 tavor_sw_cq_s::cq_buf 252 tavor_sw_cq_s::cq_mrhdl 253 tavor_sw_cq_s::cq_sync 254 tavor_sw_cq_s::cq_refcnt 255 tavor_sw_cq_s::cq_is_special 256 tavor_sw_cq_s::cq_umap_dhp)) 257 258 int tavor_cq_alloc(tavor_state_t *state, ibt_cq_hdl_t ibt_cqhdl, 259 ibt_cq_attr_t *attr_p, uint_t *actual_size, tavor_cqhdl_t *cqhdl, 260 uint_t sleepflag); 261 int tavor_cq_free(tavor_state_t *state, tavor_cqhdl_t *cqhdl, 262 uint_t sleepflag); 263 int tavor_cq_resize(tavor_state_t *state, tavor_cqhdl_t cqhdl, 264 uint_t req_size, uint_t *actual_size, uint_t sleepflag); 265 int tavor_cq_notify(tavor_state_t *state, tavor_cqhdl_t cqhdl, 266 ibt_cq_notify_flags_t flags); 267 int tavor_cq_poll(tavor_state_t *state, tavor_cqhdl_t cqhdl, ibt_wc_t *wc_p, 268 uint_t num_wc, uint_t *num_polled); 269 int tavor_cq_handler(tavor_state_t *state, tavor_eqhdl_t eq, 270 tavor_hw_eqe_t *eqe); 271 int tavor_cq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq, 272 tavor_hw_eqe_t *eqe); 273 int tavor_cq_refcnt_inc(tavor_cqhdl_t cq, uint_t is_special); 274 void tavor_cq_refcnt_dec(tavor_cqhdl_t cq); 275 tavor_cqhdl_t tavor_cqhdl_from_cqnum(tavor_state_t *state, uint_t cqnum); 276 void tavor_cq_srq_entries_flush(tavor_state_t *state, tavor_qphdl_t qp); 277 #ifdef __cplusplus 278 } 279 #endif 280 281 #endif /* _SYS_IB_ADAPTERS_TAVOR_CQ_H */ 282