1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IB_ADAPTERS_TAVOR_WR_H 28 #define _SYS_IB_ADAPTERS_TAVOR_WR_H 29 30 /* 31 * tavor_wr.h 32 * Contains all of the prototypes, #defines, and structures necessary 33 * for the Tavor Work Request Processing Routines 34 * Specifically it contains #defines, macros, and prototypes for each of 35 * building each of the various types of WQE and for managing the WRID 36 * tracking mechanisms. 37 */ 38 39 #include <sys/types.h> 40 #include <sys/conf.h> 41 #include <sys/ddi.h> 42 #include <sys/sunddi.h> 43 44 #ifdef __cplusplus 45 extern "C" { 46 #endif 47 48 /* 49 * The following macro is used to convert WQE address and size into the 50 * "wqeaddrsz" value needed in the tavor_wrid_entry_t (see below). 51 */ 52 #define TAVOR_QP_WQEADDRSZ(addr, size) \ 53 ((((uintptr_t)(addr)) & ~TAVOR_WQE_NDS_MASK) | \ 54 ((size) & TAVOR_WQE_NDS_MASK)) 55 56 /* 57 * The following macros are used to calculate pointers to the Send or Receive 58 * (or SRQ) WQEs on a given QP, respectively 59 */ 60 #define TAVOR_QP_SQ_ENTRY(qp, tail) \ 61 ((uint64_t *)((uintptr_t)((qp)->qp_sq_buf) + \ 62 ((tail) << (qp)->qp_sq_log_wqesz))) 63 #define TAVOR_QP_RQ_ENTRY(qp, tail) \ 64 ((uint64_t *)((uintptr_t)((qp)->qp_rq_buf) + \ 65 ((tail) << (qp)->qp_rq_log_wqesz))) 66 #define TAVOR_SRQ_WQ_ENTRY(srq, tail) \ 67 ((uint64_t *)((uintptr_t)((srq)->srq_wq_buf) + \ 68 ((tail) << (srq)->srq_wq_log_wqesz))) 69 70 /* 71 * The following macro is used to calculate the 'wqe_index' field during SRQ 72 * operation. This returns the index based on the WQE size, that can be used 73 * to reference WQEs in an SRQ. 74 */ 75 #define TAVOR_SRQ_WQE_INDEX(srq_base_addr, wqe_addr, log_wqesz) \ 76 (((uint32_t)(uintptr_t)wqe_addr - \ 77 (uint32_t)(uintptr_t)srq_base_addr) >> log_wqesz) 78 79 #define TAVOR_SRQ_WQE_ADDR(srq, wqe_index) \ 80 ((uint64_t *)((uintptr_t)srq->srq_wq_buf + \ 81 (wqe_index << srq->srq_wq_log_wqesz))) 82 83 /* 84 * The following macros are used to access specific fields in Directed Route 85 * MAD packets. We can extract the MgmtClass, "hop pointer", and "hop count". 86 * We can also update the "hop pointer" as appropriate. Note: Again, because 87 * of the limited amount of direct handling the Tavor hardware does on special 88 * QP request (specifically on Directed Route MADs), the driver needs to 89 * update (as necessary) the "hop pointer" value depending on whether a MAD 90 * is outbound or inbound (i.e. depending on the relationship between "hop 91 * pointer" and "hop count" in the given MAD) 92 */ 93 #define TAVOR_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass, offset, va, len) \ 94 if (((mgmtclass) == NULL) && ((offset) + (len) > 1)) { \ 95 (mgmtclass) = &((uint8_t *)(uintptr_t)(va))[1 - (offset)]; \ 96 } 97 #define TAVOR_SPECIAL_QP_DRMAD_GET_HOPPOINTER(hp, offset, va, len) \ 98 if (((hp) == NULL) && \ 99 ((offset) + (len) > 6)) { \ 100 (hp) = &((uint8_t *)(uintptr_t)(va))[6 - (offset)]; \ 101 } 102 #define TAVOR_SPECIAL_QP_DRMAD_GET_HOPCOUNT(hc, offset, va, len) \ 103 if (((hc) == NULL) && \ 104 ((offset) + (len) > 7)) { \ 105 (hc) = &((uint8_t *)(uintptr_t)(va))[7 - (offset)]; \ 106 } 107 #define TAVOR_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(mgmtclass, hp, hc) \ 108 if ((mgmtclass) == 0x81) { \ 109 if ((hp) < (hc)) { \ 110 (hp) = (hp) + 1; \ 111 } else if ((hp) > (hc)) { \ 112 (hp) = (hp) - 1; \ 113 } \ 114 } 115 116 117 /* 118 * The tavor_wrid_entry_s structure is used internally by the Tavor 119 * driver to contain all the information necessary for tracking WRIDs. 120 * Specifically, this structure contains the 64-bit WRID, the 32-bit quantity 121 * called "wr_wqeaddrsz" (which can also be found in every CQE), and the 122 * "wr_signaled_dbd" information which indicates whether a given entry was 123 * signaled or not and whether a doorbell was subsequently rung for this 124 * particular work request. Note: the latter piece of information is 125 * particularly useful during completion processing on errored CQEs. 126 */ 127 struct tavor_wrid_entry_s { 128 uint64_t wr_wrid; 129 uint32_t wr_wqeaddrsz; 130 uint32_t wr_signaled_dbd; 131 }; 132 #define TAVOR_WRID_ENTRY_SIGNALED (1 << 0) 133 #define TAVOR_WRID_ENTRY_DOORBELLED (1 << 1) 134 135 /* 136 * The tavor_sw_wqe_dbinfo_t structure is used internally by the Tavor 137 * driver to return information (from the tavor_wqe_mlx_build_nextctl() and 138 * tavor_wqe_send_build_nextctl() routines) regarding the type of Tavor 139 * doorbell necessary. 140 */ 141 typedef struct tavor_sw_wqe_dbinfo_s { 142 uint_t db_nopcode; 143 uint_t db_fence; 144 } tavor_sw_wqe_dbinfo_t; 145 146 /* 147 * The Work Queue Lock (WQL) structure. Each WQHDR (tavor_workq_hdr_t defined 148 * below) must lock access to the wridlist during any wridlist manipulation. 149 * Also, any Shared Receive Queue (SRQ) must also be able to lock the wridlist 150 * since it maintains wridlist's differently than normal QPs. This 151 * 'tavor_wq_lock_t' structure is shared and accessible through the WQ or the 152 * SRQ, and refcnt is maintained. The last entity to decrement use of the 153 * lock, also will free up the memory. 154 */ 155 struct tavor_wq_lock_s { 156 kmutex_t wql_lock; 157 uint_t wql_refcnt; 158 }; 159 160 /* 161 * The tavor_wrid_list_hdr_s structure is used internally by the Tavor driver 162 * to track all the information necessary to manage a queue of WRID entries 163 * (the tavor_wrid_entry_s struct above). 164 * It contains some information regarding the status of a given WRID list 165 * (e.g. head index, tail index, queue full condition, etc.). Note: Although 166 * some of this information is also kept by the tavor_workq_hdr_s below, what 167 * is kept here may, in fact, represent the state of an old WRID list. It 168 * could be different from what is kept in the tavor_workq_hdr_s because this 169 * WRID list may no longer be the active WRID list. If it is an active list, 170 * however, then both sets of information should be up-to-date and consistent. 171 * Several of these structures are chained together on each work queue header 172 * to form a linked list (using the "wl_next" and "wl_prev"). These structs, 173 * in turn, each have a pointer to a queue of WRID entries. They also each 174 * have a pointer to the next "reapable" entry ("wl_reap_next") which is only 175 * used when a WRID list has been retired and is ready to be freed up. 176 * Lastly, it has a backpointer to the work queue header to which the WRID 177 * list belongs (this is for proper handling on removal). 178 */ 179 struct tavor_wrid_list_hdr_s { 180 tavor_wrid_list_hdr_t *wl_next; 181 tavor_wrid_list_hdr_t *wl_prev; 182 tavor_wrid_list_hdr_t *wl_reap_next; 183 tavor_workq_hdr_t *wl_wqhdr; 184 185 tavor_wrid_entry_t *wl_wre; 186 tavor_wrid_entry_t *wl_wre_old_tail; 187 uint32_t wl_size; 188 uint32_t wl_full; 189 uint32_t wl_head; 190 uint32_t wl_tail; 191 192 /* For SRQ */ 193 uint_t wl_srq_en; 194 int wl_free_list_indx; 195 ddi_acc_handle_t wl_acchdl; 196 uint32_t *wl_srq_wq_buf; 197 uint32_t wl_srq_wq_bufsz; 198 uint64_t wl_srq_desc_off; 199 uint32_t wl_srq_log_wqesz; 200 }; 201 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock, 202 tavor_wrid_list_hdr_s::wl_next 203 tavor_wrid_list_hdr_s::wl_prev 204 tavor_wrid_list_hdr_s::wl_wqhdr)) 205 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock, 206 tavor_wrid_list_hdr_s::wl_wre 207 tavor_wrid_list_hdr_s::wl_wre_old_tail 208 tavor_wrid_list_hdr_s::wl_size 209 tavor_wrid_list_hdr_s::wl_full 210 tavor_wrid_list_hdr_s::wl_head 211 tavor_wrid_list_hdr_s::wl_tail 212 tavor_wrid_list_hdr_s::wl_srq_en 213 tavor_wrid_list_hdr_s::wl_free_list_indx 214 tavor_wrid_list_hdr_s::wl_acchdl 215 tavor_wrid_list_hdr_s::wl_srq_wq_buf 216 tavor_wrid_list_hdr_s::wl_srq_desc_off 217 tavor_wrid_list_hdr_s::wl_srq_log_wqesz)) 218 219 /* 220 * The tavor_workq_hdr_s structure is used internally by the Tavor driver to 221 * track all the information necessary to manage the work queues associated 222 * with a given completion queue. It contains much of the information 223 * regarding the status of a given work queue (e.g. head index, tail index, 224 * queue full condition, etc.). Note: This information is kept here (i.e. 225 * associated with a completion queue) rather than as part of the QP because 226 * the queue pair may potentially be destroyed while outstanding CQEs still 227 * remain on the CQ. 228 * Several of these structures are chained together on each CQ to form a 229 * linked list (using the "wq_next" and "wq_prev"). These headers, in turn, 230 * link to the containers for the individual WRID entries (managed with the 231 * tavor_wrid_list_hdr_s structs above). Note: We keep a list of these 232 * tavor_wrid_list_hdr_s because a given QP may be used, destroyed (or 233 * transition to "Reset"), and then reused. The list helps us track where 234 * to put new WRID entries and where to pull old entries from. 235 * The "wq_qpn" (QP number) and "wq_send_or_recv" (TAVOR_WR_SEND or 236 * TAVOR_WR_RECV) are used to uniquely identify the given work queue. 237 * Lookups into the work queue list (to find a given work queue) will use 238 * these two fields as identifiers. 239 */ 240 struct tavor_workq_hdr_s { 241 avl_node_t wq_avl_link; 242 uint32_t wq_qpn; 243 uint32_t wq_type; 244 245 tavor_wq_lock_t *wq_wrid_wql; 246 247 uint32_t wq_size; 248 uint32_t wq_head; 249 uint32_t wq_tail; 250 uint32_t wq_full; 251 tavor_wrid_list_hdr_t *wq_wrid_poll; 252 tavor_wrid_list_hdr_t *wq_wrid_post; 253 }; 254 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock, 255 tavor_workq_hdr_s::wq_avl_link 256 tavor_workq_hdr_s::wq_qpn 257 tavor_workq_hdr_s::wq_type 258 tavor_sw_cq_s::cq_wrid_reap_head 259 tavor_sw_cq_s::cq_wrid_reap_tail)) 260 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock, 261 tavor_workq_hdr_s::wq_size 262 tavor_workq_hdr_s::wq_head 263 tavor_workq_hdr_s::wq_tail 264 tavor_workq_hdr_s::wq_full 265 tavor_workq_hdr_s::wq_wrid_poll 266 tavor_workq_hdr_s::wq_wrid_post 267 tavor_wrid_list_hdr_s::wl_wre 268 tavor_wrid_list_hdr_s::wl_wre_old_tail 269 tavor_wrid_list_hdr_s::wl_size 270 tavor_wrid_list_hdr_s::wl_full 271 tavor_wrid_list_hdr_s::wl_head 272 tavor_wrid_list_hdr_s::wl_tail)) 273 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock, 274 tavor_wrid_list_hdr_s::wl_reap_next)) 275 _NOTE(LOCK_ORDER(tavor_sw_cq_s::cq_lock 276 tavor_sw_cq_s::cq_wrid_wqhdr_lock 277 tavor_wq_lock_s::wql_lock)) 278 #define TAVOR_WR_RECV 0x0 279 #define TAVOR_WR_SEND 0x1 280 #define TAVOR_WR_SRQ 0x2 281 282 extern int tavor_wrid_wqhdr_compare(const void *p1, const void *p2); 283 typedef struct tavor_workq_compare_s { 284 uint32_t cmp_type; 285 uint32_t cmp_qpn; 286 } tavor_workq_compare_t; 287 288 /* For Work Request posting */ 289 int tavor_post_send(tavor_state_t *state, tavor_qphdl_t qphdl, 290 ibt_send_wr_t *wr_p, uint_t num_wr, uint_t *num_posted); 291 int tavor_post_recv(tavor_state_t *state, tavor_qphdl_t qphdl, 292 ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted); 293 int tavor_post_srq(tavor_state_t *state, tavor_srqhdl_t srqhdl, 294 ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted); 295 296 /* For WRID handling */ 297 int tavor_wrid_from_reset_handling(tavor_state_t *state, tavor_qphdl_t qp); 298 void tavor_wrid_to_reset_handling(tavor_state_t *state, tavor_qphdl_t qp); 299 void tavor_wrid_add_entry(tavor_workq_hdr_t *wq, uint64_t wrid, 300 uint32_t wqeaddr_sz, uint_t signaled_dbd); 301 void tavor_wrid_add_entry_srq(tavor_srqhdl_t srq, uint64_t wrid, 302 uint_t signaled_dbd); 303 uint64_t tavor_wrid_get_entry(tavor_cqhdl_t cqhdl, tavor_hw_cqe_t *cqe, 304 tavor_wrid_entry_t *wre); 305 tavor_wq_lock_t *tavor_wrid_wql_create(tavor_state_t *state); 306 tavor_wrid_list_hdr_t *tavor_wrid_get_list(uint32_t size); 307 void tavor_wrid_list_srq_init(tavor_wrid_list_hdr_t *r_wridlist, 308 tavor_srqhdl_t srq, uint_t wq_start); 309 void tavor_wrid_cq_reap(tavor_cqhdl_t cq); 310 void tavor_wrid_cq_force_reap(tavor_cqhdl_t cq); 311 void tavor_wql_refcnt_dec(tavor_wq_lock_t *wq_lock); 312 void tavor_wql_refcnt_inc(tavor_wq_lock_t *wq_lock); 313 tavor_wrid_entry_t *tavor_wrid_find_match_srq(tavor_wrid_list_hdr_t *wq, 314 tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe); 315 316 #ifdef __cplusplus 317 } 318 #endif 319 320 #endif /* _SYS_IB_ADAPTERS_TAVOR_WR_H */ 321