xref: /illumos-gate/usr/src/uts/common/sys/ib/adapters/tavor/tavor_wr.h (revision 9e39c5ba00a55fa05777cc94b148296af305e135)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_TAVOR_WR_H
28 #define	_SYS_IB_ADAPTERS_TAVOR_WR_H
29 
30 /*
31  * tavor_wr.h
32  *    Contains all of the prototypes, #defines, and structures necessary
33  *    for the Tavor Work Request Processing Routines
34  *    Specifically it contains #defines, macros, and prototypes for each of
35  *    building each of the various types of WQE and for managing the WRID
36  *    tracking mechanisms.
37  */
38 
39 #include <sys/types.h>
40 #include <sys/conf.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
48 /*
49  * The following macro is used to convert WQE address and size into the
50  * "wqeaddrsz" value needed in the tavor_wrid_entry_t (see below).
51  */
52 #define	TAVOR_QP_WQEADDRSZ(addr, size)					\
53 	((((uintptr_t)(addr)) & ~TAVOR_WQE_NDS_MASK) |			\
54 	((size) & TAVOR_WQE_NDS_MASK))
55 
56 /*
57  * The following macros are used to calculate pointers to the Send or Receive
58  * (or SRQ) WQEs on a given QP, respectively
59  */
60 #define	TAVOR_QP_SQ_ENTRY(qp, tail)					\
61 	((uint64_t *)((uintptr_t)((qp)->qp_sq_buf) +			\
62 	((tail) << (qp)->qp_sq_log_wqesz)))
63 #define	TAVOR_QP_RQ_ENTRY(qp, tail)					\
64 	((uint64_t *)((uintptr_t)((qp)->qp_rq_buf) +			\
65 	((tail) << (qp)->qp_rq_log_wqesz)))
66 #define	TAVOR_SRQ_WQ_ENTRY(srq, tail)					\
67 	((uint64_t *)((uintptr_t)((srq)->srq_wq_buf) +			\
68 	((tail) << (srq)->srq_wq_log_wqesz)))
69 
70 /*
71  * The following macro is used to calculate the 'wqe_index' field during SRQ
72  * operation.  This returns the index based on the WQE size, that can be used
73  * to reference WQEs in an SRQ.
74  */
75 #define	TAVOR_SRQ_WQE_INDEX(srq_base_addr, wqe_addr, log_wqesz)		\
76 	(((uint32_t)(uintptr_t)wqe_addr -				\
77 	(uint32_t)(uintptr_t)srq_base_addr) >> log_wqesz)
78 
79 #define	TAVOR_SRQ_WQE_ADDR(srq, wqe_index)				\
80 	((uint64_t *)((uintptr_t)srq->srq_wq_buf +			\
81 	(wqe_index << srq->srq_wq_log_wqesz)))
82 
83 /*
84  * The following macros are used to access specific fields in Directed Route
85  * MAD packets.  We can extract the MgmtClass, "hop pointer", and "hop count".
86  * We can also update the "hop pointer" as appropriate.  Note:  Again, because
87  * of the limited amount of direct handling the Tavor hardware does on special
88  * QP request (specifically on Directed Route MADs), the driver needs to
89  * update (as necessary) the "hop pointer" value depending on whether a MAD
90  * is outbound or inbound (i.e. depending on the relationship between "hop
91  * pointer" and "hop count" in the given MAD)
92  */
93 #define	TAVOR_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass, offset, va, len) \
94 	if (((mgmtclass) == NULL) && ((offset) + (len) > 1)) {		 \
95 	    (mgmtclass) = &((uint8_t *)(uintptr_t)(va))[1 - (offset)];	 \
96 	}
97 #define	TAVOR_SPECIAL_QP_DRMAD_GET_HOPPOINTER(hp, offset, va, len)	\
98 	if (((hp) == NULL) &&					  	\
99 	    ((offset) + (len) > 6)) {					\
100 	    (hp) = &((uint8_t *)(uintptr_t)(va))[6 - (offset)];		\
101 	}
102 #define	TAVOR_SPECIAL_QP_DRMAD_GET_HOPCOUNT(hc, offset, va, len)	\
103 	if (((hc) == NULL) &&						\
104 	    ((offset) + (len) > 7)) {					\
105 	    (hc) = &((uint8_t *)(uintptr_t)(va))[7 - (offset)];		\
106 	}
107 #define	TAVOR_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(mgmtclass, hp, hc)	\
108 	if ((mgmtclass) == 0x81) {					\
109 		if ((hp) < (hc)) {					\
110 			(hp) = (hp) + 1;				\
111 		} else if ((hp) > (hc)) {				\
112 			(hp) = (hp) - 1;				\
113 		}							\
114 	}
115 
116 
117 /*
118  * The tavor_wrid_entry_s structure is used internally by the Tavor
119  * driver to contain all the information necessary for tracking WRIDs.
120  * Specifically, this structure contains the 64-bit WRID, the 32-bit quantity
121  * called "wr_wqeaddrsz" (which can also be found in every CQE), and the
122  * "wr_signaled_dbd" information which indicates whether a given entry was
123  * signaled or not and whether a doorbell was subsequently rung for this
124  * particular work request.  Note: the latter piece of information is
125  * particularly useful during completion processing on errored CQEs.
126  */
127 struct tavor_wrid_entry_s {
128 	uint64_t		wr_wrid;
129 	uint32_t		wr_wqeaddrsz;
130 	uint32_t		wr_signaled_dbd;
131 };
132 #define	TAVOR_WRID_ENTRY_SIGNALED	(1 << 0)
133 #define	TAVOR_WRID_ENTRY_DOORBELLED	(1 << 1)
134 
135 /*
136  * The tavor_sw_wqe_dbinfo_t structure is used internally by the Tavor
137  * driver to return information (from the tavor_wqe_mlx_build_nextctl() and
138  * tavor_wqe_send_build_nextctl() routines) regarding the type of Tavor
139  * doorbell necessary.
140  */
141 typedef struct tavor_sw_wqe_dbinfo_s {
142 	uint_t	db_nopcode;
143 	uint_t	db_fence;
144 } tavor_sw_wqe_dbinfo_t;
145 
146 /*
147  * The Work Queue Lock (WQL) structure.  Each WQHDR (tavor_workq_hdr_t defined
148  * below) must lock access to the wridlist during any wridlist manipulation.
149  * Also, any Shared Receive Queue (SRQ) must also be able to lock the wridlist
150  * since it maintains wridlist's differently than normal QPs.  This
151  * 'tavor_wq_lock_t' structure is shared and accessible through the WQ or the
152  * SRQ, and refcnt is maintained.  The last entity to decrement use of the
153  * lock, also will free up the memory.
154  */
155 struct tavor_wq_lock_s {
156 	kmutex_t	wql_lock;
157 	uint_t		wql_refcnt;
158 };
159 
160 /*
161  * The tavor_wrid_list_hdr_s structure is used internally by the Tavor driver
162  * to track all the information necessary to manage a queue of WRID entries
163  * (the tavor_wrid_entry_s struct above).
164  * It contains some information regarding the status of a given WRID list
165  * (e.g. head index, tail index, queue full condition, etc.).  Note:  Although
166  * some of this information is also kept by the tavor_workq_hdr_s below, what
167  * is kept here may, in fact, represent the state of an old WRID list.  It
168  * could be different from what is kept in the tavor_workq_hdr_s because this
169  * WRID list may no longer be the active WRID list.  If it is an active list,
170  * however, then both sets of information should be up-to-date and consistent.
171  * Several of these structures are chained together on each work queue header
172  * to form a linked list (using the "wl_next" and "wl_prev").  These structs,
173  * in turn, each have a pointer to a queue of WRID entries.  They also each
174  * have a pointer to the next "reapable" entry ("wl_reap_next") which is only
175  * used when a WRID list has been retired and is ready to be freed up.
176  * Lastly, it has a backpointer to the work queue header to which the WRID
177  * list belongs (this is for proper handling on removal).
178  */
179 struct tavor_wrid_list_hdr_s {
180 	tavor_wrid_list_hdr_t	*wl_next;
181 	tavor_wrid_list_hdr_t	*wl_prev;
182 	tavor_wrid_list_hdr_t	*wl_reap_next;
183 	tavor_workq_hdr_t	*wl_wqhdr;
184 
185 	tavor_wrid_entry_t	*wl_wre;
186 	tavor_wrid_entry_t	*wl_wre_old_tail;
187 	uint32_t		wl_size;
188 	uint32_t		wl_full;
189 	uint32_t		wl_head;
190 	uint32_t		wl_tail;
191 
192 	/* For SRQ */
193 	uint_t			wl_srq_en;
194 	int			wl_free_list_indx;
195 	ddi_acc_handle_t	wl_acchdl;
196 	uint32_t		*wl_srq_wq_buf;
197 	uint32_t		wl_srq_wq_bufsz;
198 	uint64_t		wl_srq_desc_off;
199 	uint32_t		wl_srq_log_wqesz;
200 };
201 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
202     tavor_wrid_list_hdr_s::wl_next
203     tavor_wrid_list_hdr_s::wl_prev
204     tavor_wrid_list_hdr_s::wl_wqhdr))
205 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock,
206     tavor_wrid_list_hdr_s::wl_wre
207     tavor_wrid_list_hdr_s::wl_wre_old_tail
208     tavor_wrid_list_hdr_s::wl_size
209     tavor_wrid_list_hdr_s::wl_full
210     tavor_wrid_list_hdr_s::wl_head
211     tavor_wrid_list_hdr_s::wl_tail
212     tavor_wrid_list_hdr_s::wl_srq_en
213     tavor_wrid_list_hdr_s::wl_free_list_indx
214     tavor_wrid_list_hdr_s::wl_acchdl
215     tavor_wrid_list_hdr_s::wl_srq_wq_buf
216     tavor_wrid_list_hdr_s::wl_srq_desc_off
217     tavor_wrid_list_hdr_s::wl_srq_log_wqesz))
218 
219 /*
220  * The tavor_workq_hdr_s structure is used internally by the Tavor driver to
221  * track all the information necessary to manage the work queues associated
222  * with a given completion queue.  It contains much of the information
223  * regarding the status of a given work queue (e.g. head index, tail index,
224  * queue full condition, etc.).  Note:  This information is kept here (i.e.
225  * associated with a completion queue) rather than as part of the QP because
226  * the queue pair may potentially be destroyed while outstanding CQEs still
227  * remain on the CQ.
228  * Several of these structures are chained together on each CQ to form a
229  * linked list (using the "wq_next" and "wq_prev").  These headers, in turn,
230  * link to the containers for the individual WRID entries (managed with the
231  * tavor_wrid_list_hdr_s structs above).  Note: We keep a list of these
232  * tavor_wrid_list_hdr_s because a given QP may be used, destroyed (or
233  * transition to "Reset"), and then reused.  The list helps us track where
234  * to put new WRID entries and where to pull old entries from.
235  * The "wq_qpn" (QP number) and "wq_send_or_recv" (TAVOR_WR_SEND or
236  * TAVOR_WR_RECV) are used to uniquely identify the given work queue.
237  * Lookups into the work queue list (to find a given work queue) will use
238  * these two fields as identifiers.
239  */
240 struct tavor_workq_hdr_s {
241 	avl_node_t		wq_avl_link;
242 	uint32_t		wq_qpn;
243 	uint32_t		wq_type;
244 
245 	tavor_wq_lock_t		*wq_wrid_wql;
246 
247 	uint32_t		wq_size;
248 	uint32_t		wq_head;
249 	uint32_t		wq_tail;
250 	uint32_t		wq_full;
251 	tavor_wrid_list_hdr_t	*wq_wrid_poll;
252 	tavor_wrid_list_hdr_t	*wq_wrid_post;
253 };
254 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
255     tavor_workq_hdr_s::wq_avl_link
256     tavor_workq_hdr_s::wq_qpn
257     tavor_workq_hdr_s::wq_type
258     tavor_sw_cq_s::cq_wrid_reap_head
259     tavor_sw_cq_s::cq_wrid_reap_tail))
260 _NOTE(MUTEX_PROTECTS_DATA(tavor_wq_lock_s::wql_lock,
261     tavor_workq_hdr_s::wq_size
262     tavor_workq_hdr_s::wq_head
263     tavor_workq_hdr_s::wq_tail
264     tavor_workq_hdr_s::wq_full
265     tavor_workq_hdr_s::wq_wrid_poll
266     tavor_workq_hdr_s::wq_wrid_post
267     tavor_wrid_list_hdr_s::wl_wre
268     tavor_wrid_list_hdr_s::wl_wre_old_tail
269     tavor_wrid_list_hdr_s::wl_size
270     tavor_wrid_list_hdr_s::wl_full
271     tavor_wrid_list_hdr_s::wl_head
272     tavor_wrid_list_hdr_s::wl_tail))
273 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_wrid_wqhdr_lock,
274     tavor_wrid_list_hdr_s::wl_reap_next))
275 _NOTE(LOCK_ORDER(tavor_sw_cq_s::cq_lock
276     tavor_sw_cq_s::cq_wrid_wqhdr_lock
277     tavor_wq_lock_s::wql_lock))
278 #define	TAVOR_WR_RECV			0x0
279 #define	TAVOR_WR_SEND			0x1
280 #define	TAVOR_WR_SRQ			0x2
281 
282 extern int tavor_wrid_wqhdr_compare(const void *p1, const void *p2);
283 typedef struct tavor_workq_compare_s {
284 	uint32_t cmp_type;
285 	uint32_t cmp_qpn;
286 } tavor_workq_compare_t;
287 
288 /* For Work Request posting */
289 int tavor_post_send(tavor_state_t *state, tavor_qphdl_t qphdl,
290     ibt_send_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
291 int tavor_post_recv(tavor_state_t *state, tavor_qphdl_t qphdl,
292     ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
293 int tavor_post_srq(tavor_state_t *state, tavor_srqhdl_t srqhdl,
294     ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
295 
296 /* For WRID handling */
297 int tavor_wrid_from_reset_handling(tavor_state_t *state, tavor_qphdl_t qp);
298 void tavor_wrid_to_reset_handling(tavor_state_t *state, tavor_qphdl_t qp);
299 void tavor_wrid_add_entry(tavor_workq_hdr_t *wq, uint64_t wrid,
300     uint32_t wqeaddr_sz, uint_t signaled_dbd);
301 void tavor_wrid_add_entry_srq(tavor_srqhdl_t srq, uint64_t wrid,
302     uint_t signaled_dbd);
303 uint64_t tavor_wrid_get_entry(tavor_cqhdl_t cqhdl, tavor_hw_cqe_t *cqe,
304     tavor_wrid_entry_t *wre);
305 tavor_wq_lock_t *tavor_wrid_wql_create(tavor_state_t *state);
306 tavor_wrid_list_hdr_t *tavor_wrid_get_list(uint32_t size);
307 void tavor_wrid_list_srq_init(tavor_wrid_list_hdr_t *r_wridlist,
308     tavor_srqhdl_t srq, uint_t wq_start);
309 void tavor_wrid_cq_reap(tavor_cqhdl_t cq);
310 void tavor_wrid_cq_force_reap(tavor_cqhdl_t cq);
311 void tavor_wql_refcnt_dec(tavor_wq_lock_t *wq_lock);
312 void tavor_wql_refcnt_inc(tavor_wq_lock_t *wq_lock);
313 tavor_wrid_entry_t *tavor_wrid_find_match_srq(tavor_wrid_list_hdr_t *wq,
314     tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe);
315 
316 #ifdef __cplusplus
317 }
318 #endif
319 
320 #endif	/* _SYS_IB_ADAPTERS_TAVOR_WR_H */
321