xref: /illumos-gate/usr/src/uts/common/sys/ib/adapters/tavor/tavor_cq.h (revision 55fea89dcaa64928bed4327112404dcb3e07b79f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_TAVOR_CQ_H
28 #define	_SYS_IB_ADAPTERS_TAVOR_CQ_H
29 
30 /*
31  * tavor_cq.h
32  *    Contains all of the prototypes, #defines, and structures necessary
33  *    for the Completion Queue Processing routines.
34  *    Specifically it contains the various completion types, flags,
35  *    structures used for managing Tavor completion queues, and prototypes
36  *    for many of the functions consumed by other parts of the Tavor driver
37  *    (including those routines directly exposed through the IBTF CI
38  *    interface).
39  */
40 
41 #include <sys/types.h>
42 #include <sys/conf.h>
43 #include <sys/ddi.h>
44 #include <sys/sunddi.h>
45 
46 #include <sys/ib/adapters/tavor/tavor_misc.h>
47 
48 #ifdef __cplusplus
49 extern "C" {
50 #endif
51 
52 /*
53  * The following defines specify the default number of Completion Queues (CQ)
54  * their maximum size.  Settings exist for the supported DDR DIMM sizes of
55  * 128MB and 256MB.  If a DIMM greater than 256 is found, then the 256MB
56  * profile is used.  See tavor_cfg.c for more discussion on config profiles.
57  *
58  * For manual configuration (not using config profiles), these values are
59  * controllable through the "tavor_log_max_cq_sz" and "tavor_log_num_cq"
60  * configuration variables, respectively. To override config profile settings
61  * the 'tavor_alt_config_enable' configuration variable must first be set.
62  *
63  * Note: We also have a define for the minimum size of a CQ.  CQs allocated
64  * with size 0, 1, 2, or 3 will always get back a CQ of size 4.  This is the
65  * smallest size that Tavor hardware and software can correctly handle.
66  */
67 #define	TAVOR_NUM_CQ_SHIFT_128		0x10
68 #define	TAVOR_NUM_CQ_SHIFT_256		0x11
69 #define	TAVOR_CQ_SZ_SHIFT		0x10
70 #define	TAVOR_CQ_SZ			(1 << TAVOR_CQ_SZ_SHIFT)
71 #define	TAVOR_CQ_MIN_SIZE		0x3
72 
73 /*
74  * Minimal configuration values.
75  */
76 #define	TAVOR_NUM_CQ_SHIFT_MIN		0xC
77 #define	TAVOR_CQ_SZ_SHIFT_MIN		0xC
78 
79 /*
80  * The following macro determines whether the contents of CQ memory (CQEs)
81  * need to be sync'd (with ddi_dma_sync()).  This decision is based on whether
82  * the CQ memory is in DDR memory (no sync) or system memory (sync required).
83  * Note: It doesn't make much sense to put CQEs in DDR memory (since they are
84  * primarily written by HW and read by the CPU), but the driver does support
85  * that possibility.  And it also supports the possibility that if a CQ in
86  * system memory is mapped DDI_DMA_CONSISTENT, it can be configured to not be
87  * sync'd because of the "sync override" parameter in the config profile.
88  */
89 #define	TAVOR_CQ_IS_SYNC_REQ(state, cqinfo)				\
90 	((((((state)->ts_cfg_profile->cp_streaming_consistent) &&	\
91 	((state)->ts_cfg_profile->cp_consistent_syncoverride))) ||      \
92 	((cqinfo).qa_location == TAVOR_QUEUE_LOCATION_INDDR))    \
93 	? 0 : 1)
94 
95 /*
96  * The following defines specify the size of the individual Completion Queue
97  * Context (CQC) entries
98  */
99 #define	TAVOR_CQC_SIZE_SHIFT		0x6
100 #define	TAVOR_CQC_SIZE			(1 << TAVOR_CQC_SIZE_SHIFT)
101 
102 /*
103  * These are the defines for the Tavor CQ completion statuses.  They are
104  * specified by the Tavor register specification.
105  */
106 #define	TAVOR_CQE_SUCCESS		0x0
107 #define	TAVOR_CQE_LOC_LEN_ERR		0x1
108 #define	TAVOR_CQE_LOC_OP_ERR		0x2
109 #define	TAVOR_CQE_LOC_EEC_ERR		0x3	/* unsupported: RD */
110 #define	TAVOR_CQE_LOC_PROT_ERR		0x4
111 #define	TAVOR_CQE_WR_FLUSHED_ERR	0x5
112 #define	TAVOR_CQE_MW_BIND_ERR		0x6
113 #define	TAVOR_CQE_BAD_RESPONSE_ERR	0x10
114 #define	TAVOR_CQE_LOCAL_ACCESS_ERR	0x11
115 #define	TAVOR_CQE_REM_INV_REQ_ERR	0x12
116 #define	TAVOR_CQE_REM_ACC_ERR		0x13
117 #define	TAVOR_CQE_REM_OP_ERR		0x14
118 #define	TAVOR_CQE_TRANS_TO_ERR		0x15
119 #define	TAVOR_CQE_RNRNAK_TO_ERR		0x16
120 #define	TAVOR_CQE_LOCAL_RDD_VIO_ERR	0x20	/* unsupported: RD */
121 #define	TAVOR_CQE_REM_INV_RD_REQ_ERR	0x21	/* unsupported: RD */
122 #define	TAVOR_CQE_EEC_REM_ABORTED_ERR	0x22	/* unsupported: RD */
123 #define	TAVOR_CQE_INV_EEC_NUM_ERR	0x23	/* unsupported: RD */
124 #define	TAVOR_CQE_INV_EEC_STATE_ERR	0x24	/* unsupported: RD */
125 
126 /*
127  * These are the defines for the Tavor CQ entry types.  They are also
128  * specified by the Tavor register specification.  They indicate what type
129  * of work request is completing (for successful completions).  Note: The
130  * "SND" or "RCV" in each define is used to indicate whether the completion
131  * work request was from the Send work queue or the Receive work queue on
132  * the associated QP.
133  */
134 #define	TAVOR_CQE_SND_RDMAWR		0x8
135 #define	TAVOR_CQE_SND_RDMAWR_IMM	0x9
136 #define	TAVOR_CQE_SND_SEND		0xA
137 #define	TAVOR_CQE_SND_SEND_IMM		0xB
138 #define	TAVOR_CQE_SND_RDMARD		0x10
139 #define	TAVOR_CQE_SND_ATOMIC_CS		0x11
140 #define	TAVOR_CQE_SND_ATOMIC_FA		0x12
141 #define	TAVOR_CQE_SND_BIND_MW		0x18
142 #define	TAVOR_CQE_RCV_RECV_IMM		0x3
143 #define	TAVOR_CQE_RCV_RECV_IMM2		0x5
144 #define	TAVOR_CQE_RCV_RECV		0x2
145 #define	TAVOR_CQE_RCV_RECV2		0x4
146 #define	TAVOR_CQE_RCV_RDMAWR_IMM	0x9
147 #define	TAVOR_CQE_RCV_RDMAWR_IMM2	0xB
148 
149 /* Define for maximum CQ number mask (CQ number is 24 bits) */
150 #define	TAVOR_CQ_MAXNUMBER_MSK		0xFFFFFF
151 
152 /*
153  * This define and the following macro are used to find an event queue for a
154  * new CQ based on its completion queue number.  Note:  This is a rather
155  * simple method that we use today.  We simply choose from one of the first
156  * 32 EQs based on the 5 least significant bits of the CQ number.
157  */
158 #define	TAVOR_CQ_TO_EQ_MASK		0x1F
159 #define	TAVOR_CQ_EQNUM_GET(cqnum)	((cqnum) & TAVOR_CQ_TO_EQ_MASK)
160 
161 /*
162  * The following macro is even simpler than the above one.  This is used to
163  * find an event queue for CQ errors for a new CQ.  In theory we could do this
164  * based on the CQ's number (as we do above).  Today, however, all CQ error
165  * events go to one specific EQ (i.e. EQ #32).
166  */
167 #define	TAVOR_CQ_ERREQNUM_GET(cqnum)	0x20
168 
169 /*
170  * The following defines are used for Tavor CQ error handling.  Note: For
171  * CQEs which correspond to error events, the Tavor device requires some
172  * special handling by software.  These defines are used to identify and
173  * extract the necessary information from each error CQE, including status
174  * code (above), doorbell count, and whether a error completion is for a
175  * send or receive work request.
176  */
177 #define	TAVOR_CQE_ERR_STATUS_SHIFT	24
178 #define	TAVOR_CQE_ERR_STATUS_MASK	0xFF
179 #define	TAVOR_CQE_ERR_DBDCNT_MASK	0xFFFF
180 #define	TAVOR_CQE_SEND_ERR_OPCODE	0xFF
181 #define	TAVOR_CQE_RECV_ERR_OPCODE	0xFE
182 #define	TAVOR_CQ_SYNC_AND_DB		0
183 #define	TAVOR_CQ_RECYCLE_ENTRY		1
184 
185 /* Defines for tracking whether a CQ is being used with special QP or not */
186 #define	TAVOR_CQ_IS_NORMAL		0
187 #define	TAVOR_CQ_IS_SPECIAL		1
188 
189 /*
190  * The tavor_sw_cq_s structure is also referred to using the "tavor_cqhdl_t"
191  * typedef (see tavor_typedef.h).  It encodes all the information necessary
192  * to track the various resources needed to allocate, initialize, poll, resize,
193  * and (later) free a completion queue (CQ).
194  *
195  * Specifically, it has a consumer index and a lock to ensure single threaded
196  * access to it.  It has pointers to the various resources allocated for the
197  * completion queue, i.e. a CQC resource and the memory for the completion
198  * queue itself.  It has flags to indicate whether the CQ requires
199  * ddi_dma_sync() ("cq_sync").  It also has a reference count and the number(s)
200  * of the EQs to which it is associated (for success and for errors).
201  *
202  * Additionally, it has a pointer to the associated MR handle (for the mapped
203  * queue memory) and a void pointer that holds the argument that should be
204  * passed back to the IBTF when events are generated on the CQ.
205  *
206  * We also have the always necessary backpointer to the resource for the
207  * CQ handle structure itself.  But we also have pointers to the "Work Request
208  * ID" processing lists (both the lock and the regular list, as well as the
209  * head and tail for the "reapable" list).  See tavor_wrid.c for more details.
210  */
211 struct tavor_sw_cq_s {
212 	kmutex_t		cq_lock;
213 	uint32_t		cq_consindx;
214 	uint32_t		cq_cqnum;
215 	tavor_hw_cqe_t		*cq_buf;
216 	tavor_mrhdl_t		cq_mrhdl;
217 	uint32_t		cq_bufsz;
218 	uint_t			cq_sync;
219 	uint_t			cq_refcnt;
220 	uint32_t		cq_eqnum;
221 	uint32_t		cq_erreqnum;
222 	uint_t			cq_is_special;
223 	uint_t			cq_is_umap;
224 	uint32_t		cq_uarpg;
225 	devmap_cookie_t		cq_umap_dhp;
226 	tavor_rsrc_t		*cq_cqcrsrcp;
227 	tavor_rsrc_t		*cq_rsrcp;
228 
229 	void			*cq_hdlrarg;
230 
231 	/* For Work Request ID processing */
232 	kmutex_t		cq_wrid_wqhdr_lock;
233 	avl_tree_t		cq_wrid_wqhdr_avl_tree;
234 	tavor_wrid_list_hdr_t	*cq_wrid_reap_head;
235 	tavor_wrid_list_hdr_t	*cq_wrid_reap_tail;
236 
237 	struct tavor_qalloc_info_s cq_cqinfo;
238 };
239 _NOTE(READ_ONLY_DATA(tavor_sw_cq_s::cq_cqnum
240     tavor_sw_cq_s::cq_eqnum
241     tavor_sw_cq_s::cq_erreqnum
242     tavor_sw_cq_s::cq_cqcrsrcp
243     tavor_sw_cq_s::cq_rsrcp
244     tavor_sw_cq_s::cq_hdlrarg
245     tavor_sw_cq_s::cq_is_umap
246     tavor_sw_cq_s::cq_uarpg))
247 _NOTE(DATA_READABLE_WITHOUT_LOCK(tavor_sw_cq_s::cq_bufsz
248     tavor_sw_cq_s::cq_cqinfo))
249 _NOTE(MUTEX_PROTECTS_DATA(tavor_sw_cq_s::cq_lock,
250     tavor_sw_cq_s::cq_consindx
251     tavor_sw_cq_s::cq_buf
252     tavor_sw_cq_s::cq_mrhdl
253     tavor_sw_cq_s::cq_sync
254     tavor_sw_cq_s::cq_refcnt
255     tavor_sw_cq_s::cq_is_special
256     tavor_sw_cq_s::cq_umap_dhp))
257 
258 int tavor_cq_alloc(tavor_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
259     ibt_cq_attr_t *attr_p, uint_t *actual_size, tavor_cqhdl_t *cqhdl,
260     uint_t sleepflag);
261 int tavor_cq_free(tavor_state_t *state, tavor_cqhdl_t *cqhdl,
262     uint_t sleepflag);
263 int tavor_cq_resize(tavor_state_t *state, tavor_cqhdl_t cqhdl,
264     uint_t req_size, uint_t *actual_size, uint_t sleepflag);
265 int tavor_cq_notify(tavor_state_t *state, tavor_cqhdl_t cqhdl,
266     ibt_cq_notify_flags_t flags);
267 int tavor_cq_poll(tavor_state_t *state, tavor_cqhdl_t cqhdl, ibt_wc_t *wc_p,
268     uint_t num_wc, uint_t *num_polled);
269 int tavor_cq_handler(tavor_state_t *state, tavor_eqhdl_t eq,
270     tavor_hw_eqe_t *eqe);
271 int tavor_cq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
272     tavor_hw_eqe_t *eqe);
273 int tavor_cq_refcnt_inc(tavor_cqhdl_t cq, uint_t is_special);
274 void tavor_cq_refcnt_dec(tavor_cqhdl_t cq);
275 tavor_cqhdl_t tavor_cqhdl_from_cqnum(tavor_state_t *state, uint_t cqnum);
276 void tavor_cq_srq_entries_flush(tavor_state_t *state, tavor_qphdl_t qp);
277 #ifdef __cplusplus
278 }
279 #endif
280 
281 #endif	/* _SYS_IB_ADAPTERS_TAVOR_CQ_H */
282