xref: /illumos-gate/usr/src/uts/common/sys/ib/adapters/hermon/hermon_srq.h (revision 9e39c5ba00a55fa05777cc94b148296af305e135)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_HERMON_SRQ_H
28 #define	_SYS_IB_ADAPTERS_HERMON_SRQ_H
29 
30 /*
31  * hermon_srq.h
32  *    Contains all of the prototypes, #defines, and structures necessary
33  *    for the Shared Receive Queue Processing routines.
34  *
35  *    (including those routines directly exposed through the IBTF CI
36  *    interface).
37  */
38 
39 #include <sys/types.h>
40 #include <sys/conf.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
48 /*
49  * The following defines the default number of Shared Receive Queues (SRQ).
50  * This value is controllable via the "hermon_log_num_srq" configuration
51  * variable.
52  * We also have a define for the minimum size of a SRQ.  SRQs allocated with
53  * size 0, 1, 2, or 3 will always get back a SRQ of size 4.
54  */
55 #define	HERMON_NUM_SRQ_SHIFT		0x10
56 #define	HERMON_SRQ_MIN_SIZE		0x4
57 
58 /*
59  * The hermon firmware currently limits an SRQ to maximum of 31 SGL
60  * per WQE (WQE size is 512 bytes or less).  With a WQE size of 256
61  * (SGL 15 or less) no problems are seen.  We set SRQ_MAX_SGL size here, for
62  * use in the config profile to be 0xF.
63  */
64 #define	HERMON_SRQ_MAX_SGL		0xF
65 
66 /*
67  * SRQ States as defined by Hermon.
68  */
69 #define	HERMON_SRQ_STATE_SW_OWNER	0xF
70 #define	HERMON_SRQ_STATE_HW_OWNER	0x0
71 #define	HERMON_SRQ_STATE_ERROR		0x1
72 
73 /*
74  * The hermon_sw_srq_s structure is also referred to using the "hermon_srqhdl_t"
75  * typedef (see hermon_typedef.h).  It encodes all the information necessary
76  * to track the various resources needed to allocate, initialize, query, modify,
77  * post, and (later) free a shared receive queue (SRQ).
78  */
79 struct hermon_sw_srq_s {
80 	kmutex_t		srq_lock;
81 	uint_t			srq_state;
82 	uint_t			srq_srqnum;
83 	hermon_pdhdl_t		srq_pdhdl;
84 	hermon_mrhdl_t		srq_mrhdl;
85 	uint_t			srq_is_umap;
86 	uint32_t		srq_uarpg;
87 	devmap_cookie_t		srq_umap_dhp;
88 
89 	ibt_srq_sizes_t		srq_real_sizes;
90 	hermon_rsrc_t		*srq_srqcrsrcp;
91 	hermon_rsrc_t		*srq_rsrcp;
92 	void			*srq_hdlrarg;
93 	uint_t			srq_refcnt;
94 
95 	/* Work Queue */
96 	hermon_workq_hdr_t	*srq_wq_wqhdr;
97 	uint32_t		*srq_wq_buf;
98 	uint32_t		srq_wq_bufsz;
99 	uint32_t		srq_wq_log_wqesz;
100 	uint32_t		srq_wq_sgl;
101 	uint32_t		srq_wq_wqecntr;
102 
103 	/* DoorBell Record information */
104 	ddi_acc_handle_t	srq_wq_dbr_acchdl;
105 	hermon_dbr_t		*srq_wq_vdbr;
106 	uint64_t		srq_wq_pdbr;
107 	uint64_t		srq_rdbr_mapoffset;	/* user mode access */
108 
109 	/* For zero-based */
110 	uint64_t		srq_desc_off;
111 
112 	/* Queue Memory for SRQ */
113 	struct hermon_qalloc_info_s	srq_wqinfo;
114 };
115 _NOTE(READ_ONLY_DATA(hermon_sw_srq_s::srq_pdhdl
116     hermon_sw_srq_s::srq_mrhdl
117     hermon_sw_srq_s::srq_srqnum
118     hermon_sw_srq_s::srq_wq_sgl
119     hermon_sw_srq_s::srq_srqcrsrcp
120     hermon_sw_srq_s::srq_rsrcp
121     hermon_sw_srq_s::srq_hdlrarg
122     hermon_sw_srq_s::srq_is_umap
123     hermon_sw_srq_s::srq_uarpg))
124 _NOTE(DATA_READABLE_WITHOUT_LOCK(hermon_sw_srq_s::srq_wq_bufsz
125     hermon_sw_srq_s::srq_wqinfo
126     hermon_sw_srq_s::srq_wq_buf
127     hermon_sw_srq_s::srq_wq_wqhdr
128     hermon_sw_srq_s::srq_desc_off))
129 _NOTE(MUTEX_PROTECTS_DATA(hermon_sw_srq_s::srq_lock,
130     hermon_sw_srq_s::srq_real_sizes
131     hermon_sw_srq_s::srq_umap_dhp))
132 
133 /*
134  * The hermon_srq_info_t structure is used internally by the Hermon driver to
135  * pass information to and from the hermon_srq_alloc() routine.  It contains
136  * placeholders for all of the potential inputs and outputs that this routine
137  * can take.
138  */
139 typedef struct hermon_srq_info_s {
140 	hermon_pdhdl_t		srqi_pd;
141 	ibt_srq_hdl_t		srqi_ibt_srqhdl;
142 	ibt_srq_sizes_t		*srqi_sizes;
143 	ibt_srq_sizes_t		*srqi_real_sizes;
144 	hermon_srqhdl_t		*srqi_srqhdl;
145 	uint_t			srqi_flags;
146 } hermon_srq_info_t;
147 
148 /*
149  * The hermon_srq_options_t structure is used in the Hermon SRQ allocation
150  * routines to provide additional option functionality.  When a NULL pointer
151  * is passed in place of a pointer to this struct, it is a way of specifying
152  * the "default" behavior.  Using this structure, however, is a way of
153  * controlling any extended behavior.
154  */
155 typedef struct hermon_srq_options_s {
156 	uint_t			srqo_wq_loc;
157 } hermon_srq_options_t;
158 
159 /*
160  * old call
161  * int hermon_srq_alloc(hermon_state_t *state, hermon_srq_info_t *srqinfo,
162  *  uint_t sleepflag, hermon_srq_options_t *op);
163  */
164 
165 int hermon_srq_alloc(hermon_state_t *state, hermon_srq_info_t *srqinfo,
166     uint_t sleepflag);
167 int hermon_srq_free(hermon_state_t *state, hermon_srqhdl_t *srqhdl,
168     uint_t sleepflag);
169 int hermon_srq_modify(hermon_state_t *state, hermon_srqhdl_t srq,
170     uint_t size, uint_t *real_size, uint_t sleepflag);
171 int hermon_srq_post(hermon_state_t *state, hermon_srqhdl_t srq,
172     ibt_recv_wr_t *wr_p, uint_t num_wr, uint_t *num_posted);
173 void hermon_srq_refcnt_inc(hermon_srqhdl_t srq);
174 void hermon_srq_refcnt_dec(hermon_srqhdl_t srq);
175 hermon_srqhdl_t hermon_srqhdl_from_srqnum(hermon_state_t *state, uint_t srqnum);
176 
177 #ifdef __cplusplus
178 }
179 #endif
180 
181 #endif	/* _SYS_IB_ADAPTERS_HERMON_SRQ_H */
182