1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * sol_uverbs_hca.c
27 *
28 * Provides the Solaris OFED User Verbs thin common hca interface for
29 * sharing of IBT client handle, device list, and asynchronous event
30 * delivery.
31 */
32 #include <sys/vfs.h>
33 #ifdef VFS_OPS
34 #include <sys/vfs_opreg.h>
35 #include <sys/vnode.h>
36 #endif
37 #include <sys/errno.h>
38 #include <sys/cred.h>
39 #include <sys/uio.h>
40 #include <sys/semaphore.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43
44 #include <sys/ib/ibtl/ibvti.h>
45 #include <sys/ib/clients/of/ofa_solaris.h>
46 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_hca.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
49
50 extern char *sol_uverbs_dbg_str;
51
52 /*
53 * Globals for managing the list of HCA's and the registered clients.
54 */
55 kmutex_t sol_uverbs_hca_lock;
56 llist_head_t sol_uverbs_hca_list;
57 llist_head_t sol_uverbs_client_list;
58 static uint32_t sol_uverbs_common_hca_initialized = 0;
59
60 typedef struct sol_uverbs_hca_client_data {
61 llist_head_t list;
62 sol_uverbs_ib_client_t *client;
63 void *data;
64 } sol_uverbs_hca_client_data_t;
65
66 static
67 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
68 sol_uverbs_ib_client_t *client);
69
70 /*
71 * Function:
72 * sol_uverbs_ib_register_client
73 * Input:
74 * client - Pointer to the client structure
75 * Output:
76 * None
77 * Returns:
78 * Zero on success, else error code.
79 * Description:
80 * The Solaris User Verbs kernel agent provides a single
81 * common view of the IBTF devices. This function allows
82 * Solaris OFA kernel implementations to share this view
83 * by registerng a client callback for notification of HCA
84 * addtion and removal. Note that when this function is
85 * called, the client will get an "add" callback for all
86 * existing devices.
87 */
88 int
sol_uverbs_ib_register_client(sol_uverbs_ib_client_t * client)89 sol_uverbs_ib_register_client(sol_uverbs_ib_client_t *client)
90 {
91 llist_head_t *entry;
92 sol_uverbs_hca_t *hca;
93
94 ASSERT(client != NULL);
95 mutex_enter(&sol_uverbs_hca_lock);
96 llist_head_init(&client->list, client);
97 llist_add_tail(&client->list, &sol_uverbs_client_list);
98 list_for_each(entry, &sol_uverbs_hca_list) {
99 hca = (sol_uverbs_hca_t *)entry->ptr;
100
101 if (client->add &&
102 !sol_uverbs_hca_add_client_context(hca, client)) {
103 client->add(hca);
104 }
105 }
106 mutex_exit(&sol_uverbs_hca_lock);
107
108 return (0);
109 }
110
111 /*
112 * Function:
113 * sol_uverbs_ib_unregister_client
114 * Input:
115 * client - Pointer to the client structure
116 * Output:
117 * None
118 * Returns:
119 * None
120 * Description:
121 * Removes a client registration previously created with
122 * the sol_uverbs_ib_register_client() call.
123 */
124 void
sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t * client)125 sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t *client)
126 {
127 llist_head_t *entry, *centry, *tmp;
128 sol_uverbs_hca_t *hca;
129 sol_uverbs_hca_client_data_t *context;
130
131 ASSERT(client != NULL);
132 mutex_enter(&sol_uverbs_hca_lock);
133
134 list_for_each(entry, &sol_uverbs_hca_list) {
135 hca = (sol_uverbs_hca_t *)entry->ptr;
136
137 ASSERT(hca != NULL);
138
139 if (client->remove) {
140 client->remove(hca);
141 }
142 mutex_enter(&hca->client_data_lock);
143 centry = hca->client_data_list.nxt;
144 tmp = centry->nxt;
145
146 while (centry != &hca->client_data_list) {
147 ASSERT(centry);
148 context = (sol_uverbs_hca_client_data_t *)centry->ptr;
149 ASSERT(context != NULL);
150
151 if (context->client == client) {
152 llist_del(centry);
153 kmem_free(context, sizeof (*context));
154 }
155 centry = tmp;
156 tmp = centry->nxt;
157 }
158 mutex_exit(&hca->client_data_lock);
159 }
160 llist_del(&client->list);
161 mutex_exit(&sol_uverbs_hca_lock);
162 }
163
164 /*
165 * Function:
166 * sol_uverbs_ib_get_client_data
167 * Input:
168 * hca - Pointer to HCA struct passed in the client
169 * add function callback.
170 * client - A pointer to the client structure.
171 * Output:
172 * None
173 * Returns:
174 * The client data, or NULL on error.
175 * Description:
176 * Returns the client data associated with the given
177 * HCA. The data is set/specified via the
178 * sol_uverbs_ib_set_client_data() function.
179 */
180 void *
sol_uverbs_ib_get_client_data(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client)181 sol_uverbs_ib_get_client_data(sol_uverbs_hca_t *hca,
182 sol_uverbs_ib_client_t *client)
183 {
184 llist_head_t *entry;
185 sol_uverbs_hca_client_data_t *context;
186 void *data = NULL;
187
188 ASSERT(hca != NULL);
189 ASSERT(client != NULL);
190
191 mutex_enter(&hca->client_data_lock);
192
193 list_for_each(entry, &hca->client_data_list) {
194 context = (sol_uverbs_hca_client_data_t *)entry->ptr;
195
196 ASSERT(context != NULL);
197
198 if (context->client == client) {
199 data = context->data;
200 break;
201 }
202 }
203 mutex_exit(&hca->client_data_lock);
204 return (data);
205 }
206
207 /*
208 * Function:
209 * sol_uverbs_ib_set_client_data
210 * Input:
211 * hca - Pointer to HCA struct passed in the client
212 * add function.
213 * client - A pointer to the client structure.
214 * data - The client data to associate with the HCA.
215 * Output:
216 * None
217 * Returns:
218 * None
219 * Description:
220 * Sets the client data associated with the given
221 * HCA.
222 */
223 void
sol_uverbs_ib_set_client_data(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client,void * data)224 sol_uverbs_ib_set_client_data(sol_uverbs_hca_t *hca,
225 sol_uverbs_ib_client_t *client, void *data)
226 {
227 llist_head_t *entry;
228 sol_uverbs_hca_client_data_t *context;
229
230 ASSERT(hca != NULL);
231 ASSERT(client != NULL);
232
233 mutex_enter(&hca->client_data_lock);
234
235 list_for_each(entry, &hca->client_data_list) {
236 context = (sol_uverbs_hca_client_data_t *)entry->ptr;
237
238 ASSERT(context != NULL);
239
240 if (context->client == client) {
241 context->data = data;
242 goto out;
243 }
244 }
245 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
246 "HCA SET CLIENT DATA: No client found for %s\n",
247 client->name != NULL ? client->name : "NULL Client Name");
248
249 out:
250 mutex_exit(&hca->client_data_lock);
251 }
252
253 /*
254 * Function:
255 * sol_uverbs_ib_register_event_handler
256 * Input:
257 * handler - Pointer to handler structure
258 * Output:
259 * None
260 * Returns:
261 * Zero
262 * Description:
263 * Register to receive ansynchronous notifications
264 * for the HCA defined in the handler struct. The notifications
265 * are delivered via the callback function defined in the handler
266 * struct.
267 */
268 int
sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t * handler)269 sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t *handler)
270 {
271 ASSERT(handler != NULL);
272 ASSERT(handler->hca != NULL);
273
274 mutex_enter(&handler->hca->event_handler_lock);
275 llist_head_init(&handler->list, handler);
276 llist_add_tail(&handler->list, &handler->hca->event_handler_list);
277 mutex_exit(&handler->hca->event_handler_lock);
278 return (0);
279 }
280
281 /*
282 * Function:
283 * sol_uverbs_ib_unregister_event_handler
284 * Input:
285 * handler - Pointer to handler structure
286 * Output:
287 * None
288 * Returns:
289 * Zero
290 * Description:
291 * Unregister a ansynchronous notification handler previously
292 * registered via the osl_uverbs_ib_register_event_handler() call.
293 */
294 int
sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t * handler)295 sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t *handler)
296 {
297 ASSERT(handler != NULL);
298 ASSERT(handler->hca != NULL);
299
300 mutex_enter(&handler->hca->event_handler_lock);
301 llist_del(&handler->list);
302 mutex_exit(&handler->hca->event_handler_lock);
303 return (0);
304 }
305
306 /*
307 * Function:
308 * sol_uverbs_common_hca_init
309 * Input:
310 * None
311 * Output:
312 * None
313 * Returns:
314 * Zero
315 * Description:
316 * Perform initialization required by the common hca client API.
317 */
318 int
sol_uverbs_common_hca_init()319 sol_uverbs_common_hca_init()
320 {
321 llist_head_init(&sol_uverbs_hca_list, NULL);
322 llist_head_init(&sol_uverbs_client_list, NULL);
323 mutex_init(&sol_uverbs_hca_lock, NULL, MUTEX_DRIVER, NULL);
324 sol_uverbs_common_hca_initialized = 1;
325 return (0);
326 }
327
328 /*
329 * Function:
330 * sol_uverbs_common_hca_fini
331 * Input:
332 * None
333 * Output:
334 * None
335 * Returns:
336 * None
337 * Description:
338 * Perform cleanup required by the common hca client API.
339 */
340 void
sol_uverbs_common_hca_fini()341 sol_uverbs_common_hca_fini()
342 {
343 ASSERT(llist_empty(&sol_uverbs_client_list));
344 sol_uverbs_common_hca_initialized = 0;
345 mutex_destroy(&sol_uverbs_hca_lock);
346 }
347
348 /*
349 * Helpers for internal use only
350 */
351 /*
352 * Function:
353 * sol_uverbs_hca_add_client_context
354 * Input:
355 * hca - Pointer to the hca struct to add a client context.
356 * client - Pointer to the client.
357 * Output:
358 * None
359 * Returns:
360 * 0 on success, else the error.
361 * Description:
362 * Create a context for the specified client and attach it to
363 * the specified hca.
364 */
365 static
sol_uverbs_hca_add_client_context(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client)366 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
367 sol_uverbs_ib_client_t *client)
368 {
369 sol_uverbs_hca_client_data_t *context;
370
371 context = kmem_zalloc(sizeof (*context), KM_NOSLEEP);
372
373 if (!context) {
374 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
375 "HCA: Couldn't allocate client context for %s",
376 client->name ? client->name : "Name is NULL");
377 return (ENOMEM);
378 }
379
380 context->client = client;
381 context->data = NULL;
382 llist_head_init(&context->list, context);
383
384 mutex_enter(&hca->client_data_lock);
385 llist_add(&context->list, &hca->client_data_list);
386 mutex_exit(&hca->client_data_lock);
387 return (0);
388 }
389
390 /*
391 * Function:
392 * sol_uverbs_ibt_hdl_to_hca
393 * Input:
394 * hca_hdl - IBT handle to an HCA.
395 * Output:
396 * None
397 * Returns:
398 * A pointer to the sol_uverbs HCA structure associated with the handle,
399 * or NULL if no associated HCA is found.
400 * Description:
401 * Given an IBT hca handle, return the user verbs HCA structure associated
402 * with that handle.
403 */
404 sol_uverbs_hca_t *
sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl)405 sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl)
406 {
407 llist_head_t *entry;
408 sol_uverbs_hca_t *hca;
409 sol_uverbs_hca_t *ret = NULL;
410
411 mutex_enter(&sol_uverbs_hca_lock);
412 list_for_each(entry, &sol_uverbs_hca_list) {
413 hca = (sol_uverbs_hca_t *)entry->ptr;
414
415 if (hca->hdl == hca_hdl) {
416 ret = hca;
417 break;
418 }
419 }
420 mutex_exit(&sol_uverbs_hca_lock);
421
422 return (ret);
423 }
424