1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 /* 26 * sol_uverbs_hca.c 27 * 28 * Provides the Solaris OFED User Verbs thin common hca interface for 29 * sharing of IBT client handle, device list, and asynchronous event 30 * delivery. 31 */ 32 #include <sys/vfs.h> 33 #ifdef VFS_OPS 34 #include <sys/vfs_opreg.h> 35 #include <sys/vnode.h> 36 #endif 37 #include <sys/errno.h> 38 #include <sys/cred.h> 39 #include <sys/uio.h> 40 #include <sys/semaphore.h> 41 #include <sys/ddi.h> 42 #include <sys/sunddi.h> 43 44 #include <sys/ib/ibtl/ibvti.h> 45 #include <sys/ib/clients/of/ofa_solaris.h> 46 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_hca.h> 48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h> 49 50 extern char *sol_uverbs_dbg_str; 51 52 /* 53 * Globals for managing the list of HCA's and the registered clients. 54 */ 55 kmutex_t sol_uverbs_hca_lock; 56 llist_head_t sol_uverbs_hca_list; 57 llist_head_t sol_uverbs_client_list; 58 static uint32_t sol_uverbs_common_hca_initialized = 0; 59 60 typedef struct sol_uverbs_hca_client_data { 61 llist_head_t list; 62 sol_uverbs_ib_client_t *client; 63 void *data; 64 } sol_uverbs_hca_client_data_t; 65 66 static 67 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca, 68 sol_uverbs_ib_client_t *client); 69 70 /* 71 * Function: 72 * sol_uverbs_ib_register_client 73 * Input: 74 * client - Pointer to the client structure 75 * Output: 76 * None 77 * Returns: 78 * Zero on success, else error code. 79 * Description: 80 * The Solaris User Verbs kernel agent provides a single 81 * common view of the IBTF devices. This function allows 82 * Solaris OFA kernel implementations to share this view 83 * by registerng a client callback for notification of HCA 84 * addtion and removal. Note that when this function is 85 * called, the client will get an "add" callback for all 86 * existing devices. 87 */ 88 int 89 sol_uverbs_ib_register_client(sol_uverbs_ib_client_t *client) 90 { 91 llist_head_t *entry; 92 sol_uverbs_hca_t *hca; 93 94 ASSERT(client != NULL); 95 mutex_enter(&sol_uverbs_hca_lock); 96 llist_head_init(&client->list, client); 97 llist_add_tail(&client->list, &sol_uverbs_client_list); 98 list_for_each(entry, &sol_uverbs_hca_list) { 99 hca = (sol_uverbs_hca_t *)entry->ptr; 100 101 if (client->add && 102 !sol_uverbs_hca_add_client_context(hca, client)) { 103 client->add(hca); 104 } 105 } 106 mutex_exit(&sol_uverbs_hca_lock); 107 108 return (0); 109 } 110 111 /* 112 * Function: 113 * sol_uverbs_ib_unregister_client 114 * Input: 115 * client - Pointer to the client structure 116 * Output: 117 * None 118 * Returns: 119 * None 120 * Description: 121 * Removes a client registration previously created with 122 * the sol_uverbs_ib_register_client() call. 123 */ 124 void 125 sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t *client) 126 { 127 llist_head_t *entry, *centry, *tmp; 128 sol_uverbs_hca_t *hca; 129 sol_uverbs_hca_client_data_t *context; 130 131 ASSERT(client != NULL); 132 mutex_enter(&sol_uverbs_hca_lock); 133 134 list_for_each(entry, &sol_uverbs_hca_list) { 135 hca = (sol_uverbs_hca_t *)entry->ptr; 136 137 ASSERT(hca != NULL); 138 139 if (client->remove) { 140 client->remove(hca); 141 } 142 mutex_enter(&hca->client_data_lock); 143 centry = hca->client_data_list.nxt; 144 tmp = centry->nxt; 145 146 while (centry != &hca->client_data_list) { 147 ASSERT(centry); 148 context = (sol_uverbs_hca_client_data_t *)centry->ptr; 149 ASSERT(context != NULL); 150 151 if (context->client == client) { 152 llist_del(centry); 153 kmem_free(context, sizeof (*context)); 154 } 155 centry = tmp; 156 tmp = centry->nxt; 157 } 158 mutex_exit(&hca->client_data_lock); 159 } 160 llist_del(&client->list); 161 mutex_exit(&sol_uverbs_hca_lock); 162 } 163 164 /* 165 * Function: 166 * sol_uverbs_ib_get_client_data 167 * Input: 168 * hca - Pointer to HCA struct passed in the client 169 * add function callback. 170 * client - A pointer to the client structure. 171 * Output: 172 * None 173 * Returns: 174 * The client data, or NULL on error. 175 * Description: 176 * Returns the client data associated with the given 177 * HCA. The data is set/specified via the 178 * sol_uverbs_ib_set_client_data() function. 179 */ 180 void * 181 sol_uverbs_ib_get_client_data(sol_uverbs_hca_t *hca, 182 sol_uverbs_ib_client_t *client) 183 { 184 llist_head_t *entry; 185 sol_uverbs_hca_client_data_t *context; 186 void *data = NULL; 187 188 ASSERT(hca != NULL); 189 ASSERT(client != NULL); 190 191 mutex_enter(&hca->client_data_lock); 192 193 list_for_each(entry, &hca->client_data_list) { 194 context = (sol_uverbs_hca_client_data_t *)entry->ptr; 195 196 ASSERT(context != NULL); 197 198 if (context->client == client) { 199 data = context->data; 200 break; 201 } 202 } 203 mutex_exit(&hca->client_data_lock); 204 return (data); 205 } 206 207 /* 208 * Function: 209 * sol_uverbs_ib_set_client_data 210 * Input: 211 * hca - Pointer to HCA struct passed in the client 212 * add function. 213 * client - A pointer to the client structure. 214 * data - The client data to associate with the HCA. 215 * Output: 216 * None 217 * Returns: 218 * None 219 * Description: 220 * Sets the client data associated with the given 221 * HCA. 222 */ 223 void 224 sol_uverbs_ib_set_client_data(sol_uverbs_hca_t *hca, 225 sol_uverbs_ib_client_t *client, void *data) 226 { 227 llist_head_t *entry; 228 sol_uverbs_hca_client_data_t *context; 229 230 ASSERT(hca != NULL); 231 ASSERT(client != NULL); 232 233 mutex_enter(&hca->client_data_lock); 234 235 list_for_each(entry, &hca->client_data_list) { 236 context = (sol_uverbs_hca_client_data_t *)entry->ptr; 237 238 ASSERT(context != NULL); 239 240 if (context->client == client) { 241 context->data = data; 242 goto out; 243 } 244 } 245 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 246 "HCA SET CLIENT DATA: No client found for %s\n", 247 client->name != NULL ? client->name : "NULL Client Name"); 248 249 out: 250 mutex_exit(&hca->client_data_lock); 251 } 252 253 /* 254 * Function: 255 * sol_uverbs_ib_register_event_handler 256 * Input: 257 * handler - Pointer to handler structure 258 * Output: 259 * None 260 * Returns: 261 * Zero 262 * Description: 263 * Register to receive ansynchronous notifications 264 * for the HCA defined in the handler struct. The notifications 265 * are delivered via the callback function defined in the handler 266 * struct. 267 */ 268 int 269 sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t *handler) 270 { 271 ASSERT(handler != NULL); 272 ASSERT(handler->hca != NULL); 273 274 mutex_enter(&handler->hca->event_handler_lock); 275 llist_head_init(&handler->list, handler); 276 llist_add_tail(&handler->list, &handler->hca->event_handler_list); 277 mutex_exit(&handler->hca->event_handler_lock); 278 return (0); 279 } 280 281 /* 282 * Function: 283 * sol_uverbs_ib_unregister_event_handler 284 * Input: 285 * handler - Pointer to handler structure 286 * Output: 287 * None 288 * Returns: 289 * Zero 290 * Description: 291 * Unregister a ansynchronous notification handler previously 292 * registered via the osl_uverbs_ib_register_event_handler() call. 293 */ 294 int 295 sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t *handler) 296 { 297 ASSERT(handler != NULL); 298 ASSERT(handler->hca != NULL); 299 300 mutex_enter(&handler->hca->event_handler_lock); 301 llist_del(&handler->list); 302 mutex_exit(&handler->hca->event_handler_lock); 303 return (0); 304 } 305 306 /* 307 * Function: 308 * sol_uverbs_common_hca_init 309 * Input: 310 * None 311 * Output: 312 * None 313 * Returns: 314 * Zero 315 * Description: 316 * Perform initialization required by the common hca client API. 317 */ 318 int 319 sol_uverbs_common_hca_init() 320 { 321 llist_head_init(&sol_uverbs_hca_list, NULL); 322 llist_head_init(&sol_uverbs_client_list, NULL); 323 mutex_init(&sol_uverbs_hca_lock, NULL, MUTEX_DRIVER, NULL); 324 sol_uverbs_common_hca_initialized = 1; 325 return (0); 326 } 327 328 /* 329 * Function: 330 * sol_uverbs_common_hca_fini 331 * Input: 332 * None 333 * Output: 334 * None 335 * Returns: 336 * None 337 * Description: 338 * Perform cleanup required by the common hca client API. 339 */ 340 void 341 sol_uverbs_common_hca_fini() 342 { 343 ASSERT(llist_empty(&sol_uverbs_client_list)); 344 sol_uverbs_common_hca_initialized = 0; 345 mutex_destroy(&sol_uverbs_hca_lock); 346 } 347 348 /* 349 * Helpers for internal use only 350 */ 351 /* 352 * Function: 353 * sol_uverbs_hca_add_client_context 354 * Input: 355 * hca - Pointer to the hca struct to add a client context. 356 * client - Pointer to the client. 357 * Output: 358 * None 359 * Returns: 360 * 0 on success, else the error. 361 * Description: 362 * Create a context for the specified client and attach it to 363 * the specified hca. 364 */ 365 static 366 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca, 367 sol_uverbs_ib_client_t *client) 368 { 369 sol_uverbs_hca_client_data_t *context; 370 371 context = kmem_zalloc(sizeof (*context), KM_NOSLEEP); 372 373 if (!context) { 374 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 375 "HCA: Couldn't allocate client context for %s", 376 client->name ? client->name : "Name is NULL"); 377 return (ENOMEM); 378 } 379 380 context->client = client; 381 context->data = NULL; 382 llist_head_init(&context->list, context); 383 384 mutex_enter(&hca->client_data_lock); 385 llist_add(&context->list, &hca->client_data_list); 386 mutex_exit(&hca->client_data_lock); 387 return (0); 388 } 389 390 /* 391 * Function: 392 * sol_uverbs_ibt_hdl_to_hca 393 * Input: 394 * hca_hdl - IBT handle to an HCA. 395 * Output: 396 * None 397 * Returns: 398 * A pointer to the sol_uverbs HCA structure associated with the handle, 399 * or NULL if no associated HCA is found. 400 * Description: 401 * Given an IBT hca handle, return the user verbs HCA structure associated 402 * with that handle. 403 */ 404 sol_uverbs_hca_t * 405 sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl) 406 { 407 llist_head_t *entry; 408 sol_uverbs_hca_t *hca; 409 sol_uverbs_hca_t *ret = NULL; 410 411 mutex_enter(&sol_uverbs_hca_lock); 412 list_for_each(entry, &sol_uverbs_hca_list) { 413 hca = (sol_uverbs_hca_t *)entry->ptr; 414 415 if (hca->hdl == hca_hdl) { 416 ret = hca; 417 break; 418 } 419 } 420 mutex_exit(&sol_uverbs_hca_lock); 421 422 return (ret); 423 } 424