1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
24 */
25
26 /*
27 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31 /*
32 *
33 * MODULE: dapl_osd.c
34 *
35 * PURPOSE: Operating System Dependent layer
36 * Description:
37 * Provide OS dependent functions with a canonical DAPL
38 * interface. Designed to be portable and hide OS specific quirks
39 * of common functions.
40 *
41 *
42 * $Id: dapl_osd.c,v 1.26 2003/07/31 14:04:18 jlentini Exp $
43 */
44
45 #include "dapl_osd.h"
46 #include "dapl.h"
47 #include "dapl_hca_util.h"
48 #include "dapl_ia_util.h"
49 #include "dapl_rmr_util.h"
50 #include "dapl_lmr_util.h"
51 #include "dapl_pz_util.h"
52 #include "dapl_ep_util.h"
53 #include "dapl_cr_util.h"
54 #include "dapl_evd_util.h"
55 #include "dapl_sp_util.h"
56 #include "dapl_adapter_util.h"
57 #include "dapl_provider.h"
58 #include "dapl_hash.h"
59 #include "dapl_debug.h"
60
61 #include <sys/time.h>
62 #include <stdlib.h> /* needed for getenv() */
63 #include <pthread.h> /* needed for pthread_atfork() */
64 #include <signal.h> /* needed for thread setup */
65
66 static void dapls_osd_fork_cleanup(void);
67
68 /*
69 * dapl_osd_init
70 *
71 * Do Linux initialization:
72 * - Set up fork handler to clean up DAPL resources in the child
73 * process after a fork().
74 *
75 * Input:
76 * none
77 *
78 * Returns:
79 * DAT_SUCCESS
80 */
81 void
dapl_os_init()82 dapl_os_init()
83 {
84 int status;
85
86 /*
87 * Set up fork control
88 */
89 status = pthread_atfork(NULL, NULL, dapls_osd_fork_cleanup);
90 if (status != 0) {
91 dapl_dbg_log(DAPL_DBG_TYPE_WARN,
92 "WARNING: pthread_atfork %d\n", status);
93 }
94 }
95
96
97 /*
98 * dapl_os_get_time
99 *
100 * Return 64 bit value of current time in microseconds.
101 *
102 * Input:
103 * loc User location to place current time
104 *
105 * Returns:
106 * DAT_SUCCESS
107 */
108
109 DAT_RETURN
dapl_os_get_time(OUT DAPL_OS_TIMEVAL * loc)110 dapl_os_get_time(
111 OUT DAPL_OS_TIMEVAL * loc)
112 {
113 struct timeval tv;
114 struct timezone tz;
115
116
117 (void) gettimeofday(&tv, &tz);
118 *loc = ((DAT_UINT64)(tv.tv_sec) * 1000000L) + (DAT_UINT64) tv.tv_usec;
119
120 return (DAT_SUCCESS);
121 }
122
123
124 /*
125 * dapl_os_get__env_bool
126 *
127 * Return boolean value of passed in environment variable: 1 if present,
128 * 0 if not
129 *
130 * Input:
131 *
132 *
133 * Returns:
134 * TRUE or FALSE
135 */
136 int
dapl_os_get_env_bool(char * env_str)137 dapl_os_get_env_bool(
138 char *env_str)
139 {
140 char *env_var;
141
142 env_var = getenv(env_str);
143 if (env_var != NULL) {
144 return (1);
145 }
146
147 return (0);
148 }
149
150
151 /*
152 * dapl_os_get_env_val
153 *
154 * Update val to value of passed in environment variable if present
155 *
156 * Input:
157 * env_str
158 * def_val default value if environment variable does not exist
159 *
160 * Returns:
161 * TRUE or FALSE
162 */
163 int
dapl_os_get_env_val(char * env_str,int def_val)164 dapl_os_get_env_val(
165 char *env_str,
166 int def_val)
167 {
168 char *env_var;
169
170 env_var = getenv(env_str);
171 if (env_var != NULL) {
172 def_val = strtol(env_var, NULL, 0);
173 }
174
175 return (def_val);
176 }
177
178
179 /*
180 * Wait object routines
181 */
182
183 /*
184 * dapl_os_wait_object_init
185 *
186 * Initialize a wait object
187 *
188 * Input:
189 * wait_obj
190 *
191 * Returns:
192 * DAT_SUCCESS
193 * DAT_INTERNAL_ERROR
194 */
195 DAT_RETURN
dapl_os_wait_object_init(IN DAPL_OS_WAIT_OBJECT * wait_obj)196 dapl_os_wait_object_init(
197 IN DAPL_OS_WAIT_OBJECT *wait_obj)
198 {
199 wait_obj->signaled = DAT_FALSE;
200 if (0 != pthread_cond_init(&wait_obj->cv, NULL)) {
201 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
202 }
203
204 /* Always returns 0. */
205 (void) pthread_mutex_init(&wait_obj->lock, NULL);
206
207 return (DAT_SUCCESS);
208 }
209
210
211 /*
212 * Wait on the supplied wait object, up to the specified time_out.
213 * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely.
214 * Timeout should be specified in micro seconds.
215 *
216 * Functional returns:
217 * DAT_SUCCESS -- another thread invoked dapl_os_wait object_wakeup
218 * DAT_INVALID_STATE -- someone else is already waiting in this wait
219 * object.
220 * only one waiter is allowed at a time.
221 * DAT_ABORT -- another thread invoked dapl_os_wait_object_destroy
222 * DAT_TIMEOUT -- the specified time limit was reached.
223 */
224
225 DAT_RETURN
dapl_os_wait_object_wait(IN DAPL_OS_WAIT_OBJECT * wait_obj,IN DAT_TIMEOUT timeout_val)226 dapl_os_wait_object_wait(
227 IN DAPL_OS_WAIT_OBJECT *wait_obj,
228 IN DAT_TIMEOUT timeout_val)
229 {
230 DAT_RETURN dat_status;
231 int pthread_status;
232 struct timespec future;
233
234 dat_status = DAT_SUCCESS;
235 pthread_status = 0;
236
237 if (timeout_val != DAT_TIMEOUT_INFINITE) {
238 struct timeval now;
239 struct timezone tz;
240 unsigned int microsecs;
241
242 (void) gettimeofday(&now, &tz);
243 microsecs = now.tv_usec + (timeout_val % 1000000);
244 if (microsecs > 1000000) {
245 now.tv_sec = now.tv_sec + timeout_val / 1000000 + 1;
246 now.tv_usec = microsecs - 1000000;
247 } else {
248 now.tv_sec = now.tv_sec + timeout_val / 1000000;
249 now.tv_usec = microsecs;
250 }
251
252 /* Convert timeval to timespec */
253 future.tv_sec = now.tv_sec;
254 future.tv_nsec = now.tv_usec * 1000;
255
256 (void) pthread_mutex_lock(&wait_obj->lock);
257 while (wait_obj->signaled == DAT_FALSE && pthread_status == 0) {
258 pthread_status = pthread_cond_timedwait(
259 &wait_obj->cv, &wait_obj->lock, &future);
260
261 /*
262 * No need to reset &future if we go around the loop;
263 * It's an absolute time.
264 */
265 }
266 /* Reset the signaled status if we were woken up. */
267 if (pthread_status == 0) {
268 wait_obj->signaled = DAT_FALSE;
269 }
270 (void) pthread_mutex_unlock(&wait_obj->lock);
271 } else {
272 (void) pthread_mutex_lock(&wait_obj->lock);
273 while (wait_obj->signaled == DAT_FALSE && pthread_status == 0) {
274 pthread_status = pthread_cond_wait(
275 &wait_obj->cv, &wait_obj->lock);
276 }
277 /* Reset the signaled status if we were woken up. */
278 if (pthread_status == 0) {
279 wait_obj->signaled = DAT_FALSE;
280 }
281 (void) pthread_mutex_unlock(&wait_obj->lock);
282 }
283
284 if (ETIMEDOUT == pthread_status) {
285 dat_status = DAT_ERROR(DAT_TIMEOUT_EXPIRED, 0);
286 } else if (0 != pthread_status) {
287 dat_status = DAT_ERROR(DAT_INTERNAL_ERROR, 0);
288 }
289
290 return (dat_status);
291 }
292
293
294 /*
295 * dapl_os_wait_object_wakeup
296 *
297 * Wakeup a thread waiting on a wait object
298 *
299 * Input:
300 * wait_obj
301 *
302 * Returns:
303 * DAT_SUCCESS
304 * DAT_INTERNAL_ERROR
305 */
306 DAT_RETURN
dapl_os_wait_object_wakeup(IN DAPL_OS_WAIT_OBJECT * wait_obj)307 dapl_os_wait_object_wakeup(
308 IN DAPL_OS_WAIT_OBJECT *wait_obj)
309 {
310 (void) pthread_mutex_lock(&wait_obj->lock);
311 wait_obj->signaled = DAT_TRUE;
312 (void) pthread_mutex_unlock(&wait_obj->lock);
313 if (0 != pthread_cond_signal(&wait_obj->cv)) {
314 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
315 }
316
317 return (DAT_SUCCESS);
318 }
319
320
321 /*
322 * dapl_os_wait_object_destroy
323 *
324 * Destroy a wait object
325 *
326 * Input:
327 * wait_obj
328 *
329 * Returns:
330 * DAT_SUCCESS
331 * DAT_INTERNAL_ERROR
332 */
333 DAT_RETURN
dapl_os_wait_object_destroy(IN DAPL_OS_WAIT_OBJECT * wait_obj)334 dapl_os_wait_object_destroy(
335 IN DAPL_OS_WAIT_OBJECT *wait_obj)
336 {
337 if (0 != pthread_cond_destroy(&wait_obj->cv)) {
338 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
339 }
340 if (0 != pthread_mutex_destroy(&wait_obj->lock)) {
341 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
342 }
343
344 return (DAT_SUCCESS);
345 }
346
347
348 /*
349 * dapls_osd_fork_cleanup
350 *
351 * Update val to value of passed in environment variable if present
352 *
353 * Input:
354 * env_str
355 * val Updated if environment variable exists
356 *
357 * Returns:
358 * TRUE or FALSE
359 */
360 void
dapls_osd_fork_cleanup(void)361 dapls_osd_fork_cleanup(void)
362 {
363 DAPL_PROVIDER_LIST_NODE *cur_node;
364 DAPL_HCA *hca_ptr;
365 DAPL_IA *ia_ptr;
366 DAPL_LMR *lmr_ptr;
367 DAPL_RMR *rmr_ptr;
368 DAPL_PZ *pz_ptr;
369 DAPL_CR *cr_ptr;
370 DAPL_EP *ep_ptr;
371 DAPL_EVD *evd_ptr;
372 DAT_EP_PARAM *param;
373 DAPL_SP *sp_ptr;
374
375 while (NULL != g_dapl_provider_list.head) {
376 cur_node = g_dapl_provider_list.head;
377 g_dapl_provider_list.head = cur_node->next;
378
379 hca_ptr = (DAPL_HCA *) cur_node->data.extension;
380
381 /*
382 * Walk the list of IA ptrs & clean up. This is purposely
383 * a destructive list walk, we really don't want to preserve
384 * any of it.
385 */
386 while (!dapl_llist_is_empty(&hca_ptr->ia_list_head)) {
387 ia_ptr = (DAPL_IA *)
388 dapl_llist_peek_head(&hca_ptr->ia_list_head);
389
390 /*
391 * The rest of the cleanup code is similar to
392 * dapl_ia_close, the big difference is that we don't
393 * release IB resources, only memory; the underlying IB
394 * subsystem doesn't deal with fork at all, so leave
395 * IB handles alone.
396 */
397 while (!dapl_llist_is_empty(&ia_ptr->rmr_list_head)) {
398 rmr_ptr = (DAPL_RMR *)
399 dapl_llist_peek_head(&ia_ptr->
400 rmr_list_head);
401 if (rmr_ptr->param.lmr_triplet.
402 virtual_address != 0) {
403 dapl_os_atomic_dec(&rmr_ptr->
404 lmr->lmr_ref_count);
405 rmr_ptr->param.lmr_triplet.
406 virtual_address = 0;
407 }
408 dapl_os_atomic_dec(&rmr_ptr->pz->pz_ref_count);
409 dapl_ia_unlink_rmr(rmr_ptr->header.owner_ia,
410 rmr_ptr);
411 dapl_rmr_dealloc(rmr_ptr);
412 }
413
414 while (!dapl_llist_is_empty(&ia_ptr->rsp_list_head)) {
415 sp_ptr = (DAPL_SP *) dapl_llist_peek_head(
416 &ia_ptr->rsp_list_head);
417 dapl_os_atomic_dec(&((DAPL_EVD *)sp_ptr->
418 evd_handle)->evd_ref_count);
419 dapls_ia_unlink_sp(ia_ptr, sp_ptr);
420 dapls_sp_free_sp(sp_ptr);
421 }
422
423 while (!dapl_llist_is_empty(&ia_ptr->ep_list_head)) {
424 ep_ptr = (DAPL_EP *) dapl_llist_peek_head(
425 &ia_ptr->ep_list_head);
426 param = &ep_ptr->param;
427 if (param->pz_handle != NULL) {
428 dapl_os_atomic_dec(&((DAPL_PZ *)param->
429 pz_handle)->pz_ref_count);
430 }
431 if (param->recv_evd_handle != NULL) {
432 dapl_os_atomic_dec(&((DAPL_EVD *)param->
433 recv_evd_handle)->evd_ref_count);
434 }
435 if (param->request_evd_handle) {
436 dapl_os_atomic_dec(&((DAPL_EVD *)param->
437 request_evd_handle)->evd_ref_count);
438 }
439 if (param->connect_evd_handle != NULL) {
440 dapl_os_atomic_dec(&((DAPL_EVD *)param->
441 connect_evd_handle)->evd_ref_count);
442 }
443
444 /* ...and free the resource */
445 dapl_ia_unlink_ep(ia_ptr, ep_ptr);
446 dapl_ep_dealloc(ep_ptr);
447 }
448
449 while (!dapl_llist_is_empty(&ia_ptr->lmr_list_head)) {
450 lmr_ptr = (DAPL_LMR *) dapl_llist_peek_head(
451 &ia_ptr->lmr_list_head);
452
453 (void) dapls_hash_remove(lmr_ptr->header.
454 owner_ia->hca_ptr->lmr_hash_table,
455 lmr_ptr->param.lmr_context, NULL);
456
457 pz_ptr = (DAPL_PZ *) lmr_ptr->param.pz_handle;
458 dapl_os_atomic_dec(&pz_ptr->pz_ref_count);
459 dapl_ia_unlink_lmr(lmr_ptr->header.owner_ia,
460 lmr_ptr);
461 dapl_lmr_dealloc(lmr_ptr);
462 }
463
464 while (!dapl_llist_is_empty(&ia_ptr->psp_list_head)) {
465 sp_ptr = (DAPL_SP *) dapl_llist_peek_head(
466 &ia_ptr->psp_list_head);
467 while (!dapl_llist_is_empty(&sp_ptr->
468 cr_list_head)) {
469 cr_ptr = (DAPL_CR *)
470 dapl_llist_peek_head(
471 &sp_ptr->cr_list_head);
472 dapl_sp_remove_cr(sp_ptr, cr_ptr);
473 dapls_cr_free(cr_ptr);
474 }
475
476 dapls_ia_unlink_sp(ia_ptr, sp_ptr);
477 dapl_os_atomic_dec(&((DAPL_EVD *)sp_ptr->
478 evd_handle)->evd_ref_count);
479 dapls_sp_free_sp(sp_ptr);
480 }
481
482 while (!dapl_llist_is_empty(&ia_ptr->pz_list_head)) {
483 pz_ptr = (DAPL_PZ *)
484 dapl_llist_peek_head(&ia_ptr->pz_list_head);
485 dapl_ia_unlink_pz(pz_ptr->header.owner_ia,
486 pz_ptr);
487 dapl_pz_dealloc(pz_ptr);
488 }
489
490 while (!dapl_llist_is_empty(&ia_ptr->evd_list_head)) {
491 evd_ptr = (DAPL_EVD *) dapl_llist_peek_head(
492 &ia_ptr->evd_list_head);
493 dapl_ia_unlink_evd(evd_ptr->header.owner_ia,
494 evd_ptr);
495 /*
496 * reset the cq_handle to avoid having it
497 * removed
498 */
499 evd_ptr->ib_cq_handle = IB_INVALID_HANDLE;
500 (void) dapls_evd_dealloc(evd_ptr);
501 }
502
503 dapl_hca_unlink_ia(ia_ptr->hca_ptr, ia_ptr);
504 /*
505 * asycn error evd was taken care of above, reset the
506 * pointer
507 */
508 ia_ptr->async_error_evd = NULL;
509 dapls_ia_free(ia_ptr);
510 } /* end while( ia_ptr != NULL ) */
511
512
513 dapl_os_free(cur_node, sizeof (DAPL_PROVIDER_LIST_NODE));
514 } /* end while (NULL != g_dapl_provider_list.head) */
515 }
516
517
518 /*
519 * Local variables:
520 * c-indent-level: 4
521 * c-basic-offset: 4
522 * tab-width: 8
523 * End:
524 */
525