1 /* 2 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 /* 37 * Abstract: 38 * Declaration of dispatcher abstraction. 39 */ 40 41 #ifndef _CL_DISPATCHER_H_ 42 #define _CL_DISPATCHER_H_ 43 44 #include <complib/cl_atomic.h> 45 #include <complib/cl_threadpool.h> 46 #include <complib/cl_qlist.h> 47 #include <complib/cl_qpool.h> 48 #include <complib/cl_spinlock.h> 49 #include <complib/cl_ptr_vector.h> 50 51 #ifdef __cplusplus 52 # define BEGIN_C_DECLS extern "C" { 53 # define END_C_DECLS } 54 #else /* !__cplusplus */ 55 # define BEGIN_C_DECLS 56 # define END_C_DECLS 57 #endif /* __cplusplus */ 58 59 BEGIN_C_DECLS 60 /****h* Component Library/Dispatcher 61 * NAME 62 * Dispatcher 63 * 64 * DESCRIPTION 65 * The Dispatcher provides a facility for message routing to 66 * asynchronous worker threads. 67 * 68 * The Dispatcher functions operate on a cl_dispatcher_t structure 69 * which should be treated as opaque and should be manipulated 70 * only through the provided functions. 71 * 72 * SEE ALSO 73 * Structures: 74 * cl_dispatcher_t 75 * 76 * Initialization/Destruction: 77 * cl_disp_construct, cl_disp_init, cl_disp_shutdown, cl_disp_destroy 78 * 79 * Manipulation: 80 * cl_disp_post, cl_disp_reset, cl_disp_wait_on 81 *********/ 82 /****s* Component Library: Dispatcher/cl_disp_msgid_t 83 * NAME 84 * cl_disp_msgid_t 85 * 86 * DESCRIPTION 87 * Defines the type of dispatcher messages. 88 * 89 * SYNOPSIS 90 */ 91 typedef uint32_t cl_disp_msgid_t; 92 /**********/ 93 94 /****s* Component Library: Dispatcher/CL_DISP_MSGID_NONE 95 * NAME 96 * CL_DISP_MSGID_NONE 97 * 98 * DESCRIPTION 99 * Defines a message value that means "no message". 100 * This value is used during registration by Dispatcher clients 101 * that do not wish to receive messages. 102 * 103 * No Dispatcher message is allowed to have this value. 104 * 105 * SYNOPSIS 106 */ 107 #define CL_DISP_MSGID_NONE 0xFFFFFFFF 108 /**********/ 109 110 /****s* Component Library: Dispatcher/CL_DISP_INVALID_HANDLE 111 * NAME 112 * CL_DISP_INVALID_HANDLE 113 * 114 * DESCRIPTION 115 * Defines the value of an invalid Dispatcher registration handle. 116 * 117 * SYNOPSIS 118 */ 119 #define CL_DISP_INVALID_HANDLE ((cl_disp_reg_handle_t)0) 120 /*********/ 121 122 /****f* Component Library: Dispatcher/cl_pfn_msgrcv_cb_t 123 * NAME 124 * cl_pfn_msgrcv_cb_t 125 * 126 * DESCRIPTION 127 * This typedef defines the prototype for client functions invoked 128 * by the Dispatcher. The Dispatcher calls the corresponding 129 * client function when delivering a message to the client. 130 * 131 * The client function must be reentrant if the user creates a 132 * Dispatcher with more than one worker thread. 133 * 134 * SYNOPSIS 135 */ 136 typedef void 137 (*cl_pfn_msgrcv_cb_t) (IN void *context, IN void *p_data); 138 /* 139 * PARAMETERS 140 * context 141 * [in] Client specific context specified in a call to 142 * cl_disp_register 143 * 144 * p_data 145 * [in] Pointer to the client specific data payload 146 * of this message. 147 * 148 * RETURN VALUE 149 * This function does not return a value. 150 * 151 * NOTES 152 * This typedef provides a function prototype reference for 153 * the function provided by Dispatcher clients as a parameter 154 * to the cl_disp_register function. 155 * 156 * SEE ALSO 157 * Dispatcher, cl_disp_register 158 *********/ 159 160 /****f* Component Library: Dispatcher/cl_pfn_msgdone_cb_t 161 * NAME 162 * cl_pfn_msgdone_cb_t 163 * 164 * DESCRIPTION 165 * This typedef defines the prototype for client functions invoked 166 * by the Dispatcher. The Dispatcher calls the corresponding 167 * client function after completing delivery of a message. 168 * 169 * The client function must be reentrant if the user creates a 170 * Dispatcher with more than one worker thread. 171 * 172 * SYNOPSIS 173 */ 174 typedef void 175 (*cl_pfn_msgdone_cb_t) (IN void *context, IN void *p_data); 176 /* 177 * PARAMETERS 178 * context 179 * [in] Client specific context specified in a call to 180 * cl_disp_post 181 * 182 * p_data 183 * [in] Pointer to the client specific data payload 184 * of this message. 185 * 186 * RETURN VALUE 187 * This function does not return a value. 188 * 189 * NOTES 190 * This typedef provides a function prototype reference for 191 * the function provided by Dispatcher clients as a parameter 192 * to the cl_disp_post function. 193 * 194 * SEE ALSO 195 * Dispatcher, cl_disp_post 196 *********/ 197 198 /****s* Component Library: Dispatcher/cl_dispatcher_t 199 * NAME 200 * cl_dispatcher_t 201 * 202 * DESCRIPTION 203 * Dispatcher structure. 204 * 205 * The Dispatcher is thread safe. 206 * 207 * The cl_dispatcher_t structure should be treated as opaque and should 208 * be manipulated only through the provided functions. 209 * 210 * SYNOPSIS 211 */ 212 typedef struct _cl_dispatcher { 213 cl_spinlock_t lock; 214 cl_ptr_vector_t reg_vec; 215 cl_qlist_t reg_list; 216 cl_thread_pool_t worker_threads; 217 cl_qlist_t msg_fifo; 218 cl_qpool_t msg_pool; 219 uint64_t last_msg_queue_time_us; 220 } cl_dispatcher_t; 221 /* 222 * FIELDS 223 * reg_vec 224 * Vector of registration info objects. Indexed by message msg_id. 225 * 226 * lock 227 * Spinlock to guard internal structures. 228 * 229 * msg_fifo 230 * FIFO of messages being processed by the Dispatcher. New 231 * messages are posted to the tail of the FIFO. Worker threads 232 * pull messages from the front. 233 * 234 * worker_threads 235 * Thread pool of worker threads to dispose of posted messages. 236 * 237 * msg_pool 238 * Pool of message objects to be processed through the FIFO. 239 * 240 * reg_count 241 * Count of the number of registrants. 242 * 243 * state 244 * Indicates the state of the object. 245 * 246 * last_msg_queue_time_us 247 * The time that the last message spent in the Q in usec 248 * 249 * SEE ALSO 250 * Dispatcher 251 *********/ 252 253 /****s* Component Library: Dispatcher/cl_disp_reg_info_t 254 * NAME 255 * cl_disp_reg_info_t 256 * 257 * DESCRIPTION 258 * Defines the dispatcher registration object structure. 259 * 260 * The cl_disp_reg_info_t structure is for internal use by the 261 * Dispatcher only. 262 * 263 * SYNOPSIS 264 */ 265 typedef struct _cl_disp_reg_info { 266 cl_list_item_t list_item; 267 cl_pfn_msgrcv_cb_t pfn_rcv_callback; 268 const void *context; 269 atomic32_t ref_cnt; 270 cl_disp_msgid_t msg_id; 271 cl_dispatcher_t *p_disp; 272 } cl_disp_reg_info_t; 273 /* 274 * FIELDS 275 * pfn_rcv_callback 276 * Client's message receive callback. 277 * 278 * context 279 * Client's context for message receive callback. 280 * 281 * rcv_thread_count 282 * Number of threads currently in the receive callback. 283 * 284 * msg_done_thread_count 285 * Number of threads currently in the message done callback. 286 * 287 * state 288 * State of this registration object. 289 * DISP_REGSTATE_INIT: initialized and inactive 290 * DISP_REGSTATE_ACTIVE: in active use 291 * DISP_REGSTATE_UNREGPEND: unregistration is pending 292 * 293 * msg_id 294 * Dispatcher message msg_id value for this registration object. 295 * 296 * p_disp 297 * Pointer to parent Dispatcher. 298 * 299 * SEE ALSO 300 *********/ 301 302 /****s* Component Library: Dispatcher/cl_disp_msg_t 303 * NAME 304 * cl_disp_msg_t 305 * 306 * DESCRIPTION 307 * Defines the dispatcher message structure. 308 * 309 * The cl_disp_msg_t structure is for internal use by the 310 * Dispatcher only. 311 * 312 * SYNOPSIS 313 */ 314 typedef struct _cl_disp_msg { 315 cl_pool_item_t item; 316 const void *p_data; 317 cl_disp_reg_info_t *p_src_reg; 318 cl_disp_reg_info_t *p_dest_reg; 319 cl_pfn_msgdone_cb_t pfn_xmt_callback; 320 uint64_t in_time; 321 const void *context; 322 } cl_disp_msg_t; 323 /* 324 * FIELDS 325 * item 326 * List & Pool linkage. Must be first element in the structure!! 327 * 328 * msg_id 329 * The message's numberic ID value. 330 * 331 * p_data 332 * Pointer to the data payload for this message. The payload 333 * is opaque to the Dispatcher. 334 * 335 * p_reg_info 336 * Pointer to the registration info of the sender. 337 * 338 * pfn_xmt_callback 339 * Client's message done callback. 340 * 341 * in_time 342 * The absolute time the message was inserted into the queue 343 * 344 * context 345 * Client's message done callback context. 346 * 347 * SEE ALSO 348 *********/ 349 350 /****s* Component Library: Dispatcher/cl_disp_reg_info_t 351 * NAME 352 * cl_disp_reg_info_t 353 * 354 * DESCRIPTION 355 * Defines the Dispatcher registration handle. This handle 356 * should be treated as opaque by the client. 357 * 358 * SYNOPSIS 359 */ 360 typedef const struct _cl_disp_reg_info *cl_disp_reg_handle_t; 361 /**********/ 362 363 /****f* Component Library: Dispatcher/cl_disp_construct 364 * NAME 365 * cl_disp_construct 366 * 367 * DESCRIPTION 368 * This function constructs a Dispatcher object. 369 * 370 * SYNOPSIS 371 */ 372 void cl_disp_construct(IN cl_dispatcher_t * const p_disp); 373 /* 374 * PARAMETERS 375 * p_disp 376 * [in] Pointer to a Dispatcher. 377 * 378 * RETURN VALUE 379 * This function does not return a value. 380 * 381 * NOTES 382 * Allows calling cl_disp_init and cl_disp_destroy. 383 * 384 * SEE ALSO 385 * Dispatcher, cl_disp_init, cl_disp_destroy 386 *********/ 387 388 /****f* Component Library: Dispatcher/cl_disp_init 389 * NAME 390 * cl_disp_init 391 * 392 * DESCRIPTION 393 * This function initializes a Dispatcher object. 394 * 395 * SYNOPSIS 396 */ 397 cl_status_t 398 cl_disp_init(IN cl_dispatcher_t * const p_disp, 399 IN const uint32_t thread_count, IN const char *const name); 400 /* 401 * PARAMETERS 402 * p_disp 403 * [in] Pointer to a Dispatcher. 404 * 405 * thread_count 406 * [in] The number of worker threads to create in this Dispatcher. 407 * A value of 0 causes the Dispatcher to create one worker thread 408 * per CPU in the system. When the Dispatcher is created with 409 * only one thread, the Dispatcher guarantees to deliver posted 410 * messages in order. When the Dispatcher is created with more 411 * than one thread, messages may be delivered out of order. 412 * 413 * name 414 * [in] Name to associate with the threads. The name may be up to 16 415 * characters, including a terminating null character. All threads 416 * created in the Dispatcher have the same name. 417 * 418 * RETURN VALUE 419 * CL_SUCCESS if the operation is successful. 420 * 421 * SEE ALSO 422 * Dispatcher, cl_disp_destoy, cl_disp_register, cl_disp_unregister, 423 * cl_disp_post 424 *********/ 425 426 /****f* Component Library: Dispatcher/cl_disp_shutdown 427 * NAME 428 * cl_disp_shutdown 429 * 430 * DESCRIPTION 431 * This function shutdown a Dispatcher object. So it unreg all messages and 432 * clears the fifo and waits for the threads to exit 433 * 434 * SYNOPSIS 435 */ 436 void cl_disp_shutdown(IN cl_dispatcher_t * const p_disp); 437 /* 438 * PARAMETERS 439 * p_disp 440 * [in] Pointer to a Dispatcher. 441 * 442 * RETURN VALUE 443 * This function does not return a value. 444 * 445 * NOTES 446 * This function does not returns until all worker threads 447 * have exited client callback functions and been successfully 448 * shutdowned. 449 * 450 * SEE ALSO 451 * Dispatcher, cl_disp_construct, cl_disp_init 452 *********/ 453 454 /****f* Component Library: Dispatcher/cl_disp_destroy 455 * NAME 456 * cl_disp_destroy 457 * 458 * DESCRIPTION 459 * This function destroys a Dispatcher object. 460 * 461 * SYNOPSIS 462 */ 463 void cl_disp_destroy(IN cl_dispatcher_t * const p_disp); 464 /* 465 * PARAMETERS 466 * p_disp 467 * [in] Pointer to a Dispatcher. 468 * 469 * RETURN VALUE 470 * This function does not return a value. 471 * 472 * SEE ALSO 473 * Dispatcher, cl_disp_construct, cl_disp_init 474 *********/ 475 476 /****f* Component Library: Dispatcher/cl_disp_register 477 * NAME 478 * cl_disp_register 479 * 480 * DESCRIPTION 481 * This function registers a client with a Dispatcher object. 482 * 483 * SYNOPSIS 484 */ 485 cl_disp_reg_handle_t 486 cl_disp_register(IN cl_dispatcher_t * const p_disp, 487 IN const cl_disp_msgid_t msg_id, 488 IN cl_pfn_msgrcv_cb_t pfn_callback OPTIONAL, 489 IN const void *const context); 490 /* 491 * PARAMETERS 492 * p_disp 493 * [in] Pointer to a Dispatcher. 494 * 495 * msg_id 496 * [in] Numberic message ID for which the client is registering. 497 * If the client does not wish to receive any messages, 498 * (a send-only client) then the caller should set this value 499 * to CL_DISP_MSGID_NONE. For efficiency, numeric message msg_id 500 * values should start with 0 and should be contiguous, or nearly so. 501 * 502 * pfn_callback 503 * [in] Message receive callback. The Dispatcher calls this 504 * function after receiving a posted message with the 505 * appropriate message msg_id value. Send-only clients may specify 506 * NULL for this value. 507 * 508 * context 509 * [in] Client context value passed to the cl_pfn_msgrcv_cb_t 510 * function. 511 * 512 * RETURN VALUE 513 * On success a Dispatcher registration handle. 514 * CL_CL_DISP_INVALID_HANDLE otherwise. 515 * 516 * SEE ALSO 517 * Dispatcher, cl_disp_unregister, cl_disp_post 518 *********/ 519 520 /****f* Component Library: Dispatcher/cl_disp_unregister 521 * NAME 522 * cl_disp_unregister 523 * 524 * DESCRIPTION 525 * This function unregisters a client from a Dispatcher. 526 * 527 * SYNOPSIS 528 */ 529 void cl_disp_unregister(IN const cl_disp_reg_handle_t handle); 530 /* 531 * PARAMETERS 532 * handle 533 * [in] cl_disp_reg_handle_t value return by cl_disp_register. 534 * 535 * RETURN VALUE 536 * This function does not return a value. 537 * 538 * NOTES 539 * This function will not return until worker threads have exited 540 * the callback functions for this client. Do not invoke this 541 * function from a callback. 542 * 543 * SEE ALSO 544 * Dispatcher, cl_disp_register 545 *********/ 546 547 /****f* Component Library: Dispatcher/cl_disp_post 548 * NAME 549 * cl_disp_post 550 * 551 * DESCRIPTION 552 * This function posts a message to a Dispatcher object. 553 * 554 * SYNOPSIS 555 */ 556 cl_status_t 557 cl_disp_post(IN const cl_disp_reg_handle_t handle, 558 IN const cl_disp_msgid_t msg_id, 559 IN const void *const p_data, 560 IN cl_pfn_msgdone_cb_t pfn_callback OPTIONAL, 561 IN const void *const context); 562 /* 563 * PARAMETERS 564 * handle 565 * [in] cl_disp_reg_handle_t value return by cl_disp_register. 566 * 567 * msg_id 568 * [in] Numeric message msg_id value associated with this message. 569 * 570 * p_data 571 * [in] Data payload for this message. 572 * 573 * pfn_callback 574 * [in] Pointer to a cl_pfn_msgdone_cb_t function. 575 * The Dispatcher calls this function after the message has been 576 * processed by the recipient. 577 * The caller may pass NULL for this value, which indicates no 578 * message done callback is necessary. 579 * 580 * context 581 * [in] Client context value passed to the cl_pfn_msgdone_cb_t 582 * function. 583 * 584 * RETURN VALUE 585 * CL_SUCCESS if the message was successfully queued in the Dispatcher. 586 * 587 * NOTES 588 * The caller must not modify the memory pointed to by p_data until 589 * the Dispatcher call the pfn_callback function. 590 * 591 * SEE ALSO 592 * Dispatcher 593 *********/ 594 595 /****f* Component Library: Dispatcher/cl_disp_get_queue_status 596 * NAME 597 * cl_disp_get_queue_status 598 * 599 * DESCRIPTION 600 * This function posts a message to a Dispatcher object. 601 * 602 * SYNOPSIS 603 */ 604 void 605 cl_disp_get_queue_status(IN const cl_disp_reg_handle_t handle, 606 OUT uint32_t * p_num_queued_msgs, 607 OUT uint64_t * p_last_msg_queue_time_ms); 608 /* 609 * PARAMETERS 610 * handle 611 * [in] cl_disp_reg_handle_t value return by cl_disp_register. 612 * 613 * p_last_msg_queue_time_ms 614 * [out] pointer to a variable to hold the time the last popped up message 615 * spent in the queue 616 * 617 * p_num_queued_msgs 618 * [out] number of messages in the queue 619 * 620 * RETURN VALUE 621 * Thr time the last popped up message stayed in the queue, in msec 622 * 623 * NOTES 624 * Extarnel Locking is not required. 625 * 626 * SEE ALSO 627 * Dispatcher 628 *********/ 629 630 END_C_DECLS 631 #endif /* !defined(_CL_DISPATCHER_H_) */ 632