1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * Copyright 2012 Milan Jurik. All rights reserved. 25 */ 26 27 28 /* 29 * Overview of the RSM Kernel Agent: 30 * --------------------------------- 31 * 32 * rsm.c constitutes the implementation of the RSM kernel agent. The RSM 33 * kernel agent is a pseudo device driver which makes use of the RSMPI 34 * interface on behalf of the RSMAPI user library. 35 * 36 * The kernel agent functionality can be categorized into the following 37 * components: 38 * 1. Driver Infrastructure 39 * 2. Export/Import Segment Management 40 * 3. Internal resource allocation/deallocation 41 * 42 * The driver infrastructure includes the basic module loading entry points 43 * like _init, _info, _fini to load, unload and report information about 44 * the driver module. The driver infrastructure also includes the 45 * autoconfiguration entry points namely, attach, detach and getinfo for 46 * the device autoconfiguration. 47 * 48 * The kernel agent is a pseudo character device driver and exports 49 * a cb_ops structure which defines the driver entry points for character 50 * device access. This includes the open and close entry points. The 51 * other entry points provided include ioctl, devmap and segmap and chpoll. 52 * read and write entry points are not used since the device is memory 53 * mapped. Also ddi_prop_op is used for the prop_op entry point. 54 * 55 * The ioctl entry point supports a number of commands, which are used by 56 * the RSMAPI library in order to export and import segments. These 57 * commands include commands for binding and rebinding the physical pages 58 * allocated to the virtual address range, publishing the export segment, 59 * unpublishing and republishing an export segment, creating an 60 * import segment and a virtual connection from this import segment to 61 * an export segment, performing scatter-gather data transfer, barrier 62 * operations. 63 * 64 * 65 * Export and Import segments: 66 * --------------------------- 67 * 68 * In order to create an RSM export segment a process allocates a range in its 69 * virtual address space for the segment using standard Solaris interfaces. 70 * The process then calls RSMAPI, which in turn makes an ioctl call to the 71 * RSM kernel agent for an allocation of physical memory pages and for 72 * creation of the export segment by binding these pages to the virtual 73 * address range. These pages are locked in memory so that remote accesses 74 * are always applied to the correct page. Then the RSM segment is published, 75 * again via RSMAPI making an ioctl to the RSM kernel agent, and a segment id 76 * is assigned to it. 77 * 78 * In order to import a published RSM segment, RSMAPI creates an import 79 * segment and forms a virtual connection across the interconnect to the 80 * export segment, via an ioctl into the kernel agent with the connect 81 * command. The import segment setup is completed by mapping the 82 * local device memory into the importers virtual address space. The 83 * mapping of the import segment is handled by the segmap/devmap 84 * infrastructure described as follows. 85 * 86 * Segmap and Devmap interfaces: 87 * 88 * The RSM kernel agent allows device memory to be directly accessed by user 89 * threads via memory mapping. In order to do so, the RSM kernel agent 90 * supports the devmap and segmap entry points. 91 * 92 * The segmap entry point(rsm_segmap) is responsible for setting up a memory 93 * mapping as requested by mmap. The devmap entry point(rsm_devmap) is 94 * responsible for exporting the device memory to the user applications. 95 * rsm_segmap calls RSMPI rsm_map to allocate device memory. Then the 96 * control is transfered to the devmap_setup call which calls rsm_devmap. 97 * 98 * rsm_devmap validates the user mapping to the device or kernel memory 99 * and passes the information to the system for setting up the mapping. The 100 * actual setting up of the mapping is done by devmap_devmem_setup(for 101 * device memory) or devmap_umem_setup(for kernel memory). Callbacks are 102 * registered for device context management via the devmap_devmem_setup 103 * or devmap_umem_setup calls. The callbacks are rsmmap_map, rsmmap_unmap, 104 * rsmmap_access, rsmmap_dup. The callbacks are called when a new mapping 105 * is created, a mapping is freed, a mapping is accessed or an existing 106 * mapping is duplicated respectively. These callbacks allow the RSM kernel 107 * agent to maintain state information associated with the mappings. 108 * The state information is mainly in the form of a cookie list for the import 109 * segment for which mapping has been done. 110 * 111 * Forced disconnect of import segments: 112 * 113 * When an exported segment is unpublished, the exporter sends a forced 114 * disconnect message to all its importers. The importer segments are 115 * unloaded and disconnected. This involves unloading the original 116 * mappings and remapping to a preallocated kernel trash page. This is 117 * done by devmap_umem_remap. The trash/dummy page is a kernel page, 118 * preallocated by the kernel agent during attach using ddi_umem_alloc with 119 * the DDI_UMEM_TRASH flag set. This avoids a core dump in the application 120 * due to unloading of the original mappings. 121 * 122 * Additionally every segment has a mapping generation number associated 123 * with it. This is an entry in the barrier generation page, created 124 * during attach time. This mapping generation number for the import 125 * segments is incremented on a force disconnect to notify the application 126 * of the force disconnect. On this notification, the application needs 127 * to reconnect the segment to establish a new legitimate mapping. 128 * 129 * 130 * Locks used in the kernel agent: 131 * ------------------------------- 132 * 133 * The kernel agent uses a variety of mutexes and condition variables for 134 * mutual exclusion of the shared data structures and for synchronization 135 * between the various threads. Some of the locks are described as follows. 136 * 137 * Each resource structure, which represents either an export/import segment 138 * has a lock associated with it. The lock is the resource mutex, rsmrc_lock. 139 * This is used directly by RSMRC_LOCK and RSMRC_UNLOCK macros and in the 140 * rsmseglock_acquire and rsmseglock_release macros. An additional 141 * lock called the rsmsi_lock is used for the shared import data structure 142 * that is relevant for resources representing import segments. There is 143 * also a condition variable associated with the resource called s_cv. This 144 * is used to wait for events like the segment state change etc. 145 * 146 * The resource structures are allocated from a pool of resource structures, 147 * called rsm_resource. This pool is protected via a reader-writer lock, 148 * called rsmrc_lock. 149 * 150 * There are two separate hash tables, one for the export segments and 151 * one for the import segments. The export segments are inserted into the 152 * export segment hash table only after they have been published and the 153 * import segments are inserted in the import segments list only after they 154 * have successfully connected to an exported segment. These tables are 155 * protected via reader-writer locks. 156 * 157 * Debug Support in the kernel agent: 158 * ---------------------------------- 159 * 160 * Debugging support in the kernel agent is provided by the following 161 * macros. 162 * 163 * DBG_PRINTF((category, level, message)) is a macro which logs a debug 164 * message to the kernel agents debug buffer, rsmka_dbg. This debug buffer 165 * can be viewed in kmdb as *rsmka_dbg/s. The message is logged based 166 * on the definition of the category and level. All messages that belong to 167 * the specified category(rsmdbg_category) and are of an equal or greater 168 * severity than the specified level(rsmdbg_level) are logged. The message 169 * is a string which uses the same formatting rules as the strings used in 170 * printf. 171 * 172 * The category defines which component of the kernel agent has logged this 173 * message. There are a number of categories that have been defined such as 174 * RSM_KERNEL_AGENT, RSM_OPS, RSM_IMPORT, RSM_EXPORT etc. A macro, 175 * DBG_ADDCATEGORY is used to add in another category to the currently 176 * specified category value so that the component using this new category 177 * can also effectively log debug messages. Thus, the category of a specific 178 * message is some combination of the available categories and we can define 179 * sub-categories if we want a finer level of granularity. 180 * 181 * The level defines the severity of the message. Different level values are 182 * defined, with RSM_ERR being the most severe and RSM_DEBUG_VERBOSE being 183 * the least severe(debug level is 0). 184 * 185 * DBG_DEFINE and DBG_DEFINE_STR are macros provided to declare a debug 186 * variable or a string respectively. 187 * 188 * 189 * NOTES: 190 * 191 * Special Fork and Exec Handling: 192 * ------------------------------- 193 * 194 * The backing physical pages of an exported segment are always locked down. 195 * Thus, there are two cases in which a process having exported segments 196 * will cause a cpu to hang: (1) the process invokes exec; (2) a process 197 * forks and invokes exit before the duped file descriptors for the export 198 * segments are closed in the child process. The hang is caused because the 199 * address space release algorithm in Solaris VM subsystem is based on a 200 * non-blocking loop which does not terminate while segments are locked 201 * down. In addition to this, Solaris VM subsystem lacks a callback 202 * mechanism to the rsm kernel agent to allow unlocking these export 203 * segment pages. 204 * 205 * In order to circumvent this problem, the kernel agent does the following. 206 * The Solaris VM subsystem keeps memory segments in increasing order of 207 * virtual addressses. Thus a special page(special_exit_offset) is allocated 208 * by the kernel agent and is mmapped into the heap area of the process address 209 * space(the mmap is done by the RSMAPI library). During the mmap processing 210 * of this special page by the devmap infrastructure, a callback(the same 211 * devmap context management callbacks discussed above) is registered for an 212 * unmap. 213 * 214 * As discussed above, this page is processed by the Solaris address space 215 * release code before any of the exported segments pages(which are allocated 216 * from high memory). It is during this processing that the unmap callback gets 217 * called and this callback is responsible for force destroying the exported 218 * segments and thus eliminating the problem of locked pages. 219 * 220 * Flow-control: 221 * ------------ 222 * 223 * A credit based flow control algorithm is used for messages whose 224 * processing cannot be done in the interrupt context because it might 225 * involve invoking rsmpi calls, or might take a long time to complete 226 * or might need to allocate resources. The algorithm operates on a per 227 * path basis. To send a message the pathend needs to have a credit and 228 * it consumes one for every message that is flow controlled. On the 229 * receiving pathend the message is put on a msgbuf_queue and a task is 230 * dispatched on the worker thread - recv_taskq where it is processed. 231 * After processing the message, the receiving pathend dequeues the message, 232 * and if it has processed > RSMIPC_LOTSFREE_MSGBUFS messages sends 233 * credits to the sender pathend. 234 * 235 * RSM_DRTEST: 236 * ----------- 237 * 238 * This is used to enable the DR testing using a test driver on test 239 * platforms which do not supported DR. 240 * 241 */ 242 243 #include <sys/types.h> 244 #include <sys/param.h> 245 #include <sys/user.h> 246 #include <sys/buf.h> 247 #include <sys/systm.h> 248 #include <sys/cred.h> 249 #include <sys/vm.h> 250 #include <sys/uio.h> 251 #include <vm/seg.h> 252 #include <vm/page.h> 253 #include <sys/stat.h> 254 255 #include <sys/time.h> 256 #include <sys/errno.h> 257 258 #include <sys/file.h> 259 #include <sys/uio.h> 260 #include <sys/proc.h> 261 #include <sys/mman.h> 262 #include <sys/open.h> 263 #include <sys/atomic.h> 264 #include <sys/mem_config.h> 265 266 267 #include <sys/ddi.h> 268 #include <sys/devops.h> 269 #include <sys/ddidevmap.h> 270 #include <sys/sunddi.h> 271 #include <sys/esunddi.h> 272 #include <sys/ddi_impldefs.h> 273 274 #include <sys/kmem.h> 275 #include <sys/conf.h> 276 #include <sys/devops.h> 277 #include <sys/ddi_impldefs.h> 278 279 #include <sys/modctl.h> 280 281 #include <sys/policy.h> 282 #include <sys/types.h> 283 #include <sys/conf.h> 284 #include <sys/param.h> 285 286 #include <sys/taskq.h> 287 288 #include <sys/rsm/rsm_common.h> 289 #include <sys/rsm/rsmapi_common.h> 290 #include <sys/rsm/rsm.h> 291 #include <rsm_in.h> 292 #include <sys/rsm/rsmka_path_int.h> 293 #include <sys/rsm/rsmpi.h> 294 295 #include <sys/modctl.h> 296 #include <sys/debug.h> 297 298 #include <sys/tuneable.h> 299 300 #ifdef RSM_DRTEST 301 extern int rsm_kphysm_setup_func_register(kphysm_setup_vector_t *vec, 302 void *arg); 303 extern void rsm_kphysm_setup_func_unregister(kphysm_setup_vector_t *vec, 304 void *arg); 305 #endif 306 307 extern void dbg_printf(int category, int level, char *fmt, ...); 308 extern void rsmka_pathmanager_init(); 309 extern void rsmka_pathmanager_cleanup(); 310 extern void rele_sendq_token(sendq_token_t *); 311 extern rsm_addr_t get_remote_hwaddr(adapter_t *, rsm_node_id_t); 312 extern rsm_node_id_t get_remote_nodeid(adapter_t *, rsm_addr_t); 313 extern int rsmka_topology_ioctl(caddr_t, int, int); 314 315 extern pri_t maxclsyspri; 316 extern work_queue_t work_queue; 317 extern kmutex_t ipc_info_lock; 318 extern kmutex_t ipc_info_cvlock; 319 extern kcondvar_t ipc_info_cv; 320 extern kmutex_t path_hold_cvlock; 321 extern kcondvar_t path_hold_cv; 322 323 extern kmutex_t rsmka_buf_lock; 324 325 extern path_t *rsm_find_path(char *, int, rsm_addr_t); 326 extern adapter_t *rsmka_lookup_adapter(char *, int); 327 extern sendq_token_t *rsmka_get_sendq_token(rsm_node_id_t, sendq_token_t *); 328 extern boolean_t rsmka_do_path_active(path_t *, int); 329 extern boolean_t rsmka_check_node_alive(rsm_node_id_t); 330 extern void rsmka_release_adapter(adapter_t *); 331 extern void rsmka_enqueue_msgbuf(path_t *path, void *data); 332 extern void rsmka_dequeue_msgbuf(path_t *path); 333 extern msgbuf_elem_t *rsmka_gethead_msgbuf(path_t *path); 334 /* lint -w2 */ 335 336 static int rsm_open(dev_t *, int, int, cred_t *); 337 static int rsm_close(dev_t, int, int, cred_t *); 338 static int rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, 339 cred_t *credp, int *rvalp); 340 static int rsm_devmap(dev_t, devmap_cookie_t, offset_t, size_t, size_t *, 341 uint_t); 342 static int rsm_segmap(dev_t, off_t, struct as *, caddr_t *, off_t, uint_t, 343 uint_t, uint_t, cred_t *); 344 static int rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp, 345 struct pollhead **phpp); 346 347 static int rsm_info(dev_info_t *, ddi_info_cmd_t, void *, void **); 348 static int rsm_attach(dev_info_t *, ddi_attach_cmd_t); 349 static int rsm_detach(dev_info_t *, ddi_detach_cmd_t); 350 351 static int rsmipc_send(rsm_node_id_t, rsmipc_request_t *, rsmipc_reply_t *); 352 static void rsm_force_unload(rsm_node_id_t, rsm_memseg_id_t, boolean_t); 353 static void rsm_send_importer_disconnects(rsm_memseg_id_t, rsm_node_id_t); 354 static void rsm_send_republish(rsm_memseg_id_t, rsmapi_access_entry_t *, int, 355 rsm_permission_t); 356 static void rsm_export_force_destroy(ddi_umem_cookie_t *); 357 static void rsmacl_free(rsmapi_access_entry_t *, int); 358 static void rsmpiacl_free(rsm_access_entry_t *, int); 359 360 static int rsm_inc_pgcnt(pgcnt_t); 361 static void rsm_dec_pgcnt(pgcnt_t); 362 static void rsm_free_mapinfo(rsm_mapinfo_t *mapinfop); 363 static rsm_mapinfo_t *rsm_get_mapinfo(rsmseg_t *, off_t, size_t, off_t *, 364 size_t *); 365 static void exporter_quiesce(); 366 static void rsmseg_suspend(rsmseg_t *, int *); 367 static void rsmsegshare_suspend(rsmseg_t *); 368 static int rsmseg_resume(rsmseg_t *, void **); 369 static int rsmsegshare_resume(rsmseg_t *); 370 371 static struct cb_ops rsm_cb_ops = { 372 rsm_open, /* open */ 373 rsm_close, /* close */ 374 nodev, /* strategy */ 375 nodev, /* print */ 376 nodev, /* dump */ 377 nodev, /* read */ 378 nodev, /* write */ 379 rsm_ioctl, /* ioctl */ 380 rsm_devmap, /* devmap */ 381 NULL, /* mmap */ 382 rsm_segmap, /* segmap */ 383 rsm_chpoll, /* poll */ 384 ddi_prop_op, /* cb_prop_op */ 385 0, /* streamtab */ 386 D_NEW|D_MP|D_DEVMAP, /* Driver compatibility flag */ 387 0, 388 0, 389 0 390 }; 391 392 static struct dev_ops rsm_ops = { 393 DEVO_REV, /* devo_rev, */ 394 0, /* refcnt */ 395 rsm_info, /* get_dev_info */ 396 nulldev, /* identify */ 397 nulldev, /* probe */ 398 rsm_attach, /* attach */ 399 rsm_detach, /* detach */ 400 nodev, /* reset */ 401 &rsm_cb_ops, /* driver operations */ 402 (struct bus_ops *)0, /* bus operations */ 403 0, 404 ddi_quiesce_not_needed, /* quiesce */ 405 }; 406 407 /* 408 * Module linkage information for the kernel. 409 */ 410 411 static struct modldrv modldrv = { 412 &mod_driverops, /* Type of module. This one is a pseudo driver */ 413 "Remote Shared Memory Driver", 414 &rsm_ops, /* driver ops */ 415 }; 416 417 static struct modlinkage modlinkage = { 418 MODREV_1, 419 (void *)&modldrv, 420 0, 421 0, 422 0 423 }; 424 425 static void rsm_dr_callback_post_add(void *arg, pgcnt_t delta); 426 static int rsm_dr_callback_pre_del(void *arg, pgcnt_t delta); 427 static void rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled); 428 429 static kphysm_setup_vector_t rsm_dr_callback_vec = { 430 KPHYSM_SETUP_VECTOR_VERSION, 431 rsm_dr_callback_post_add, 432 rsm_dr_callback_pre_del, 433 rsm_dr_callback_post_del 434 }; 435 436 /* This flag can be changed to 0 to help with PIT testing */ 437 int rsmka_modunloadok = 1; 438 int no_reply_cnt = 0; 439 440 uint64_t rsm_ctrlmsg_errcnt = 0; 441 uint64_t rsm_ipcsend_errcnt = 0; 442 443 #define MAX_NODES 64 444 445 static struct rsm_driver_data rsm_drv_data; 446 static struct rsmresource_table rsm_resource; 447 448 static void rsmresource_insert(minor_t, rsmresource_t *, rsm_resource_type_t); 449 static void rsmresource_destroy(void); 450 static int rsmresource_alloc(minor_t *); 451 static rsmresource_t *rsmresource_free(minor_t rnum); 452 static int rsm_closeconnection(rsmseg_t *seg, void **cookie); 453 static int rsm_unpublish(rsmseg_t *seg, int mode); 454 static int rsm_unbind(rsmseg_t *seg); 455 static uint_t rsmhash(rsm_memseg_id_t key); 456 static void rsmhash_alloc(rsmhash_table_t *rhash, int size); 457 static void rsmhash_free(rsmhash_table_t *rhash, int size); 458 static void *rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval); 459 static void **rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval); 460 static int rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid, 461 void *cookie); 462 int rsm_disconnect(rsmseg_t *seg); 463 void rsmseg_unload(rsmseg_t *); 464 void rsm_suspend_complete(rsm_node_id_t src_node, int flag); 465 466 rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd, 467 rsm_intr_q_op_t opcode, rsm_addr_t src, 468 void *data, size_t size, rsm_intr_hand_arg_t arg); 469 470 static void rsm_intr_callback(void *, rsm_addr_t, rsm_intr_hand_arg_t); 471 472 rsm_node_id_t my_nodeid; 473 474 /* cookie, va, offsets and length for the barrier */ 475 static rsm_gnum_t *bar_va; 476 static ddi_umem_cookie_t bar_cookie; 477 static off_t barrier_offset; 478 static size_t barrier_size; 479 static int max_segs; 480 481 /* cookie for the trash memory */ 482 static ddi_umem_cookie_t remap_cookie; 483 484 static rsm_memseg_id_t rsm_nextavail_segmentid; 485 486 extern taskq_t *work_taskq; 487 extern char *taskq_name; 488 489 static dev_info_t *rsm_dip; /* private copy of devinfo pointer */ 490 491 static rsmhash_table_t rsm_export_segs; /* list of exported segs */ 492 rsmhash_table_t rsm_import_segs; /* list of imported segs */ 493 static rsmhash_table_t rsm_event_queues; /* list of event queues */ 494 495 static rsm_ipc_t rsm_ipc; /* ipc info */ 496 497 /* list of nodes to which RSMIPC_MSG_SUSPEND has been sent */ 498 static list_head_t rsm_suspend_list; 499 500 /* list of descriptors for remote importers */ 501 static importers_table_t importer_list; 502 503 kmutex_t rsm_suspend_cvlock; 504 kcondvar_t rsm_suspend_cv; 505 506 static kmutex_t rsm_lock; 507 508 adapter_t loopback_adapter; 509 rsm_controller_attr_t loopback_attr; 510 511 int rsmipc_send_controlmsg(path_t *path, int msgtype); 512 513 void rsmka_init_loopback(); 514 515 int rsmka_null_seg_create( 516 rsm_controller_handle_t, 517 rsm_memseg_export_handle_t *, 518 size_t, 519 uint_t, 520 rsm_memory_local_t *, 521 rsm_resource_callback_t, 522 rsm_resource_callback_arg_t); 523 524 int rsmka_null_seg_destroy( 525 rsm_memseg_export_handle_t); 526 527 int rsmka_null_bind( 528 rsm_memseg_export_handle_t, 529 off_t, 530 rsm_memory_local_t *, 531 rsm_resource_callback_t, 532 rsm_resource_callback_arg_t); 533 534 int rsmka_null_unbind( 535 rsm_memseg_export_handle_t, 536 off_t, 537 size_t); 538 539 int rsmka_null_rebind( 540 rsm_memseg_export_handle_t, 541 off_t, 542 rsm_memory_local_t *, 543 rsm_resource_callback_t, 544 rsm_resource_callback_arg_t); 545 546 int rsmka_null_publish( 547 rsm_memseg_export_handle_t, 548 rsm_access_entry_t [], 549 uint_t, 550 rsm_memseg_id_t, 551 rsm_resource_callback_t, 552 rsm_resource_callback_arg_t); 553 554 555 int rsmka_null_republish( 556 rsm_memseg_export_handle_t, 557 rsm_access_entry_t [], 558 uint_t, 559 rsm_resource_callback_t, 560 rsm_resource_callback_arg_t); 561 562 int rsmka_null_unpublish( 563 rsm_memseg_export_handle_t); 564 565 rsm_ops_t null_rsmpi_ops; 566 567 /* 568 * data and locks to keep track of total amount of exported memory 569 */ 570 static pgcnt_t rsm_pgcnt; 571 static pgcnt_t rsm_pgcnt_max; /* max allowed */ 572 static kmutex_t rsm_pgcnt_lock; 573 574 static int rsm_enable_dr; 575 576 static char loopback_str[] = "loopback"; 577 578 int rsm_hash_size; 579 580 /* 581 * The locking model is as follows: 582 * 583 * Local operations: 584 * find resource - grab reader lock on resouce list 585 * insert rc - grab writer lock 586 * delete rc - grab writer lock and resource mutex 587 * read/write - no lock 588 * 589 * Remote invocations: 590 * find resource - grab read lock and resource mutex 591 * 592 * State: 593 * resource state - grab resource mutex 594 */ 595 596 int 597 _init(void) 598 { 599 int e; 600 601 e = mod_install(&modlinkage); 602 if (e != 0) { 603 return (e); 604 } 605 606 mutex_init(&rsm_lock, NULL, MUTEX_DRIVER, NULL); 607 608 mutex_init(&rsmka_buf_lock, NULL, MUTEX_DEFAULT, NULL); 609 610 611 rw_init(&rsm_resource.rsmrc_lock, NULL, RW_DRIVER, NULL); 612 613 rsm_hash_size = RSM_HASHSZ; 614 615 rw_init(&rsm_export_segs.rsmhash_rw, NULL, RW_DRIVER, NULL); 616 617 rw_init(&rsm_import_segs.rsmhash_rw, NULL, RW_DRIVER, NULL); 618 619 mutex_init(&importer_list.lock, NULL, MUTEX_DRIVER, NULL); 620 621 mutex_init(&rsm_ipc.lock, NULL, MUTEX_DRIVER, NULL); 622 cv_init(&rsm_ipc.cv, NULL, CV_DRIVER, 0); 623 624 mutex_init(&rsm_suspend_cvlock, NULL, MUTEX_DRIVER, NULL); 625 cv_init(&rsm_suspend_cv, NULL, CV_DRIVER, 0); 626 627 mutex_init(&rsm_drv_data.drv_lock, NULL, MUTEX_DRIVER, NULL); 628 cv_init(&rsm_drv_data.drv_cv, NULL, CV_DRIVER, 0); 629 630 rsm_ipc.count = RSMIPC_SZ; 631 rsm_ipc.wanted = 0; 632 rsm_ipc.sequence = 0; 633 634 (void) mutex_init(&rsm_pgcnt_lock, NULL, MUTEX_DRIVER, NULL); 635 636 for (e = 0; e < RSMIPC_SZ; e++) { 637 rsmipc_slot_t *slot = &rsm_ipc.slots[e]; 638 639 RSMIPC_SET(slot, RSMIPC_FREE); 640 mutex_init(&slot->rsmipc_lock, NULL, MUTEX_DRIVER, NULL); 641 cv_init(&slot->rsmipc_cv, NULL, CV_DRIVER, 0); 642 } 643 644 /* 645 * Initialize the suspend message list 646 */ 647 rsm_suspend_list.list_head = NULL; 648 mutex_init(&rsm_suspend_list.list_lock, NULL, MUTEX_DRIVER, NULL); 649 650 /* 651 * It is assumed here that configuration data is available 652 * during system boot since _init may be called at that time. 653 */ 654 655 rsmka_pathmanager_init(); 656 657 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE, 658 "rsm: _init done\n")); 659 660 return (DDI_SUCCESS); 661 662 } 663 664 int 665 _info(struct modinfo *modinfop) 666 { 667 668 return (mod_info(&modlinkage, modinfop)); 669 } 670 671 int 672 _fini(void) 673 { 674 int e; 675 676 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE, 677 "rsm: _fini enter\n")); 678 679 /* 680 * The rsmka_modunloadok flag is simply used to help with 681 * the PIT testing. Make this flag 0 to disallow modunload. 682 */ 683 if (rsmka_modunloadok == 0) 684 return (EBUSY); 685 686 /* rsm_detach will be called as a result of mod_remove */ 687 e = mod_remove(&modlinkage); 688 if (e) { 689 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_ERR, 690 "Unable to fini RSM %x\n", e)); 691 return (e); 692 } 693 694 rsmka_pathmanager_cleanup(); 695 696 rw_destroy(&rsm_resource.rsmrc_lock); 697 698 rw_destroy(&rsm_export_segs.rsmhash_rw); 699 rw_destroy(&rsm_import_segs.rsmhash_rw); 700 rw_destroy(&rsm_event_queues.rsmhash_rw); 701 702 mutex_destroy(&importer_list.lock); 703 704 mutex_destroy(&rsm_ipc.lock); 705 cv_destroy(&rsm_ipc.cv); 706 707 (void) mutex_destroy(&rsm_suspend_list.list_lock); 708 709 (void) mutex_destroy(&rsm_pgcnt_lock); 710 711 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE, "_fini done\n")); 712 713 return (DDI_SUCCESS); 714 715 } 716 717 /*ARGSUSED1*/ 718 static int 719 rsm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 720 { 721 minor_t rnum; 722 int percent; 723 int ret; 724 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI); 725 726 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach enter\n")); 727 728 switch (cmd) { 729 case DDI_ATTACH: 730 break; 731 case DDI_RESUME: 732 default: 733 DBG_PRINTF((category, RSM_ERR, 734 "rsm:rsm_attach - cmd not supported\n")); 735 return (DDI_FAILURE); 736 } 737 738 if (rsm_dip != NULL) { 739 DBG_PRINTF((category, RSM_ERR, 740 "rsm:rsm_attach - supports only " 741 "one instance\n")); 742 return (DDI_FAILURE); 743 } 744 745 rsm_enable_dr = ddi_prop_get_int(DDI_DEV_T_ANY, devi, 746 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, 747 "enable-dynamic-reconfiguration", 1); 748 749 mutex_enter(&rsm_drv_data.drv_lock); 750 rsm_drv_data.drv_state = RSM_DRV_REG_PROCESSING; 751 mutex_exit(&rsm_drv_data.drv_lock); 752 753 if (rsm_enable_dr) { 754 #ifdef RSM_DRTEST 755 ret = rsm_kphysm_setup_func_register(&rsm_dr_callback_vec, 756 (void *)NULL); 757 #else 758 ret = kphysm_setup_func_register(&rsm_dr_callback_vec, 759 (void *)NULL); 760 #endif 761 if (ret != 0) { 762 mutex_exit(&rsm_drv_data.drv_lock); 763 cmn_err(CE_CONT, "rsm:rsm_attach - Dynamic " 764 "reconfiguration setup failed\n"); 765 return (DDI_FAILURE); 766 } 767 } 768 769 mutex_enter(&rsm_drv_data.drv_lock); 770 ASSERT(rsm_drv_data.drv_state == RSM_DRV_REG_PROCESSING); 771 rsm_drv_data.drv_state = RSM_DRV_OK; 772 cv_broadcast(&rsm_drv_data.drv_cv); 773 mutex_exit(&rsm_drv_data.drv_lock); 774 775 /* 776 * page_list_read_lock(); 777 * xx_setup(); 778 * page_list_read_unlock(); 779 */ 780 781 rsm_hash_size = ddi_prop_get_int(DDI_DEV_T_ANY, devi, 782 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, 783 "segment-hashtable-size", RSM_HASHSZ); 784 if (rsm_hash_size == 0) { 785 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 786 "rsm: segment-hashtable-size in rsm.conf " 787 "must be greater than 0, defaulting to 128\n")); 788 rsm_hash_size = RSM_HASHSZ; 789 } 790 791 DBG_PRINTF((category, RSM_DEBUG, "rsm_attach rsm_hash_size: %d\n", 792 rsm_hash_size)); 793 794 rsm_pgcnt = 0; 795 796 percent = ddi_prop_get_int(DDI_DEV_T_ANY, devi, 797 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, 798 "max-exported-memory", 0); 799 if (percent < 0) { 800 DBG_PRINTF((category, RSM_ERR, 801 "rsm:rsm_attach not enough memory available to " 802 "export, or max-exported-memory set incorrectly.\n")); 803 return (DDI_FAILURE); 804 } 805 /* 0 indicates no fixed upper limit. maxmem is the max */ 806 /* available pageable physical mem */ 807 rsm_pgcnt_max = (percent*maxmem)/100; 808 809 if (rsm_pgcnt_max > 0) { 810 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 811 "rsm: Available physical memory = %lu pages, " 812 "Max exportable memory = %lu pages", 813 maxmem, rsm_pgcnt_max)); 814 } 815 816 /* 817 * Create minor number 818 */ 819 if (rsmresource_alloc(&rnum) != RSM_SUCCESS) { 820 DBG_PRINTF((category, RSM_ERR, 821 "rsm: rsm_attach - Unable to get " 822 "minor number\n")); 823 return (DDI_FAILURE); 824 } 825 826 ASSERT(rnum == RSM_DRIVER_MINOR); 827 828 if (ddi_create_minor_node(devi, DRIVER_NAME, S_IFCHR, 829 rnum, DDI_PSEUDO, NULL) == DDI_FAILURE) { 830 DBG_PRINTF((category, RSM_ERR, 831 "rsm: rsm_attach - unable to allocate " 832 "minor #\n")); 833 return (DDI_FAILURE); 834 } 835 836 rsm_dip = devi; 837 /* 838 * Allocate the hashtables 839 */ 840 rsmhash_alloc(&rsm_export_segs, rsm_hash_size); 841 rsmhash_alloc(&rsm_import_segs, rsm_hash_size); 842 843 importer_list.bucket = (importing_token_t **) 844 kmem_zalloc(rsm_hash_size * sizeof (importing_token_t *), KM_SLEEP); 845 846 /* 847 * Allocate a resource struct 848 */ 849 { 850 rsmresource_t *p; 851 852 p = (rsmresource_t *)kmem_zalloc(sizeof (*p), KM_SLEEP); 853 854 mutex_init(&p->rsmrc_lock, NULL, MUTEX_DRIVER, (void *) NULL); 855 856 rsmresource_insert(rnum, p, RSM_RESOURCE_BAR); 857 } 858 859 /* 860 * Based on the rsm.conf property max-segments, determine the maximum 861 * number of segments that can be exported/imported. This is then used 862 * to determine the size for barrier failure pages. 863 */ 864 865 /* First get the max number of segments from the rsm.conf file */ 866 max_segs = ddi_prop_get_int(DDI_DEV_T_ANY, devi, 867 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, 868 "max-segments", 0); 869 if (max_segs == 0) { 870 /* Use default number of segments */ 871 max_segs = RSM_MAX_NUM_SEG; 872 } 873 874 /* 875 * Based on the max number of segments allowed, determine the barrier 876 * page size. add 1 to max_segs since the barrier page itself uses 877 * a slot 878 */ 879 barrier_size = roundup((max_segs + 1) * sizeof (rsm_gnum_t), 880 PAGESIZE); 881 882 /* 883 * allocation of the barrier failure page 884 */ 885 bar_va = (rsm_gnum_t *)ddi_umem_alloc(barrier_size, 886 DDI_UMEM_SLEEP, &bar_cookie); 887 888 /* 889 * Set the barrier_offset 890 */ 891 barrier_offset = 0; 892 893 /* 894 * Allocate a trash memory and get a cookie for it. This will be used 895 * when remapping segments during force disconnects. Allocate the 896 * trash memory with a large size which is page aligned. 897 */ 898 (void) ddi_umem_alloc((size_t)TRASHSIZE, 899 DDI_UMEM_TRASH, &remap_cookie); 900 901 /* initialize user segment id allocation variable */ 902 rsm_nextavail_segmentid = (rsm_memseg_id_t)RSM_USER_APP_ID_BASE; 903 904 /* 905 * initialize the null_rsmpi_ops vector and the loopback adapter 906 */ 907 rsmka_init_loopback(); 908 909 910 ddi_report_dev(devi); 911 912 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach done\n")); 913 914 return (DDI_SUCCESS); 915 } 916 917 /* 918 * The call to mod_remove in the _fine routine will cause the system 919 * to call rsm_detach 920 */ 921 /*ARGSUSED*/ 922 static int 923 rsm_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 924 { 925 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI); 926 927 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach enter\n")); 928 929 switch (cmd) { 930 case DDI_DETACH: 931 break; 932 default: 933 DBG_PRINTF((category, RSM_ERR, 934 "rsm:rsm_detach - cmd %x not supported\n", 935 cmd)); 936 return (DDI_FAILURE); 937 } 938 939 mutex_enter(&rsm_drv_data.drv_lock); 940 while (rsm_drv_data.drv_state != RSM_DRV_OK) 941 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 942 rsm_drv_data.drv_state = RSM_DRV_UNREG_PROCESSING; 943 mutex_exit(&rsm_drv_data.drv_lock); 944 945 /* 946 * Unregister the DR callback functions 947 */ 948 if (rsm_enable_dr) { 949 #ifdef RSM_DRTEST 950 rsm_kphysm_setup_func_unregister(&rsm_dr_callback_vec, 951 (void *)NULL); 952 #else 953 kphysm_setup_func_unregister(&rsm_dr_callback_vec, 954 (void *)NULL); 955 #endif 956 } 957 958 mutex_enter(&rsm_drv_data.drv_lock); 959 ASSERT(rsm_drv_data.drv_state == RSM_DRV_UNREG_PROCESSING); 960 rsm_drv_data.drv_state = RSM_DRV_NEW; 961 mutex_exit(&rsm_drv_data.drv_lock); 962 963 ASSERT(rsm_suspend_list.list_head == NULL); 964 965 /* 966 * Release all resources, seglist, controller, ... 967 */ 968 969 /* remove intersend queues */ 970 /* remove registered services */ 971 972 973 ddi_remove_minor_node(dip, DRIVER_NAME); 974 rsm_dip = NULL; 975 976 /* 977 * Free minor zero resource 978 */ 979 { 980 rsmresource_t *p; 981 982 p = rsmresource_free(RSM_DRIVER_MINOR); 983 if (p) { 984 mutex_destroy(&p->rsmrc_lock); 985 kmem_free((void *)p, sizeof (*p)); 986 } 987 } 988 989 /* 990 * Free resource table 991 */ 992 993 rsmresource_destroy(); 994 995 /* 996 * Free the hash tables 997 */ 998 rsmhash_free(&rsm_export_segs, rsm_hash_size); 999 rsmhash_free(&rsm_import_segs, rsm_hash_size); 1000 1001 kmem_free((void *)importer_list.bucket, 1002 rsm_hash_size * sizeof (importing_token_t *)); 1003 importer_list.bucket = NULL; 1004 1005 1006 /* free barrier page */ 1007 if (bar_cookie != NULL) { 1008 ddi_umem_free(bar_cookie); 1009 } 1010 bar_va = NULL; 1011 bar_cookie = NULL; 1012 1013 /* 1014 * Free the memory allocated for the trash 1015 */ 1016 if (remap_cookie != NULL) { 1017 ddi_umem_free(remap_cookie); 1018 } 1019 remap_cookie = NULL; 1020 1021 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach done\n")); 1022 1023 return (DDI_SUCCESS); 1024 } 1025 1026 /*ARGSUSED*/ 1027 static int 1028 rsm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 1029 { 1030 register int error; 1031 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI); 1032 1033 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info enter\n")); 1034 1035 switch (infocmd) { 1036 case DDI_INFO_DEVT2DEVINFO: 1037 if (rsm_dip == NULL) 1038 error = DDI_FAILURE; 1039 else { 1040 *result = (void *)rsm_dip; 1041 error = DDI_SUCCESS; 1042 } 1043 break; 1044 case DDI_INFO_DEVT2INSTANCE: 1045 *result = (void *)0; 1046 error = DDI_SUCCESS; 1047 break; 1048 default: 1049 error = DDI_FAILURE; 1050 } 1051 1052 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info done\n")); 1053 return (error); 1054 } 1055 1056 adapter_t * 1057 rsm_getadapter(rsm_ioctlmsg_t *msg, int mode) 1058 { 1059 adapter_t *adapter; 1060 char adapter_devname[MAXNAMELEN]; 1061 int instance; 1062 DBG_DEFINE(category, 1063 RSM_KERNEL_AGENT | RSM_IMPORT | RSM_EXPORT | RSM_IOCTL); 1064 1065 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter enter\n")); 1066 1067 instance = msg->cnum; 1068 1069 if ((msg->cname_len <= 0) || (msg->cname_len > MAXNAMELEN)) { 1070 return (NULL); 1071 } 1072 1073 if (ddi_copyin(msg->cname, adapter_devname, msg->cname_len, mode)) 1074 return (NULL); 1075 1076 if (strcmp(adapter_devname, "loopback") == 0) 1077 return (&loopback_adapter); 1078 1079 adapter = rsmka_lookup_adapter(adapter_devname, instance); 1080 1081 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter done\n")); 1082 1083 return (adapter); 1084 } 1085 1086 1087 /* 1088 * *********************** Resource Number Management ******************** 1089 * All resources are stored in a simple hash table. The table is an array 1090 * of pointers to resource blks. Each blk contains: 1091 * base - base number of this blk 1092 * used - number of used slots in this blk. 1093 * blks - array of pointers to resource items. 1094 * An entry in a resource blk is empty if it's NULL. 1095 * 1096 * We start with no resource array. Each time we run out of slots, we 1097 * reallocate a new larger array and copy the pointer to the new array and 1098 * a new resource blk is allocated and added to the hash table. 1099 * 1100 * The resource control block contains: 1101 * root - array of pointer of resource blks 1102 * sz - current size of array. 1103 * len - last valid entry in array. 1104 * 1105 * A search operation based on a resource number is as follows: 1106 * index = rnum / RESOURCE_BLKSZ; 1107 * ASSERT(index < resource_block.len); 1108 * ASSERT(index < resource_block.sz); 1109 * offset = rnum % RESOURCE_BLKSZ; 1110 * ASSERT(offset >= resource_block.root[index]->base); 1111 * ASSERT(offset < resource_block.root[index]->base + RESOURCE_BLKSZ); 1112 * return resource_block.root[index]->blks[offset]; 1113 * 1114 * A resource blk is freed with its used count reachs zero. 1115 */ 1116 static int 1117 rsmresource_alloc(minor_t *rnum) 1118 { 1119 1120 /* search for available resource slot */ 1121 int i, j, empty = -1; 1122 rsmresource_blk_t *blk; 1123 1124 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1125 "rsmresource_alloc enter\n")); 1126 1127 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER); 1128 1129 /* Try to find an empty slot */ 1130 for (i = 0; i < rsm_resource.rsmrc_len; i++) { 1131 blk = rsm_resource.rsmrc_root[i]; 1132 if (blk != NULL && blk->rsmrcblk_avail > 0) { 1133 /* found an empty slot in this blk */ 1134 for (j = 0; j < RSMRC_BLKSZ; j++) { 1135 if (blk->rsmrcblk_blks[j] == NULL) { 1136 *rnum = (minor_t) 1137 (j + (i * RSMRC_BLKSZ)); 1138 /* 1139 * obey gen page limits 1140 */ 1141 if (*rnum >= max_segs + 1) { 1142 if (empty < 0) { 1143 rw_exit(&rsm_resource. 1144 rsmrc_lock); 1145 DBG_PRINTF(( 1146 RSM_KERNEL_ALL, 1147 RSM_ERR, 1148 "rsmresource" 1149 "_alloc failed:" 1150 "not enough res" 1151 "%d\n", *rnum)); 1152 return (RSMERR_INSUFFICIENT_RESOURCES); 1153 } else { 1154 /* use empty slot */ 1155 break; 1156 } 1157 1158 } 1159 1160 blk->rsmrcblk_blks[j] = RSMRC_RESERVED; 1161 blk->rsmrcblk_avail--; 1162 rw_exit(&rsm_resource.rsmrc_lock); 1163 DBG_PRINTF((RSM_KERNEL_ALL, 1164 RSM_DEBUG_VERBOSE, 1165 "rsmresource_alloc done\n")); 1166 return (RSM_SUCCESS); 1167 } 1168 } 1169 } else if (blk == NULL && empty < 0) { 1170 /* remember first empty slot */ 1171 empty = i; 1172 } 1173 } 1174 1175 /* Couldn't find anything, allocate a new blk */ 1176 /* 1177 * Do we need to reallocate the root array 1178 */ 1179 if (empty < 0) { 1180 if (rsm_resource.rsmrc_len == rsm_resource.rsmrc_sz) { 1181 /* 1182 * Allocate new array and copy current stuff into it 1183 */ 1184 rsmresource_blk_t **p; 1185 uint_t newsz = (uint_t)rsm_resource.rsmrc_sz + 1186 RSMRC_BLKSZ; 1187 /* 1188 * Don't allocate more that max valid rnum 1189 */ 1190 if (rsm_resource.rsmrc_len*RSMRC_BLKSZ >= 1191 max_segs + 1) { 1192 rw_exit(&rsm_resource.rsmrc_lock); 1193 return (RSMERR_INSUFFICIENT_RESOURCES); 1194 } 1195 1196 p = (rsmresource_blk_t **)kmem_zalloc( 1197 newsz * sizeof (*p), 1198 KM_SLEEP); 1199 1200 if (rsm_resource.rsmrc_root) { 1201 uint_t oldsz; 1202 1203 oldsz = (uint_t)(rsm_resource.rsmrc_sz * 1204 (int)sizeof (*p)); 1205 1206 /* 1207 * Copy old data into new space and 1208 * free old stuff 1209 */ 1210 bcopy(rsm_resource.rsmrc_root, p, oldsz); 1211 kmem_free(rsm_resource.rsmrc_root, oldsz); 1212 } 1213 1214 rsm_resource.rsmrc_root = p; 1215 rsm_resource.rsmrc_sz = (int)newsz; 1216 } 1217 1218 empty = rsm_resource.rsmrc_len; 1219 rsm_resource.rsmrc_len++; 1220 } 1221 1222 /* 1223 * Allocate a new blk 1224 */ 1225 blk = (rsmresource_blk_t *)kmem_zalloc(sizeof (*blk), KM_SLEEP); 1226 ASSERT(rsm_resource.rsmrc_root[empty] == NULL); 1227 rsm_resource.rsmrc_root[empty] = blk; 1228 blk->rsmrcblk_avail = RSMRC_BLKSZ - 1; 1229 1230 /* 1231 * Allocate slot 1232 */ 1233 1234 *rnum = (minor_t)(empty * RSMRC_BLKSZ); 1235 1236 /* 1237 * watch out not to exceed bounds of barrier page 1238 */ 1239 if (*rnum >= max_segs + 1) { 1240 rw_exit(&rsm_resource.rsmrc_lock); 1241 DBG_PRINTF((RSM_KERNEL_ALL, RSM_ERR, 1242 "rsmresource_alloc failed %d\n", *rnum)); 1243 1244 return (RSMERR_INSUFFICIENT_RESOURCES); 1245 } 1246 blk->rsmrcblk_blks[0] = RSMRC_RESERVED; 1247 1248 1249 rw_exit(&rsm_resource.rsmrc_lock); 1250 1251 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1252 "rsmresource_alloc done\n")); 1253 1254 return (RSM_SUCCESS); 1255 } 1256 1257 static rsmresource_t * 1258 rsmresource_free(minor_t rnum) 1259 { 1260 1261 /* search for available resource slot */ 1262 int i, j; 1263 rsmresource_blk_t *blk; 1264 rsmresource_t *p; 1265 1266 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1267 "rsmresource_free enter\n")); 1268 1269 i = (int)(rnum / RSMRC_BLKSZ); 1270 j = (int)(rnum % RSMRC_BLKSZ); 1271 1272 if (i >= rsm_resource.rsmrc_len) { 1273 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1274 "rsmresource_free done\n")); 1275 return (NULL); 1276 } 1277 1278 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER); 1279 1280 ASSERT(rsm_resource.rsmrc_root); 1281 ASSERT(i < rsm_resource.rsmrc_len); 1282 ASSERT(i < rsm_resource.rsmrc_sz); 1283 blk = rsm_resource.rsmrc_root[i]; 1284 if (blk == NULL) { 1285 rw_exit(&rsm_resource.rsmrc_lock); 1286 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1287 "rsmresource_free done\n")); 1288 return (NULL); 1289 } 1290 1291 ASSERT(blk->rsmrcblk_blks[j]); /* reserved or full */ 1292 1293 p = blk->rsmrcblk_blks[j]; 1294 if (p == RSMRC_RESERVED) { 1295 p = NULL; 1296 } 1297 1298 blk->rsmrcblk_blks[j] = NULL; 1299 blk->rsmrcblk_avail++; 1300 if (blk->rsmrcblk_avail == RSMRC_BLKSZ) { 1301 /* free this blk */ 1302 kmem_free(blk, sizeof (*blk)); 1303 rsm_resource.rsmrc_root[i] = NULL; 1304 } 1305 1306 rw_exit(&rsm_resource.rsmrc_lock); 1307 1308 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1309 "rsmresource_free done\n")); 1310 1311 return (p); 1312 } 1313 1314 static rsmresource_t * 1315 rsmresource_lookup(minor_t rnum, int lock) 1316 { 1317 int i, j; 1318 rsmresource_blk_t *blk; 1319 rsmresource_t *p; 1320 1321 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1322 "rsmresource_lookup enter\n")); 1323 1324 /* Find resource and lock it in READER mode */ 1325 /* search for available resource slot */ 1326 1327 i = (int)(rnum / RSMRC_BLKSZ); 1328 j = (int)(rnum % RSMRC_BLKSZ); 1329 1330 if (i >= rsm_resource.rsmrc_len) { 1331 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1332 "rsmresource_lookup done\n")); 1333 return (NULL); 1334 } 1335 1336 rw_enter(&rsm_resource.rsmrc_lock, RW_READER); 1337 1338 blk = rsm_resource.rsmrc_root[i]; 1339 if (blk != NULL) { 1340 ASSERT(i < rsm_resource.rsmrc_len); 1341 ASSERT(i < rsm_resource.rsmrc_sz); 1342 1343 p = blk->rsmrcblk_blks[j]; 1344 if (lock == RSM_LOCK) { 1345 if (p != RSMRC_RESERVED) { 1346 mutex_enter(&p->rsmrc_lock); 1347 } else { 1348 p = NULL; 1349 } 1350 } 1351 } else { 1352 p = NULL; 1353 } 1354 rw_exit(&rsm_resource.rsmrc_lock); 1355 1356 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1357 "rsmresource_lookup done\n")); 1358 1359 return (p); 1360 } 1361 1362 static void 1363 rsmresource_insert(minor_t rnum, rsmresource_t *p, rsm_resource_type_t type) 1364 { 1365 /* Find resource and lock it in READER mode */ 1366 /* Caller can upgrade if need be */ 1367 /* search for available resource slot */ 1368 int i, j; 1369 rsmresource_blk_t *blk; 1370 1371 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1372 "rsmresource_insert enter\n")); 1373 1374 i = (int)(rnum / RSMRC_BLKSZ); 1375 j = (int)(rnum % RSMRC_BLKSZ); 1376 1377 p->rsmrc_type = type; 1378 p->rsmrc_num = rnum; 1379 1380 rw_enter(&rsm_resource.rsmrc_lock, RW_READER); 1381 1382 ASSERT(rsm_resource.rsmrc_root); 1383 ASSERT(i < rsm_resource.rsmrc_len); 1384 ASSERT(i < rsm_resource.rsmrc_sz); 1385 1386 blk = rsm_resource.rsmrc_root[i]; 1387 ASSERT(blk); 1388 1389 ASSERT(blk->rsmrcblk_blks[j] == RSMRC_RESERVED); 1390 1391 blk->rsmrcblk_blks[j] = p; 1392 1393 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1394 "rsmresource_insert done\n")); 1395 1396 rw_exit(&rsm_resource.rsmrc_lock); 1397 } 1398 1399 static void 1400 rsmresource_destroy() 1401 { 1402 int i, j; 1403 1404 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1405 "rsmresource_destroy enter\n")); 1406 1407 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER); 1408 1409 for (i = 0; i < rsm_resource.rsmrc_len; i++) { 1410 rsmresource_blk_t *blk; 1411 1412 blk = rsm_resource.rsmrc_root[i]; 1413 if (blk == NULL) { 1414 continue; 1415 } 1416 for (j = 0; j < RSMRC_BLKSZ; j++) { 1417 if (blk->rsmrcblk_blks[j] != NULL) { 1418 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1419 "Not null slot %d, %lx\n", j, 1420 (size_t)blk->rsmrcblk_blks[j])); 1421 } 1422 } 1423 kmem_free(blk, sizeof (*blk)); 1424 rsm_resource.rsmrc_root[i] = NULL; 1425 } 1426 if (rsm_resource.rsmrc_root) { 1427 i = rsm_resource.rsmrc_sz * (int)sizeof (rsmresource_blk_t *); 1428 kmem_free(rsm_resource.rsmrc_root, (uint_t)i); 1429 rsm_resource.rsmrc_root = NULL; 1430 rsm_resource.rsmrc_len = 0; 1431 rsm_resource.rsmrc_sz = 0; 1432 } 1433 1434 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE, 1435 "rsmresource_destroy done\n")); 1436 1437 rw_exit(&rsm_resource.rsmrc_lock); 1438 } 1439 1440 1441 /* ******************** Generic Key Hash Table Management ********* */ 1442 static rsmresource_t * 1443 rsmhash_lookup(rsmhash_table_t *rhash, rsm_memseg_id_t key, 1444 rsm_resource_state_t state) 1445 { 1446 rsmresource_t *p; 1447 uint_t hashval; 1448 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1449 1450 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup enter\n")); 1451 1452 hashval = rsmhash(key); 1453 1454 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_lookup %u=%d\n", 1455 key, hashval)); 1456 1457 rw_enter(&rhash->rsmhash_rw, RW_READER); 1458 1459 p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval); 1460 1461 for (; p; p = p->rsmrc_next) { 1462 if (p->rsmrc_key == key) { 1463 /* acquire resource lock */ 1464 RSMRC_LOCK(p); 1465 break; 1466 } 1467 } 1468 1469 rw_exit(&rhash->rsmhash_rw); 1470 1471 if (p != NULL && p->rsmrc_state != state) { 1472 /* state changed, release lock and return null */ 1473 RSMRC_UNLOCK(p); 1474 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 1475 "rsmhash_lookup done: state changed\n")); 1476 return (NULL); 1477 } 1478 1479 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup done\n")); 1480 1481 return (p); 1482 } 1483 1484 static void 1485 rsmhash_rm(rsmhash_table_t *rhash, rsmresource_t *rcelm) 1486 { 1487 rsmresource_t *p, **back; 1488 uint_t hashval; 1489 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1490 1491 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm enter\n")); 1492 1493 hashval = rsmhash(rcelm->rsmrc_key); 1494 1495 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_rm %u=%d\n", 1496 rcelm->rsmrc_key, hashval)); 1497 1498 /* 1499 * It's ok not to find the segment. 1500 */ 1501 rw_enter(&rhash->rsmhash_rw, RW_WRITER); 1502 1503 back = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval); 1504 1505 for (; (p = *back) != NULL; back = &p->rsmrc_next) { 1506 if (p == rcelm) { 1507 *back = rcelm->rsmrc_next; 1508 break; 1509 } 1510 } 1511 1512 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm done\n")); 1513 1514 rw_exit(&rhash->rsmhash_rw); 1515 } 1516 1517 static int 1518 rsmhash_add(rsmhash_table_t *rhash, rsmresource_t *new, rsm_memseg_id_t key, 1519 int dup_check, rsm_resource_state_t state) 1520 { 1521 rsmresource_t *p = NULL, **bktp; 1522 uint_t hashval; 1523 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1524 1525 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add enter\n")); 1526 1527 /* lock table */ 1528 rw_enter(&rhash->rsmhash_rw, RW_WRITER); 1529 1530 /* 1531 * If the current resource state is other than the state passed in 1532 * then the resource is (probably) already on the list. eg. for an 1533 * import segment if the state is not RSM_STATE_NEW then it's on the 1534 * list already. 1535 */ 1536 RSMRC_LOCK(new); 1537 if (new->rsmrc_state != state) { 1538 RSMRC_UNLOCK(new); 1539 rw_exit(&rhash->rsmhash_rw); 1540 return (RSMERR_BAD_SEG_HNDL); 1541 } 1542 1543 hashval = rsmhash(key); 1544 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_add %d\n", hashval)); 1545 1546 if (dup_check) { 1547 /* 1548 * Used for checking export segments; don't want to have 1549 * the same key used for multiple segments. 1550 */ 1551 1552 p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval); 1553 1554 for (; p; p = p->rsmrc_next) { 1555 if (p->rsmrc_key == key) { 1556 RSMRC_UNLOCK(new); 1557 break; 1558 } 1559 } 1560 } 1561 1562 if (p == NULL) { 1563 /* Key doesn't exist, add it */ 1564 1565 bktp = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval); 1566 1567 new->rsmrc_key = key; 1568 new->rsmrc_next = *bktp; 1569 *bktp = new; 1570 } 1571 1572 rw_exit(&rhash->rsmhash_rw); 1573 1574 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add done\n")); 1575 1576 return (p == NULL ? RSM_SUCCESS : RSMERR_SEGID_IN_USE); 1577 } 1578 1579 /* 1580 * XOR each byte of the key. 1581 */ 1582 static uint_t 1583 rsmhash(rsm_memseg_id_t key) 1584 { 1585 uint_t hash = key; 1586 1587 hash ^= (key >> 8); 1588 hash ^= (key >> 16); 1589 hash ^= (key >> 24); 1590 1591 return (hash % rsm_hash_size); 1592 1593 } 1594 1595 /* 1596 * generic function to get a specific bucket 1597 */ 1598 static void * 1599 rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval) 1600 { 1601 1602 if (rhash->bucket == NULL) 1603 return (NULL); 1604 else 1605 return ((void *)rhash->bucket[hashval]); 1606 } 1607 1608 /* 1609 * generic function to get a specific bucket's address 1610 */ 1611 static void ** 1612 rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval) 1613 { 1614 if (rhash->bucket == NULL) 1615 return (NULL); 1616 else 1617 return ((void **)&(rhash->bucket[hashval])); 1618 } 1619 1620 /* 1621 * generic function to alloc a hash table 1622 */ 1623 static void 1624 rsmhash_alloc(rsmhash_table_t *rhash, int size) 1625 { 1626 rhash->bucket = (rsmresource_t **) 1627 kmem_zalloc(size * sizeof (rsmresource_t *), KM_SLEEP); 1628 } 1629 1630 /* 1631 * generic function to free a hash table 1632 */ 1633 static void 1634 rsmhash_free(rsmhash_table_t *rhash, int size) 1635 { 1636 1637 kmem_free((void *)rhash->bucket, size * sizeof (caddr_t)); 1638 rhash->bucket = NULL; 1639 1640 } 1641 /* *********************** Exported Segment Key Management ************ */ 1642 1643 #define rsmexport_add(new, key) \ 1644 rsmhash_add(&rsm_export_segs, (rsmresource_t *)new, key, 1, \ 1645 RSM_STATE_BIND) 1646 1647 #define rsmexport_rm(arg) \ 1648 rsmhash_rm(&rsm_export_segs, (rsmresource_t *)(arg)) 1649 1650 #define rsmexport_lookup(key) \ 1651 (rsmseg_t *)rsmhash_lookup(&rsm_export_segs, key, RSM_STATE_EXPORT) 1652 1653 /* ************************** Import Segment List Management ********** */ 1654 1655 /* 1656 * Add segment to import list. This will be useful for paging and loopback 1657 * segment unloading. 1658 */ 1659 #define rsmimport_add(arg, key) \ 1660 rsmhash_add(&rsm_import_segs, (rsmresource_t *)(arg), (key), 0, \ 1661 RSM_STATE_NEW) 1662 1663 #define rsmimport_rm(arg) \ 1664 rsmhash_rm(&rsm_import_segs, (rsmresource_t *)(arg)) 1665 1666 /* 1667 * #define rsmimport_lookup(key) \ 1668 * (rsmseg_t *)rsmhash_lookup(&rsm_import_segs, (key), RSM_STATE_CONNECT) 1669 */ 1670 1671 /* 1672 * increase the ref count and make the import segment point to the 1673 * shared data structure. Return a pointer to the share data struct 1674 * and the shared data struct is locked upon return 1675 */ 1676 static rsm_import_share_t * 1677 rsmshare_get(rsm_memseg_id_t key, rsm_node_id_t node, adapter_t *adapter, 1678 rsmseg_t *segp) 1679 { 1680 uint_t hash; 1681 rsmresource_t *p; 1682 rsm_import_share_t *shdatap; 1683 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1684 1685 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get enter\n")); 1686 1687 hash = rsmhash(key); 1688 /* lock table */ 1689 rw_enter(&rsm_import_segs.rsmhash_rw, RW_WRITER); 1690 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmshare_get:key=%u, hash=%d\n", 1691 key, hash)); 1692 1693 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hash); 1694 1695 for (; p; p = p->rsmrc_next) { 1696 /* 1697 * Look for an entry that is importing the same exporter 1698 * with the share data structure allocated. 1699 */ 1700 if ((p->rsmrc_key == key) && 1701 (p->rsmrc_node == node) && 1702 (p->rsmrc_adapter == adapter) && 1703 (((rsmseg_t *)p)->s_share != NULL)) { 1704 shdatap = ((rsmseg_t *)p)->s_share; 1705 break; 1706 } 1707 } 1708 1709 if (p == NULL) { 1710 /* we are the first importer, create the shared data struct */ 1711 shdatap = kmem_zalloc(sizeof (rsm_import_share_t), KM_SLEEP); 1712 shdatap->rsmsi_state = RSMSI_STATE_NEW; 1713 shdatap->rsmsi_segid = key; 1714 shdatap->rsmsi_node = node; 1715 mutex_init(&shdatap->rsmsi_lock, NULL, MUTEX_DRIVER, NULL); 1716 cv_init(&shdatap->rsmsi_cv, NULL, CV_DRIVER, 0); 1717 } 1718 1719 rsmseglock_acquire(segp); 1720 1721 /* we grab the shared lock before returning from this function */ 1722 mutex_enter(&shdatap->rsmsi_lock); 1723 1724 shdatap->rsmsi_refcnt++; 1725 segp->s_share = shdatap; 1726 1727 rsmseglock_release(segp); 1728 1729 rw_exit(&rsm_import_segs.rsmhash_rw); 1730 1731 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get done\n")); 1732 1733 return (shdatap); 1734 } 1735 1736 /* 1737 * the shared data structure should be locked before calling 1738 * rsmsharecv_signal(). 1739 * Change the state and signal any waiting segments. 1740 */ 1741 void 1742 rsmsharecv_signal(rsmseg_t *seg, int oldstate, int newstate) 1743 { 1744 ASSERT(rsmsharelock_held(seg)); 1745 1746 if (seg->s_share->rsmsi_state == oldstate) { 1747 seg->s_share->rsmsi_state = newstate; 1748 cv_broadcast(&seg->s_share->rsmsi_cv); 1749 } 1750 } 1751 1752 /* 1753 * Add to the hash table 1754 */ 1755 static void 1756 importer_list_add(rsm_node_id_t node, rsm_memseg_id_t key, rsm_addr_t hwaddr, 1757 void *cookie) 1758 { 1759 1760 importing_token_t *head; 1761 importing_token_t *new_token; 1762 int index; 1763 1764 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1765 1766 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add enter\n")); 1767 1768 new_token = kmem_zalloc(sizeof (importing_token_t), KM_SLEEP); 1769 new_token->importing_node = node; 1770 new_token->key = key; 1771 new_token->import_segment_cookie = cookie; 1772 new_token->importing_adapter_hwaddr = hwaddr; 1773 1774 index = rsmhash(key); 1775 1776 mutex_enter(&importer_list.lock); 1777 1778 head = importer_list.bucket[index]; 1779 importer_list.bucket[index] = new_token; 1780 new_token->next = head; 1781 mutex_exit(&importer_list.lock); 1782 1783 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add done\n")); 1784 } 1785 1786 static void 1787 importer_list_rm(rsm_node_id_t node, rsm_memseg_id_t key, void *cookie) 1788 { 1789 1790 importing_token_t *prev, *token = NULL; 1791 int index; 1792 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1793 1794 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm enter\n")); 1795 1796 index = rsmhash(key); 1797 1798 mutex_enter(&importer_list.lock); 1799 1800 token = importer_list.bucket[index]; 1801 1802 prev = token; 1803 while (token != NULL) { 1804 if (token->importing_node == node && 1805 token->import_segment_cookie == cookie) { 1806 if (prev == token) 1807 importer_list.bucket[index] = token->next; 1808 else 1809 prev->next = token->next; 1810 kmem_free((void *)token, sizeof (*token)); 1811 break; 1812 } else { 1813 prev = token; 1814 token = token->next; 1815 } 1816 } 1817 1818 mutex_exit(&importer_list.lock); 1819 1820 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm done\n")); 1821 1822 1823 } 1824 1825 /* **************************Segment Structure Management ************* */ 1826 1827 /* 1828 * Free segment structure 1829 */ 1830 static void 1831 rsmseg_free(rsmseg_t *seg) 1832 { 1833 1834 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1835 1836 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free enter\n")); 1837 1838 /* need to take seglock here to avoid race with rsmmap_unmap() */ 1839 rsmseglock_acquire(seg); 1840 if (seg->s_ckl != NULL) { 1841 /* Segment is still busy */ 1842 seg->s_state = RSM_STATE_END; 1843 rsmseglock_release(seg); 1844 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 1845 "rsmseg_free done\n")); 1846 return; 1847 } 1848 1849 rsmseglock_release(seg); 1850 1851 ASSERT(seg->s_state == RSM_STATE_END || seg->s_state == RSM_STATE_NEW); 1852 1853 /* 1854 * If it's an importer decrement the refcount 1855 * and if its down to zero free the shared data structure. 1856 * This is where failures during rsm_connect() are unrefcounted 1857 */ 1858 if (seg->s_share != NULL) { 1859 1860 ASSERT(seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT); 1861 1862 rsmsharelock_acquire(seg); 1863 1864 ASSERT(seg->s_share->rsmsi_refcnt > 0); 1865 1866 seg->s_share->rsmsi_refcnt--; 1867 1868 if (seg->s_share->rsmsi_refcnt == 0) { 1869 rsmsharelock_release(seg); 1870 mutex_destroy(&seg->s_share->rsmsi_lock); 1871 cv_destroy(&seg->s_share->rsmsi_cv); 1872 kmem_free((void *)(seg->s_share), 1873 sizeof (rsm_import_share_t)); 1874 } else { 1875 rsmsharelock_release(seg); 1876 } 1877 /* 1878 * The following needs to be done after any 1879 * rsmsharelock calls which use seg->s_share. 1880 */ 1881 seg->s_share = NULL; 1882 } 1883 1884 cv_destroy(&seg->s_cv); 1885 mutex_destroy(&seg->s_lock); 1886 rsmacl_free(seg->s_acl, seg->s_acl_len); 1887 rsmpiacl_free(seg->s_acl_in, seg->s_acl_len); 1888 if (seg->s_adapter) 1889 rsmka_release_adapter(seg->s_adapter); 1890 1891 kmem_free((void *)seg, sizeof (*seg)); 1892 1893 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free done\n")); 1894 1895 } 1896 1897 1898 static rsmseg_t * 1899 rsmseg_alloc(minor_t num, struct cred *cred) 1900 { 1901 rsmseg_t *new; 1902 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 1903 1904 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc enter\n")); 1905 /* 1906 * allocate memory for new segment. This should be a segkmem cache. 1907 */ 1908 new = (rsmseg_t *)kmem_zalloc(sizeof (*new), KM_SLEEP); 1909 1910 new->s_state = RSM_STATE_NEW; 1911 new->s_minor = num; 1912 new->s_acl_len = 0; 1913 new->s_cookie = NULL; 1914 new->s_adapter = NULL; 1915 1916 new->s_mode = 0777 & ~PTOU((ttoproc(curthread)))->u_cmask; 1917 /* we don't have a key yet, will set at export/connect */ 1918 new->s_uid = crgetuid(cred); 1919 new->s_gid = crgetgid(cred); 1920 1921 mutex_init(&new->s_lock, NULL, MUTEX_DRIVER, (void *)NULL); 1922 cv_init(&new->s_cv, NULL, CV_DRIVER, 0); 1923 1924 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc done\n")); 1925 1926 return (new); 1927 } 1928 1929 /* ******************************** Driver Open/Close/Poll *************** */ 1930 1931 /*ARGSUSED1*/ 1932 static int 1933 rsm_open(dev_t *devp, int flag, int otyp, struct cred *cred) 1934 { 1935 minor_t rnum; 1936 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI); 1937 1938 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open enter\n")); 1939 /* 1940 * Char only 1941 */ 1942 if (otyp != OTYP_CHR) { 1943 DBG_PRINTF((category, RSM_ERR, "rsm_open: bad otyp\n")); 1944 return (EINVAL); 1945 } 1946 1947 /* 1948 * Only zero can be opened, clones are used for resources. 1949 */ 1950 if (getminor(*devp) != RSM_DRIVER_MINOR) { 1951 DBG_PRINTF((category, RSM_ERR, 1952 "rsm_open: bad minor %d\n", getminor(*devp))); 1953 return (ENODEV); 1954 } 1955 1956 if ((flag & FEXCL) != 0 && secpolicy_excl_open(cred) != 0) { 1957 DBG_PRINTF((category, RSM_ERR, "rsm_open: bad perm\n")); 1958 return (EPERM); 1959 } 1960 1961 if (!(flag & FWRITE)) { 1962 /* 1963 * The library function _rsm_librsm_init calls open for 1964 * /dev/rsm with flag set to O_RDONLY. We want a valid 1965 * file descriptor to be returned for minor device zero. 1966 */ 1967 1968 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 1969 "rsm_open RDONLY done\n")); 1970 return (DDI_SUCCESS); 1971 } 1972 1973 /* 1974 * - allocate new minor number and segment. 1975 * - add segment to list of all segments. 1976 * - set minordev data to segment 1977 * - update devp argument to new device 1978 * - update s_cred to cred; make sure you do crhold(cred); 1979 */ 1980 1981 /* allocate a new resource number */ 1982 if (rsmresource_alloc(&rnum) == RSM_SUCCESS) { 1983 /* 1984 * We will bind this minor to a specific resource in first 1985 * ioctl 1986 */ 1987 *devp = makedevice(getmajor(*devp), rnum); 1988 } else { 1989 return (EAGAIN); 1990 } 1991 1992 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open done\n")); 1993 return (DDI_SUCCESS); 1994 } 1995 1996 static void 1997 rsmseg_close(rsmseg_t *seg, int force_flag) 1998 { 1999 int e = RSM_SUCCESS; 2000 2001 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI); 2002 2003 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close enter\n")); 2004 2005 rsmseglock_acquire(seg); 2006 if (!force_flag && (seg->s_hdr.rsmrc_type == 2007 RSM_RESOURCE_EXPORT_SEGMENT)) { 2008 /* 2009 * If we are processing rsm_close wait for force_destroy 2010 * processing to complete since force_destroy processing 2011 * needs to finish first before we can free the segment. 2012 * force_destroy is only for export segments 2013 */ 2014 while (seg->s_flags & RSM_FORCE_DESTROY_WAIT) { 2015 cv_wait(&seg->s_cv, &seg->s_lock); 2016 } 2017 } 2018 rsmseglock_release(seg); 2019 2020 /* It's ok to read the state without a lock */ 2021 switch (seg->s_state) { 2022 case RSM_STATE_EXPORT: 2023 case RSM_STATE_EXPORT_QUIESCING: 2024 case RSM_STATE_EXPORT_QUIESCED: 2025 e = rsm_unpublish(seg, 1); 2026 /* FALLTHRU */ 2027 case RSM_STATE_BIND_QUIESCED: 2028 /* FALLTHRU */ 2029 case RSM_STATE_BIND: 2030 e = rsm_unbind(seg); 2031 if (e != RSM_SUCCESS && force_flag == 1) 2032 return; 2033 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT); 2034 /* FALLTHRU */ 2035 case RSM_STATE_NEW_QUIESCED: 2036 rsmseglock_acquire(seg); 2037 seg->s_state = RSM_STATE_NEW; 2038 cv_broadcast(&seg->s_cv); 2039 rsmseglock_release(seg); 2040 break; 2041 case RSM_STATE_NEW: 2042 break; 2043 case RSM_STATE_ZOMBIE: 2044 /* 2045 * Segments in this state have been removed off the 2046 * exported segments list and have been unpublished 2047 * and unbind. These segments have been removed during 2048 * a callback to the rsm_export_force_destroy, which 2049 * is called for the purpose of unlocking these 2050 * exported memory segments when a process exits but 2051 * leaves the segments locked down since rsm_close is 2052 * is not called for the segments. This can happen 2053 * when a process calls fork or exec and then exits. 2054 * Once the segments are in the ZOMBIE state, all that 2055 * remains is to destroy them when rsm_close is called. 2056 * This is done here. Thus, for such segments the 2057 * the state is changed to new so that later in this 2058 * function rsmseg_free is called. 2059 */ 2060 rsmseglock_acquire(seg); 2061 seg->s_state = RSM_STATE_NEW; 2062 rsmseglock_release(seg); 2063 break; 2064 case RSM_STATE_MAP_QUIESCE: 2065 case RSM_STATE_ACTIVE: 2066 /* Disconnect will handle the unmap */ 2067 case RSM_STATE_CONN_QUIESCE: 2068 case RSM_STATE_CONNECT: 2069 case RSM_STATE_DISCONNECT: 2070 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 2071 (void) rsm_disconnect(seg); 2072 break; 2073 case RSM_STATE_MAPPING: 2074 /*FALLTHRU*/ 2075 case RSM_STATE_END: 2076 DBG_PRINTF((category, RSM_ERR, 2077 "Invalid segment state %d in rsm_close\n", seg->s_state)); 2078 break; 2079 default: 2080 DBG_PRINTF((category, RSM_ERR, 2081 "Invalid segment state %d in rsm_close\n", seg->s_state)); 2082 break; 2083 } 2084 2085 /* 2086 * check state. 2087 * - make sure you do crfree(s_cred); 2088 * release segment and minor number 2089 */ 2090 ASSERT(seg->s_state == RSM_STATE_NEW); 2091 2092 /* 2093 * The export_force_destroy callback is created to unlock 2094 * the exported segments of a process 2095 * when the process does a fork or exec and then exits calls this 2096 * function with the force flag set to 1 which indicates that the 2097 * segment state must be converted to ZOMBIE. This state means that the 2098 * segments still exist and have been unlocked and most importantly the 2099 * only operation allowed is to destroy them on an rsm_close. 2100 */ 2101 if (force_flag) { 2102 rsmseglock_acquire(seg); 2103 seg->s_state = RSM_STATE_ZOMBIE; 2104 rsmseglock_release(seg); 2105 } else { 2106 rsmseg_free(seg); 2107 } 2108 2109 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close done\n")); 2110 } 2111 2112 static int 2113 rsm_close(dev_t dev, int flag, int otyp, cred_t *cred) 2114 { 2115 minor_t rnum = getminor(dev); 2116 rsmresource_t *res; 2117 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI); 2118 2119 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close enter\n")); 2120 2121 flag = flag; cred = cred; 2122 2123 if (otyp != OTYP_CHR) 2124 return (EINVAL); 2125 2126 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rnum = %d\n", rnum)); 2127 2128 /* 2129 * At this point we are the last reference to the resource. 2130 * Free resource number from resource table. 2131 * It's ok to remove number before we free the segment. 2132 * We need to lock the resource to protect against remote calls. 2133 */ 2134 if (rnum == RSM_DRIVER_MINOR || 2135 (res = rsmresource_free(rnum)) == NULL) { 2136 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n")); 2137 return (DDI_SUCCESS); 2138 } 2139 2140 switch (res->rsmrc_type) { 2141 case RSM_RESOURCE_EXPORT_SEGMENT: 2142 case RSM_RESOURCE_IMPORT_SEGMENT: 2143 rsmseg_close((rsmseg_t *)res, 0); 2144 break; 2145 case RSM_RESOURCE_BAR: 2146 DBG_PRINTF((category, RSM_ERR, "bad resource in rsm_close\n")); 2147 break; 2148 default: 2149 break; 2150 } 2151 2152 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n")); 2153 2154 return (DDI_SUCCESS); 2155 } 2156 2157 /* 2158 * rsm_inc_pgcnt 2159 * 2160 * Description: increment rsm page counter. 2161 * 2162 * Parameters: pgcnt_t pnum; number of pages to be used 2163 * 2164 * Returns: RSM_SUCCESS if memory limit not exceeded 2165 * ENOSPC if memory limit exceeded. In this case, the 2166 * page counter remains unchanged. 2167 * 2168 */ 2169 static int 2170 rsm_inc_pgcnt(pgcnt_t pnum) 2171 { 2172 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2173 if (rsm_pgcnt_max == 0) { /* no upper limit has been set */ 2174 return (RSM_SUCCESS); 2175 } 2176 2177 mutex_enter(&rsm_pgcnt_lock); 2178 2179 if (rsm_pgcnt + pnum > rsm_pgcnt_max) { 2180 /* ensure that limits have not been exceeded */ 2181 mutex_exit(&rsm_pgcnt_lock); 2182 return (RSMERR_INSUFFICIENT_MEM); 2183 } 2184 2185 rsm_pgcnt += pnum; 2186 DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt incr to %d.\n", 2187 rsm_pgcnt)); 2188 mutex_exit(&rsm_pgcnt_lock); 2189 2190 return (RSM_SUCCESS); 2191 } 2192 2193 /* 2194 * rsm_dec_pgcnt 2195 * 2196 * Description: decrement rsm page counter. 2197 * 2198 * Parameters: pgcnt_t pnum; number of pages freed 2199 * 2200 */ 2201 static void 2202 rsm_dec_pgcnt(pgcnt_t pnum) 2203 { 2204 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2205 2206 if (rsm_pgcnt_max == 0) { /* no upper limit has been set */ 2207 return; 2208 } 2209 2210 mutex_enter(&rsm_pgcnt_lock); 2211 ASSERT(rsm_pgcnt >= pnum); 2212 rsm_pgcnt -= pnum; 2213 DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt decr to %d.\n", 2214 rsm_pgcnt)); 2215 mutex_exit(&rsm_pgcnt_lock); 2216 } 2217 2218 static struct umem_callback_ops rsm_as_ops = { 2219 UMEM_CALLBACK_VERSION, /* version number */ 2220 rsm_export_force_destroy, 2221 }; 2222 2223 static int 2224 rsm_bind_pages(ddi_umem_cookie_t *cookie, caddr_t vaddr, size_t len, 2225 proc_t *procp) 2226 { 2227 int error = RSM_SUCCESS; 2228 ulong_t pnum; 2229 struct umem_callback_ops *callbackops = &rsm_as_ops; 2230 2231 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2232 2233 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages enter\n")); 2234 2235 /* 2236 * Make sure vaddr and len are aligned on a page boundary 2237 */ 2238 if ((uintptr_t)vaddr & (PAGESIZE - 1)) { 2239 return (RSMERR_BAD_ADDR); 2240 } 2241 2242 if (len & (PAGESIZE - 1)) { 2243 return (RSMERR_BAD_LENGTH); 2244 } 2245 2246 /* 2247 * Find number of pages 2248 */ 2249 pnum = btopr(len); 2250 error = rsm_inc_pgcnt(pnum); 2251 if (error != RSM_SUCCESS) { 2252 DBG_PRINTF((category, RSM_ERR, 2253 "rsm_bind_pages:mem limit exceeded\n")); 2254 return (RSMERR_INSUFFICIENT_MEM); 2255 } 2256 2257 error = umem_lockmemory(vaddr, len, 2258 DDI_UMEMLOCK_WRITE|DDI_UMEMLOCK_READ|DDI_UMEMLOCK_LONGTERM, 2259 cookie, 2260 callbackops, procp); 2261 2262 if (error) { 2263 rsm_dec_pgcnt(pnum); 2264 DBG_PRINTF((category, RSM_ERR, 2265 "rsm_bind_pages:ddi_umem_lock failed\n")); 2266 /* 2267 * ddi_umem_lock, in the case of failure, returns one of 2268 * the following three errors. These are translated into 2269 * the RSMERR namespace and returned. 2270 */ 2271 if (error == EFAULT) 2272 return (RSMERR_BAD_ADDR); 2273 else if (error == EACCES) 2274 return (RSMERR_PERM_DENIED); 2275 else 2276 return (RSMERR_INSUFFICIENT_MEM); 2277 } 2278 2279 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages done\n")); 2280 2281 return (error); 2282 2283 } 2284 2285 static int 2286 rsm_unbind_pages(rsmseg_t *seg) 2287 { 2288 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2289 2290 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages enter\n")); 2291 2292 ASSERT(rsmseglock_held(seg)); 2293 2294 if (seg->s_cookie != NULL) { 2295 /* unlock address range */ 2296 ddi_umem_unlock(seg->s_cookie); 2297 rsm_dec_pgcnt(btopr(seg->s_len)); 2298 seg->s_cookie = NULL; 2299 } 2300 2301 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages done\n")); 2302 2303 return (RSM_SUCCESS); 2304 } 2305 2306 2307 static int 2308 rsm_bind(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode) 2309 { 2310 int e; 2311 adapter_t *adapter; 2312 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2313 2314 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind enter\n")); 2315 2316 adapter = rsm_getadapter(msg, mode); 2317 if (adapter == NULL) { 2318 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2319 "rsm_bind done:no adapter\n")); 2320 return (RSMERR_CTLR_NOT_PRESENT); 2321 } 2322 2323 /* lock address range */ 2324 if (msg->vaddr == NULL) { 2325 rsmka_release_adapter(adapter); 2326 DBG_PRINTF((category, RSM_ERR, 2327 "rsm: rsm_bind done: invalid vaddr\n")); 2328 return (RSMERR_BAD_ADDR); 2329 } 2330 if (msg->len <= 0) { 2331 rsmka_release_adapter(adapter); 2332 DBG_PRINTF((category, RSM_ERR, 2333 "rsm_bind: invalid length\n")); 2334 return (RSMERR_BAD_LENGTH); 2335 } 2336 2337 /* Lock segment */ 2338 rsmseglock_acquire(seg); 2339 2340 while (seg->s_state == RSM_STATE_NEW_QUIESCED) { 2341 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 2342 DBG_PRINTF((category, RSM_DEBUG, 2343 "rsm_bind done: cv_wait INTERRUPTED")); 2344 rsmka_release_adapter(adapter); 2345 rsmseglock_release(seg); 2346 return (RSMERR_INTERRUPTED); 2347 } 2348 } 2349 2350 ASSERT(seg->s_state == RSM_STATE_NEW); 2351 2352 ASSERT(seg->s_cookie == NULL); 2353 2354 e = rsm_bind_pages(&seg->s_cookie, msg->vaddr, msg->len, curproc); 2355 if (e == RSM_SUCCESS) { 2356 seg->s_flags |= RSM_USER_MEMORY; 2357 if (msg->perm & RSM_ALLOW_REBIND) { 2358 seg->s_flags |= RSMKA_ALLOW_UNBIND_REBIND; 2359 } 2360 if (msg->perm & RSM_CREATE_SEG_DONTWAIT) { 2361 seg->s_flags |= RSMKA_SET_RESOURCE_DONTWAIT; 2362 } 2363 seg->s_region.r_vaddr = msg->vaddr; 2364 /* 2365 * Set the s_pid value in the segment structure. This is used 2366 * to identify exported segments belonging to a particular 2367 * process so that when the process exits, these segments can 2368 * be unlocked forcefully even if rsm_close is not called on 2369 * process exit since there maybe other processes referencing 2370 * them (for example on a fork or exec). 2371 * The s_pid value is also used to authenticate the process 2372 * doing a publish or unpublish on the export segment. Only 2373 * the creator of the export segment has a right to do a 2374 * publish or unpublish and unbind on the segment. 2375 */ 2376 seg->s_pid = ddi_get_pid(); 2377 seg->s_len = msg->len; 2378 seg->s_state = RSM_STATE_BIND; 2379 seg->s_adapter = adapter; 2380 seg->s_proc = curproc; 2381 } else { 2382 rsmka_release_adapter(adapter); 2383 DBG_PRINTF((category, RSM_WARNING, 2384 "unable to lock down pages\n")); 2385 } 2386 2387 msg->rnum = seg->s_minor; 2388 /* Unlock segment */ 2389 rsmseglock_release(seg); 2390 2391 if (e == RSM_SUCCESS) { 2392 /* copyout the resource number */ 2393 #ifdef _MULTI_DATAMODEL 2394 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 2395 rsm_ioctlmsg32_t msg32; 2396 2397 msg32.rnum = msg->rnum; 2398 if (ddi_copyout((caddr_t)&msg32.rnum, 2399 (caddr_t)&((rsm_ioctlmsg32_t *)dataptr)->rnum, 2400 sizeof (minor_t), mode)) { 2401 rsmka_release_adapter(adapter); 2402 e = RSMERR_BAD_ADDR; 2403 } 2404 } 2405 #endif 2406 if (ddi_copyout((caddr_t)&msg->rnum, 2407 (caddr_t)&((rsm_ioctlmsg_t *)dataptr)->rnum, 2408 sizeof (minor_t), mode)) { 2409 rsmka_release_adapter(adapter); 2410 e = RSMERR_BAD_ADDR; 2411 } 2412 } 2413 2414 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind done\n")); 2415 2416 return (e); 2417 } 2418 2419 static void 2420 rsm_remap_local_importers(rsm_node_id_t src_nodeid, 2421 rsm_memseg_id_t ex_segid, 2422 ddi_umem_cookie_t cookie) 2423 2424 { 2425 rsmresource_t *p = NULL; 2426 rsmhash_table_t *rhash = &rsm_import_segs; 2427 uint_t index; 2428 2429 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE, 2430 "rsm_remap_local_importers enter\n")); 2431 2432 index = rsmhash(ex_segid); 2433 2434 rw_enter(&rhash->rsmhash_rw, RW_READER); 2435 2436 p = rsmhash_getbkt(rhash, index); 2437 2438 for (; p; p = p->rsmrc_next) { 2439 rsmseg_t *seg = (rsmseg_t *)p; 2440 rsmseglock_acquire(seg); 2441 /* 2442 * Change the s_cookie value of only the local importers 2443 * which have been mapped (in state RSM_STATE_ACTIVE). 2444 * Note that there is no need to change the s_cookie value 2445 * if the imported segment is in RSM_STATE_MAPPING since 2446 * eventually the s_cookie will be updated via the mapping 2447 * functionality. 2448 */ 2449 if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid) && 2450 (seg->s_state == RSM_STATE_ACTIVE)) { 2451 seg->s_cookie = cookie; 2452 } 2453 rsmseglock_release(seg); 2454 } 2455 rw_exit(&rhash->rsmhash_rw); 2456 2457 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE, 2458 "rsm_remap_local_importers done\n")); 2459 } 2460 2461 static int 2462 rsm_rebind(rsmseg_t *seg, rsm_ioctlmsg_t *msg) 2463 { 2464 int e; 2465 adapter_t *adapter; 2466 ddi_umem_cookie_t cookie; 2467 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2468 2469 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind enter\n")); 2470 2471 /* Check for permissions to rebind */ 2472 if (!(seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND)) { 2473 return (RSMERR_REBIND_NOT_ALLOWED); 2474 } 2475 2476 if (seg->s_pid != ddi_get_pid() && 2477 ddi_get_pid() != 0) { 2478 DBG_PRINTF((category, RSM_ERR, "rsm_rebind: Not owner\n")); 2479 return (RSMERR_NOT_CREATOR); 2480 } 2481 2482 /* 2483 * We will not be allowing partial rebind and hence length passed 2484 * in must be same as segment length 2485 */ 2486 if (msg->vaddr == NULL) { 2487 DBG_PRINTF((category, RSM_ERR, 2488 "rsm_rebind done: null msg->vaddr\n")); 2489 return (RSMERR_BAD_ADDR); 2490 } 2491 if (msg->len != seg->s_len) { 2492 DBG_PRINTF((category, RSM_ERR, 2493 "rsm_rebind: invalid length\n")); 2494 return (RSMERR_BAD_LENGTH); 2495 } 2496 2497 /* Lock segment */ 2498 rsmseglock_acquire(seg); 2499 2500 while ((seg->s_state == RSM_STATE_BIND_QUIESCED) || 2501 (seg->s_state == RSM_STATE_EXPORT_QUIESCING) || 2502 (seg->s_state == RSM_STATE_EXPORT_QUIESCED)) { 2503 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 2504 rsmseglock_release(seg); 2505 DBG_PRINTF((category, RSM_DEBUG, 2506 "rsm_rebind done: cv_wait INTERRUPTED")); 2507 return (RSMERR_INTERRUPTED); 2508 } 2509 } 2510 2511 /* verify segment state */ 2512 if ((seg->s_state != RSM_STATE_BIND) && 2513 (seg->s_state != RSM_STATE_EXPORT)) { 2514 /* Unlock segment */ 2515 rsmseglock_release(seg); 2516 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2517 "rsm_rebind done: invalid state\n")); 2518 return (RSMERR_BAD_SEG_HNDL); 2519 } 2520 2521 ASSERT(seg->s_cookie != NULL); 2522 2523 if (msg->vaddr == seg->s_region.r_vaddr) { 2524 rsmseglock_release(seg); 2525 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n")); 2526 return (RSM_SUCCESS); 2527 } 2528 2529 e = rsm_bind_pages(&cookie, msg->vaddr, msg->len, curproc); 2530 if (e == RSM_SUCCESS) { 2531 struct buf *xbuf; 2532 dev_t sdev = 0; 2533 rsm_memory_local_t mem; 2534 2535 xbuf = ddi_umem_iosetup(cookie, 0, msg->len, B_WRITE, 2536 sdev, 0, NULL, DDI_UMEM_SLEEP); 2537 ASSERT(xbuf != NULL); 2538 2539 mem.ms_type = RSM_MEM_BUF; 2540 mem.ms_bp = xbuf; 2541 2542 adapter = seg->s_adapter; 2543 e = adapter->rsmpi_ops->rsm_rebind( 2544 seg->s_handle.out, 0, &mem, 2545 RSM_RESOURCE_DONTWAIT, NULL); 2546 2547 if (e == RSM_SUCCESS) { 2548 /* 2549 * unbind the older pages, and unload local importers; 2550 * but don't disconnect importers 2551 */ 2552 (void) rsm_unbind_pages(seg); 2553 seg->s_cookie = cookie; 2554 seg->s_region.r_vaddr = msg->vaddr; 2555 rsm_remap_local_importers(my_nodeid, seg->s_segid, 2556 cookie); 2557 } else { 2558 /* 2559 * Unbind the pages associated with "cookie" by the 2560 * rsm_bind_pages calls prior to this. This is 2561 * similar to what is done in the rsm_unbind_pages 2562 * routine for the seg->s_cookie. 2563 */ 2564 ddi_umem_unlock(cookie); 2565 rsm_dec_pgcnt(btopr(msg->len)); 2566 DBG_PRINTF((category, RSM_ERR, 2567 "rsm_rebind failed with %d\n", e)); 2568 } 2569 /* 2570 * At present there is no dependency on the existence of xbuf. 2571 * So we can free it here. If in the future this changes, it can 2572 * be freed sometime during the segment destroy. 2573 */ 2574 freerbuf(xbuf); 2575 } 2576 2577 /* Unlock segment */ 2578 rsmseglock_release(seg); 2579 2580 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n")); 2581 2582 return (e); 2583 } 2584 2585 static int 2586 rsm_unbind(rsmseg_t *seg) 2587 { 2588 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2589 2590 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind enter\n")); 2591 2592 rsmseglock_acquire(seg); 2593 2594 /* verify segment state */ 2595 if ((seg->s_state != RSM_STATE_BIND) && 2596 (seg->s_state != RSM_STATE_BIND_QUIESCED)) { 2597 rsmseglock_release(seg); 2598 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2599 "rsm_unbind: invalid state\n")); 2600 return (RSMERR_BAD_SEG_HNDL); 2601 } 2602 2603 /* unlock current range */ 2604 (void) rsm_unbind_pages(seg); 2605 2606 if (seg->s_state == RSM_STATE_BIND) { 2607 seg->s_state = RSM_STATE_NEW; 2608 } else if (seg->s_state == RSM_STATE_BIND_QUIESCED) { 2609 seg->s_state = RSM_STATE_NEW_QUIESCED; 2610 } 2611 2612 rsmseglock_release(seg); 2613 2614 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind done\n")); 2615 2616 return (RSM_SUCCESS); 2617 } 2618 2619 /* **************************** Exporter Access List Management ******* */ 2620 static void 2621 rsmacl_free(rsmapi_access_entry_t *acl, int acl_len) 2622 { 2623 int acl_sz; 2624 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2625 2626 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free enter\n")); 2627 2628 /* acl could be NULL */ 2629 2630 if (acl != NULL && acl_len > 0) { 2631 acl_sz = acl_len * sizeof (rsmapi_access_entry_t); 2632 kmem_free((void *)acl, acl_sz); 2633 } 2634 2635 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free done\n")); 2636 } 2637 2638 static void 2639 rsmpiacl_free(rsm_access_entry_t *acl, int acl_len) 2640 { 2641 int acl_sz; 2642 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2643 2644 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free enter\n")); 2645 2646 if (acl != NULL && acl_len > 0) { 2647 acl_sz = acl_len * sizeof (rsm_access_entry_t); 2648 kmem_free((void *)acl, acl_sz); 2649 } 2650 2651 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free done\n")); 2652 2653 } 2654 2655 static int 2656 rsmacl_build(rsm_ioctlmsg_t *msg, int mode, 2657 rsmapi_access_entry_t **list, int *len, int loopback) 2658 { 2659 rsmapi_access_entry_t *acl; 2660 int acl_len; 2661 int i; 2662 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2663 2664 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build enter\n")); 2665 2666 *len = 0; 2667 *list = NULL; 2668 2669 acl_len = msg->acl_len; 2670 if ((loopback && acl_len > 1) || (acl_len < 0) || 2671 (acl_len > MAX_NODES)) { 2672 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2673 "rsmacl_build done: acl invalid\n")); 2674 return (RSMERR_BAD_ACL); 2675 } 2676 2677 if (acl_len > 0 && acl_len <= MAX_NODES) { 2678 size_t acl_size = acl_len * sizeof (rsmapi_access_entry_t); 2679 2680 acl = kmem_alloc(acl_size, KM_SLEEP); 2681 2682 if (ddi_copyin((caddr_t)msg->acl, (caddr_t)acl, 2683 acl_size, mode)) { 2684 kmem_free((void *) acl, acl_size); 2685 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2686 "rsmacl_build done: BAD_ADDR\n")); 2687 return (RSMERR_BAD_ADDR); 2688 } 2689 2690 /* 2691 * Verify access list 2692 */ 2693 for (i = 0; i < acl_len; i++) { 2694 if (acl[i].ae_node > MAX_NODES || 2695 (loopback && (acl[i].ae_node != my_nodeid)) || 2696 acl[i].ae_permission > RSM_ACCESS_TRUSTED) { 2697 /* invalid entry */ 2698 kmem_free((void *) acl, acl_size); 2699 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2700 "rsmacl_build done: EINVAL\n")); 2701 return (RSMERR_BAD_ACL); 2702 } 2703 } 2704 2705 *len = acl_len; 2706 *list = acl; 2707 } 2708 2709 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build done\n")); 2710 2711 return (DDI_SUCCESS); 2712 } 2713 2714 static int 2715 rsmpiacl_create(rsmapi_access_entry_t *src, rsm_access_entry_t **dest, 2716 int acl_len, adapter_t *adapter) 2717 { 2718 rsm_access_entry_t *acl; 2719 rsm_addr_t hwaddr; 2720 int i; 2721 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2722 2723 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create enter\n")); 2724 2725 if (src != NULL) { 2726 size_t acl_size = acl_len * sizeof (rsm_access_entry_t); 2727 acl = kmem_alloc(acl_size, KM_SLEEP); 2728 2729 /* 2730 * translate access list 2731 */ 2732 for (i = 0; i < acl_len; i++) { 2733 if (src[i].ae_node == my_nodeid) { 2734 acl[i].ae_addr = adapter->hwaddr; 2735 } else { 2736 hwaddr = get_remote_hwaddr(adapter, 2737 src[i].ae_node); 2738 if ((int64_t)hwaddr < 0) { 2739 /* invalid hwaddr */ 2740 kmem_free((void *) acl, acl_size); 2741 DBG_PRINTF((category, 2742 RSM_DEBUG_VERBOSE, 2743 "rsmpiacl_create done:" 2744 "EINVAL hwaddr\n")); 2745 return (RSMERR_INTERNAL_ERROR); 2746 } 2747 acl[i].ae_addr = hwaddr; 2748 } 2749 /* rsmpi understands only RSM_PERM_XXXX */ 2750 acl[i].ae_permission = 2751 src[i].ae_permission & RSM_PERM_RDWR; 2752 } 2753 *dest = acl; 2754 } else { 2755 *dest = NULL; 2756 } 2757 2758 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create done\n")); 2759 2760 return (RSM_SUCCESS); 2761 } 2762 2763 static int 2764 rsmsegacl_validate(rsmipc_request_t *req, rsm_node_id_t rnode, 2765 rsmipc_reply_t *reply) 2766 { 2767 2768 int i; 2769 rsmseg_t *seg; 2770 rsm_memseg_id_t key = req->rsmipc_key; 2771 rsm_permission_t perm = req->rsmipc_perm; 2772 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2773 2774 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2775 "rsmsegacl_validate enter\n")); 2776 2777 /* 2778 * Find segment and grab its lock. The reason why we grab the segment 2779 * lock in side the search is to avoid the race when the segment is 2780 * being deleted and we already have a pointer to it. 2781 */ 2782 seg = rsmexport_lookup(key); 2783 if (!seg) { 2784 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2785 "rsmsegacl_validate done: %u ENXIO\n", key)); 2786 return (RSMERR_SEG_NOT_PUBLISHED); 2787 } 2788 2789 ASSERT(rsmseglock_held(seg)); 2790 ASSERT(seg->s_state == RSM_STATE_EXPORT); 2791 2792 /* 2793 * We implement a 2-level protection scheme. 2794 * First, we check if local/remote host has access rights. 2795 * Second, we check if the user has access rights. 2796 * 2797 * This routine only validates the rnode access_list 2798 */ 2799 if (seg->s_acl_len > 0) { 2800 /* 2801 * Check host access list 2802 */ 2803 ASSERT(seg->s_acl != NULL); 2804 for (i = 0; i < seg->s_acl_len; i++) { 2805 if (seg->s_acl[i].ae_node == rnode) { 2806 perm &= seg->s_acl[i].ae_permission; 2807 goto found; 2808 } 2809 } 2810 /* rnode is not found in the list */ 2811 rsmseglock_release(seg); 2812 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 2813 "rsmsegacl_validate done: EPERM\n")); 2814 return (RSMERR_SEG_NOT_PUBLISHED_TO_NODE); 2815 } else { 2816 /* use default owner creation umask */ 2817 perm &= seg->s_mode; 2818 } 2819 2820 found: 2821 /* update perm for this node */ 2822 reply->rsmipc_mode = perm; 2823 reply->rsmipc_uid = seg->s_uid; 2824 reply->rsmipc_gid = seg->s_gid; 2825 reply->rsmipc_segid = seg->s_segid; 2826 reply->rsmipc_seglen = seg->s_len; 2827 2828 /* 2829 * Perm of requesting node is valid; source will validate user 2830 */ 2831 rsmseglock_release(seg); 2832 2833 /* 2834 * Add the importer to the list right away, if connect fails 2835 * the importer will ask the exporter to remove it. 2836 */ 2837 importer_list_add(rnode, key, req->rsmipc_adapter_hwaddr, 2838 req->rsmipc_segment_cookie); 2839 2840 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegacl_validate done\n")); 2841 2842 return (RSM_SUCCESS); 2843 } 2844 2845 2846 /* ************************** Exporter Calls ************************* */ 2847 2848 static int 2849 rsm_publish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode) 2850 { 2851 int e; 2852 int acl_len; 2853 rsmapi_access_entry_t *acl; 2854 rsm_access_entry_t *rsmpi_acl; 2855 rsm_memory_local_t mem; 2856 struct buf *xbuf; 2857 dev_t sdev = 0; 2858 adapter_t *adapter; 2859 rsm_memseg_id_t segment_id = 0; 2860 int loopback_flag = 0; 2861 int create_flags = 0; 2862 rsm_resource_callback_t callback_flag; 2863 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 2864 2865 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish enter\n")); 2866 2867 if (seg->s_adapter == &loopback_adapter) 2868 loopback_flag = 1; 2869 2870 if (seg->s_pid != ddi_get_pid() && 2871 ddi_get_pid() != 0) { 2872 DBG_PRINTF((category, RSM_ERR, 2873 "rsm_publish: Not creator\n")); 2874 return (RSMERR_NOT_CREATOR); 2875 } 2876 2877 /* 2878 * Get per node access list 2879 */ 2880 e = rsmacl_build(msg, mode, &acl, &acl_len, loopback_flag); 2881 if (e != DDI_SUCCESS) { 2882 DBG_PRINTF((category, RSM_ERR, 2883 "rsm_publish done: rsmacl_build failed\n")); 2884 return (e); 2885 } 2886 2887 /* 2888 * The application provided msg->key is used for resolving a 2889 * segment id according to the following: 2890 * key = 0 Kernel Agent selects the segment id 2891 * key <= RSM_DLPI_ID_END Reserved for system usage except 2892 * RSMLIB range 2893 * key < RSM_USER_APP_ID_BASE segment id = key 2894 * key >= RSM_USER_APP_ID_BASE Reserved for KA selections 2895 * 2896 * rsm_nextavail_segmentid is initialized to 0x80000000 and 2897 * overflows to zero after 0x80000000 allocations. 2898 * An algorithm is needed which allows reinitialization and provides 2899 * for reallocation after overflow. For now, ENOMEM is returned 2900 * once the overflow condition has occurred. 2901 */ 2902 if (msg->key == 0) { 2903 mutex_enter(&rsm_lock); 2904 segment_id = rsm_nextavail_segmentid; 2905 if (segment_id != 0) { 2906 rsm_nextavail_segmentid++; 2907 mutex_exit(&rsm_lock); 2908 } else { 2909 mutex_exit(&rsm_lock); 2910 DBG_PRINTF((category, RSM_ERR, 2911 "rsm_publish done: no more keys avlbl\n")); 2912 return (RSMERR_INSUFFICIENT_RESOURCES); 2913 } 2914 } else if BETWEEN(msg->key, RSM_RSMLIB_ID_BASE, RSM_RSMLIB_ID_END) 2915 /* range reserved for internal use by base/ndi libraries */ 2916 segment_id = msg->key; 2917 else if (msg->key <= RSM_DLPI_ID_END) 2918 return (RSMERR_RESERVED_SEGID); 2919 else if (msg->key <= (uint_t)RSM_USER_APP_ID_BASE -1) 2920 segment_id = msg->key; 2921 else { 2922 DBG_PRINTF((category, RSM_ERR, 2923 "rsm_publish done: invalid key %u\n", msg->key)); 2924 return (RSMERR_RESERVED_SEGID); 2925 } 2926 2927 /* Add key to exportlist; The segment lock is held on success */ 2928 e = rsmexport_add(seg, segment_id); 2929 if (e) { 2930 rsmacl_free(acl, acl_len); 2931 DBG_PRINTF((category, RSM_ERR, 2932 "rsm_publish done: export_add failed: %d\n", e)); 2933 return (e); 2934 } 2935 2936 seg->s_segid = segment_id; 2937 2938 if ((seg->s_state != RSM_STATE_BIND) && 2939 (seg->s_state != RSM_STATE_BIND_QUIESCED)) { 2940 /* state changed since then, free acl and return */ 2941 rsmseglock_release(seg); 2942 rsmexport_rm(seg); 2943 rsmacl_free(acl, acl_len); 2944 DBG_PRINTF((category, RSM_ERR, 2945 "rsm_publish done: segment in wrong state: %d\n", 2946 seg->s_state)); 2947 return (RSMERR_BAD_SEG_HNDL); 2948 } 2949 2950 /* 2951 * If this is for a local memory handle and permissions are zero, 2952 * then the surrogate segment is very large and we want to skip 2953 * allocation of DVMA space. 2954 * 2955 * Careful! If the user didn't use an ACL list, acl will be a NULL 2956 * pointer. Check that before dereferencing it. 2957 */ 2958 if (acl != (rsmapi_access_entry_t *)NULL) { 2959 if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0) 2960 goto skipdriver; 2961 } 2962 2963 /* create segment */ 2964 xbuf = ddi_umem_iosetup(seg->s_cookie, 0, seg->s_len, B_WRITE, 2965 sdev, 0, NULL, DDI_UMEM_SLEEP); 2966 ASSERT(xbuf != NULL); 2967 2968 mem.ms_type = RSM_MEM_BUF; 2969 mem.ms_bp = xbuf; 2970 2971 /* This call includes a bind operations */ 2972 2973 adapter = seg->s_adapter; 2974 /* 2975 * create a acl list with hwaddr for RSMPI publish 2976 */ 2977 e = rsmpiacl_create(acl, &rsmpi_acl, acl_len, adapter); 2978 2979 if (e != RSM_SUCCESS) { 2980 rsmseglock_release(seg); 2981 rsmexport_rm(seg); 2982 rsmacl_free(acl, acl_len); 2983 freerbuf(xbuf); 2984 DBG_PRINTF((category, RSM_ERR, 2985 "rsm_publish done: rsmpiacl_create failed: %d\n", e)); 2986 return (e); 2987 } 2988 2989 if (seg->s_state == RSM_STATE_BIND) { 2990 /* create segment */ 2991 2992 /* This call includes a bind operations */ 2993 2994 if (seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND) { 2995 create_flags = RSM_ALLOW_UNBIND_REBIND; 2996 } 2997 2998 if (seg->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) { 2999 callback_flag = RSM_RESOURCE_DONTWAIT; 3000 } else { 3001 callback_flag = RSM_RESOURCE_SLEEP; 3002 } 3003 3004 e = adapter->rsmpi_ops->rsm_seg_create( 3005 adapter->rsmpi_handle, 3006 &seg->s_handle.out, seg->s_len, 3007 create_flags, &mem, 3008 callback_flag, NULL); 3009 /* 3010 * At present there is no dependency on the existence of xbuf. 3011 * So we can free it here. If in the future this changes, it can 3012 * be freed sometime during the segment destroy. 3013 */ 3014 freerbuf(xbuf); 3015 3016 if (e != RSM_SUCCESS) { 3017 rsmseglock_release(seg); 3018 rsmexport_rm(seg); 3019 rsmacl_free(acl, acl_len); 3020 rsmpiacl_free(rsmpi_acl, acl_len); 3021 DBG_PRINTF((category, RSM_ERR, 3022 "rsm_publish done: export_create failed: %d\n", e)); 3023 /* 3024 * The following assertion ensures that the two errors 3025 * related to the length and its alignment do not occur 3026 * since they have been checked during export_create 3027 */ 3028 ASSERT(e != RSMERR_BAD_MEM_ALIGNMENT && 3029 e != RSMERR_BAD_LENGTH); 3030 if (e == RSMERR_NOT_MEM) 3031 e = RSMERR_INSUFFICIENT_MEM; 3032 3033 return (e); 3034 } 3035 /* export segment, this should create an IMMU mapping */ 3036 e = adapter->rsmpi_ops->rsm_publish( 3037 seg->s_handle.out, 3038 rsmpi_acl, acl_len, 3039 seg->s_segid, 3040 RSM_RESOURCE_DONTWAIT, NULL); 3041 3042 if (e != RSM_SUCCESS) { 3043 adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out); 3044 rsmseglock_release(seg); 3045 rsmexport_rm(seg); 3046 rsmacl_free(acl, acl_len); 3047 rsmpiacl_free(rsmpi_acl, acl_len); 3048 DBG_PRINTF((category, RSM_ERR, 3049 "rsm_publish done: export_publish failed: %d\n", 3050 e)); 3051 return (e); 3052 } 3053 } 3054 3055 seg->s_acl_in = rsmpi_acl; 3056 3057 skipdriver: 3058 /* defer s_acl/s_acl_len -> avoid crash in rsmseg_free */ 3059 seg->s_acl_len = acl_len; 3060 seg->s_acl = acl; 3061 3062 if (seg->s_state == RSM_STATE_BIND) { 3063 seg->s_state = RSM_STATE_EXPORT; 3064 } else if (seg->s_state == RSM_STATE_BIND_QUIESCED) { 3065 seg->s_state = RSM_STATE_EXPORT_QUIESCED; 3066 cv_broadcast(&seg->s_cv); 3067 } 3068 3069 rsmseglock_release(seg); 3070 3071 /* 3072 * If the segment id was solicited, then return it in 3073 * the original incoming message. 3074 */ 3075 if (msg->key == 0) { 3076 msg->key = segment_id; 3077 #ifdef _MULTI_DATAMODEL 3078 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 3079 rsm_ioctlmsg32_t msg32; 3080 3081 msg32.key = msg->key; 3082 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3083 "rsm_publish done\n")); 3084 return (ddi_copyout((caddr_t)&msg32, 3085 (caddr_t)dataptr, sizeof (msg32), mode)); 3086 } 3087 #endif 3088 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3089 "rsm_publish done\n")); 3090 return (ddi_copyout((caddr_t)msg, 3091 (caddr_t)dataptr, sizeof (*msg), mode)); 3092 } 3093 3094 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish done\n")); 3095 return (DDI_SUCCESS); 3096 } 3097 3098 /* 3099 * This function modifies the access control list of an already published 3100 * segment. There is no effect on import segments which are already 3101 * connected. 3102 */ 3103 static int 3104 rsm_republish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int mode) 3105 { 3106 rsmapi_access_entry_t *new_acl, *old_acl, *tmp_acl; 3107 rsm_access_entry_t *rsmpi_new_acl, *rsmpi_old_acl; 3108 int new_acl_len, old_acl_len, tmp_acl_len; 3109 int e, i; 3110 adapter_t *adapter; 3111 int loopback_flag = 0; 3112 rsm_memseg_id_t key; 3113 rsm_permission_t permission; 3114 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 3115 3116 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish enter\n")); 3117 3118 if ((seg->s_state != RSM_STATE_EXPORT) && 3119 (seg->s_state != RSM_STATE_EXPORT_QUIESCED) && 3120 (seg->s_state != RSM_STATE_EXPORT_QUIESCING)) 3121 return (RSMERR_SEG_NOT_PUBLISHED); 3122 3123 if (seg->s_pid != ddi_get_pid() && 3124 ddi_get_pid() != 0) { 3125 DBG_PRINTF((category, RSM_ERR, 3126 "rsm_republish: Not owner\n")); 3127 return (RSMERR_NOT_CREATOR); 3128 } 3129 3130 if (seg->s_adapter == &loopback_adapter) 3131 loopback_flag = 1; 3132 3133 /* 3134 * Build new list first 3135 */ 3136 e = rsmacl_build(msg, mode, &new_acl, &new_acl_len, loopback_flag); 3137 if (e) { 3138 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3139 "rsm_republish done: rsmacl_build failed %d", e)); 3140 return (e); 3141 } 3142 3143 /* Lock segment */ 3144 rsmseglock_acquire(seg); 3145 /* 3146 * a republish is in progress - REPUBLISH message is being 3147 * sent to the importers so wait for it to complete OR 3148 * wait till DR completes 3149 */ 3150 while (((seg->s_state == RSM_STATE_EXPORT) && 3151 (seg->s_flags & RSM_REPUBLISH_WAIT)) || 3152 (seg->s_state == RSM_STATE_EXPORT_QUIESCED) || 3153 (seg->s_state == RSM_STATE_EXPORT_QUIESCING)) { 3154 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 3155 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3156 "rsm_republish done: cv_wait INTERRUPTED")); 3157 rsmseglock_release(seg); 3158 rsmacl_free(new_acl, new_acl_len); 3159 return (RSMERR_INTERRUPTED); 3160 } 3161 } 3162 3163 /* recheck if state is valid */ 3164 if (seg->s_state != RSM_STATE_EXPORT) { 3165 rsmseglock_release(seg); 3166 rsmacl_free(new_acl, new_acl_len); 3167 return (RSMERR_SEG_NOT_PUBLISHED); 3168 } 3169 3170 key = seg->s_key; 3171 old_acl = seg->s_acl; 3172 old_acl_len = seg->s_acl_len; 3173 3174 seg->s_acl = new_acl; 3175 seg->s_acl_len = new_acl_len; 3176 3177 /* 3178 * This call will only be meaningful if and when the interconnect 3179 * layer makes use of the access list 3180 */ 3181 adapter = seg->s_adapter; 3182 /* 3183 * create a acl list with hwaddr for RSMPI publish 3184 */ 3185 e = rsmpiacl_create(new_acl, &rsmpi_new_acl, new_acl_len, adapter); 3186 3187 if (e != RSM_SUCCESS) { 3188 seg->s_acl = old_acl; 3189 seg->s_acl_len = old_acl_len; 3190 rsmseglock_release(seg); 3191 rsmacl_free(new_acl, new_acl_len); 3192 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3193 "rsm_republish done: rsmpiacl_create failed %d", e)); 3194 return (e); 3195 } 3196 rsmpi_old_acl = seg->s_acl_in; 3197 seg->s_acl_in = rsmpi_new_acl; 3198 3199 e = adapter->rsmpi_ops->rsm_republish(seg->s_handle.out, 3200 seg->s_acl_in, seg->s_acl_len, 3201 RSM_RESOURCE_DONTWAIT, NULL); 3202 3203 if (e != RSM_SUCCESS) { 3204 seg->s_acl = old_acl; 3205 seg->s_acl_in = rsmpi_old_acl; 3206 seg->s_acl_len = old_acl_len; 3207 rsmseglock_release(seg); 3208 rsmacl_free(new_acl, new_acl_len); 3209 rsmpiacl_free(rsmpi_new_acl, new_acl_len); 3210 3211 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3212 "rsm_republish done: rsmpi republish failed %d\n", e)); 3213 return (e); 3214 } 3215 3216 /* create a tmp copy of the new acl */ 3217 tmp_acl_len = new_acl_len; 3218 if (tmp_acl_len > 0) { 3219 tmp_acl = kmem_zalloc(new_acl_len*sizeof (*tmp_acl), KM_SLEEP); 3220 for (i = 0; i < tmp_acl_len; i++) { 3221 tmp_acl[i].ae_node = new_acl[i].ae_node; 3222 tmp_acl[i].ae_permission = new_acl[i].ae_permission; 3223 } 3224 /* 3225 * The default permission of a node which was in the old 3226 * ACL but not in the new ACL is 0 ie no access. 3227 */ 3228 permission = 0; 3229 } else { 3230 /* 3231 * NULL acl means all importers can connect and 3232 * default permission will be owner creation umask 3233 */ 3234 tmp_acl = NULL; 3235 permission = seg->s_mode; 3236 } 3237 3238 /* make other republishers to wait for republish to complete */ 3239 seg->s_flags |= RSM_REPUBLISH_WAIT; 3240 3241 rsmseglock_release(seg); 3242 3243 /* send the new perms to the importing nodes */ 3244 rsm_send_republish(key, tmp_acl, tmp_acl_len, permission); 3245 3246 rsmseglock_acquire(seg); 3247 seg->s_flags &= ~RSM_REPUBLISH_WAIT; 3248 /* wake up any one waiting for republish to complete */ 3249 cv_broadcast(&seg->s_cv); 3250 rsmseglock_release(seg); 3251 3252 rsmacl_free(tmp_acl, tmp_acl_len); 3253 rsmacl_free(old_acl, old_acl_len); 3254 rsmpiacl_free(rsmpi_old_acl, old_acl_len); 3255 3256 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish done\n")); 3257 return (DDI_SUCCESS); 3258 } 3259 3260 static int 3261 rsm_unpublish(rsmseg_t *seg, int mode) 3262 { 3263 rsmapi_access_entry_t *acl; 3264 rsm_access_entry_t *rsmpi_acl; 3265 int acl_len; 3266 int e; 3267 adapter_t *adapter; 3268 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT); 3269 3270 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish enter\n")); 3271 3272 if (seg->s_pid != ddi_get_pid() && 3273 ddi_get_pid() != 0) { 3274 DBG_PRINTF((category, RSM_ERR, 3275 "rsm_unpublish: Not creator\n")); 3276 return (RSMERR_NOT_CREATOR); 3277 } 3278 3279 rsmseglock_acquire(seg); 3280 /* 3281 * wait for QUIESCING to complete here before rsmexport_rm 3282 * is called because the SUSPEND_COMPLETE mesg which changes 3283 * the seg state from EXPORT_QUIESCING to EXPORT_QUIESCED and 3284 * signals the cv_wait needs to find it in the hashtable. 3285 */ 3286 while ((seg->s_state == RSM_STATE_EXPORT_QUIESCING) || 3287 ((seg->s_state == RSM_STATE_EXPORT) && (seg->s_rdmacnt > 0))) { 3288 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 3289 rsmseglock_release(seg); 3290 DBG_PRINTF((category, RSM_ERR, 3291 "rsm_unpublish done: cv_wait INTR qscing" 3292 "getv/putv in progress")); 3293 return (RSMERR_INTERRUPTED); 3294 } 3295 } 3296 3297 /* verify segment state */ 3298 if ((seg->s_state != RSM_STATE_EXPORT) && 3299 (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) { 3300 rsmseglock_release(seg); 3301 DBG_PRINTF((category, RSM_ERR, 3302 "rsm_unpublish done: bad state %x\n", seg->s_state)); 3303 return (RSMERR_SEG_NOT_PUBLISHED); 3304 } 3305 3306 rsmseglock_release(seg); 3307 3308 rsmexport_rm(seg); 3309 3310 rsm_send_importer_disconnects(seg->s_segid, my_nodeid); 3311 3312 rsmseglock_acquire(seg); 3313 /* 3314 * wait for republish to complete 3315 */ 3316 while ((seg->s_state == RSM_STATE_EXPORT) && 3317 (seg->s_flags & RSM_REPUBLISH_WAIT)) { 3318 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 3319 DBG_PRINTF((category, RSM_ERR, 3320 "rsm_unpublish done: cv_wait INTR repubing")); 3321 rsmseglock_release(seg); 3322 return (RSMERR_INTERRUPTED); 3323 } 3324 } 3325 3326 if ((seg->s_state != RSM_STATE_EXPORT) && 3327 (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) { 3328 DBG_PRINTF((category, RSM_ERR, 3329 "rsm_unpublish done: invalid state")); 3330 rsmseglock_release(seg); 3331 return (RSMERR_SEG_NOT_PUBLISHED); 3332 } 3333 3334 /* 3335 * check for putv/get surrogate segment which was not published 3336 * to the driver. 3337 * 3338 * Be certain to see if there is an ACL first! If this segment was 3339 * not published with an ACL, acl will be a null pointer. Check 3340 * that before dereferencing it. 3341 */ 3342 acl = seg->s_acl; 3343 if (acl != (rsmapi_access_entry_t *)NULL) { 3344 if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0) 3345 goto bypass; 3346 } 3347 3348 /* The RSMPI unpublish/destroy has been done if seg is QUIESCED */ 3349 if (seg->s_state == RSM_STATE_EXPORT_QUIESCED) 3350 goto bypass; 3351 3352 adapter = seg->s_adapter; 3353 for (;;) { 3354 if (seg->s_state != RSM_STATE_EXPORT) { 3355 rsmseglock_release(seg); 3356 DBG_PRINTF((category, RSM_ERR, 3357 "rsm_unpublish done: bad state %x\n", 3358 seg->s_state)); 3359 return (RSMERR_SEG_NOT_PUBLISHED); 3360 } 3361 3362 /* unpublish from adapter */ 3363 e = adapter->rsmpi_ops->rsm_unpublish(seg->s_handle.out); 3364 3365 if (e == RSM_SUCCESS) { 3366 break; 3367 } 3368 3369 if (e == RSMERR_SEG_IN_USE && mode == 1) { 3370 /* 3371 * wait for unpublish to succeed, it's busy. 3372 */ 3373 seg->s_flags |= RSM_EXPORT_WAIT; 3374 3375 /* wait for a max of 1 ms - this is an empirical */ 3376 /* value that was found by some minimal testing */ 3377 /* can be fine tuned when we have better numbers */ 3378 /* A long term fix would be to send cv_signal */ 3379 /* from the intr callback routine */ 3380 /* currently nobody signals this wait */ 3381 (void) cv_reltimedwait(&seg->s_cv, &seg->s_lock, 3382 drv_usectohz(1000), TR_CLOCK_TICK); 3383 3384 DBG_PRINTF((category, RSM_ERR, 3385 "rsm_unpublish: SEG_IN_USE\n")); 3386 3387 seg->s_flags &= ~RSM_EXPORT_WAIT; 3388 } else { 3389 if (mode == 1) { 3390 DBG_PRINTF((category, RSM_ERR, 3391 "rsm:rsmpi unpublish err %x\n", e)); 3392 seg->s_state = RSM_STATE_BIND; 3393 } 3394 rsmseglock_release(seg); 3395 return (e); 3396 } 3397 } 3398 3399 /* Free segment */ 3400 e = adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out); 3401 3402 if (e != RSM_SUCCESS) { 3403 DBG_PRINTF((category, RSM_ERR, 3404 "rsm_unpublish: rsmpi destroy key=%x failed %x\n", 3405 seg->s_key, e)); 3406 } 3407 3408 bypass: 3409 acl = seg->s_acl; 3410 rsmpi_acl = seg->s_acl_in; 3411 acl_len = seg->s_acl_len; 3412 3413 seg->s_acl = NULL; 3414 seg->s_acl_in = NULL; 3415 seg->s_acl_len = 0; 3416 3417 if (seg->s_state == RSM_STATE_EXPORT) { 3418 seg->s_state = RSM_STATE_BIND; 3419 } else if (seg->s_state == RSM_STATE_EXPORT_QUIESCED) { 3420 seg->s_state = RSM_STATE_BIND_QUIESCED; 3421 cv_broadcast(&seg->s_cv); 3422 } 3423 3424 rsmseglock_release(seg); 3425 3426 rsmacl_free(acl, acl_len); 3427 rsmpiacl_free(rsmpi_acl, acl_len); 3428 3429 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish done\n")); 3430 3431 return (DDI_SUCCESS); 3432 } 3433 3434 /* 3435 * Called from rsm_unpublish to force an unload and disconnection of all 3436 * importers of the unpublished segment. 3437 * 3438 * First build the list of segments requiring a force disconnect, then 3439 * send a request for each. 3440 */ 3441 static void 3442 rsm_send_importer_disconnects(rsm_memseg_id_t ex_segid, 3443 rsm_node_id_t ex_nodeid) 3444 { 3445 rsmipc_request_t request; 3446 importing_token_t *prev_token, *token, *tmp_token, *tokp; 3447 importing_token_t *force_disconnect_list = NULL; 3448 int index; 3449 3450 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 3451 "rsm_send_importer_disconnects enter\n")); 3452 3453 index = rsmhash(ex_segid); 3454 3455 mutex_enter(&importer_list.lock); 3456 3457 prev_token = NULL; 3458 token = importer_list.bucket[index]; 3459 3460 while (token != NULL) { 3461 if (token->key == ex_segid) { 3462 /* 3463 * take it off the importer list and add it 3464 * to the force disconnect list. 3465 */ 3466 if (prev_token == NULL) 3467 importer_list.bucket[index] = token->next; 3468 else 3469 prev_token->next = token->next; 3470 tmp_token = token; 3471 token = token->next; 3472 if (force_disconnect_list == NULL) { 3473 force_disconnect_list = tmp_token; 3474 tmp_token->next = NULL; 3475 } else { 3476 tokp = force_disconnect_list; 3477 /* 3478 * make sure that the tmp_token's node 3479 * is not already on the force disconnect 3480 * list. 3481 */ 3482 while (tokp != NULL) { 3483 if (tokp->importing_node == 3484 tmp_token->importing_node) { 3485 break; 3486 } 3487 tokp = tokp->next; 3488 } 3489 if (tokp == NULL) { 3490 tmp_token->next = 3491 force_disconnect_list; 3492 force_disconnect_list = tmp_token; 3493 } else { 3494 kmem_free((void *)tmp_token, 3495 sizeof (*token)); 3496 } 3497 } 3498 3499 } else { 3500 prev_token = token; 3501 token = token->next; 3502 } 3503 } 3504 mutex_exit(&importer_list.lock); 3505 3506 token = force_disconnect_list; 3507 while (token != NULL) { 3508 if (token->importing_node == my_nodeid) { 3509 rsm_force_unload(ex_nodeid, ex_segid, 3510 DISCONNECT); 3511 } else { 3512 request.rsmipc_hdr.rsmipc_type = 3513 RSMIPC_MSG_DISCONNECT; 3514 request.rsmipc_key = token->key; 3515 for (;;) { 3516 if (rsmipc_send(token->importing_node, 3517 &request, 3518 RSM_NO_REPLY) == RSM_SUCCESS) { 3519 break; 3520 } else { 3521 delay(drv_usectohz(10000)); 3522 } 3523 } 3524 } 3525 tmp_token = token; 3526 token = token->next; 3527 kmem_free((void *)tmp_token, sizeof (*token)); 3528 } 3529 3530 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 3531 "rsm_send_importer_disconnects done\n")); 3532 } 3533 3534 /* 3535 * This function is used as a callback for unlocking the pages locked 3536 * down by a process which then does a fork or an exec. 3537 * It marks the export segments corresponding to umem cookie given by 3538 * the *arg to be in a ZOMBIE state(by calling rsmseg_close to be 3539 * destroyed later when an rsm_close occurs). 3540 */ 3541 static void 3542 rsm_export_force_destroy(ddi_umem_cookie_t *ck) 3543 { 3544 rsmresource_blk_t *blk; 3545 rsmresource_t *p; 3546 rsmseg_t *eseg = NULL; 3547 int i, j; 3548 int found = 0; 3549 3550 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 3551 "rsm_export_force_destroy enter\n")); 3552 3553 /* 3554 * Walk the resource list and locate the export segment (either 3555 * in the BIND or the EXPORT state) which corresponds to the 3556 * ddi_umem_cookie_t being freed up, and call rsmseg_close. 3557 * Change the state to ZOMBIE by calling rsmseg_close with the 3558 * force_flag argument (the second argument) set to 1. Also, 3559 * unpublish and unbind the segment, but don't free it. Free it 3560 * only on a rsm_close call for the segment. 3561 */ 3562 rw_enter(&rsm_resource.rsmrc_lock, RW_READER); 3563 3564 for (i = 0; i < rsm_resource.rsmrc_len; i++) { 3565 blk = rsm_resource.rsmrc_root[i]; 3566 if (blk == NULL) { 3567 continue; 3568 } 3569 3570 for (j = 0; j < RSMRC_BLKSZ; j++) { 3571 p = blk->rsmrcblk_blks[j]; 3572 if ((p != NULL) && (p != RSMRC_RESERVED) && 3573 (p->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT)) { 3574 eseg = (rsmseg_t *)p; 3575 if (eseg->s_cookie != ck) 3576 continue; /* continue searching */ 3577 /* 3578 * Found the segment, set flag to indicate 3579 * force destroy processing is in progress 3580 */ 3581 rsmseglock_acquire(eseg); 3582 eseg->s_flags |= RSM_FORCE_DESTROY_WAIT; 3583 rsmseglock_release(eseg); 3584 found = 1; 3585 break; 3586 } 3587 } 3588 3589 if (found) 3590 break; 3591 } 3592 3593 rw_exit(&rsm_resource.rsmrc_lock); 3594 3595 if (found) { 3596 ASSERT(eseg != NULL); 3597 /* call rsmseg_close with force flag set to 1 */ 3598 rsmseg_close(eseg, 1); 3599 /* 3600 * force destroy processing done, clear flag and signal any 3601 * thread waiting in rsmseg_close. 3602 */ 3603 rsmseglock_acquire(eseg); 3604 eseg->s_flags &= ~RSM_FORCE_DESTROY_WAIT; 3605 cv_broadcast(&eseg->s_cv); 3606 rsmseglock_release(eseg); 3607 } 3608 3609 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 3610 "rsm_export_force_destroy done\n")); 3611 } 3612 3613 /* ******************************* Remote Calls *********************** */ 3614 static void 3615 rsm_intr_segconnect(rsm_node_id_t src, rsmipc_request_t *req) 3616 { 3617 rsmipc_reply_t reply; 3618 DBG_DEFINE(category, 3619 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3620 3621 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3622 "rsm_intr_segconnect enter\n")); 3623 3624 reply.rsmipc_status = (short)rsmsegacl_validate(req, src, &reply); 3625 3626 reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY; 3627 reply.rsmipc_hdr.rsmipc_cookie = req->rsmipc_hdr.rsmipc_cookie; 3628 3629 (void) rsmipc_send(src, NULL, &reply); 3630 3631 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3632 "rsm_intr_segconnect done\n")); 3633 } 3634 3635 3636 /* 3637 * When an exported segment is unpublished the exporter sends an ipc 3638 * message (RSMIPC_MSG_DISCONNECT) to all importers. The recv ipc dispatcher 3639 * calls this function. The import list is scanned; segments which match the 3640 * exported segment id are unloaded and disconnected. 3641 * 3642 * Will also be called from rsm_rebind with disconnect_flag FALSE. 3643 * 3644 */ 3645 static void 3646 rsm_force_unload(rsm_node_id_t src_nodeid, 3647 rsm_memseg_id_t ex_segid, 3648 boolean_t disconnect_flag) 3649 3650 { 3651 rsmresource_t *p = NULL; 3652 rsmhash_table_t *rhash = &rsm_import_segs; 3653 uint_t index; 3654 DBG_DEFINE(category, 3655 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3656 3657 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload enter\n")); 3658 3659 index = rsmhash(ex_segid); 3660 3661 rw_enter(&rhash->rsmhash_rw, RW_READER); 3662 3663 p = rsmhash_getbkt(rhash, index); 3664 3665 for (; p; p = p->rsmrc_next) { 3666 rsmseg_t *seg = (rsmseg_t *)p; 3667 if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid)) { 3668 /* 3669 * In order to make rsmseg_unload and rsm_force_unload 3670 * thread safe, acquire the segment lock here. 3671 * rsmseg_unload is responsible for releasing the lock. 3672 * rsmseg_unload releases the lock just before a call 3673 * to rsmipc_send or in case of an early exit which 3674 * occurs if the segment was in the state 3675 * RSM_STATE_CONNECTING or RSM_STATE_NEW. 3676 */ 3677 rsmseglock_acquire(seg); 3678 if (disconnect_flag) 3679 seg->s_flags |= RSM_FORCE_DISCONNECT; 3680 rsmseg_unload(seg); 3681 } 3682 } 3683 rw_exit(&rhash->rsmhash_rw); 3684 3685 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload done\n")); 3686 } 3687 3688 static void 3689 rsm_intr_reply(rsmipc_msghdr_t *msg) 3690 { 3691 /* 3692 * Find slot for cookie in reply. 3693 * Match sequence with sequence in cookie 3694 * If no match; return 3695 * Try to grap lock of slot, if locked return 3696 * copy data into reply slot area 3697 * signal waiter 3698 */ 3699 rsmipc_slot_t *slot; 3700 rsmipc_cookie_t *cookie; 3701 void *data = (void *) msg; 3702 size_t size = sizeof (rsmipc_reply_t); 3703 DBG_DEFINE(category, 3704 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3705 3706 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply enter\n")); 3707 3708 cookie = &msg->rsmipc_cookie; 3709 if (cookie->ic.index >= RSMIPC_SZ) { 3710 DBG_PRINTF((category, RSM_ERR, 3711 "rsm: rsm_intr_reply bad cookie %d\n", cookie->ic.index)); 3712 return; 3713 } 3714 3715 ASSERT(cookie->ic.index < RSMIPC_SZ); 3716 slot = &rsm_ipc.slots[cookie->ic.index]; 3717 mutex_enter(&slot->rsmipc_lock); 3718 if (slot->rsmipc_cookie.value == cookie->value) { 3719 /* found a match */ 3720 if (RSMIPC_GET(slot, RSMIPC_PENDING)) { 3721 bcopy(data, slot->rsmipc_data, size); 3722 RSMIPC_CLEAR(slot, RSMIPC_PENDING); 3723 cv_signal(&slot->rsmipc_cv); 3724 } 3725 } else { 3726 DBG_PRINTF((category, RSM_DEBUG, 3727 "rsm: rsm_intr_reply mismatched reply %d\n", 3728 cookie->ic.index)); 3729 } 3730 mutex_exit(&slot->rsmipc_lock); 3731 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply done\n")); 3732 } 3733 3734 /* 3735 * This function gets dispatched on the worker thread when we receive 3736 * the SQREADY message. This function sends the SQREADY_ACK message. 3737 */ 3738 static void 3739 rsm_sqready_ack_deferred(void *arg) 3740 { 3741 path_t *path = (path_t *)arg; 3742 DBG_DEFINE(category, 3743 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3744 3745 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3746 "rsm_sqready_ack_deferred enter\n")); 3747 3748 mutex_enter(&path->mutex); 3749 3750 /* 3751 * If path is not active no point in sending the ACK 3752 * because the whole SQREADY protocol will again start 3753 * when the path becomes active. 3754 */ 3755 if (path->state != RSMKA_PATH_ACTIVE) { 3756 /* 3757 * decrement the path refcnt incremented in rsm_proc_sqready 3758 */ 3759 PATH_RELE_NOLOCK(path); 3760 mutex_exit(&path->mutex); 3761 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3762 "rsm_sqready_ack_deferred done:!ACTIVE\n")); 3763 return; 3764 } 3765 3766 /* send an SQREADY_ACK message */ 3767 (void) rsmipc_send_controlmsg(path, RSMIPC_MSG_SQREADY_ACK); 3768 3769 /* initialize credits to the max level */ 3770 path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES; 3771 3772 /* wake up any send that is waiting for credits */ 3773 cv_broadcast(&path->sendq_token.sendq_cv); 3774 3775 /* 3776 * decrement the path refcnt since we incremented it in 3777 * rsm_proc_sqready 3778 */ 3779 PATH_RELE_NOLOCK(path); 3780 3781 mutex_exit(&path->mutex); 3782 3783 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3784 "rsm_sqready_ack_deferred done\n")); 3785 } 3786 3787 /* 3788 * Process the SQREADY message 3789 */ 3790 static void 3791 rsm_proc_sqready(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr, 3792 rsm_intr_hand_arg_t arg) 3793 { 3794 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg; 3795 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg; 3796 path_t *path; 3797 DBG_DEFINE(category, 3798 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3799 3800 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready enter\n")); 3801 3802 /* look up the path - incr the path refcnt */ 3803 path = rsm_find_path(hdlr_argp->adapter_name, 3804 hdlr_argp->adapter_instance, src_hwaddr); 3805 3806 /* 3807 * No path exists or path is not active - drop the message 3808 */ 3809 if (path == NULL) { 3810 DBG_PRINTF((category, RSM_DEBUG, 3811 "rsm_proc_sqready done: msg dropped no path\n")); 3812 return; 3813 } 3814 3815 mutex_exit(&path->mutex); 3816 3817 /* drain any tasks from the previous incarnation */ 3818 taskq_wait(path->recv_taskq); 3819 3820 mutex_enter(&path->mutex); 3821 /* 3822 * If we'd sent an SQREADY message and were waiting for SQREADY_ACK 3823 * in the meanwhile we received an SQREADY message, blindly reset 3824 * the WAIT_FOR_SQACK flag because we'll just send SQREADY_ACK 3825 * and forget about the SQREADY that we sent. 3826 */ 3827 path->flags &= ~RSMKA_WAIT_FOR_SQACK; 3828 3829 if (path->state != RSMKA_PATH_ACTIVE) { 3830 /* decr refcnt and drop the mutex */ 3831 PATH_RELE_NOLOCK(path); 3832 mutex_exit(&path->mutex); 3833 DBG_PRINTF((category, RSM_DEBUG, 3834 "rsm_proc_sqready done: msg dropped path !ACTIVE\n")); 3835 return; 3836 } 3837 3838 DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready:path=%lx " 3839 " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr)); 3840 3841 /* 3842 * The sender's local incarnation number is our remote incarnation 3843 * number save it in the path data structure 3844 */ 3845 path->remote_incn = msg->rsmipc_local_incn; 3846 path->sendq_token.msgbuf_avail = 0; 3847 path->procmsg_cnt = 0; 3848 3849 /* 3850 * path is active - dispatch task to send SQREADY_ACK - remember 3851 * RSMPI calls can't be done in interrupt context 3852 * 3853 * We can use the recv_taskq to send because the remote endpoint 3854 * cannot start sending messages till it receives SQREADY_ACK hence 3855 * at this point there are no tasks on recv_taskq. 3856 * 3857 * The path refcnt will be decremented in rsm_sqready_ack_deferred. 3858 */ 3859 (void) taskq_dispatch(path->recv_taskq, 3860 rsm_sqready_ack_deferred, path, KM_NOSLEEP); 3861 3862 mutex_exit(&path->mutex); 3863 3864 3865 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready done\n")); 3866 } 3867 3868 /* 3869 * Process the SQREADY_ACK message 3870 */ 3871 static void 3872 rsm_proc_sqready_ack(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr, 3873 rsm_intr_hand_arg_t arg) 3874 { 3875 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg; 3876 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg; 3877 path_t *path; 3878 DBG_DEFINE(category, 3879 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 3880 3881 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3882 "rsm_proc_sqready_ack enter\n")); 3883 3884 /* look up the path - incr the path refcnt */ 3885 path = rsm_find_path(hdlr_argp->adapter_name, 3886 hdlr_argp->adapter_instance, src_hwaddr); 3887 3888 /* 3889 * drop the message if - no path exists or path is not active 3890 * or if its not waiting for SQREADY_ACK message 3891 */ 3892 if (path == NULL) { 3893 DBG_PRINTF((category, RSM_DEBUG, 3894 "rsm_proc_sqready_ack done: msg dropped no path\n")); 3895 return; 3896 } 3897 3898 if ((path->state != RSMKA_PATH_ACTIVE) || 3899 !(path->flags & RSMKA_WAIT_FOR_SQACK)) { 3900 /* decrement the refcnt */ 3901 PATH_RELE_NOLOCK(path); 3902 mutex_exit(&path->mutex); 3903 DBG_PRINTF((category, RSM_DEBUG, 3904 "rsm_proc_sqready_ack done: msg dropped\n")); 3905 return; 3906 } 3907 3908 /* 3909 * Check if this message is in response to the last RSMIPC_MSG_SQREADY 3910 * sent, if not drop it. 3911 */ 3912 if (path->local_incn != msghdr->rsmipc_incn) { 3913 /* decrement the refcnt */ 3914 PATH_RELE_NOLOCK(path); 3915 mutex_exit(&path->mutex); 3916 DBG_PRINTF((category, RSM_DEBUG, 3917 "rsm_proc_sqready_ack done: msg old incn %lld\n", 3918 msghdr->rsmipc_incn)); 3919 return; 3920 } 3921 3922 DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready_ack:path=%lx " 3923 " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr)); 3924 3925 /* 3926 * clear the WAIT_FOR_SQACK flag since we have recvd the ack 3927 */ 3928 path->flags &= ~RSMKA_WAIT_FOR_SQACK; 3929 3930 /* save the remote sendq incn number */ 3931 path->remote_incn = msg->rsmipc_local_incn; 3932 3933 /* initialize credits to the max level */ 3934 path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES; 3935 3936 /* wake up any send that is waiting for credits */ 3937 cv_broadcast(&path->sendq_token.sendq_cv); 3938 3939 /* decrement the refcnt */ 3940 PATH_RELE_NOLOCK(path); 3941 3942 mutex_exit(&path->mutex); 3943 3944 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 3945 "rsm_proc_sqready_ack done\n")); 3946 } 3947 3948 /* 3949 * process the RSMIPC_MSG_CREDIT message 3950 */ 3951 static void 3952 rsm_add_credits(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr, 3953 rsm_intr_hand_arg_t arg) 3954 { 3955 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg; 3956 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg; 3957 path_t *path; 3958 DBG_DEFINE(category, 3959 RSM_KERNEL_AGENT | RSM_FUNC_ALL | 3960 RSM_INTR_CALLBACK | RSM_FLOWCONTROL); 3961 3962 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits enter\n")); 3963 3964 /* look up the path - incr the path refcnt */ 3965 path = rsm_find_path(hdlr_argp->adapter_name, 3966 hdlr_argp->adapter_instance, src_hwaddr); 3967 3968 if (path == NULL) { 3969 DBG_PRINTF((category, RSM_DEBUG, 3970 "rsm_add_credits enter: path not found\n")); 3971 return; 3972 } 3973 3974 /* the path is not active - discard credits */ 3975 if (path->state != RSMKA_PATH_ACTIVE) { 3976 PATH_RELE_NOLOCK(path); 3977 mutex_exit(&path->mutex); 3978 DBG_PRINTF((category, RSM_DEBUG, 3979 "rsm_add_credits enter:path=%lx !ACTIVE\n", path)); 3980 return; 3981 } 3982 3983 /* 3984 * Check if these credits are for current incarnation of the path. 3985 */ 3986 if (path->local_incn != msghdr->rsmipc_incn) { 3987 /* decrement the refcnt */ 3988 PATH_RELE_NOLOCK(path); 3989 mutex_exit(&path->mutex); 3990 DBG_PRINTF((category, RSM_DEBUG, 3991 "rsm_add_credits enter: old incn %lld\n", 3992 msghdr->rsmipc_incn)); 3993 return; 3994 } 3995 3996 DBG_PRINTF((category, RSM_DEBUG, 3997 "rsm_add_credits:path=%lx new-creds=%d " 3998 "curr credits=%d src=%lx:%llx\n", path, msg->rsmipc_credits, 3999 path->sendq_token.msgbuf_avail, msghdr->rsmipc_src, 4000 src_hwaddr)); 4001 4002 4003 /* add credits to the path's sendq */ 4004 path->sendq_token.msgbuf_avail += msg->rsmipc_credits; 4005 4006 ASSERT(path->sendq_token.msgbuf_avail <= RSMIPC_MAX_MESSAGES); 4007 4008 /* wake up any send that is waiting for credits */ 4009 cv_broadcast(&path->sendq_token.sendq_cv); 4010 4011 /* decrement the refcnt */ 4012 PATH_RELE_NOLOCK(path); 4013 4014 mutex_exit(&path->mutex); 4015 4016 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits done\n")); 4017 } 4018 4019 static void 4020 rsm_intr_event(rsmipc_request_t *msg) 4021 { 4022 rsmseg_t *seg; 4023 rsmresource_t *p; 4024 rsm_node_id_t src_node; 4025 DBG_DEFINE(category, 4026 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4027 4028 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event enter\n")); 4029 4030 src_node = msg->rsmipc_hdr.rsmipc_src; 4031 4032 if ((seg = msg->rsmipc_segment_cookie) != NULL) { 4033 /* This is for an import segment */ 4034 uint_t hashval = rsmhash(msg->rsmipc_key); 4035 4036 rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER); 4037 4038 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval); 4039 4040 for (; p; p = p->rsmrc_next) { 4041 if ((p->rsmrc_key == msg->rsmipc_key) && 4042 (p->rsmrc_node == src_node)) { 4043 seg = (rsmseg_t *)p; 4044 rsmseglock_acquire(seg); 4045 4046 atomic_inc_32(&seg->s_pollevent); 4047 4048 if (seg->s_pollflag & RSM_SEGMENT_POLL) 4049 pollwakeup(&seg->s_poll, POLLRDNORM); 4050 4051 rsmseglock_release(seg); 4052 } 4053 } 4054 4055 rw_exit(&rsm_import_segs.rsmhash_rw); 4056 } else { 4057 /* This is for an export segment */ 4058 seg = rsmexport_lookup(msg->rsmipc_key); 4059 if (!seg) { 4060 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4061 "rsm_intr_event done: exp seg not found\n")); 4062 return; 4063 } 4064 4065 ASSERT(rsmseglock_held(seg)); 4066 4067 atomic_inc_32(&seg->s_pollevent); 4068 4069 /* 4070 * We must hold the segment lock here, or else the segment 4071 * can be freed while pollwakeup is using it. This implies 4072 * that we MUST NOT grab the segment lock during rsm_chpoll, 4073 * as outlined in the chpoll(2) man page. 4074 */ 4075 if (seg->s_pollflag & RSM_SEGMENT_POLL) 4076 pollwakeup(&seg->s_poll, POLLRDNORM); 4077 4078 rsmseglock_release(seg); 4079 } 4080 4081 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event done\n")); 4082 } 4083 4084 /* 4085 * The exporter did a republish and changed the ACL - this change is only 4086 * visible to new importers. 4087 */ 4088 static void 4089 importer_update(rsm_node_id_t src_node, rsm_memseg_id_t key, 4090 rsm_permission_t perm) 4091 { 4092 4093 rsmresource_t *p; 4094 rsmseg_t *seg; 4095 uint_t hashval = rsmhash(key); 4096 DBG_DEFINE(category, 4097 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4098 4099 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update enter\n")); 4100 4101 rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER); 4102 4103 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval); 4104 4105 for (; p; p = p->rsmrc_next) { 4106 /* 4107 * find the importer and update the permission in the shared 4108 * data structure. Any new importers will use the new perms 4109 */ 4110 if ((p->rsmrc_key == key) && (p->rsmrc_node == src_node)) { 4111 seg = (rsmseg_t *)p; 4112 4113 rsmseglock_acquire(seg); 4114 rsmsharelock_acquire(seg); 4115 seg->s_share->rsmsi_mode = perm; 4116 rsmsharelock_release(seg); 4117 rsmseglock_release(seg); 4118 4119 break; 4120 } 4121 } 4122 4123 rw_exit(&rsm_import_segs.rsmhash_rw); 4124 4125 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update done\n")); 4126 } 4127 4128 void 4129 rsm_suspend_complete(rsm_node_id_t src_node, int flag) 4130 { 4131 int done = 1; /* indicate all SUSPENDS have been acked */ 4132 list_element_t *elem; 4133 DBG_DEFINE(category, 4134 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4135 4136 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4137 "rsm_suspend_complete enter\n")); 4138 4139 mutex_enter(&rsm_suspend_list.list_lock); 4140 4141 if (rsm_suspend_list.list_head == NULL) { 4142 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4143 "rsm_suspend_complete done: suspend_list is empty\n")); 4144 mutex_exit(&rsm_suspend_list.list_lock); 4145 return; 4146 } 4147 4148 elem = rsm_suspend_list.list_head; 4149 while (elem != NULL) { 4150 if (elem->nodeid == src_node) { 4151 /* clear the pending flag for the node */ 4152 elem->flags &= ~RSM_SUSPEND_ACKPENDING; 4153 elem->flags |= flag; 4154 } 4155 4156 if (done && (elem->flags & RSM_SUSPEND_ACKPENDING)) 4157 done = 0; /* still some nodes have not yet ACKED */ 4158 4159 elem = elem->next; 4160 } 4161 4162 mutex_exit(&rsm_suspend_list.list_lock); 4163 4164 if (!done) { 4165 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4166 "rsm_suspend_complete done: acks pending\n")); 4167 return; 4168 } 4169 /* 4170 * Now that we are done with suspending all the remote importers 4171 * time to quiesce the local exporters 4172 */ 4173 exporter_quiesce(); 4174 4175 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4176 "rsm_suspend_complete done\n")); 4177 } 4178 4179 static void 4180 exporter_quiesce() 4181 { 4182 int i, e; 4183 rsmresource_t *current; 4184 rsmseg_t *seg; 4185 adapter_t *adapter; 4186 DBG_DEFINE(category, 4187 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4188 4189 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce enter\n")); 4190 /* 4191 * The importers send a SUSPEND_COMPLETE to the exporter node 4192 * Unpublish, unbind the export segment and 4193 * move the segments to the EXPORT_QUIESCED state 4194 */ 4195 4196 rw_enter(&rsm_export_segs.rsmhash_rw, RW_READER); 4197 4198 for (i = 0; i < rsm_hash_size; i++) { 4199 current = rsm_export_segs.bucket[i]; 4200 while (current != NULL) { 4201 seg = (rsmseg_t *)current; 4202 rsmseglock_acquire(seg); 4203 if (current->rsmrc_state == 4204 RSM_STATE_EXPORT_QUIESCING) { 4205 adapter = seg->s_adapter; 4206 /* 4207 * some local memory handles are not published 4208 * check if it was published 4209 */ 4210 if ((seg->s_acl == NULL) || 4211 (seg->s_acl[0].ae_node != my_nodeid) || 4212 (seg->s_acl[0].ae_permission != 0)) { 4213 4214 e = adapter->rsmpi_ops->rsm_unpublish( 4215 seg->s_handle.out); 4216 DBG_PRINTF((category, RSM_DEBUG, 4217 "exporter_quiesce:unpub %d\n", e)); 4218 4219 e = adapter->rsmpi_ops->rsm_seg_destroy( 4220 seg->s_handle.out); 4221 4222 DBG_PRINTF((category, RSM_DEBUG, 4223 "exporter_quiesce:destroy %d\n", 4224 e)); 4225 } 4226 4227 (void) rsm_unbind_pages(seg); 4228 seg->s_state = RSM_STATE_EXPORT_QUIESCED; 4229 cv_broadcast(&seg->s_cv); 4230 } 4231 rsmseglock_release(seg); 4232 current = current->rsmrc_next; 4233 } 4234 } 4235 rw_exit(&rsm_export_segs.rsmhash_rw); 4236 4237 /* 4238 * All the local segments we are done with the pre-del processing 4239 * - time to move to PREDEL_COMPLETED. 4240 */ 4241 4242 mutex_enter(&rsm_drv_data.drv_lock); 4243 4244 ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED); 4245 4246 rsm_drv_data.drv_state = RSM_DRV_PREDEL_COMPLETED; 4247 4248 cv_broadcast(&rsm_drv_data.drv_cv); 4249 4250 mutex_exit(&rsm_drv_data.drv_lock); 4251 4252 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce done\n")); 4253 } 4254 4255 static void 4256 importer_suspend(rsm_node_id_t src_node) 4257 { 4258 int i; 4259 int susp_flg; /* true means already suspended */ 4260 int num_importers; 4261 rsmresource_t *p = NULL, *curp; 4262 rsmhash_table_t *rhash = &rsm_import_segs; 4263 rsmseg_t *seg; 4264 rsmipc_request_t request; 4265 DBG_DEFINE(category, 4266 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4267 4268 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend enter\n")); 4269 4270 rw_enter(&rhash->rsmhash_rw, RW_READER); 4271 for (i = 0; i < rsm_hash_size; i++) { 4272 p = rhash->bucket[i]; 4273 4274 /* 4275 * Suspend all importers with same <node, key> pair. 4276 * After the last one of the shared importers has been 4277 * suspended - suspend the shared mappings/connection. 4278 */ 4279 for (; p; p = p->rsmrc_next) { 4280 rsmseg_t *first = (rsmseg_t *)p; 4281 if ((first->s_node != src_node) || 4282 (first->s_state == RSM_STATE_DISCONNECT)) 4283 continue; /* go to next entry */ 4284 /* 4285 * search the rest of the bucket for 4286 * other siblings (imprtrs with the same key) 4287 * of "first" and suspend them. 4288 * All importers with same key fall in 4289 * the same bucket. 4290 */ 4291 num_importers = 0; 4292 for (curp = p; curp; curp = curp->rsmrc_next) { 4293 seg = (rsmseg_t *)curp; 4294 4295 rsmseglock_acquire(seg); 4296 4297 if ((seg->s_node != first->s_node) || 4298 (seg->s_key != first->s_key) || 4299 (seg->s_state == RSM_STATE_DISCONNECT)) { 4300 /* 4301 * either not a peer segment or its a 4302 * disconnected segment - skip it 4303 */ 4304 rsmseglock_release(seg); 4305 continue; 4306 } 4307 4308 rsmseg_suspend(seg, &susp_flg); 4309 4310 if (susp_flg) { /* seg already suspended */ 4311 rsmseglock_release(seg); 4312 break; /* the inner for loop */ 4313 } 4314 4315 num_importers++; 4316 rsmsharelock_acquire(seg); 4317 /* 4318 * we've processed all importers that are 4319 * siblings of "first" 4320 */ 4321 if (num_importers == 4322 seg->s_share->rsmsi_refcnt) { 4323 rsmsharelock_release(seg); 4324 rsmseglock_release(seg); 4325 break; 4326 } 4327 rsmsharelock_release(seg); 4328 rsmseglock_release(seg); 4329 } 4330 4331 /* 4332 * All the importers with the same key and 4333 * nodeid as "first" have been suspended. 4334 * Now suspend the shared connect/mapping. 4335 * This is done only once. 4336 */ 4337 if (!susp_flg) { 4338 rsmsegshare_suspend(seg); 4339 } 4340 } 4341 } 4342 4343 rw_exit(&rhash->rsmhash_rw); 4344 4345 /* send an ACK for SUSPEND message */ 4346 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND_DONE; 4347 (void) rsmipc_send(src_node, &request, RSM_NO_REPLY); 4348 4349 4350 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend done\n")); 4351 4352 } 4353 4354 static void 4355 rsmseg_suspend(rsmseg_t *seg, int *susp_flg) 4356 { 4357 int recheck_state; 4358 rsmcookie_t *hdl; 4359 DBG_DEFINE(category, 4360 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4361 4362 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4363 "rsmseg_suspend enter: key=%u\n", seg->s_key)); 4364 4365 *susp_flg = 0; 4366 4367 ASSERT(rsmseglock_held(seg)); 4368 /* wait if putv/getv is in progress */ 4369 while (seg->s_rdmacnt > 0) 4370 cv_wait(&seg->s_cv, &seg->s_lock); 4371 4372 do { 4373 recheck_state = 0; 4374 4375 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4376 "rsmseg_suspend:segment %x state=%d\n", 4377 seg->s_key, seg->s_state)); 4378 4379 switch (seg->s_state) { 4380 case RSM_STATE_NEW: 4381 /* not a valid state */ 4382 break; 4383 case RSM_STATE_CONNECTING: 4384 seg->s_state = RSM_STATE_ABORT_CONNECT; 4385 break; 4386 case RSM_STATE_ABORT_CONNECT: 4387 break; 4388 case RSM_STATE_CONNECT: 4389 seg->s_handle.in = NULL; 4390 seg->s_state = RSM_STATE_CONN_QUIESCE; 4391 break; 4392 case RSM_STATE_MAPPING: 4393 /* wait until segment leaves the mapping state */ 4394 while (seg->s_state == RSM_STATE_MAPPING) 4395 cv_wait(&seg->s_cv, &seg->s_lock); 4396 recheck_state = 1; 4397 break; 4398 case RSM_STATE_ACTIVE: 4399 /* unload the mappings */ 4400 if (seg->s_ckl != NULL) { 4401 hdl = seg->s_ckl; 4402 for (; hdl != NULL; hdl = hdl->c_next) { 4403 (void) devmap_unload(hdl->c_dhp, 4404 hdl->c_off, hdl->c_len); 4405 } 4406 } 4407 seg->s_mapinfo = NULL; 4408 seg->s_state = RSM_STATE_MAP_QUIESCE; 4409 break; 4410 case RSM_STATE_CONN_QUIESCE: 4411 /* FALLTHRU */ 4412 case RSM_STATE_MAP_QUIESCE: 4413 /* rsmseg_suspend already done for seg */ 4414 *susp_flg = 1; 4415 break; 4416 case RSM_STATE_DISCONNECT: 4417 break; 4418 default: 4419 ASSERT(0); /* invalid state */ 4420 } 4421 } while (recheck_state); 4422 4423 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_suspend done\n")); 4424 } 4425 4426 static void 4427 rsmsegshare_suspend(rsmseg_t *seg) 4428 { 4429 int e; 4430 adapter_t *adapter; 4431 rsm_import_share_t *sharedp; 4432 DBG_DEFINE(category, 4433 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4434 4435 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4436 "rsmsegshare_suspend enter\n")); 4437 4438 rsmseglock_acquire(seg); 4439 rsmsharelock_acquire(seg); 4440 4441 sharedp = seg->s_share; 4442 adapter = seg->s_adapter; 4443 switch (sharedp->rsmsi_state) { 4444 case RSMSI_STATE_NEW: 4445 break; 4446 case RSMSI_STATE_CONNECTING: 4447 sharedp->rsmsi_state = RSMSI_STATE_ABORT_CONNECT; 4448 break; 4449 case RSMSI_STATE_ABORT_CONNECT: 4450 break; 4451 case RSMSI_STATE_CONNECTED: 4452 /* do the rsmpi disconnect */ 4453 if (sharedp->rsmsi_node != my_nodeid) { 4454 e = adapter->rsmpi_ops-> 4455 rsm_disconnect(sharedp->rsmsi_handle); 4456 4457 DBG_PRINTF((category, RSM_DEBUG, 4458 "rsm:rsmpi disconnect seg=%x:err=%d\n", 4459 sharedp->rsmsi_segid, e)); 4460 } 4461 4462 sharedp->rsmsi_handle = NULL; 4463 4464 sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE; 4465 break; 4466 case RSMSI_STATE_CONN_QUIESCE: 4467 break; 4468 case RSMSI_STATE_MAPPED: 4469 /* do the rsmpi unmap and disconnect */ 4470 if (sharedp->rsmsi_node != my_nodeid) { 4471 e = adapter->rsmpi_ops->rsm_unmap(seg->s_handle.in); 4472 4473 DBG_PRINTF((category, RSM_DEBUG, 4474 "rsmshare_suspend: rsmpi unmap %d\n", e)); 4475 4476 e = adapter->rsmpi_ops-> 4477 rsm_disconnect(sharedp->rsmsi_handle); 4478 DBG_PRINTF((category, RSM_DEBUG, 4479 "rsm:rsmpi disconnect seg=%x:err=%d\n", 4480 sharedp->rsmsi_segid, e)); 4481 } 4482 4483 sharedp->rsmsi_handle = NULL; 4484 4485 sharedp->rsmsi_state = RSMSI_STATE_MAP_QUIESCE; 4486 break; 4487 case RSMSI_STATE_MAP_QUIESCE: 4488 break; 4489 case RSMSI_STATE_DISCONNECTED: 4490 break; 4491 default: 4492 ASSERT(0); /* invalid state */ 4493 } 4494 4495 rsmsharelock_release(seg); 4496 rsmseglock_release(seg); 4497 4498 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4499 "rsmsegshare_suspend done\n")); 4500 } 4501 4502 /* 4503 * This should get called on receiving a RESUME message or from 4504 * the pathmanger if the node undergoing DR dies. 4505 */ 4506 static void 4507 importer_resume(rsm_node_id_t src_node) 4508 { 4509 int i; 4510 rsmresource_t *p = NULL; 4511 rsmhash_table_t *rhash = &rsm_import_segs; 4512 void *cookie; 4513 DBG_DEFINE(category, 4514 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4515 4516 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume enter\n")); 4517 4518 rw_enter(&rhash->rsmhash_rw, RW_READER); 4519 4520 for (i = 0; i < rsm_hash_size; i++) { 4521 p = rhash->bucket[i]; 4522 4523 for (; p; p = p->rsmrc_next) { 4524 rsmseg_t *seg = (rsmseg_t *)p; 4525 4526 rsmseglock_acquire(seg); 4527 4528 /* process only importers of node undergoing DR */ 4529 if (seg->s_node != src_node) { 4530 rsmseglock_release(seg); 4531 continue; 4532 } 4533 4534 if (rsmseg_resume(seg, &cookie) != RSM_SUCCESS) { 4535 rsmipc_request_t request; 4536 /* 4537 * rsmpi map/connect failed 4538 * inform the exporter so that it can 4539 * remove the importer. 4540 */ 4541 request.rsmipc_hdr.rsmipc_type = 4542 RSMIPC_MSG_NOTIMPORTING; 4543 request.rsmipc_key = seg->s_segid; 4544 request.rsmipc_segment_cookie = cookie; 4545 rsmseglock_release(seg); 4546 (void) rsmipc_send(seg->s_node, &request, 4547 RSM_NO_REPLY); 4548 } else { 4549 rsmseglock_release(seg); 4550 } 4551 } 4552 } 4553 4554 rw_exit(&rhash->rsmhash_rw); 4555 4556 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume done\n")); 4557 } 4558 4559 static int 4560 rsmseg_resume(rsmseg_t *seg, void **cookie) 4561 { 4562 int e; 4563 int retc; 4564 off_t dev_offset; 4565 size_t maplen; 4566 uint_t maxprot; 4567 rsm_mapinfo_t *p; 4568 rsmcookie_t *hdl; 4569 rsm_import_share_t *sharedp; 4570 DBG_DEFINE(category, 4571 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4572 4573 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4574 "rsmseg_resume enter: key=%u\n", seg->s_key)); 4575 4576 *cookie = NULL; 4577 4578 ASSERT(rsmseglock_held(seg)); 4579 4580 if ((seg->s_state != RSM_STATE_CONN_QUIESCE) && 4581 (seg->s_state != RSM_STATE_MAP_QUIESCE)) { 4582 return (RSM_SUCCESS); 4583 } 4584 4585 sharedp = seg->s_share; 4586 4587 rsmsharelock_acquire(seg); 4588 4589 /* resume the shared connection and/or mapping */ 4590 retc = rsmsegshare_resume(seg); 4591 4592 if (seg->s_state == RSM_STATE_CONN_QUIESCE) { 4593 /* shared state can either be connected or mapped */ 4594 if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) || 4595 (sharedp->rsmsi_state == RSMSI_STATE_MAPPED)) { 4596 ASSERT(retc == RSM_SUCCESS); 4597 seg->s_handle.in = sharedp->rsmsi_handle; 4598 rsmsharelock_release(seg); 4599 seg->s_state = RSM_STATE_CONNECT; 4600 4601 } else { /* error in rsmpi connect during resume */ 4602 seg->s_handle.in = NULL; 4603 seg->s_state = RSM_STATE_DISCONNECT; 4604 4605 sharedp->rsmsi_refcnt--; 4606 cookie = (void *)sharedp->rsmsi_cookie; 4607 4608 if (sharedp->rsmsi_refcnt == 0) { 4609 ASSERT(sharedp->rsmsi_mapcnt == 0); 4610 rsmsharelock_release(seg); 4611 4612 /* clean up the shared data structure */ 4613 mutex_destroy(&sharedp->rsmsi_lock); 4614 cv_destroy(&sharedp->rsmsi_cv); 4615 kmem_free((void *)(sharedp), 4616 sizeof (rsm_import_share_t)); 4617 4618 } else { 4619 rsmsharelock_release(seg); 4620 } 4621 /* 4622 * The following needs to be done after any 4623 * rsmsharelock calls which use seg->s_share. 4624 */ 4625 seg->s_share = NULL; 4626 } 4627 4628 /* signal any waiting segment */ 4629 cv_broadcast(&seg->s_cv); 4630 4631 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4632 "rsmseg_resume done:state=%d\n", seg->s_state)); 4633 return (retc); 4634 } 4635 4636 ASSERT(seg->s_state == RSM_STATE_MAP_QUIESCE); 4637 4638 /* Setup protections for remap */ 4639 maxprot = PROT_USER; 4640 if (seg->s_mode & RSM_PERM_READ) { 4641 maxprot |= PROT_READ; 4642 } 4643 if (seg->s_mode & RSM_PERM_WRITE) { 4644 maxprot |= PROT_WRITE; 4645 } 4646 4647 if (sharedp->rsmsi_state != RSMSI_STATE_MAPPED) { 4648 /* error in rsmpi connect or map during resume */ 4649 4650 /* remap to trash page */ 4651 ASSERT(seg->s_ckl != NULL); 4652 4653 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) { 4654 e = devmap_umem_remap(hdl->c_dhp, rsm_dip, 4655 remap_cookie, hdl->c_off, hdl->c_len, 4656 maxprot, 0, NULL); 4657 4658 DBG_PRINTF((category, RSM_ERR, 4659 "rsmseg_resume:remap=%d\n", e)); 4660 } 4661 4662 seg->s_handle.in = NULL; 4663 seg->s_state = RSM_STATE_DISCONNECT; 4664 4665 sharedp->rsmsi_refcnt--; 4666 4667 sharedp->rsmsi_mapcnt--; 4668 seg->s_mapinfo = NULL; 4669 4670 if (sharedp->rsmsi_refcnt == 0) { 4671 ASSERT(sharedp->rsmsi_mapcnt == 0); 4672 rsmsharelock_release(seg); 4673 4674 /* clean up the shared data structure */ 4675 mutex_destroy(&sharedp->rsmsi_lock); 4676 cv_destroy(&sharedp->rsmsi_cv); 4677 kmem_free((void *)(sharedp), 4678 sizeof (rsm_import_share_t)); 4679 4680 } else { 4681 rsmsharelock_release(seg); 4682 } 4683 /* 4684 * The following needs to be done after any 4685 * rsmsharelock calls which use seg->s_share. 4686 */ 4687 seg->s_share = NULL; 4688 4689 /* signal any waiting segment */ 4690 cv_broadcast(&seg->s_cv); 4691 4692 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4693 "rsmseg_resume done:seg=%x,err=%d\n", 4694 seg->s_key, retc)); 4695 return (retc); 4696 4697 } 4698 4699 seg->s_handle.in = sharedp->rsmsi_handle; 4700 4701 if (seg->s_node == my_nodeid) { /* loopback */ 4702 ASSERT(seg->s_mapinfo == NULL); 4703 4704 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) { 4705 e = devmap_umem_remap(hdl->c_dhp, 4706 rsm_dip, seg->s_cookie, 4707 hdl->c_off, hdl->c_len, 4708 maxprot, 0, NULL); 4709 4710 DBG_PRINTF((category, RSM_ERR, 4711 "rsmseg_resume:remap=%d\n", e)); 4712 } 4713 } else { /* remote exporter */ 4714 /* remap to the new rsmpi maps */ 4715 seg->s_mapinfo = sharedp->rsmsi_mapinfo; 4716 4717 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) { 4718 p = rsm_get_mapinfo(seg, hdl->c_off, hdl->c_len, 4719 &dev_offset, &maplen); 4720 e = devmap_devmem_remap(hdl->c_dhp, 4721 p->dip, p->dev_register, dev_offset, 4722 maplen, maxprot, 0, NULL); 4723 4724 DBG_PRINTF((category, RSM_ERR, 4725 "rsmseg_resume:remap=%d\n", e)); 4726 } 4727 } 4728 4729 rsmsharelock_release(seg); 4730 4731 seg->s_state = RSM_STATE_ACTIVE; 4732 cv_broadcast(&seg->s_cv); 4733 4734 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_resume done\n")); 4735 4736 return (retc); 4737 } 4738 4739 static int 4740 rsmsegshare_resume(rsmseg_t *seg) 4741 { 4742 int e = RSM_SUCCESS; 4743 adapter_t *adapter; 4744 rsm_import_share_t *sharedp; 4745 DBG_DEFINE(category, 4746 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4747 4748 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume enter\n")); 4749 4750 ASSERT(rsmseglock_held(seg)); 4751 ASSERT(rsmsharelock_held(seg)); 4752 4753 sharedp = seg->s_share; 4754 4755 /* 4756 * If we are not in a xxxx_QUIESCE state that means shared 4757 * connect/mapping processing has been already been done 4758 * so return success. 4759 */ 4760 if ((sharedp->rsmsi_state != RSMSI_STATE_CONN_QUIESCE) && 4761 (sharedp->rsmsi_state != RSMSI_STATE_MAP_QUIESCE)) { 4762 return (RSM_SUCCESS); 4763 } 4764 4765 adapter = seg->s_adapter; 4766 4767 if (sharedp->rsmsi_node != my_nodeid) { 4768 rsm_addr_t hwaddr; 4769 hwaddr = get_remote_hwaddr(adapter, sharedp->rsmsi_node); 4770 4771 e = adapter->rsmpi_ops->rsm_connect( 4772 adapter->rsmpi_handle, hwaddr, 4773 sharedp->rsmsi_segid, &sharedp->rsmsi_handle); 4774 4775 DBG_PRINTF((category, RSM_DEBUG, 4776 "rsmsegshare_resume:rsmpi connect seg=%x:err=%d\n", 4777 sharedp->rsmsi_segid, e)); 4778 4779 if (e != RSM_SUCCESS) { 4780 /* when do we send the NOT_IMPORTING message */ 4781 sharedp->rsmsi_handle = NULL; 4782 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED; 4783 /* signal any waiting segment */ 4784 cv_broadcast(&sharedp->rsmsi_cv); 4785 return (e); 4786 } 4787 } 4788 4789 if (sharedp->rsmsi_state == RSMSI_STATE_CONN_QUIESCE) { 4790 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED; 4791 /* signal any waiting segment */ 4792 cv_broadcast(&sharedp->rsmsi_cv); 4793 return (e); 4794 } 4795 4796 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE); 4797 4798 /* do the rsmpi map of the whole segment here */ 4799 if (sharedp->rsmsi_node != my_nodeid) { 4800 size_t mapped_len; 4801 rsm_mapinfo_t *p; 4802 4803 /* 4804 * We need to do rsmpi maps with <off, lens> identical to 4805 * the old mapinfo list because the segment mapping handles 4806 * dhp and such need the fragmentation of rsmpi maps to be 4807 * identical to what it was during the mmap of the segment 4808 */ 4809 p = sharedp->rsmsi_mapinfo; 4810 4811 while (p != NULL) { 4812 mapped_len = 0; 4813 4814 e = adapter->rsmpi_ops->rsm_map( 4815 sharedp->rsmsi_handle, p->start_offset, 4816 p->individual_len, &mapped_len, 4817 &p->dip, &p->dev_register, &p->dev_offset, 4818 NULL, NULL); 4819 4820 if (e != 0) { 4821 DBG_PRINTF((category, RSM_ERR, 4822 "rsmsegshare_resume: rsmpi map err=%d\n", 4823 e)); 4824 break; 4825 } 4826 4827 if (mapped_len != p->individual_len) { 4828 DBG_PRINTF((category, RSM_ERR, 4829 "rsmsegshare_resume: rsmpi maplen" 4830 "< reqlen=%lx\n", mapped_len)); 4831 e = RSMERR_BAD_LENGTH; 4832 break; 4833 } 4834 4835 p = p->next; 4836 4837 } 4838 4839 4840 if (e != RSM_SUCCESS) { /* rsmpi map failed */ 4841 int err; 4842 /* Check if this is the first rsm_map */ 4843 if (p != sharedp->rsmsi_mapinfo) { 4844 /* 4845 * A single rsm_unmap undoes multiple rsm_maps. 4846 */ 4847 (void) seg->s_adapter->rsmpi_ops-> 4848 rsm_unmap(sharedp->rsmsi_handle); 4849 } 4850 4851 rsm_free_mapinfo(sharedp->rsmsi_mapinfo); 4852 sharedp->rsmsi_mapinfo = NULL; 4853 4854 err = adapter->rsmpi_ops-> 4855 rsm_disconnect(sharedp->rsmsi_handle); 4856 4857 DBG_PRINTF((category, RSM_DEBUG, 4858 "rsmsegshare_resume:disconn seg=%x:err=%d\n", 4859 sharedp->rsmsi_segid, err)); 4860 4861 sharedp->rsmsi_handle = NULL; 4862 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED; 4863 4864 /* signal the waiting segments */ 4865 cv_broadcast(&sharedp->rsmsi_cv); 4866 DBG_PRINTF((category, RSM_DEBUG, 4867 "rsmsegshare_resume done: rsmpi map err\n")); 4868 return (e); 4869 } 4870 } 4871 4872 sharedp->rsmsi_state = RSMSI_STATE_MAPPED; 4873 4874 /* signal any waiting segment */ 4875 cv_broadcast(&sharedp->rsmsi_cv); 4876 4877 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume done\n")); 4878 4879 return (e); 4880 } 4881 4882 /* 4883 * this is the routine that gets called by recv_taskq which is the 4884 * thread that processes messages that are flow-controlled. 4885 */ 4886 static void 4887 rsm_intr_proc_deferred(void *arg) 4888 { 4889 path_t *path = (path_t *)arg; 4890 rsmipc_request_t *msg; 4891 rsmipc_msghdr_t *msghdr; 4892 rsm_node_id_t src_node; 4893 msgbuf_elem_t *head; 4894 int e; 4895 DBG_DEFINE(category, 4896 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4897 4898 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4899 "rsm_intr_proc_deferred enter\n")); 4900 4901 mutex_enter(&path->mutex); 4902 4903 /* use the head of the msgbuf_queue */ 4904 head = rsmka_gethead_msgbuf(path); 4905 4906 mutex_exit(&path->mutex); 4907 4908 msg = (rsmipc_request_t *)&(head->msg); 4909 msghdr = (rsmipc_msghdr_t *)msg; 4910 4911 src_node = msghdr->rsmipc_src; 4912 4913 /* 4914 * messages that need to send a reply should check the message version 4915 * before processing the message. And all messages that need to 4916 * send a reply should be processed here by the worker thread. 4917 */ 4918 switch (msghdr->rsmipc_type) { 4919 case RSMIPC_MSG_SEGCONNECT: 4920 if (msghdr->rsmipc_version != RSM_VERSION) { 4921 rsmipc_reply_t reply; 4922 reply.rsmipc_status = RSMERR_BAD_DRIVER_VERSION; 4923 reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY; 4924 reply.rsmipc_hdr.rsmipc_cookie = msghdr->rsmipc_cookie; 4925 (void) rsmipc_send(msghdr->rsmipc_src, NULL, &reply); 4926 } else { 4927 rsm_intr_segconnect(src_node, msg); 4928 } 4929 break; 4930 case RSMIPC_MSG_DISCONNECT: 4931 rsm_force_unload(src_node, msg->rsmipc_key, DISCONNECT); 4932 break; 4933 case RSMIPC_MSG_SUSPEND: 4934 importer_suspend(src_node); 4935 break; 4936 case RSMIPC_MSG_SUSPEND_DONE: 4937 rsm_suspend_complete(src_node, 0); 4938 break; 4939 case RSMIPC_MSG_RESUME: 4940 importer_resume(src_node); 4941 break; 4942 default: 4943 ASSERT(0); 4944 } 4945 4946 mutex_enter(&path->mutex); 4947 4948 rsmka_dequeue_msgbuf(path); 4949 4950 /* incr procmsg_cnt can be at most RSMIPC_MAX_MESSAGES */ 4951 if (path->procmsg_cnt < RSMIPC_MAX_MESSAGES) 4952 path->procmsg_cnt++; 4953 4954 ASSERT(path->procmsg_cnt <= RSMIPC_MAX_MESSAGES); 4955 4956 /* No need to send credits if path is going down */ 4957 if ((path->state == RSMKA_PATH_ACTIVE) && 4958 (path->procmsg_cnt >= RSMIPC_LOTSFREE_MSGBUFS)) { 4959 /* 4960 * send credits and reset procmsg_cnt if success otherwise 4961 * credits will be sent after processing the next message 4962 */ 4963 e = rsmipc_send_controlmsg(path, RSMIPC_MSG_CREDIT); 4964 if (e == 0) 4965 path->procmsg_cnt = 0; 4966 else 4967 DBG_PRINTF((category, RSM_ERR, 4968 "rsm_intr_proc_deferred:send credits err=%d\n", e)); 4969 } 4970 4971 /* 4972 * decrement the path refcnt since we incremented it in 4973 * rsm_intr_callback_dispatch 4974 */ 4975 PATH_RELE_NOLOCK(path); 4976 4977 mutex_exit(&path->mutex); 4978 4979 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4980 "rsm_intr_proc_deferred done\n")); 4981 } 4982 4983 /* 4984 * Flow-controlled messages are enqueued and dispatched onto a taskq here 4985 */ 4986 static void 4987 rsm_intr_callback_dispatch(void *data, rsm_addr_t src_hwaddr, 4988 rsm_intr_hand_arg_t arg) 4989 { 4990 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg; 4991 path_t *path; 4992 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data; 4993 DBG_DEFINE(category, 4994 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 4995 4996 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 4997 "rsm_intr_callback_dispatch enter\n")); 4998 ASSERT(data && hdlr_argp); 4999 5000 /* look up the path - incr the path refcnt */ 5001 path = rsm_find_path(hdlr_argp->adapter_name, 5002 hdlr_argp->adapter_instance, src_hwaddr); 5003 5004 /* the path has been removed - drop this message */ 5005 if (path == NULL) { 5006 DBG_PRINTF((category, RSM_DEBUG, 5007 "rsm_intr_callback_dispatch done: msg dropped\n")); 5008 return; 5009 } 5010 /* the path is not active - don't accept new messages */ 5011 if (path->state != RSMKA_PATH_ACTIVE) { 5012 PATH_RELE_NOLOCK(path); 5013 mutex_exit(&path->mutex); 5014 DBG_PRINTF((category, RSM_DEBUG, 5015 "rsm_intr_callback_dispatch done: msg dropped" 5016 " path=%lx !ACTIVE\n", path)); 5017 return; 5018 } 5019 5020 /* 5021 * Check if this message was sent to an older incarnation 5022 * of the path/sendq. 5023 */ 5024 if (path->local_incn != msghdr->rsmipc_incn) { 5025 /* decrement the refcnt */ 5026 PATH_RELE_NOLOCK(path); 5027 mutex_exit(&path->mutex); 5028 DBG_PRINTF((category, RSM_DEBUG, 5029 "rsm_intr_callback_dispatch done: old incn %lld\n", 5030 msghdr->rsmipc_incn)); 5031 return; 5032 } 5033 5034 /* copy and enqueue msg on the path's msgbuf queue */ 5035 rsmka_enqueue_msgbuf(path, data); 5036 5037 /* 5038 * schedule task to process messages - ignore retval from 5039 * task_dispatch because we sender cannot send more than 5040 * what receiver can handle. 5041 */ 5042 (void) taskq_dispatch(path->recv_taskq, 5043 rsm_intr_proc_deferred, path, KM_NOSLEEP); 5044 5045 mutex_exit(&path->mutex); 5046 5047 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5048 "rsm_intr_callback_dispatch done\n")); 5049 } 5050 5051 /* 5052 * This procedure is called from rsm_srv_func when a remote node creates a 5053 * a send queue. This event is used as a hint that an earlier failed 5054 * attempt to create a send queue to that remote node may now succeed and 5055 * should be retried. Indication of an earlier failed attempt is provided 5056 * by the RSMKA_SQCREATE_PENDING flag. 5057 */ 5058 static void 5059 rsm_sqcreateop_callback(rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg) 5060 { 5061 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg; 5062 path_t *path; 5063 DBG_DEFINE(category, 5064 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 5065 5066 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5067 "rsm_sqcreateop_callback enter\n")); 5068 5069 /* look up the path - incr the path refcnt */ 5070 path = rsm_find_path(hdlr_argp->adapter_name, 5071 hdlr_argp->adapter_instance, src_hwaddr); 5072 5073 if (path == NULL) { 5074 DBG_PRINTF((category, RSM_DEBUG, 5075 "rsm_sqcreateop_callback done: no path\n")); 5076 return; 5077 } 5078 5079 if ((path->state == RSMKA_PATH_UP) && 5080 (path->flags & RSMKA_SQCREATE_PENDING)) { 5081 /* 5082 * previous attempt to create sendq had failed, retry 5083 * it and move to RSMKA_PATH_ACTIVE state if successful. 5084 * the refcnt will be decremented in the do_deferred_work 5085 */ 5086 (void) rsmka_do_path_active(path, RSMKA_NO_SLEEP); 5087 } else { 5088 /* decrement the refcnt */ 5089 PATH_RELE_NOLOCK(path); 5090 } 5091 mutex_exit(&path->mutex); 5092 5093 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5094 "rsm_sqcreateop_callback done\n")); 5095 } 5096 5097 static void 5098 rsm_intr_callback(void *data, rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg) 5099 { 5100 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data; 5101 rsmipc_request_t *msg = (rsmipc_request_t *)data; 5102 rsmipc_controlmsg_t *ctrlmsg = (rsmipc_controlmsg_t *)data; 5103 rsm_node_id_t src_node; 5104 DBG_DEFINE(category, 5105 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 5106 5107 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback enter:" 5108 "src=%d, type=%d\n", msghdr->rsmipc_src, 5109 msghdr->rsmipc_type)); 5110 5111 /* 5112 * Check for the version number in the msg header. If it is not 5113 * RSM_VERSION, drop the message. In the future, we need to manage 5114 * incompatible version numbers in some way 5115 */ 5116 if (msghdr->rsmipc_version != RSM_VERSION) { 5117 DBG_PRINTF((category, RSM_ERR, "wrong KA version\n")); 5118 /* 5119 * Drop requests that don't have a reply right here 5120 * Request with reply will send a BAD_VERSION reply 5121 * when they get processed by the worker thread. 5122 */ 5123 if (msghdr->rsmipc_type != RSMIPC_MSG_SEGCONNECT) { 5124 return; 5125 } 5126 5127 } 5128 5129 src_node = msghdr->rsmipc_src; 5130 5131 switch (msghdr->rsmipc_type) { 5132 case RSMIPC_MSG_SEGCONNECT: 5133 case RSMIPC_MSG_DISCONNECT: 5134 case RSMIPC_MSG_SUSPEND: 5135 case RSMIPC_MSG_SUSPEND_DONE: 5136 case RSMIPC_MSG_RESUME: 5137 /* 5138 * These message types are handled by a worker thread using 5139 * the flow-control algorithm. 5140 * Any message processing that does one or more of the 5141 * following should be handled in a worker thread. 5142 * - allocates resources and might sleep 5143 * - makes RSMPI calls down to the interconnect driver 5144 * this by defn include requests with reply. 5145 * - takes a long duration of time 5146 */ 5147 rsm_intr_callback_dispatch(data, src_hwaddr, arg); 5148 break; 5149 case RSMIPC_MSG_NOTIMPORTING: 5150 importer_list_rm(src_node, msg->rsmipc_key, 5151 msg->rsmipc_segment_cookie); 5152 break; 5153 case RSMIPC_MSG_SQREADY: 5154 rsm_proc_sqready(data, src_hwaddr, arg); 5155 break; 5156 case RSMIPC_MSG_SQREADY_ACK: 5157 rsm_proc_sqready_ack(data, src_hwaddr, arg); 5158 break; 5159 case RSMIPC_MSG_CREDIT: 5160 rsm_add_credits(ctrlmsg, src_hwaddr, arg); 5161 break; 5162 case RSMIPC_MSG_REPLY: 5163 rsm_intr_reply(msghdr); 5164 break; 5165 case RSMIPC_MSG_BELL: 5166 rsm_intr_event(msg); 5167 break; 5168 case RSMIPC_MSG_IMPORTING: 5169 importer_list_add(src_node, msg->rsmipc_key, 5170 msg->rsmipc_adapter_hwaddr, 5171 msg->rsmipc_segment_cookie); 5172 break; 5173 case RSMIPC_MSG_REPUBLISH: 5174 importer_update(src_node, msg->rsmipc_key, msg->rsmipc_perm); 5175 break; 5176 default: 5177 DBG_PRINTF((category, RSM_DEBUG, 5178 "rsm_intr_callback: bad msg %lx type %d data %lx\n", 5179 (size_t)msg, (int)(msghdr->rsmipc_type), (size_t)data)); 5180 } 5181 5182 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback done\n")); 5183 5184 } 5185 5186 rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd, 5187 rsm_intr_q_op_t opcode, rsm_addr_t src, 5188 void *data, size_t size, rsm_intr_hand_arg_t arg) 5189 { 5190 DBG_DEFINE(category, 5191 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 5192 5193 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func enter\n")); 5194 5195 switch (opcode) { 5196 case RSM_INTR_Q_OP_CREATE: 5197 DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_CREATE\n")); 5198 rsm_sqcreateop_callback(src, arg); 5199 break; 5200 case RSM_INTR_Q_OP_DESTROY: 5201 DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_DESTROY\n")); 5202 break; 5203 case RSM_INTR_Q_OP_RECEIVE: 5204 rsm_intr_callback(data, src, arg); 5205 break; 5206 default: 5207 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5208 "rsm_srv_func: unknown opcode = %x\n", opcode)); 5209 } 5210 5211 chd = chd; 5212 size = size; 5213 5214 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func done\n")); 5215 5216 return (RSM_INTR_HAND_CLAIMED); 5217 } 5218 5219 /* *************************** IPC slots ************************* */ 5220 static rsmipc_slot_t * 5221 rsmipc_alloc() 5222 { 5223 int i; 5224 rsmipc_slot_t *slot; 5225 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 5226 5227 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc enter\n")); 5228 5229 /* try to find a free slot, if not wait */ 5230 mutex_enter(&rsm_ipc.lock); 5231 5232 while (rsm_ipc.count == 0) { 5233 rsm_ipc.wanted = 1; 5234 cv_wait(&rsm_ipc.cv, &rsm_ipc.lock); 5235 } 5236 5237 /* An empty slot is available, find it */ 5238 slot = &rsm_ipc.slots[0]; 5239 for (i = 0; i < RSMIPC_SZ; i++, slot++) { 5240 if (RSMIPC_GET(slot, RSMIPC_FREE)) { 5241 RSMIPC_CLEAR(slot, RSMIPC_FREE); 5242 break; 5243 } 5244 } 5245 5246 ASSERT(i < RSMIPC_SZ); 5247 rsm_ipc.count--; /* one less is available */ 5248 rsm_ipc.sequence++; /* new sequence */ 5249 5250 slot->rsmipc_cookie.ic.sequence = (uint_t)rsm_ipc.sequence; 5251 slot->rsmipc_cookie.ic.index = (uint_t)i; 5252 5253 mutex_exit(&rsm_ipc.lock); 5254 5255 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc done\n")); 5256 5257 return (slot); 5258 } 5259 5260 static void 5261 rsmipc_free(rsmipc_slot_t *slot) 5262 { 5263 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 5264 5265 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free enter\n")); 5266 5267 ASSERT(MUTEX_HELD(&slot->rsmipc_lock)); 5268 ASSERT(&rsm_ipc.slots[slot->rsmipc_cookie.ic.index] == slot); 5269 5270 mutex_enter(&rsm_ipc.lock); 5271 5272 RSMIPC_SET(slot, RSMIPC_FREE); 5273 5274 slot->rsmipc_cookie.ic.sequence = 0; 5275 5276 mutex_exit(&slot->rsmipc_lock); 5277 rsm_ipc.count++; 5278 ASSERT(rsm_ipc.count <= RSMIPC_SZ); 5279 if (rsm_ipc.wanted) { 5280 rsm_ipc.wanted = 0; 5281 cv_broadcast(&rsm_ipc.cv); 5282 } 5283 5284 mutex_exit(&rsm_ipc.lock); 5285 5286 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free done\n")); 5287 } 5288 5289 static int 5290 rsmipc_send(rsm_node_id_t dest, rsmipc_request_t *req, rsmipc_reply_t *reply) 5291 { 5292 int e = 0; 5293 int credit_check = 0; 5294 int retry_cnt = 0; 5295 int min_retry_cnt = 10; 5296 rsm_send_t is; 5297 rsmipc_slot_t *rslot; 5298 adapter_t *adapter; 5299 path_t *path; 5300 sendq_token_t *sendq_token; 5301 sendq_token_t *used_sendq_token = NULL; 5302 rsm_send_q_handle_t ipc_handle; 5303 DBG_DEFINE(category, 5304 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 5305 5306 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send enter:dest=%d", 5307 dest)); 5308 5309 /* 5310 * Check if this is a local case 5311 */ 5312 if (dest == my_nodeid) { 5313 switch (req->rsmipc_hdr.rsmipc_type) { 5314 case RSMIPC_MSG_SEGCONNECT: 5315 reply->rsmipc_status = (short)rsmsegacl_validate( 5316 req, dest, reply); 5317 break; 5318 case RSMIPC_MSG_BELL: 5319 req->rsmipc_hdr.rsmipc_src = dest; 5320 rsm_intr_event(req); 5321 break; 5322 case RSMIPC_MSG_IMPORTING: 5323 importer_list_add(dest, req->rsmipc_key, 5324 req->rsmipc_adapter_hwaddr, 5325 req->rsmipc_segment_cookie); 5326 break; 5327 case RSMIPC_MSG_NOTIMPORTING: 5328 importer_list_rm(dest, req->rsmipc_key, 5329 req->rsmipc_segment_cookie); 5330 break; 5331 case RSMIPC_MSG_REPUBLISH: 5332 importer_update(dest, req->rsmipc_key, 5333 req->rsmipc_perm); 5334 break; 5335 case RSMIPC_MSG_SUSPEND: 5336 importer_suspend(dest); 5337 break; 5338 case RSMIPC_MSG_SUSPEND_DONE: 5339 rsm_suspend_complete(dest, 0); 5340 break; 5341 case RSMIPC_MSG_RESUME: 5342 importer_resume(dest); 5343 break; 5344 default: 5345 ASSERT(0); 5346 } 5347 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5348 "rsmipc_send done\n")); 5349 return (0); 5350 } 5351 5352 if (dest >= MAX_NODES) { 5353 DBG_PRINTF((category, RSM_ERR, 5354 "rsm: rsmipc_send bad node number %x\n", dest)); 5355 return (RSMERR_REMOTE_NODE_UNREACHABLE); 5356 } 5357 5358 /* 5359 * Oh boy! we are going remote. 5360 */ 5361 5362 /* 5363 * identify if we need to have credits to send this message 5364 * - only selected requests are flow controlled 5365 */ 5366 if (req != NULL) { 5367 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5368 "rsmipc_send:request type=%d\n", 5369 req->rsmipc_hdr.rsmipc_type)); 5370 5371 switch (req->rsmipc_hdr.rsmipc_type) { 5372 case RSMIPC_MSG_SEGCONNECT: 5373 case RSMIPC_MSG_DISCONNECT: 5374 case RSMIPC_MSG_IMPORTING: 5375 case RSMIPC_MSG_SUSPEND: 5376 case RSMIPC_MSG_SUSPEND_DONE: 5377 case RSMIPC_MSG_RESUME: 5378 credit_check = 1; 5379 break; 5380 default: 5381 credit_check = 0; 5382 } 5383 } 5384 5385 again: 5386 if (retry_cnt++ == min_retry_cnt) { 5387 /* backoff before further retries for 10ms */ 5388 delay(drv_usectohz(10000)); 5389 retry_cnt = 0; /* reset retry_cnt */ 5390 } 5391 sendq_token = rsmka_get_sendq_token(dest, used_sendq_token); 5392 if (sendq_token == NULL) { 5393 DBG_PRINTF((category, RSM_ERR, 5394 "rsm: rsmipc_send no device to reach node %d\n", dest)); 5395 return (RSMERR_REMOTE_NODE_UNREACHABLE); 5396 } 5397 5398 if ((sendq_token == used_sendq_token) && 5399 ((e == RSMERR_CONN_ABORTED) || (e == RSMERR_TIMEOUT) || 5400 (e == RSMERR_COMM_ERR_MAYBE_DELIVERED))) { 5401 rele_sendq_token(sendq_token); 5402 DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send done=%d\n", e)); 5403 return (RSMERR_CONN_ABORTED); 5404 } else 5405 used_sendq_token = sendq_token; 5406 5407 /* lint -save -e413 */ 5408 path = SQ_TOKEN_TO_PATH(sendq_token); 5409 adapter = path->local_adapter; 5410 /* lint -restore */ 5411 ipc_handle = sendq_token->rsmpi_sendq_handle; 5412 5413 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5414 "rsmipc_send: path=%lx sendq_hdl=%lx\n", path, ipc_handle)); 5415 5416 if (reply == NULL) { 5417 /* Send request without ack */ 5418 /* 5419 * Set the rsmipc_version number in the msghdr for KA 5420 * communication versioning 5421 */ 5422 req->rsmipc_hdr.rsmipc_version = RSM_VERSION; 5423 req->rsmipc_hdr.rsmipc_src = my_nodeid; 5424 /* 5425 * remote endpoints incn should match the value in our 5426 * path's remote_incn field. No need to grab any lock 5427 * since we have refcnted the path in rsmka_get_sendq_token 5428 */ 5429 req->rsmipc_hdr.rsmipc_incn = path->remote_incn; 5430 5431 is.is_data = (void *)req; 5432 is.is_size = sizeof (*req); 5433 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP; 5434 is.is_wait = 0; 5435 5436 if (credit_check) { 5437 mutex_enter(&path->mutex); 5438 /* 5439 * wait till we recv credits or path goes down. If path 5440 * goes down rsm_send will fail and we handle the error 5441 * then 5442 */ 5443 while ((sendq_token->msgbuf_avail == 0) && 5444 (path->state == RSMKA_PATH_ACTIVE)) { 5445 e = cv_wait_sig(&sendq_token->sendq_cv, 5446 &path->mutex); 5447 if (e == 0) { 5448 mutex_exit(&path->mutex); 5449 no_reply_cnt++; 5450 rele_sendq_token(sendq_token); 5451 DBG_PRINTF((category, RSM_DEBUG, 5452 "rsmipc_send done: " 5453 "cv_wait INTERRUPTED")); 5454 return (RSMERR_INTERRUPTED); 5455 } 5456 } 5457 5458 /* 5459 * path is not active retry on another path. 5460 */ 5461 if (path->state != RSMKA_PATH_ACTIVE) { 5462 mutex_exit(&path->mutex); 5463 rele_sendq_token(sendq_token); 5464 e = RSMERR_CONN_ABORTED; 5465 DBG_PRINTF((category, RSM_ERR, 5466 "rsm: rsmipc_send: path !ACTIVE")); 5467 goto again; 5468 } 5469 5470 ASSERT(sendq_token->msgbuf_avail > 0); 5471 5472 /* 5473 * reserve a msgbuf 5474 */ 5475 sendq_token->msgbuf_avail--; 5476 5477 mutex_exit(&path->mutex); 5478 5479 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, 5480 NULL); 5481 5482 if (e != RSM_SUCCESS) { 5483 mutex_enter(&path->mutex); 5484 /* 5485 * release the reserved msgbuf since 5486 * the send failed 5487 */ 5488 sendq_token->msgbuf_avail++; 5489 cv_broadcast(&sendq_token->sendq_cv); 5490 mutex_exit(&path->mutex); 5491 } 5492 } else 5493 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, 5494 NULL); 5495 5496 no_reply_cnt++; 5497 rele_sendq_token(sendq_token); 5498 if (e != RSM_SUCCESS) { 5499 DBG_PRINTF((category, RSM_ERR, 5500 "rsm: rsmipc_send no reply send" 5501 " err = %d no reply count = %d\n", 5502 e, no_reply_cnt)); 5503 ASSERT(e != RSMERR_QUEUE_FENCE_UP && 5504 e != RSMERR_BAD_BARRIER_HNDL); 5505 atomic_inc_64(&rsm_ipcsend_errcnt); 5506 goto again; 5507 } else { 5508 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5509 "rsmipc_send done\n")); 5510 return (e); 5511 } 5512 5513 } 5514 5515 if (req == NULL) { 5516 /* Send reply - No flow control is done for reply */ 5517 /* 5518 * Set the version in the msg header for KA communication 5519 * versioning 5520 */ 5521 reply->rsmipc_hdr.rsmipc_version = RSM_VERSION; 5522 reply->rsmipc_hdr.rsmipc_src = my_nodeid; 5523 /* incn number is not used for reply msgs currently */ 5524 reply->rsmipc_hdr.rsmipc_incn = path->remote_incn; 5525 5526 is.is_data = (void *)reply; 5527 is.is_size = sizeof (*reply); 5528 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP; 5529 is.is_wait = 0; 5530 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL); 5531 rele_sendq_token(sendq_token); 5532 if (e != RSM_SUCCESS) { 5533 DBG_PRINTF((category, RSM_ERR, 5534 "rsm: rsmipc_send reply send" 5535 " err = %d\n", e)); 5536 atomic_inc_64(&rsm_ipcsend_errcnt); 5537 goto again; 5538 } else { 5539 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5540 "rsmipc_send done\n")); 5541 return (e); 5542 } 5543 } 5544 5545 /* Reply needed */ 5546 rslot = rsmipc_alloc(); /* allocate a new ipc slot */ 5547 5548 mutex_enter(&rslot->rsmipc_lock); 5549 5550 rslot->rsmipc_data = (void *)reply; 5551 RSMIPC_SET(rslot, RSMIPC_PENDING); 5552 5553 while (RSMIPC_GET(rslot, RSMIPC_PENDING)) { 5554 /* 5555 * Set the rsmipc_version number in the msghdr for KA 5556 * communication versioning 5557 */ 5558 req->rsmipc_hdr.rsmipc_version = RSM_VERSION; 5559 req->rsmipc_hdr.rsmipc_src = my_nodeid; 5560 req->rsmipc_hdr.rsmipc_cookie = rslot->rsmipc_cookie; 5561 /* 5562 * remote endpoints incn should match the value in our 5563 * path's remote_incn field. No need to grab any lock 5564 * since we have refcnted the path in rsmka_get_sendq_token 5565 */ 5566 req->rsmipc_hdr.rsmipc_incn = path->remote_incn; 5567 5568 is.is_data = (void *)req; 5569 is.is_size = sizeof (*req); 5570 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP; 5571 is.is_wait = 0; 5572 if (credit_check) { 5573 5574 mutex_enter(&path->mutex); 5575 /* 5576 * wait till we recv credits or path goes down. If path 5577 * goes down rsm_send will fail and we handle the error 5578 * then. 5579 */ 5580 while ((sendq_token->msgbuf_avail == 0) && 5581 (path->state == RSMKA_PATH_ACTIVE)) { 5582 e = cv_wait_sig(&sendq_token->sendq_cv, 5583 &path->mutex); 5584 if (e == 0) { 5585 mutex_exit(&path->mutex); 5586 RSMIPC_CLEAR(rslot, RSMIPC_PENDING); 5587 rsmipc_free(rslot); 5588 rele_sendq_token(sendq_token); 5589 DBG_PRINTF((category, RSM_DEBUG, 5590 "rsmipc_send done: " 5591 "cv_wait INTERRUPTED")); 5592 return (RSMERR_INTERRUPTED); 5593 } 5594 } 5595 5596 /* 5597 * path is not active retry on another path. 5598 */ 5599 if (path->state != RSMKA_PATH_ACTIVE) { 5600 mutex_exit(&path->mutex); 5601 RSMIPC_CLEAR(rslot, RSMIPC_PENDING); 5602 rsmipc_free(rslot); 5603 rele_sendq_token(sendq_token); 5604 e = RSMERR_CONN_ABORTED; 5605 DBG_PRINTF((category, RSM_ERR, 5606 "rsm: rsmipc_send: path !ACTIVE")); 5607 goto again; 5608 } 5609 5610 ASSERT(sendq_token->msgbuf_avail > 0); 5611 5612 /* 5613 * reserve a msgbuf 5614 */ 5615 sendq_token->msgbuf_avail--; 5616 5617 mutex_exit(&path->mutex); 5618 5619 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, 5620 NULL); 5621 5622 if (e != RSM_SUCCESS) { 5623 mutex_enter(&path->mutex); 5624 /* 5625 * release the reserved msgbuf since 5626 * the send failed 5627 */ 5628 sendq_token->msgbuf_avail++; 5629 cv_broadcast(&sendq_token->sendq_cv); 5630 mutex_exit(&path->mutex); 5631 } 5632 } else 5633 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, 5634 NULL); 5635 5636 if (e != RSM_SUCCESS) { 5637 DBG_PRINTF((category, RSM_ERR, 5638 "rsm: rsmipc_send rsmpi send err = %d\n", e)); 5639 RSMIPC_CLEAR(rslot, RSMIPC_PENDING); 5640 rsmipc_free(rslot); 5641 rele_sendq_token(sendq_token); 5642 atomic_inc_64(&rsm_ipcsend_errcnt); 5643 goto again; 5644 } 5645 5646 /* wait for a reply signal, a SIGINT, or 5 sec. timeout */ 5647 e = cv_reltimedwait_sig(&rslot->rsmipc_cv, &rslot->rsmipc_lock, 5648 drv_usectohz(5000000), TR_CLOCK_TICK); 5649 if (e < 0) { 5650 /* timed out - retry */ 5651 e = RSMERR_TIMEOUT; 5652 } else if (e == 0) { 5653 /* signalled - return error */ 5654 e = RSMERR_INTERRUPTED; 5655 break; 5656 } else { 5657 e = RSM_SUCCESS; 5658 } 5659 } 5660 5661 RSMIPC_CLEAR(rslot, RSMIPC_PENDING); 5662 rsmipc_free(rslot); 5663 rele_sendq_token(sendq_token); 5664 5665 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send done=%d\n", e)); 5666 return (e); 5667 } 5668 5669 static int 5670 rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid, void *cookie) 5671 { 5672 rsmipc_request_t request; 5673 5674 /* 5675 * inform the exporter to delete this importer 5676 */ 5677 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING; 5678 request.rsmipc_key = segid; 5679 request.rsmipc_segment_cookie = cookie; 5680 return (rsmipc_send(dest, &request, RSM_NO_REPLY)); 5681 } 5682 5683 static void 5684 rsm_send_republish(rsm_memseg_id_t segid, rsmapi_access_entry_t *acl, 5685 int acl_len, rsm_permission_t default_permission) 5686 { 5687 int i; 5688 importing_token_t *token; 5689 rsmipc_request_t request; 5690 republish_token_t *republish_list = NULL; 5691 republish_token_t *rp; 5692 rsm_permission_t permission; 5693 int index; 5694 5695 /* 5696 * send the new access mode to all the nodes that have imported 5697 * this segment. 5698 * If the new acl does not have a node that was present in 5699 * the old acl a access permission of 0 is sent. 5700 */ 5701 5702 index = rsmhash(segid); 5703 5704 /* 5705 * create a list of node/permissions to send the republish message 5706 */ 5707 mutex_enter(&importer_list.lock); 5708 5709 token = importer_list.bucket[index]; 5710 while (token != NULL) { 5711 if (segid == token->key) { 5712 permission = default_permission; 5713 5714 for (i = 0; i < acl_len; i++) { 5715 if (token->importing_node == acl[i].ae_node) { 5716 permission = acl[i].ae_permission; 5717 break; 5718 } 5719 } 5720 rp = kmem_zalloc(sizeof (republish_token_t), KM_SLEEP); 5721 5722 rp->key = segid; 5723 rp->importing_node = token->importing_node; 5724 rp->permission = permission; 5725 rp->next = republish_list; 5726 republish_list = rp; 5727 } 5728 token = token->next; 5729 } 5730 5731 mutex_exit(&importer_list.lock); 5732 5733 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPUBLISH; 5734 request.rsmipc_key = segid; 5735 5736 while (republish_list != NULL) { 5737 request.rsmipc_perm = republish_list->permission; 5738 (void) rsmipc_send(republish_list->importing_node, 5739 &request, RSM_NO_REPLY); 5740 rp = republish_list; 5741 republish_list = republish_list->next; 5742 kmem_free(rp, sizeof (republish_token_t)); 5743 } 5744 } 5745 5746 static void 5747 rsm_send_suspend() 5748 { 5749 int i, e; 5750 rsmipc_request_t request; 5751 list_element_t *tokp; 5752 list_element_t *head = NULL; 5753 importing_token_t *token; 5754 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 5755 "rsm_send_suspend enter\n")); 5756 5757 /* 5758 * create a list of node to send the suspend message 5759 * 5760 * Currently the whole importer list is scanned and we obtain 5761 * all the nodes - this basically gets all nodes that at least 5762 * import one segment from the local node. 5763 * 5764 * no need to grab the rsm_suspend_list lock here since we are 5765 * single threaded when suspend is called. 5766 */ 5767 5768 mutex_enter(&importer_list.lock); 5769 for (i = 0; i < rsm_hash_size; i++) { 5770 5771 token = importer_list.bucket[i]; 5772 5773 while (token != NULL) { 5774 5775 tokp = head; 5776 5777 /* 5778 * make sure that the token's node 5779 * is not already on the suspend list 5780 */ 5781 while (tokp != NULL) { 5782 if (tokp->nodeid == token->importing_node) { 5783 break; 5784 } 5785 tokp = tokp->next; 5786 } 5787 5788 if (tokp == NULL) { /* not in suspend list */ 5789 tokp = kmem_zalloc(sizeof (list_element_t), 5790 KM_SLEEP); 5791 tokp->nodeid = token->importing_node; 5792 tokp->next = head; 5793 head = tokp; 5794 } 5795 5796 token = token->next; 5797 } 5798 } 5799 mutex_exit(&importer_list.lock); 5800 5801 if (head == NULL) { /* no importers so go ahead and quiesce segments */ 5802 exporter_quiesce(); 5803 return; 5804 } 5805 5806 mutex_enter(&rsm_suspend_list.list_lock); 5807 ASSERT(rsm_suspend_list.list_head == NULL); 5808 /* 5809 * update the suspend list righaway so that if a node dies the 5810 * pathmanager can set the NODE dead flag 5811 */ 5812 rsm_suspend_list.list_head = head; 5813 mutex_exit(&rsm_suspend_list.list_lock); 5814 5815 tokp = head; 5816 5817 while (tokp != NULL) { 5818 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND; 5819 e = rsmipc_send(tokp->nodeid, &request, RSM_NO_REPLY); 5820 /* 5821 * Error in rsmipc_send currently happens due to inaccessibility 5822 * of the remote node. 5823 */ 5824 if (e == RSM_SUCCESS) { /* send failed - don't wait for ack */ 5825 tokp->flags |= RSM_SUSPEND_ACKPENDING; 5826 } 5827 5828 tokp = tokp->next; 5829 } 5830 5831 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE, 5832 "rsm_send_suspend done\n")); 5833 5834 } 5835 5836 static void 5837 rsm_send_resume() 5838 { 5839 rsmipc_request_t request; 5840 list_element_t *elem, *head; 5841 5842 /* 5843 * save the suspend list so that we know where to send 5844 * the resume messages and make the suspend list head 5845 * NULL. 5846 */ 5847 mutex_enter(&rsm_suspend_list.list_lock); 5848 head = rsm_suspend_list.list_head; 5849 rsm_suspend_list.list_head = NULL; 5850 mutex_exit(&rsm_suspend_list.list_lock); 5851 5852 while (head != NULL) { 5853 elem = head; 5854 head = head->next; 5855 5856 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_RESUME; 5857 5858 (void) rsmipc_send(elem->nodeid, &request, RSM_NO_REPLY); 5859 5860 kmem_free((void *)elem, sizeof (list_element_t)); 5861 5862 } 5863 5864 } 5865 5866 /* 5867 * This function takes path and sends a message using the sendq 5868 * corresponding to it. The RSMIPC_MSG_SQREADY, RSMIPC_MSG_SQREADY_ACK 5869 * and RSMIPC_MSG_CREDIT are sent using this function. 5870 */ 5871 int 5872 rsmipc_send_controlmsg(path_t *path, int msgtype) 5873 { 5874 int e; 5875 int retry_cnt = 0; 5876 int min_retry_cnt = 10; 5877 adapter_t *adapter; 5878 rsm_send_t is; 5879 rsm_send_q_handle_t ipc_handle; 5880 rsmipc_controlmsg_t msg; 5881 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_FLOWCONTROL); 5882 5883 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5884 "rsmipc_send_controlmsg enter\n")); 5885 5886 ASSERT(MUTEX_HELD(&path->mutex)); 5887 5888 adapter = path->local_adapter; 5889 5890 DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send_controlmsg:path=%lx " 5891 "msgtype=%d %lx:%llx->%lx:%llx procmsg=%d\n", path, msgtype, 5892 my_nodeid, adapter->hwaddr, path->remote_node, 5893 path->remote_hwaddr, path->procmsg_cnt)); 5894 5895 if (path->state != RSMKA_PATH_ACTIVE) { 5896 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5897 "rsmipc_send_controlmsg done: ! RSMKA_PATH_ACTIVE")); 5898 return (1); 5899 } 5900 5901 ipc_handle = path->sendq_token.rsmpi_sendq_handle; 5902 5903 msg.rsmipc_hdr.rsmipc_version = RSM_VERSION; 5904 msg.rsmipc_hdr.rsmipc_src = my_nodeid; 5905 msg.rsmipc_hdr.rsmipc_type = msgtype; 5906 msg.rsmipc_hdr.rsmipc_incn = path->remote_incn; 5907 5908 if (msgtype == RSMIPC_MSG_CREDIT) 5909 msg.rsmipc_credits = path->procmsg_cnt; 5910 5911 msg.rsmipc_local_incn = path->local_incn; 5912 5913 msg.rsmipc_adapter_hwaddr = adapter->hwaddr; 5914 /* incr the sendq, path refcnt */ 5915 PATH_HOLD_NOLOCK(path); 5916 SENDQ_TOKEN_HOLD(path); 5917 5918 do { 5919 /* drop the path lock before doing the rsm_send */ 5920 mutex_exit(&path->mutex); 5921 5922 is.is_data = (void *)&msg; 5923 is.is_size = sizeof (msg); 5924 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP; 5925 is.is_wait = 0; 5926 5927 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL); 5928 5929 ASSERT(e != RSMERR_QUEUE_FENCE_UP && 5930 e != RSMERR_BAD_BARRIER_HNDL); 5931 5932 mutex_enter(&path->mutex); 5933 5934 if (e == RSM_SUCCESS) { 5935 break; 5936 } 5937 /* error counter for statistics */ 5938 atomic_inc_64(&rsm_ctrlmsg_errcnt); 5939 5940 DBG_PRINTF((category, RSM_ERR, 5941 "rsmipc_send_controlmsg:rsm_send error=%d", e)); 5942 5943 if (++retry_cnt == min_retry_cnt) { /* backoff before retry */ 5944 (void) cv_reltimedwait(&path->sendq_token.sendq_cv, 5945 &path->mutex, drv_usectohz(10000), TR_CLOCK_TICK); 5946 retry_cnt = 0; 5947 } 5948 } while (path->state == RSMKA_PATH_ACTIVE); 5949 5950 /* decrement the sendq,path refcnt that we incr before rsm_send */ 5951 SENDQ_TOKEN_RELE(path); 5952 PATH_RELE_NOLOCK(path); 5953 5954 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 5955 "rsmipc_send_controlmsg done=%d", e)); 5956 return (e); 5957 } 5958 5959 /* 5960 * Called from rsm_force_unload and path_importer_disconnect. The memory 5961 * mapping for the imported segment is removed and the segment is 5962 * disconnected at the interconnect layer if disconnect_flag is TRUE. 5963 * rsm_force_unload will get disconnect_flag TRUE from rsm_intr_callback 5964 * and FALSE from rsm_rebind. 5965 * 5966 * When subsequent accesses cause page faulting, the dummy page is mapped 5967 * to resolve the fault, and the mapping generation number is incremented 5968 * so that the application can be notified on a close barrier operation. 5969 * 5970 * It is important to note that the caller of rsmseg_unload is responsible for 5971 * acquiring the segment lock before making a call to rsmseg_unload. This is 5972 * required to make the caller and rsmseg_unload thread safe. The segment lock 5973 * will be released by the rsmseg_unload function. 5974 */ 5975 void 5976 rsmseg_unload(rsmseg_t *im_seg) 5977 { 5978 rsmcookie_t *hdl; 5979 void *shared_cookie; 5980 rsmipc_request_t request; 5981 uint_t maxprot; 5982 5983 DBG_DEFINE(category, 5984 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK); 5985 5986 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload enter\n")); 5987 5988 ASSERT(im_seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 5989 5990 /* wait until segment leaves the mapping state */ 5991 while (im_seg->s_state == RSM_STATE_MAPPING) 5992 cv_wait(&im_seg->s_cv, &im_seg->s_lock); 5993 /* 5994 * An unload is only necessary if the segment is connected. However, 5995 * if the segment was on the import list in state RSM_STATE_CONNECTING 5996 * then a connection was in progress. Change to RSM_STATE_NEW 5997 * here to cause an early exit from the connection process. 5998 */ 5999 if (im_seg->s_state == RSM_STATE_NEW) { 6000 rsmseglock_release(im_seg); 6001 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6002 "rsmseg_unload done: RSM_STATE_NEW\n")); 6003 return; 6004 } else if (im_seg->s_state == RSM_STATE_CONNECTING) { 6005 im_seg->s_state = RSM_STATE_ABORT_CONNECT; 6006 rsmsharelock_acquire(im_seg); 6007 im_seg->s_share->rsmsi_state = RSMSI_STATE_ABORT_CONNECT; 6008 rsmsharelock_release(im_seg); 6009 rsmseglock_release(im_seg); 6010 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6011 "rsmseg_unload done: RSM_STATE_CONNECTING\n")); 6012 return; 6013 } 6014 6015 if (im_seg->s_flags & RSM_FORCE_DISCONNECT) { 6016 if (im_seg->s_ckl != NULL) { 6017 int e; 6018 /* Setup protections for remap */ 6019 maxprot = PROT_USER; 6020 if (im_seg->s_mode & RSM_PERM_READ) { 6021 maxprot |= PROT_READ; 6022 } 6023 if (im_seg->s_mode & RSM_PERM_WRITE) { 6024 maxprot |= PROT_WRITE; 6025 } 6026 hdl = im_seg->s_ckl; 6027 for (; hdl != NULL; hdl = hdl->c_next) { 6028 e = devmap_umem_remap(hdl->c_dhp, rsm_dip, 6029 remap_cookie, 6030 hdl->c_off, hdl->c_len, 6031 maxprot, 0, NULL); 6032 6033 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6034 "remap returns %d\n", e)); 6035 } 6036 } 6037 6038 (void) rsm_closeconnection(im_seg, &shared_cookie); 6039 6040 if (shared_cookie != NULL) { 6041 /* 6042 * inform the exporting node so this import 6043 * can be deleted from the list of importers. 6044 */ 6045 request.rsmipc_hdr.rsmipc_type = 6046 RSMIPC_MSG_NOTIMPORTING; 6047 request.rsmipc_key = im_seg->s_segid; 6048 request.rsmipc_segment_cookie = shared_cookie; 6049 rsmseglock_release(im_seg); 6050 (void) rsmipc_send(im_seg->s_node, &request, 6051 RSM_NO_REPLY); 6052 } else { 6053 rsmseglock_release(im_seg); 6054 } 6055 } 6056 else 6057 rsmseglock_release(im_seg); 6058 6059 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload done\n")); 6060 6061 } 6062 6063 /* ****************************** Importer Calls ************************ */ 6064 6065 static int 6066 rsm_access(uid_t owner, gid_t group, int perm, int mode, const struct cred *cr) 6067 { 6068 int shifts = 0; 6069 6070 if (crgetuid(cr) != owner) { 6071 shifts += 3; 6072 if (!groupmember(group, cr)) 6073 shifts += 3; 6074 } 6075 6076 mode &= ~(perm << shifts); 6077 6078 if (mode == 0) 6079 return (0); 6080 6081 return (secpolicy_rsm_access(cr, owner, mode)); 6082 } 6083 6084 6085 static int 6086 rsm_connect(rsmseg_t *seg, rsm_ioctlmsg_t *msg, cred_t *cred, 6087 intptr_t dataptr, int mode) 6088 { 6089 int e; 6090 int recheck_state = 0; 6091 void *shared_cookie; 6092 rsmipc_request_t request; 6093 rsmipc_reply_t reply; 6094 rsm_permission_t access; 6095 adapter_t *adapter; 6096 rsm_addr_t addr = 0; 6097 rsm_import_share_t *sharedp; 6098 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT); 6099 6100 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect enter\n")); 6101 6102 adapter = rsm_getadapter(msg, mode); 6103 if (adapter == NULL) { 6104 DBG_PRINTF((category, RSM_ERR, 6105 "rsm_connect done:ENODEV adapter=NULL\n")); 6106 return (RSMERR_CTLR_NOT_PRESENT); 6107 } 6108 6109 if ((adapter == &loopback_adapter) && (msg->nodeid != my_nodeid)) { 6110 rsmka_release_adapter(adapter); 6111 DBG_PRINTF((category, RSM_ERR, 6112 "rsm_connect done:ENODEV loopback\n")); 6113 return (RSMERR_CTLR_NOT_PRESENT); 6114 } 6115 6116 6117 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 6118 ASSERT(seg->s_state == RSM_STATE_NEW); 6119 6120 /* 6121 * Translate perm to access 6122 */ 6123 if (msg->perm & ~RSM_PERM_RDWR) { 6124 rsmka_release_adapter(adapter); 6125 DBG_PRINTF((category, RSM_ERR, 6126 "rsm_connect done:EINVAL invalid perms\n")); 6127 return (RSMERR_BAD_PERMS); 6128 } 6129 access = 0; 6130 if (msg->perm & RSM_PERM_READ) 6131 access |= RSM_ACCESS_READ; 6132 if (msg->perm & RSM_PERM_WRITE) 6133 access |= RSM_ACCESS_WRITE; 6134 6135 seg->s_node = msg->nodeid; 6136 6137 /* 6138 * Adding to the import list locks the segment; release the segment 6139 * lock so we can get the reply for the send. 6140 */ 6141 e = rsmimport_add(seg, msg->key); 6142 if (e) { 6143 rsmka_release_adapter(adapter); 6144 DBG_PRINTF((category, RSM_ERR, 6145 "rsm_connect done:rsmimport_add failed %d\n", e)); 6146 return (e); 6147 } 6148 seg->s_state = RSM_STATE_CONNECTING; 6149 6150 /* 6151 * Set the s_adapter field here so as to have a valid comparison of 6152 * the adapter and the s_adapter value during rsmshare_get. For 6153 * any error, set s_adapter to NULL before doing a release_adapter 6154 */ 6155 seg->s_adapter = adapter; 6156 6157 rsmseglock_release(seg); 6158 6159 /* 6160 * get the pointer to the shared data structure; the 6161 * shared data is locked and refcount has been incremented 6162 */ 6163 sharedp = rsmshare_get(msg->key, msg->nodeid, adapter, seg); 6164 6165 ASSERT(rsmsharelock_held(seg)); 6166 6167 do { 6168 /* flag indicates whether we need to recheck the state */ 6169 recheck_state = 0; 6170 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6171 "rsm_connect:RSMSI_STATE=%d\n", sharedp->rsmsi_state)); 6172 switch (sharedp->rsmsi_state) { 6173 case RSMSI_STATE_NEW: 6174 sharedp->rsmsi_state = RSMSI_STATE_CONNECTING; 6175 break; 6176 case RSMSI_STATE_CONNECTING: 6177 /* FALLTHRU */ 6178 case RSMSI_STATE_CONN_QUIESCE: 6179 /* FALLTHRU */ 6180 case RSMSI_STATE_MAP_QUIESCE: 6181 /* wait for the state to change */ 6182 while ((sharedp->rsmsi_state == 6183 RSMSI_STATE_CONNECTING) || 6184 (sharedp->rsmsi_state == 6185 RSMSI_STATE_CONN_QUIESCE) || 6186 (sharedp->rsmsi_state == 6187 RSMSI_STATE_MAP_QUIESCE)) { 6188 if (cv_wait_sig(&sharedp->rsmsi_cv, 6189 &sharedp->rsmsi_lock) == 0) { 6190 /* signalled - clean up and return */ 6191 rsmsharelock_release(seg); 6192 rsmimport_rm(seg); 6193 seg->s_adapter = NULL; 6194 rsmka_release_adapter(adapter); 6195 seg->s_state = RSM_STATE_NEW; 6196 DBG_PRINTF((category, RSM_ERR, 6197 "rsm_connect done: INTERRUPTED\n")); 6198 return (RSMERR_INTERRUPTED); 6199 } 6200 } 6201 /* 6202 * the state changed, loop back and check what it is 6203 */ 6204 recheck_state = 1; 6205 break; 6206 case RSMSI_STATE_ABORT_CONNECT: 6207 /* exit the loop and clean up further down */ 6208 break; 6209 case RSMSI_STATE_CONNECTED: 6210 /* already connected, good - fall through */ 6211 case RSMSI_STATE_MAPPED: 6212 /* already mapped, wow - fall through */ 6213 /* access validation etc is done further down */ 6214 break; 6215 case RSMSI_STATE_DISCONNECTED: 6216 /* disconnected - so reconnect now */ 6217 sharedp->rsmsi_state = RSMSI_STATE_CONNECTING; 6218 break; 6219 default: 6220 ASSERT(0); /* Invalid State */ 6221 } 6222 } while (recheck_state); 6223 6224 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) { 6225 /* we are the first to connect */ 6226 rsmsharelock_release(seg); 6227 6228 if (msg->nodeid != my_nodeid) { 6229 addr = get_remote_hwaddr(adapter, msg->nodeid); 6230 6231 if ((int64_t)addr < 0) { 6232 rsmsharelock_acquire(seg); 6233 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, 6234 RSMSI_STATE_NEW); 6235 rsmsharelock_release(seg); 6236 rsmimport_rm(seg); 6237 seg->s_adapter = NULL; 6238 rsmka_release_adapter(adapter); 6239 seg->s_state = RSM_STATE_NEW; 6240 DBG_PRINTF((category, RSM_ERR, 6241 "rsm_connect done: hwaddr<0\n")); 6242 return (RSMERR_INTERNAL_ERROR); 6243 } 6244 } else { 6245 addr = adapter->hwaddr; 6246 } 6247 6248 /* 6249 * send request to node [src, dest, key, msgid] and get back 6250 * [status, msgid, cookie] 6251 */ 6252 request.rsmipc_key = msg->key; 6253 /* 6254 * we need the s_mode of the exporter so pass 6255 * RSM_ACCESS_TRUSTED 6256 */ 6257 request.rsmipc_perm = RSM_ACCESS_TRUSTED; 6258 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SEGCONNECT; 6259 request.rsmipc_adapter_hwaddr = addr; 6260 request.rsmipc_segment_cookie = sharedp; 6261 6262 e = (int)rsmipc_send(msg->nodeid, &request, &reply); 6263 if (e) { 6264 rsmsharelock_acquire(seg); 6265 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, 6266 RSMSI_STATE_NEW); 6267 rsmsharelock_release(seg); 6268 rsmimport_rm(seg); 6269 seg->s_adapter = NULL; 6270 rsmka_release_adapter(adapter); 6271 seg->s_state = RSM_STATE_NEW; 6272 DBG_PRINTF((category, RSM_ERR, 6273 "rsm_connect done:rsmipc_send failed %d\n", e)); 6274 return (e); 6275 } 6276 6277 if (reply.rsmipc_status != RSM_SUCCESS) { 6278 rsmsharelock_acquire(seg); 6279 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, 6280 RSMSI_STATE_NEW); 6281 rsmsharelock_release(seg); 6282 rsmimport_rm(seg); 6283 seg->s_adapter = NULL; 6284 rsmka_release_adapter(adapter); 6285 seg->s_state = RSM_STATE_NEW; 6286 DBG_PRINTF((category, RSM_ERR, 6287 "rsm_connect done:rsmipc_send reply err %d\n", 6288 reply.rsmipc_status)); 6289 return (reply.rsmipc_status); 6290 } 6291 6292 rsmsharelock_acquire(seg); 6293 /* store the information recvd into the shared data struct */ 6294 sharedp->rsmsi_mode = reply.rsmipc_mode; 6295 sharedp->rsmsi_uid = reply.rsmipc_uid; 6296 sharedp->rsmsi_gid = reply.rsmipc_gid; 6297 sharedp->rsmsi_seglen = reply.rsmipc_seglen; 6298 sharedp->rsmsi_cookie = sharedp; 6299 } 6300 6301 rsmsharelock_release(seg); 6302 6303 /* 6304 * Get the segment lock and check for a force disconnect 6305 * from the export side which would have changed the state 6306 * back to RSM_STATE_NEW. Once the segment lock is acquired a 6307 * force disconnect will be held off until the connection 6308 * has completed. 6309 */ 6310 rsmseglock_acquire(seg); 6311 rsmsharelock_acquire(seg); 6312 ASSERT(seg->s_state == RSM_STATE_CONNECTING || 6313 seg->s_state == RSM_STATE_ABORT_CONNECT); 6314 6315 shared_cookie = sharedp->rsmsi_cookie; 6316 6317 if ((seg->s_state == RSM_STATE_ABORT_CONNECT) || 6318 (sharedp->rsmsi_state == RSMSI_STATE_ABORT_CONNECT)) { 6319 seg->s_state = RSM_STATE_NEW; 6320 seg->s_adapter = NULL; 6321 rsmsharelock_release(seg); 6322 rsmseglock_release(seg); 6323 rsmimport_rm(seg); 6324 rsmka_release_adapter(adapter); 6325 6326 rsmsharelock_acquire(seg); 6327 if (!(sharedp->rsmsi_flags & RSMSI_FLAGS_ABORTDONE)) { 6328 /* 6329 * set a flag indicating abort handling has been 6330 * done 6331 */ 6332 sharedp->rsmsi_flags |= RSMSI_FLAGS_ABORTDONE; 6333 rsmsharelock_release(seg); 6334 /* send a message to exporter - only once */ 6335 (void) rsm_send_notimporting(msg->nodeid, 6336 msg->key, shared_cookie); 6337 rsmsharelock_acquire(seg); 6338 /* 6339 * wake up any waiting importers and inform that 6340 * connection has been aborted 6341 */ 6342 cv_broadcast(&sharedp->rsmsi_cv); 6343 } 6344 rsmsharelock_release(seg); 6345 6346 DBG_PRINTF((category, RSM_ERR, 6347 "rsm_connect done: RSM_STATE_ABORT_CONNECT\n")); 6348 return (RSMERR_INTERRUPTED); 6349 } 6350 6351 6352 /* 6353 * We need to verify that this process has access 6354 */ 6355 e = rsm_access(sharedp->rsmsi_uid, sharedp->rsmsi_gid, 6356 access & sharedp->rsmsi_mode, 6357 (int)(msg->perm & RSM_PERM_RDWR), cred); 6358 if (e) { 6359 rsmsharelock_release(seg); 6360 seg->s_state = RSM_STATE_NEW; 6361 seg->s_adapter = NULL; 6362 rsmseglock_release(seg); 6363 rsmimport_rm(seg); 6364 rsmka_release_adapter(adapter); 6365 /* 6366 * No need to lock segment it has been removed 6367 * from the hash table 6368 */ 6369 rsmsharelock_acquire(seg); 6370 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) { 6371 rsmsharelock_release(seg); 6372 /* this is the first importer */ 6373 6374 (void) rsm_send_notimporting(msg->nodeid, msg->key, 6375 shared_cookie); 6376 rsmsharelock_acquire(seg); 6377 sharedp->rsmsi_state = RSMSI_STATE_NEW; 6378 cv_broadcast(&sharedp->rsmsi_cv); 6379 } 6380 rsmsharelock_release(seg); 6381 6382 DBG_PRINTF((category, RSM_ERR, 6383 "rsm_connect done: ipcaccess failed\n")); 6384 return (RSMERR_PERM_DENIED); 6385 } 6386 6387 /* update state and cookie */ 6388 seg->s_segid = sharedp->rsmsi_segid; 6389 seg->s_len = sharedp->rsmsi_seglen; 6390 seg->s_mode = access & sharedp->rsmsi_mode; 6391 seg->s_pid = ddi_get_pid(); 6392 seg->s_mapinfo = NULL; 6393 6394 if (seg->s_node != my_nodeid) { 6395 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) { 6396 e = adapter->rsmpi_ops->rsm_connect( 6397 adapter->rsmpi_handle, 6398 addr, seg->s_segid, &sharedp->rsmsi_handle); 6399 6400 if (e != RSM_SUCCESS) { 6401 seg->s_state = RSM_STATE_NEW; 6402 seg->s_adapter = NULL; 6403 rsmsharelock_release(seg); 6404 rsmseglock_release(seg); 6405 rsmimport_rm(seg); 6406 rsmka_release_adapter(adapter); 6407 /* 6408 * inform the exporter to delete this importer 6409 */ 6410 (void) rsm_send_notimporting(msg->nodeid, 6411 msg->key, shared_cookie); 6412 6413 /* 6414 * Now inform any waiting importers to 6415 * retry connect. This needs to be done 6416 * after sending notimporting so that 6417 * the notimporting is sent before a waiting 6418 * importer sends a segconnect while retrying 6419 * 6420 * No need to lock segment it has been removed 6421 * from the hash table 6422 */ 6423 6424 rsmsharelock_acquire(seg); 6425 sharedp->rsmsi_state = RSMSI_STATE_NEW; 6426 cv_broadcast(&sharedp->rsmsi_cv); 6427 rsmsharelock_release(seg); 6428 6429 DBG_PRINTF((category, RSM_ERR, 6430 "rsm_connect error %d\n", e)); 6431 if (e == RSMERR_SEG_NOT_PUBLISHED_TO_RSM_ADDR) 6432 return ( 6433 RSMERR_SEG_NOT_PUBLISHED_TO_NODE); 6434 else if ((e == RSMERR_RSM_ADDR_UNREACHABLE) || 6435 (e == RSMERR_UNKNOWN_RSM_ADDR)) 6436 return (RSMERR_REMOTE_NODE_UNREACHABLE); 6437 else 6438 return (e); 6439 } 6440 6441 } 6442 seg->s_handle.in = sharedp->rsmsi_handle; 6443 6444 } 6445 6446 seg->s_state = RSM_STATE_CONNECT; 6447 6448 6449 seg->s_flags &= ~RSM_IMPORT_DUMMY; /* clear dummy flag */ 6450 if (bar_va) { 6451 /* increment generation number on barrier page */ 6452 atomic_inc_16(bar_va + seg->s_hdr.rsmrc_num); 6453 /* return user off into barrier page where status will be */ 6454 msg->off = (int)seg->s_hdr.rsmrc_num; 6455 msg->gnum = bar_va[msg->off]; /* gnum race */ 6456 } else { 6457 msg->off = 0; 6458 msg->gnum = 0; /* gnum race */ 6459 } 6460 6461 msg->len = (int)sharedp->rsmsi_seglen; 6462 msg->rnum = seg->s_minor; 6463 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, RSMSI_STATE_CONNECTED); 6464 rsmsharelock_release(seg); 6465 rsmseglock_release(seg); 6466 6467 /* Return back to user the segment size & perm in case it's needed */ 6468 6469 #ifdef _MULTI_DATAMODEL 6470 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 6471 rsm_ioctlmsg32_t msg32; 6472 6473 if (msg->len > UINT_MAX) 6474 msg32.len = RSM_MAXSZ_PAGE_ALIGNED; 6475 else 6476 msg32.len = msg->len; 6477 msg32.off = msg->off; 6478 msg32.perm = msg->perm; 6479 msg32.gnum = msg->gnum; 6480 msg32.rnum = msg->rnum; 6481 6482 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6483 "rsm_connect done\n")); 6484 6485 if (ddi_copyout((caddr_t)&msg32, (caddr_t)dataptr, 6486 sizeof (msg32), mode)) 6487 return (RSMERR_BAD_ADDR); 6488 else 6489 return (RSM_SUCCESS); 6490 } 6491 #endif 6492 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect done\n")); 6493 6494 if (ddi_copyout((caddr_t)msg, (caddr_t)dataptr, sizeof (*msg), 6495 mode)) 6496 return (RSMERR_BAD_ADDR); 6497 else 6498 return (RSM_SUCCESS); 6499 } 6500 6501 static int 6502 rsm_unmap(rsmseg_t *seg) 6503 { 6504 int err; 6505 adapter_t *adapter; 6506 rsm_import_share_t *sharedp; 6507 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT); 6508 6509 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6510 "rsm_unmap enter %u\n", seg->s_segid)); 6511 6512 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 6513 6514 /* assert seg is locked */ 6515 ASSERT(rsmseglock_held(seg)); 6516 ASSERT(seg->s_state != RSM_STATE_MAPPING); 6517 6518 if ((seg->s_state != RSM_STATE_ACTIVE) && 6519 (seg->s_state != RSM_STATE_MAP_QUIESCE)) { 6520 /* segment unmap has already been done */ 6521 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n")); 6522 return (RSM_SUCCESS); 6523 } 6524 6525 sharedp = seg->s_share; 6526 6527 rsmsharelock_acquire(seg); 6528 6529 /* 6530 * - shared data struct is in MAPPED or MAP_QUIESCE state 6531 */ 6532 6533 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED || 6534 sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE); 6535 6536 /* 6537 * Unmap pages - previously rsm_memseg_import_unmap was called only if 6538 * the segment cookie list was NULL; but it is always NULL when 6539 * called from rsmmap_unmap and won't be NULL when called for 6540 * a force disconnect - so the check for NULL cookie list was removed 6541 */ 6542 6543 ASSERT(sharedp->rsmsi_mapcnt > 0); 6544 6545 sharedp->rsmsi_mapcnt--; 6546 6547 if (sharedp->rsmsi_mapcnt == 0) { 6548 if (sharedp->rsmsi_state == RSMSI_STATE_MAPPED) { 6549 /* unmap the shared RSMPI mapping */ 6550 adapter = seg->s_adapter; 6551 if (seg->s_node != my_nodeid) { 6552 ASSERT(sharedp->rsmsi_handle != NULL); 6553 err = adapter->rsmpi_ops-> 6554 rsm_unmap(sharedp->rsmsi_handle); 6555 DBG_PRINTF((category, RSM_DEBUG, 6556 "rsm_unmap: rsmpi unmap %d\n", err)); 6557 rsm_free_mapinfo(sharedp->rsmsi_mapinfo); 6558 sharedp->rsmsi_mapinfo = NULL; 6559 } 6560 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED; 6561 } else { /* MAP_QUIESCE --munmap()--> CONN_QUIESCE */ 6562 sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE; 6563 } 6564 } 6565 6566 rsmsharelock_release(seg); 6567 6568 /* 6569 * The s_cookie field is used to store the cookie returned from the 6570 * ddi_umem_lock when binding the pages for an export segment. This 6571 * is the primary use of the s_cookie field and does not normally 6572 * pertain to any importing segment except in the loopback case. 6573 * For the loopback case, the import segment and export segment are 6574 * on the same node, the s_cookie field of the segment structure for 6575 * the importer is initialized to the s_cookie field in the exported 6576 * segment during the map operation and is used during the call to 6577 * devmap_umem_setup for the import mapping. 6578 * Thus, during unmap, we simply need to set s_cookie to NULL to 6579 * indicate that the mapping no longer exists. 6580 */ 6581 seg->s_cookie = NULL; 6582 6583 seg->s_mapinfo = NULL; 6584 6585 if (seg->s_state == RSM_STATE_ACTIVE) 6586 seg->s_state = RSM_STATE_CONNECT; 6587 else 6588 seg->s_state = RSM_STATE_CONN_QUIESCE; 6589 6590 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n")); 6591 6592 return (RSM_SUCCESS); 6593 } 6594 6595 /* 6596 * cookie returned here if not null indicates that it is 6597 * the last importer and it can be used in the RSMIPC_NOT_IMPORTING 6598 * message. 6599 */ 6600 static int 6601 rsm_closeconnection(rsmseg_t *seg, void **cookie) 6602 { 6603 int e; 6604 adapter_t *adapter; 6605 rsm_import_share_t *sharedp; 6606 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT); 6607 6608 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6609 "rsm_closeconnection enter\n")); 6610 6611 *cookie = (void *)NULL; 6612 6613 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 6614 6615 /* assert seg is locked */ 6616 ASSERT(rsmseglock_held(seg)); 6617 6618 if (seg->s_state == RSM_STATE_DISCONNECT) { 6619 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6620 "rsm_closeconnection done: already disconnected\n")); 6621 return (RSM_SUCCESS); 6622 } 6623 6624 /* wait for all putv/getv ops to get done */ 6625 while (seg->s_rdmacnt > 0) { 6626 cv_wait(&seg->s_cv, &seg->s_lock); 6627 } 6628 6629 (void) rsm_unmap(seg); 6630 6631 ASSERT(seg->s_state == RSM_STATE_CONNECT || 6632 seg->s_state == RSM_STATE_CONN_QUIESCE); 6633 6634 adapter = seg->s_adapter; 6635 sharedp = seg->s_share; 6636 6637 ASSERT(sharedp != NULL); 6638 6639 rsmsharelock_acquire(seg); 6640 6641 /* 6642 * Disconnect on adapter 6643 * 6644 * The current algorithm is stateless, I don't have to contact 6645 * server when I go away. He only gives me permissions. Of course, 6646 * the adapters will talk to terminate the connect. 6647 * 6648 * disconnect is needed only if we are CONNECTED not in CONN_QUIESCE 6649 */ 6650 if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) && 6651 (sharedp->rsmsi_node != my_nodeid)) { 6652 6653 if (sharedp->rsmsi_refcnt == 1) { 6654 /* this is the last importer */ 6655 ASSERT(sharedp->rsmsi_mapcnt == 0); 6656 6657 e = adapter->rsmpi_ops-> 6658 rsm_disconnect(sharedp->rsmsi_handle); 6659 if (e != RSM_SUCCESS) { 6660 DBG_PRINTF((category, RSM_DEBUG, 6661 "rsm:disconnect failed seg=%x:err=%d\n", 6662 seg->s_key, e)); 6663 } 6664 } 6665 } 6666 6667 seg->s_handle.in = NULL; 6668 6669 sharedp->rsmsi_refcnt--; 6670 6671 if (sharedp->rsmsi_refcnt == 0) { 6672 *cookie = (void *)sharedp->rsmsi_cookie; 6673 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED; 6674 sharedp->rsmsi_handle = NULL; 6675 rsmsharelock_release(seg); 6676 6677 /* clean up the shared data structure */ 6678 mutex_destroy(&sharedp->rsmsi_lock); 6679 cv_destroy(&sharedp->rsmsi_cv); 6680 kmem_free((void *)(sharedp), sizeof (rsm_import_share_t)); 6681 6682 } else { 6683 rsmsharelock_release(seg); 6684 } 6685 6686 /* increment generation number on barrier page */ 6687 if (bar_va) { 6688 atomic_inc_16(bar_va + seg->s_hdr.rsmrc_num); 6689 } 6690 6691 /* 6692 * The following needs to be done after any 6693 * rsmsharelock calls which use seg->s_share. 6694 */ 6695 seg->s_share = NULL; 6696 6697 seg->s_state = RSM_STATE_DISCONNECT; 6698 /* signal anyone waiting in the CONN_QUIESCE state */ 6699 cv_broadcast(&seg->s_cv); 6700 6701 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6702 "rsm_closeconnection done\n")); 6703 6704 return (RSM_SUCCESS); 6705 } 6706 6707 int 6708 rsm_disconnect(rsmseg_t *seg) 6709 { 6710 rsmipc_request_t request; 6711 void *shared_cookie; 6712 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT); 6713 6714 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect enter\n")); 6715 6716 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 6717 6718 /* assert seg isn't locked */ 6719 ASSERT(!rsmseglock_held(seg)); 6720 6721 6722 /* Remove segment from imported list */ 6723 rsmimport_rm(seg); 6724 6725 /* acquire the segment */ 6726 rsmseglock_acquire(seg); 6727 6728 /* wait until segment leaves the mapping state */ 6729 while (seg->s_state == RSM_STATE_MAPPING) 6730 cv_wait(&seg->s_cv, &seg->s_lock); 6731 6732 if (seg->s_state == RSM_STATE_DISCONNECT) { 6733 seg->s_state = RSM_STATE_NEW; 6734 rsmseglock_release(seg); 6735 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6736 "rsm_disconnect done: already disconnected\n")); 6737 return (RSM_SUCCESS); 6738 } 6739 6740 (void) rsm_closeconnection(seg, &shared_cookie); 6741 6742 /* update state */ 6743 seg->s_state = RSM_STATE_NEW; 6744 6745 if (shared_cookie != NULL) { 6746 /* 6747 * This is the last importer so inform the exporting node 6748 * so this import can be deleted from the list of importers. 6749 */ 6750 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING; 6751 request.rsmipc_key = seg->s_segid; 6752 request.rsmipc_segment_cookie = shared_cookie; 6753 rsmseglock_release(seg); 6754 (void) rsmipc_send(seg->s_node, &request, RSM_NO_REPLY); 6755 } else { 6756 rsmseglock_release(seg); 6757 } 6758 6759 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect done\n")); 6760 6761 return (DDI_SUCCESS); 6762 } 6763 6764 /*ARGSUSED*/ 6765 static int 6766 rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp, 6767 struct pollhead **phpp) 6768 { 6769 minor_t rnum; 6770 rsmresource_t *res; 6771 rsmseg_t *seg; 6772 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 6773 6774 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll enter\n")); 6775 6776 /* find minor, no lock */ 6777 rnum = getminor(dev); 6778 res = rsmresource_lookup(rnum, RSM_NOLOCK); 6779 6780 /* poll is supported only for export/import segments */ 6781 if ((res == NULL) || (res == RSMRC_RESERVED) || 6782 (res->rsmrc_type == RSM_RESOURCE_BAR)) { 6783 return (ENXIO); 6784 } 6785 6786 *reventsp = 0; 6787 6788 /* 6789 * An exported segment must be in state RSM_STATE_EXPORT; an 6790 * imported segment must be in state RSM_STATE_ACTIVE. 6791 */ 6792 seg = (rsmseg_t *)res; 6793 6794 if (seg->s_pollevent) { 6795 *reventsp = POLLRDNORM; 6796 } else if (!anyyet) { 6797 /* cannot take segment lock here */ 6798 *phpp = &seg->s_poll; 6799 seg->s_pollflag |= RSM_SEGMENT_POLL; 6800 } 6801 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll done\n")); 6802 return (0); 6803 } 6804 6805 6806 6807 /* ************************* IOCTL Commands ********************* */ 6808 6809 static rsmseg_t * 6810 rsmresource_seg(rsmresource_t *res, minor_t rnum, cred_t *credp, 6811 rsm_resource_type_t type) 6812 { 6813 /* get segment from resource handle */ 6814 rsmseg_t *seg; 6815 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL); 6816 6817 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg enter\n")); 6818 6819 6820 if (res != RSMRC_RESERVED) { 6821 seg = (rsmseg_t *)res; 6822 } else { 6823 /* Allocate segment now and bind it */ 6824 seg = rsmseg_alloc(rnum, credp); 6825 6826 /* 6827 * if DR pre-processing is going on or DR is in progress 6828 * then the new export segments should be in the NEW_QSCD state 6829 */ 6830 if (type == RSM_RESOURCE_EXPORT_SEGMENT) { 6831 mutex_enter(&rsm_drv_data.drv_lock); 6832 if ((rsm_drv_data.drv_state == 6833 RSM_DRV_PREDEL_STARTED) || 6834 (rsm_drv_data.drv_state == 6835 RSM_DRV_PREDEL_COMPLETED) || 6836 (rsm_drv_data.drv_state == 6837 RSM_DRV_DR_IN_PROGRESS)) { 6838 seg->s_state = RSM_STATE_NEW_QUIESCED; 6839 } 6840 mutex_exit(&rsm_drv_data.drv_lock); 6841 } 6842 6843 rsmresource_insert(rnum, (rsmresource_t *)seg, type); 6844 } 6845 6846 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg done\n")); 6847 6848 return (seg); 6849 } 6850 6851 static int 6852 rsmexport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg, 6853 int mode, cred_t *credp) 6854 { 6855 int error; 6856 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL); 6857 6858 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl enter\n")); 6859 6860 arg = arg; 6861 credp = credp; 6862 6863 ASSERT(seg != NULL); 6864 6865 switch (cmd) { 6866 case RSM_IOCTL_BIND: 6867 error = rsm_bind(seg, msg, arg, mode); 6868 break; 6869 case RSM_IOCTL_REBIND: 6870 error = rsm_rebind(seg, msg); 6871 break; 6872 case RSM_IOCTL_UNBIND: 6873 error = ENOTSUP; 6874 break; 6875 case RSM_IOCTL_PUBLISH: 6876 error = rsm_publish(seg, msg, arg, mode); 6877 break; 6878 case RSM_IOCTL_REPUBLISH: 6879 error = rsm_republish(seg, msg, mode); 6880 break; 6881 case RSM_IOCTL_UNPUBLISH: 6882 error = rsm_unpublish(seg, 1); 6883 break; 6884 default: 6885 error = EINVAL; 6886 break; 6887 } 6888 6889 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl done: %d\n", 6890 error)); 6891 6892 return (error); 6893 } 6894 static int 6895 rsmimport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg, 6896 int mode, cred_t *credp) 6897 { 6898 int error; 6899 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 6900 6901 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl enter\n")); 6902 6903 ASSERT(seg); 6904 6905 switch (cmd) { 6906 case RSM_IOCTL_CONNECT: 6907 error = rsm_connect(seg, msg, credp, arg, mode); 6908 break; 6909 default: 6910 error = EINVAL; 6911 break; 6912 } 6913 6914 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl done: %d\n", 6915 error)); 6916 return (error); 6917 } 6918 6919 static int 6920 rsmbar_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg, 6921 int mode) 6922 { 6923 int e; 6924 adapter_t *adapter; 6925 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 6926 6927 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmbar_ioctl enter\n")); 6928 6929 6930 if ((seg->s_flags & RSM_IMPORT_DUMMY) != 0) { 6931 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6932 "rsmbar_ioctl done: RSM_IMPORT_DUMMY\n")); 6933 return (RSMERR_CONN_ABORTED); 6934 } else if (seg->s_node == my_nodeid) { 6935 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6936 "rsmbar_ioctl done: loopback\n")); 6937 return (RSM_SUCCESS); 6938 } 6939 6940 adapter = seg->s_adapter; 6941 6942 switch (cmd) { 6943 case RSM_IOCTL_BAR_CHECK: 6944 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6945 "rsmbar_ioctl done: RSM_BAR_CHECK %d\n", bar_va)); 6946 return (bar_va ? RSM_SUCCESS : EINVAL); 6947 case RSM_IOCTL_BAR_OPEN: 6948 e = adapter->rsmpi_ops-> 6949 rsm_open_barrier_ctrl(adapter->rsmpi_handle, &msg->bar); 6950 break; 6951 case RSM_IOCTL_BAR_ORDER: 6952 e = adapter->rsmpi_ops->rsm_order_barrier(&msg->bar); 6953 break; 6954 case RSM_IOCTL_BAR_CLOSE: 6955 e = adapter->rsmpi_ops->rsm_close_barrier(&msg->bar); 6956 break; 6957 default: 6958 e = EINVAL; 6959 break; 6960 } 6961 6962 if (e == RSM_SUCCESS) { 6963 #ifdef _MULTI_DATAMODEL 6964 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 6965 rsm_ioctlmsg32_t msg32; 6966 int i; 6967 6968 for (i = 0; i < 4; i++) { 6969 msg32.bar.comp[i].u64 = msg->bar.comp[i].u64; 6970 } 6971 6972 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6973 "rsmbar_ioctl done\n")); 6974 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg, 6975 sizeof (msg32), mode)) 6976 return (RSMERR_BAD_ADDR); 6977 else 6978 return (RSM_SUCCESS); 6979 } 6980 #endif 6981 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6982 "rsmbar_ioctl done\n")); 6983 if (ddi_copyout((caddr_t)&msg->bar, (caddr_t)arg, 6984 sizeof (*msg), mode)) 6985 return (RSMERR_BAD_ADDR); 6986 else 6987 return (RSM_SUCCESS); 6988 } 6989 6990 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 6991 "rsmbar_ioctl done: error=%d\n", e)); 6992 6993 return (e); 6994 } 6995 6996 /* 6997 * Ring the doorbell of the export segment to which this segment is 6998 * connected. 6999 */ 7000 static int 7001 exportbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/) 7002 { 7003 int e = 0; 7004 rsmipc_request_t request; 7005 7006 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 7007 7008 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exportbell_ioctl enter\n")); 7009 7010 request.rsmipc_key = seg->s_segid; 7011 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL; 7012 request.rsmipc_segment_cookie = NULL; 7013 e = rsmipc_send(seg->s_node, &request, RSM_NO_REPLY); 7014 7015 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7016 "exportbell_ioctl done: %d\n", e)); 7017 7018 return (e); 7019 } 7020 7021 /* 7022 * Ring the doorbells of all segments importing this segment 7023 */ 7024 static int 7025 importbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/) 7026 { 7027 importing_token_t *token = NULL; 7028 rsmipc_request_t request; 7029 int index; 7030 7031 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL); 7032 7033 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importbell_ioctl enter\n")); 7034 7035 ASSERT(seg->s_state != RSM_STATE_NEW && 7036 seg->s_state != RSM_STATE_NEW_QUIESCED); 7037 7038 request.rsmipc_key = seg->s_segid; 7039 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL; 7040 7041 index = rsmhash(seg->s_segid); 7042 7043 token = importer_list.bucket[index]; 7044 7045 while (token != NULL) { 7046 if (seg->s_key == token->key) { 7047 request.rsmipc_segment_cookie = 7048 token->import_segment_cookie; 7049 (void) rsmipc_send(token->importing_node, 7050 &request, RSM_NO_REPLY); 7051 } 7052 token = token->next; 7053 } 7054 7055 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7056 "importbell_ioctl done\n")); 7057 return (RSM_SUCCESS); 7058 } 7059 7060 static int 7061 rsm_consumeevent_copyin(caddr_t arg, rsm_consume_event_msg_t *msgp, 7062 rsm_poll_event_t **eventspp, int mode) 7063 { 7064 rsm_poll_event_t *evlist = NULL; 7065 size_t evlistsz; 7066 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL); 7067 7068 #ifdef _MULTI_DATAMODEL 7069 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7070 int i; 7071 rsm_consume_event_msg32_t cemsg32 = {0}; 7072 rsm_poll_event32_t event32[RSM_MAX_POLLFDS]; 7073 rsm_poll_event32_t *evlist32; 7074 size_t evlistsz32; 7075 7076 /* copyin the ioctl message */ 7077 if (ddi_copyin(arg, (caddr_t)&cemsg32, 7078 sizeof (rsm_consume_event_msg32_t), mode)) { 7079 DBG_PRINTF((category, RSM_ERR, 7080 "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n")); 7081 return (RSMERR_BAD_ADDR); 7082 } 7083 msgp->seglist = (caddr_t)(uintptr_t)cemsg32.seglist; 7084 msgp->numents = (int)cemsg32.numents; 7085 7086 evlistsz32 = sizeof (rsm_poll_event32_t) * msgp->numents; 7087 /* 7088 * If numents is large alloc events list on heap otherwise 7089 * use the address of array that was passed in. 7090 */ 7091 if (msgp->numents > RSM_MAX_POLLFDS) { 7092 if (msgp->numents > max_segs) { /* validate numents */ 7093 DBG_PRINTF((category, RSM_ERR, 7094 "consumeevent_copyin: " 7095 "RSMERR_BAD_ARGS_ERRORS\n")); 7096 return (RSMERR_BAD_ARGS_ERRORS); 7097 } 7098 evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP); 7099 } else { 7100 evlist32 = event32; 7101 } 7102 7103 /* copyin the seglist into the rsm_poll_event32_t array */ 7104 if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)evlist32, 7105 evlistsz32, mode)) { 7106 if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) { 7107 kmem_free(evlist32, evlistsz32); 7108 } 7109 DBG_PRINTF((category, RSM_ERR, 7110 "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n")); 7111 return (RSMERR_BAD_ADDR); 7112 } 7113 7114 /* evlist and evlistsz are based on rsm_poll_event_t type */ 7115 evlistsz = sizeof (rsm_poll_event_t)* msgp->numents; 7116 7117 if (msgp->numents > RSM_MAX_POLLFDS) { 7118 evlist = kmem_zalloc(evlistsz, KM_SLEEP); 7119 *eventspp = evlist; 7120 } else { 7121 evlist = *eventspp; 7122 } 7123 /* 7124 * copy the rsm_poll_event32_t array to the rsm_poll_event_t 7125 * array 7126 */ 7127 for (i = 0; i < msgp->numents; i++) { 7128 evlist[i].rnum = evlist32[i].rnum; 7129 evlist[i].fdsidx = evlist32[i].fdsidx; 7130 evlist[i].revent = evlist32[i].revent; 7131 } 7132 /* free the temp 32-bit event list */ 7133 if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) { 7134 kmem_free(evlist32, evlistsz32); 7135 } 7136 7137 return (RSM_SUCCESS); 7138 } 7139 #endif 7140 /* copyin the ioctl message */ 7141 if (ddi_copyin(arg, (caddr_t)msgp, sizeof (rsm_consume_event_msg_t), 7142 mode)) { 7143 DBG_PRINTF((category, RSM_ERR, 7144 "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n")); 7145 return (RSMERR_BAD_ADDR); 7146 } 7147 /* 7148 * If numents is large alloc events list on heap otherwise 7149 * use the address of array that was passed in. 7150 */ 7151 if (msgp->numents > RSM_MAX_POLLFDS) { 7152 if (msgp->numents > max_segs) { /* validate numents */ 7153 DBG_PRINTF((category, RSM_ERR, 7154 "consumeevent_copyin: RSMERR_BAD_ARGS_ERRORS\n")); 7155 return (RSMERR_BAD_ARGS_ERRORS); 7156 } 7157 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents; 7158 evlist = kmem_zalloc(evlistsz, KM_SLEEP); 7159 *eventspp = evlist; 7160 } 7161 7162 /* copyin the seglist */ 7163 if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)(*eventspp), 7164 sizeof (rsm_poll_event_t)*msgp->numents, mode)) { 7165 if (evlist) { 7166 kmem_free(evlist, evlistsz); 7167 *eventspp = NULL; 7168 } 7169 DBG_PRINTF((category, RSM_ERR, 7170 "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n")); 7171 return (RSMERR_BAD_ADDR); 7172 } 7173 7174 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7175 "consumeevent_copyin done\n")); 7176 return (RSM_SUCCESS); 7177 } 7178 7179 static int 7180 rsm_consumeevent_copyout(rsm_consume_event_msg_t *msgp, 7181 rsm_poll_event_t *eventsp, int mode) 7182 { 7183 size_t evlistsz; 7184 int err = RSM_SUCCESS; 7185 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL); 7186 7187 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7188 "consumeevent_copyout enter: numents(%d) eventsp(%p)\n", 7189 msgp->numents, eventsp)); 7190 7191 #ifdef _MULTI_DATAMODEL 7192 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7193 int i; 7194 rsm_poll_event32_t event32[RSM_MAX_POLLFDS]; 7195 rsm_poll_event32_t *evlist32; 7196 size_t evlistsz32; 7197 7198 evlistsz32 = sizeof (rsm_poll_event32_t)*msgp->numents; 7199 if (msgp->numents > RSM_MAX_POLLFDS) { 7200 evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP); 7201 } else { 7202 evlist32 = event32; 7203 } 7204 7205 /* 7206 * copy the rsm_poll_event_t array to the rsm_poll_event32_t 7207 * array 7208 */ 7209 for (i = 0; i < msgp->numents; i++) { 7210 evlist32[i].rnum = eventsp[i].rnum; 7211 evlist32[i].fdsidx = eventsp[i].fdsidx; 7212 evlist32[i].revent = eventsp[i].revent; 7213 } 7214 7215 if (ddi_copyout((caddr_t)evlist32, (caddr_t)msgp->seglist, 7216 evlistsz32, mode)) { 7217 err = RSMERR_BAD_ADDR; 7218 } 7219 7220 if (msgp->numents > RSM_MAX_POLLFDS) { 7221 if (evlist32) { /* free the temp 32-bit event list */ 7222 kmem_free(evlist32, evlistsz32); 7223 } 7224 /* 7225 * eventsp and evlistsz are based on rsm_poll_event_t 7226 * type 7227 */ 7228 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents; 7229 /* event list on the heap and needs to be freed here */ 7230 if (eventsp) { 7231 kmem_free(eventsp, evlistsz); 7232 } 7233 } 7234 7235 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7236 "consumeevent_copyout done: err=%d\n", err)); 7237 return (err); 7238 } 7239 #endif 7240 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents; 7241 7242 if (ddi_copyout((caddr_t)eventsp, (caddr_t)msgp->seglist, evlistsz, 7243 mode)) { 7244 err = RSMERR_BAD_ADDR; 7245 } 7246 7247 if ((msgp->numents > RSM_MAX_POLLFDS) && eventsp) { 7248 /* event list on the heap and needs to be freed here */ 7249 kmem_free(eventsp, evlistsz); 7250 } 7251 7252 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7253 "consumeevent_copyout done: err=%d\n", err)); 7254 return (err); 7255 } 7256 7257 static int 7258 rsm_consumeevent_ioctl(caddr_t arg, int mode) 7259 { 7260 int rc; 7261 int i; 7262 minor_t rnum; 7263 rsm_consume_event_msg_t msg = {0}; 7264 rsmseg_t *seg; 7265 rsm_poll_event_t *event_list; 7266 rsm_poll_event_t events[RSM_MAX_POLLFDS]; 7267 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL); 7268 7269 event_list = events; 7270 7271 if ((rc = rsm_consumeevent_copyin(arg, &msg, &event_list, mode)) != 7272 RSM_SUCCESS) { 7273 return (rc); 7274 } 7275 7276 for (i = 0; i < msg.numents; i++) { 7277 rnum = event_list[i].rnum; 7278 event_list[i].revent = 0; 7279 /* get the segment structure */ 7280 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK); 7281 if (seg) { 7282 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7283 "consumeevent_ioctl: rnum(%d) seg(%p)\n", rnum, 7284 seg)); 7285 if (seg->s_pollevent) { 7286 /* consume the event */ 7287 atomic_dec_32(&seg->s_pollevent); 7288 event_list[i].revent = POLLRDNORM; 7289 } 7290 rsmseglock_release(seg); 7291 } 7292 } 7293 7294 if ((rc = rsm_consumeevent_copyout(&msg, event_list, mode)) != 7295 RSM_SUCCESS) { 7296 return (rc); 7297 } 7298 7299 return (RSM_SUCCESS); 7300 } 7301 7302 static int 7303 iovec_copyin(caddr_t user_vec, rsmka_iovec_t *iovec, int count, int mode) 7304 { 7305 int size; 7306 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 7307 7308 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin enter\n")); 7309 7310 #ifdef _MULTI_DATAMODEL 7311 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7312 rsmka_iovec32_t *iovec32, *iovec32_base; 7313 int i; 7314 7315 size = count * sizeof (rsmka_iovec32_t); 7316 iovec32_base = iovec32 = kmem_zalloc(size, KM_SLEEP); 7317 if (ddi_copyin((caddr_t)user_vec, 7318 (caddr_t)iovec32, size, mode)) { 7319 kmem_free(iovec32, size); 7320 DBG_PRINTF((category, RSM_DEBUG, 7321 "iovec_copyin: returning RSMERR_BAD_ADDR\n")); 7322 return (RSMERR_BAD_ADDR); 7323 } 7324 7325 for (i = 0; i < count; i++, iovec++, iovec32++) { 7326 iovec->io_type = (int)iovec32->io_type; 7327 if (iovec->io_type == RSM_HANDLE_TYPE) 7328 iovec->local.segid = (rsm_memseg_id_t) 7329 iovec32->local; 7330 else 7331 iovec->local.vaddr = 7332 (caddr_t)(uintptr_t)iovec32->local; 7333 iovec->local_offset = (size_t)iovec32->local_offset; 7334 iovec->remote_offset = (size_t)iovec32->remote_offset; 7335 iovec->transfer_len = (size_t)iovec32->transfer_len; 7336 7337 } 7338 kmem_free(iovec32_base, size); 7339 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7340 "iovec_copyin done\n")); 7341 return (DDI_SUCCESS); 7342 } 7343 #endif 7344 7345 size = count * sizeof (rsmka_iovec_t); 7346 if (ddi_copyin((caddr_t)user_vec, (caddr_t)iovec, size, mode)) { 7347 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7348 "iovec_copyin done: RSMERR_BAD_ADDR\n")); 7349 return (RSMERR_BAD_ADDR); 7350 } 7351 7352 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin done\n")); 7353 7354 return (DDI_SUCCESS); 7355 } 7356 7357 7358 static int 7359 sgio_copyin(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode) 7360 { 7361 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 7362 7363 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin enter\n")); 7364 7365 #ifdef _MULTI_DATAMODEL 7366 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7367 rsmka_scat_gath32_t sg_io32; 7368 7369 if (ddi_copyin(arg, (caddr_t)&sg_io32, sizeof (sg_io32), 7370 mode)) { 7371 DBG_PRINTF((category, RSM_DEBUG, 7372 "sgio_copyin done: returning EFAULT\n")); 7373 return (RSMERR_BAD_ADDR); 7374 } 7375 sg_io->local_nodeid = (rsm_node_id_t)sg_io32.local_nodeid; 7376 sg_io->io_request_count = (size_t)sg_io32.io_request_count; 7377 sg_io->io_residual_count = (size_t)sg_io32.io_residual_count; 7378 sg_io->flags = (size_t)sg_io32.flags; 7379 sg_io->remote_handle = (rsm_memseg_import_handle_t) 7380 (uintptr_t)sg_io32.remote_handle; 7381 sg_io->iovec = (rsmka_iovec_t *)(uintptr_t)sg_io32.iovec; 7382 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7383 "sgio_copyin done\n")); 7384 return (DDI_SUCCESS); 7385 } 7386 #endif 7387 if (ddi_copyin(arg, (caddr_t)sg_io, sizeof (rsmka_scat_gath_t), 7388 mode)) { 7389 DBG_PRINTF((category, RSM_DEBUG, 7390 "sgio_copyin done: returning EFAULT\n")); 7391 return (RSMERR_BAD_ADDR); 7392 } 7393 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin done\n")); 7394 return (DDI_SUCCESS); 7395 } 7396 7397 static int 7398 sgio_resid_copyout(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode) 7399 { 7400 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 7401 7402 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7403 "sgio_resid_copyout enter\n")); 7404 7405 #ifdef _MULTI_DATAMODEL 7406 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7407 rsmka_scat_gath32_t sg_io32; 7408 7409 sg_io32.io_residual_count = sg_io->io_residual_count; 7410 sg_io32.flags = sg_io->flags; 7411 7412 if (ddi_copyout((caddr_t)&sg_io32.io_residual_count, 7413 (caddr_t)&((rsmka_scat_gath32_t *)arg)->io_residual_count, 7414 sizeof (uint32_t), mode)) { 7415 7416 DBG_PRINTF((category, RSM_ERR, 7417 "sgio_resid_copyout error: rescnt\n")); 7418 return (RSMERR_BAD_ADDR); 7419 } 7420 7421 if (ddi_copyout((caddr_t)&sg_io32.flags, 7422 (caddr_t)&((rsmka_scat_gath32_t *)arg)->flags, 7423 sizeof (uint32_t), mode)) { 7424 7425 DBG_PRINTF((category, RSM_ERR, 7426 "sgio_resid_copyout error: flags\n")); 7427 return (RSMERR_BAD_ADDR); 7428 } 7429 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7430 "sgio_resid_copyout done\n")); 7431 return (DDI_SUCCESS); 7432 } 7433 #endif 7434 if (ddi_copyout((caddr_t)&sg_io->io_residual_count, 7435 (caddr_t)&((rsmka_scat_gath_t *)arg)->io_residual_count, 7436 sizeof (ulong_t), mode)) { 7437 7438 DBG_PRINTF((category, RSM_ERR, 7439 "sgio_resid_copyout error:rescnt\n")); 7440 return (RSMERR_BAD_ADDR); 7441 } 7442 7443 if (ddi_copyout((caddr_t)&sg_io->flags, 7444 (caddr_t)&((rsmka_scat_gath_t *)arg)->flags, 7445 sizeof (uint_t), mode)) { 7446 7447 DBG_PRINTF((category, RSM_ERR, 7448 "sgio_resid_copyout error:flags\n")); 7449 return (RSMERR_BAD_ADDR); 7450 } 7451 7452 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_resid_copyout done\n")); 7453 return (DDI_SUCCESS); 7454 } 7455 7456 7457 static int 7458 rsm_iovec_ioctl(dev_t dev, caddr_t arg, int cmd, int mode, cred_t *credp) 7459 { 7460 rsmka_scat_gath_t sg_io; 7461 rsmka_iovec_t ka_iovec_arr[RSM_MAX_IOVLEN]; 7462 rsmka_iovec_t *ka_iovec; 7463 rsmka_iovec_t *ka_iovec_start; 7464 rsmpi_scat_gath_t rsmpi_sg_io; 7465 rsmpi_iovec_t iovec_arr[RSM_MAX_IOVLEN]; 7466 rsmpi_iovec_t *iovec; 7467 rsmpi_iovec_t *iovec_start = NULL; 7468 rsmapi_access_entry_t *acl; 7469 rsmresource_t *res; 7470 minor_t rnum; 7471 rsmseg_t *im_seg, *ex_seg; 7472 int e; 7473 int error = 0; 7474 uint_t i; 7475 uint_t iov_proc = 0; /* num of iovecs processed */ 7476 size_t size = 0; 7477 size_t ka_size; 7478 7479 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL); 7480 7481 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_iovec_ioctl enter\n")); 7482 7483 credp = credp; 7484 7485 /* 7486 * Copyin the scatter/gather structure and build new structure 7487 * for rsmpi. 7488 */ 7489 e = sgio_copyin(arg, &sg_io, mode); 7490 if (e != DDI_SUCCESS) { 7491 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7492 "rsm_iovec_ioctl done: sgio_copyin %d\n", e)); 7493 return (e); 7494 } 7495 7496 if (sg_io.io_request_count > RSM_MAX_SGIOREQS) { 7497 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7498 "rsm_iovec_ioctl done: request_count(%d) too large\n", 7499 sg_io.io_request_count)); 7500 return (RSMERR_BAD_SGIO); 7501 } 7502 7503 rsmpi_sg_io.io_request_count = sg_io.io_request_count; 7504 rsmpi_sg_io.io_residual_count = sg_io.io_request_count; 7505 rsmpi_sg_io.io_segflg = 0; 7506 7507 /* Allocate memory and copyin io vector array */ 7508 if (sg_io.io_request_count > RSM_MAX_IOVLEN) { 7509 ka_size = sg_io.io_request_count * sizeof (rsmka_iovec_t); 7510 ka_iovec_start = ka_iovec = kmem_zalloc(ka_size, KM_SLEEP); 7511 } else { 7512 ka_iovec_start = ka_iovec = ka_iovec_arr; 7513 } 7514 e = iovec_copyin((caddr_t)sg_io.iovec, ka_iovec, 7515 sg_io.io_request_count, mode); 7516 if (e != DDI_SUCCESS) { 7517 if (sg_io.io_request_count > RSM_MAX_IOVLEN) 7518 kmem_free(ka_iovec, ka_size); 7519 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7520 "rsm_iovec_ioctl done: iovec_copyin %d\n", e)); 7521 return (e); 7522 } 7523 7524 /* get the import segment descriptor */ 7525 rnum = getminor(dev); 7526 res = rsmresource_lookup(rnum, RSM_LOCK); 7527 7528 /* 7529 * The following sequence of locking may (or MAY NOT) cause a 7530 * deadlock but this is currently not addressed here since the 7531 * implementation will be changed to incorporate the use of 7532 * reference counting for both the import and the export segments. 7533 */ 7534 7535 /* rsmseglock_acquire(im_seg) done in rsmresource_lookup */ 7536 7537 im_seg = (rsmseg_t *)res; 7538 7539 if (im_seg == NULL) { 7540 if (sg_io.io_request_count > RSM_MAX_IOVLEN) 7541 kmem_free(ka_iovec, ka_size); 7542 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7543 "rsm_iovec_ioctl done: rsmresource_lookup failed\n")); 7544 return (EINVAL); 7545 } 7546 /* putv/getv supported is supported only on import segments */ 7547 if (im_seg->s_type != RSM_RESOURCE_IMPORT_SEGMENT) { 7548 rsmseglock_release(im_seg); 7549 if (sg_io.io_request_count > RSM_MAX_IOVLEN) 7550 kmem_free(ka_iovec, ka_size); 7551 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7552 "rsm_iovec_ioctl done: not an import segment\n")); 7553 return (EINVAL); 7554 } 7555 7556 /* 7557 * wait for a remote DR to complete ie. for segments to get UNQUIESCED 7558 * as well as wait for a local DR to complete. 7559 */ 7560 while ((im_seg->s_state == RSM_STATE_CONN_QUIESCE) || 7561 (im_seg->s_state == RSM_STATE_MAP_QUIESCE) || 7562 (im_seg->s_flags & RSM_DR_INPROGRESS)) { 7563 if (cv_wait_sig(&im_seg->s_cv, &im_seg->s_lock) == 0) { 7564 DBG_PRINTF((category, RSM_DEBUG, 7565 "rsm_iovec_ioctl done: cv_wait INTR")); 7566 rsmseglock_release(im_seg); 7567 return (RSMERR_INTERRUPTED); 7568 } 7569 } 7570 7571 if ((im_seg->s_state != RSM_STATE_CONNECT) && 7572 (im_seg->s_state != RSM_STATE_ACTIVE)) { 7573 7574 ASSERT(im_seg->s_state == RSM_STATE_DISCONNECT || 7575 im_seg->s_state == RSM_STATE_NEW); 7576 7577 DBG_PRINTF((category, RSM_DEBUG, 7578 "rsm_iovec_ioctl done: im_seg not conn/map")); 7579 rsmseglock_release(im_seg); 7580 e = RSMERR_BAD_SGIO; 7581 goto out; 7582 } 7583 7584 im_seg->s_rdmacnt++; 7585 rsmseglock_release(im_seg); 7586 7587 /* 7588 * Allocate and set up the io vector for rsmpi 7589 */ 7590 if (sg_io.io_request_count > RSM_MAX_IOVLEN) { 7591 size = sg_io.io_request_count * sizeof (rsmpi_iovec_t); 7592 iovec_start = iovec = kmem_zalloc(size, KM_SLEEP); 7593 } else { 7594 iovec_start = iovec = iovec_arr; 7595 } 7596 7597 rsmpi_sg_io.iovec = iovec; 7598 for (iov_proc = 0; iov_proc < sg_io.io_request_count; iov_proc++) { 7599 if (ka_iovec->io_type == RSM_HANDLE_TYPE) { 7600 ex_seg = rsmexport_lookup(ka_iovec->local.segid); 7601 7602 if (ex_seg == NULL) { 7603 e = RSMERR_BAD_SGIO; 7604 break; 7605 } 7606 ASSERT(ex_seg->s_state == RSM_STATE_EXPORT); 7607 7608 acl = ex_seg->s_acl; 7609 if (acl[0].ae_permission == 0) { 7610 struct buf *xbuf; 7611 dev_t sdev = 0; 7612 7613 xbuf = ddi_umem_iosetup(ex_seg->s_cookie, 7614 0, ex_seg->s_len, B_WRITE, 7615 sdev, 0, NULL, DDI_UMEM_SLEEP); 7616 7617 ASSERT(xbuf != NULL); 7618 7619 iovec->local_mem.ms_type = RSM_MEM_BUF; 7620 iovec->local_mem.ms_memory.bp = xbuf; 7621 } else { 7622 iovec->local_mem.ms_type = RSM_MEM_HANDLE; 7623 iovec->local_mem.ms_memory.handle = 7624 ex_seg->s_handle.out; 7625 } 7626 ex_seg->s_rdmacnt++; /* refcnt the handle */ 7627 rsmseglock_release(ex_seg); 7628 } else { 7629 iovec->local_mem.ms_type = RSM_MEM_VADDR; 7630 iovec->local_mem.ms_memory.vr.vaddr = 7631 ka_iovec->local.vaddr; 7632 } 7633 7634 iovec->local_offset = ka_iovec->local_offset; 7635 iovec->remote_handle = im_seg->s_handle.in; 7636 iovec->remote_offset = ka_iovec->remote_offset; 7637 iovec->transfer_length = ka_iovec->transfer_len; 7638 iovec++; 7639 ka_iovec++; 7640 } 7641 7642 if (iov_proc < sg_io.io_request_count) { 7643 /* error while processing handle */ 7644 rsmseglock_acquire(im_seg); 7645 im_seg->s_rdmacnt--; /* decrement the refcnt for importseg */ 7646 if (im_seg->s_rdmacnt == 0) { 7647 cv_broadcast(&im_seg->s_cv); 7648 } 7649 rsmseglock_release(im_seg); 7650 goto out; 7651 } 7652 7653 /* call rsmpi */ 7654 if (cmd == RSM_IOCTL_PUTV) 7655 e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_putv( 7656 im_seg->s_adapter->rsmpi_handle, 7657 &rsmpi_sg_io); 7658 else if (cmd == RSM_IOCTL_GETV) 7659 e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_getv( 7660 im_seg->s_adapter->rsmpi_handle, 7661 &rsmpi_sg_io); 7662 else { 7663 e = EINVAL; 7664 DBG_PRINTF((category, RSM_DEBUG, 7665 "iovec_ioctl: bad command = %x\n", cmd)); 7666 } 7667 7668 7669 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7670 "rsm_iovec_ioctl RSMPI oper done %d\n", e)); 7671 7672 sg_io.io_residual_count = rsmpi_sg_io.io_residual_count; 7673 7674 /* 7675 * Check for implicit signal post flag and do the signal 7676 * post if needed 7677 */ 7678 if (sg_io.flags & RSM_IMPLICIT_SIGPOST && 7679 e == RSM_SUCCESS) { 7680 rsmipc_request_t request; 7681 7682 request.rsmipc_key = im_seg->s_segid; 7683 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL; 7684 request.rsmipc_segment_cookie = NULL; 7685 e = rsmipc_send(im_seg->s_node, &request, RSM_NO_REPLY); 7686 /* 7687 * Reset the implicit signal post flag to 0 to indicate 7688 * that the signal post has been done and need not be 7689 * done in the RSMAPI library 7690 */ 7691 sg_io.flags &= ~RSM_IMPLICIT_SIGPOST; 7692 } 7693 7694 rsmseglock_acquire(im_seg); 7695 im_seg->s_rdmacnt--; 7696 if (im_seg->s_rdmacnt == 0) { 7697 cv_broadcast(&im_seg->s_cv); 7698 } 7699 rsmseglock_release(im_seg); 7700 error = sgio_resid_copyout(arg, &sg_io, mode); 7701 out: 7702 iovec = iovec_start; 7703 ka_iovec = ka_iovec_start; 7704 for (i = 0; i < iov_proc; i++) { 7705 if (ka_iovec->io_type == RSM_HANDLE_TYPE) { 7706 ex_seg = rsmexport_lookup(ka_iovec->local.segid); 7707 7708 ASSERT(ex_seg != NULL); 7709 ASSERT(ex_seg->s_state == RSM_STATE_EXPORT); 7710 7711 ex_seg->s_rdmacnt--; /* unrefcnt the handle */ 7712 if (ex_seg->s_rdmacnt == 0) { 7713 cv_broadcast(&ex_seg->s_cv); 7714 } 7715 rsmseglock_release(ex_seg); 7716 } 7717 7718 ASSERT(iovec != NULL); /* true if iov_proc > 0 */ 7719 7720 /* 7721 * At present there is no dependency on the existence of xbufs 7722 * created by ddi_umem_iosetup for each of the iovecs. So we 7723 * can these xbufs here. 7724 */ 7725 if (iovec->local_mem.ms_type == RSM_MEM_BUF) { 7726 freerbuf(iovec->local_mem.ms_memory.bp); 7727 } 7728 7729 iovec++; 7730 ka_iovec++; 7731 } 7732 7733 if (sg_io.io_request_count > RSM_MAX_IOVLEN) { 7734 if (iovec_start) 7735 kmem_free(iovec_start, size); 7736 kmem_free(ka_iovec_start, ka_size); 7737 } 7738 7739 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7740 "rsm_iovec_ioctl done %d\n", e)); 7741 /* if RSMPI call fails return that else return copyout's retval */ 7742 return ((e != RSM_SUCCESS) ? e : error); 7743 7744 } 7745 7746 7747 static int 7748 rsmaddr_ioctl(int cmd, rsm_ioctlmsg_t *msg, int mode) 7749 { 7750 adapter_t *adapter; 7751 rsm_addr_t addr; 7752 rsm_node_id_t node; 7753 int rval = DDI_SUCCESS; 7754 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL); 7755 7756 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmaddr_ioctl enter\n")); 7757 7758 adapter = rsm_getadapter(msg, mode); 7759 if (adapter == NULL) { 7760 DBG_PRINTF((category, RSM_DEBUG, 7761 "rsmaddr_ioctl done: adapter not found\n")); 7762 return (RSMERR_CTLR_NOT_PRESENT); 7763 } 7764 7765 switch (cmd) { 7766 case RSM_IOCTL_MAP_TO_ADDR: /* nodeid to hwaddr mapping */ 7767 /* returns the hwaddr in msg->hwaddr */ 7768 if (msg->nodeid == my_nodeid) { 7769 msg->hwaddr = adapter->hwaddr; 7770 } else { 7771 addr = get_remote_hwaddr(adapter, msg->nodeid); 7772 if ((int64_t)addr < 0) { 7773 rval = RSMERR_INTERNAL_ERROR; 7774 } else { 7775 msg->hwaddr = addr; 7776 } 7777 } 7778 break; 7779 case RSM_IOCTL_MAP_TO_NODEID: /* hwaddr to nodeid mapping */ 7780 /* returns the nodeid in msg->nodeid */ 7781 if (msg->hwaddr == adapter->hwaddr) { 7782 msg->nodeid = my_nodeid; 7783 } else { 7784 node = get_remote_nodeid(adapter, msg->hwaddr); 7785 if ((int)node < 0) { 7786 rval = RSMERR_INTERNAL_ERROR; 7787 } else { 7788 msg->nodeid = (rsm_node_id_t)node; 7789 } 7790 } 7791 break; 7792 default: 7793 rval = EINVAL; 7794 break; 7795 } 7796 7797 rsmka_release_adapter(adapter); 7798 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7799 "rsmaddr_ioctl done: %d\n", rval)); 7800 return (rval); 7801 } 7802 7803 static int 7804 rsm_ddi_copyin(caddr_t arg, rsm_ioctlmsg_t *msg, int mode) 7805 { 7806 DBG_DEFINE(category, 7807 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI); 7808 7809 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin enter\n")); 7810 7811 #ifdef _MULTI_DATAMODEL 7812 7813 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7814 rsm_ioctlmsg32_t msg32; 7815 int i; 7816 7817 if (ddi_copyin(arg, (caddr_t)&msg32, sizeof (msg32), mode)) { 7818 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7819 "rsm_ddi_copyin done: EFAULT\n")); 7820 return (RSMERR_BAD_ADDR); 7821 } 7822 msg->len = msg32.len; 7823 msg->vaddr = (caddr_t)(uintptr_t)msg32.vaddr; 7824 msg->arg = (caddr_t)(uintptr_t)msg32.arg; 7825 msg->key = msg32.key; 7826 msg->acl_len = msg32.acl_len; 7827 msg->acl = (rsmapi_access_entry_t *)(uintptr_t)msg32.acl; 7828 msg->cnum = msg32.cnum; 7829 msg->cname = (caddr_t)(uintptr_t)msg32.cname; 7830 msg->cname_len = msg32.cname_len; 7831 msg->nodeid = msg32.nodeid; 7832 msg->hwaddr = msg32.hwaddr; 7833 msg->perm = msg32.perm; 7834 for (i = 0; i < 4; i++) { 7835 msg->bar.comp[i].u64 = msg32.bar.comp[i].u64; 7836 } 7837 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7838 "rsm_ddi_copyin done\n")); 7839 return (RSM_SUCCESS); 7840 } 7841 #endif 7842 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin done\n")); 7843 if (ddi_copyin(arg, (caddr_t)msg, sizeof (*msg), mode)) 7844 return (RSMERR_BAD_ADDR); 7845 else 7846 return (RSM_SUCCESS); 7847 } 7848 7849 static int 7850 rsmattr_ddi_copyout(adapter_t *adapter, caddr_t arg, int mode) 7851 { 7852 rsmka_int_controller_attr_t rsm_cattr; 7853 DBG_DEFINE(category, 7854 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI); 7855 7856 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7857 "rsmattr_ddi_copyout enter\n")); 7858 /* 7859 * need to copy appropriate data from rsm_controller_attr_t 7860 * to rsmka_int_controller_attr_t 7861 */ 7862 #ifdef _MULTI_DATAMODEL 7863 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 7864 rsmka_int_controller_attr32_t rsm_cattr32; 7865 7866 rsm_cattr32.attr_direct_access_sizes = 7867 adapter->rsm_attr.attr_direct_access_sizes; 7868 rsm_cattr32.attr_atomic_sizes = 7869 adapter->rsm_attr.attr_atomic_sizes; 7870 rsm_cattr32.attr_page_size = 7871 adapter->rsm_attr.attr_page_size; 7872 if (adapter->rsm_attr.attr_max_export_segment_size > 7873 UINT_MAX) 7874 rsm_cattr32.attr_max_export_segment_size = 7875 RSM_MAXSZ_PAGE_ALIGNED; 7876 else 7877 rsm_cattr32.attr_max_export_segment_size = 7878 adapter->rsm_attr.attr_max_export_segment_size; 7879 if (adapter->rsm_attr.attr_tot_export_segment_size > 7880 UINT_MAX) 7881 rsm_cattr32.attr_tot_export_segment_size = 7882 RSM_MAXSZ_PAGE_ALIGNED; 7883 else 7884 rsm_cattr32.attr_tot_export_segment_size = 7885 adapter->rsm_attr.attr_tot_export_segment_size; 7886 if (adapter->rsm_attr.attr_max_export_segments > 7887 UINT_MAX) 7888 rsm_cattr32.attr_max_export_segments = 7889 UINT_MAX; 7890 else 7891 rsm_cattr32.attr_max_export_segments = 7892 adapter->rsm_attr.attr_max_export_segments; 7893 if (adapter->rsm_attr.attr_max_import_map_size > 7894 UINT_MAX) 7895 rsm_cattr32.attr_max_import_map_size = 7896 RSM_MAXSZ_PAGE_ALIGNED; 7897 else 7898 rsm_cattr32.attr_max_import_map_size = 7899 adapter->rsm_attr.attr_max_import_map_size; 7900 if (adapter->rsm_attr.attr_tot_import_map_size > 7901 UINT_MAX) 7902 rsm_cattr32.attr_tot_import_map_size = 7903 RSM_MAXSZ_PAGE_ALIGNED; 7904 else 7905 rsm_cattr32.attr_tot_import_map_size = 7906 adapter->rsm_attr.attr_tot_import_map_size; 7907 if (adapter->rsm_attr.attr_max_import_segments > 7908 UINT_MAX) 7909 rsm_cattr32.attr_max_import_segments = 7910 UINT_MAX; 7911 else 7912 rsm_cattr32.attr_max_import_segments = 7913 adapter->rsm_attr.attr_max_import_segments; 7914 rsm_cattr32.attr_controller_addr = 7915 adapter->rsm_attr.attr_controller_addr; 7916 7917 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7918 "rsmattr_ddi_copyout done\n")); 7919 if (ddi_copyout((caddr_t)&rsm_cattr32, arg, 7920 sizeof (rsmka_int_controller_attr32_t), mode)) { 7921 return (RSMERR_BAD_ADDR); 7922 } 7923 else 7924 return (RSM_SUCCESS); 7925 } 7926 #endif 7927 rsm_cattr.attr_direct_access_sizes = 7928 adapter->rsm_attr.attr_direct_access_sizes; 7929 rsm_cattr.attr_atomic_sizes = 7930 adapter->rsm_attr.attr_atomic_sizes; 7931 rsm_cattr.attr_page_size = 7932 adapter->rsm_attr.attr_page_size; 7933 rsm_cattr.attr_max_export_segment_size = 7934 adapter->rsm_attr.attr_max_export_segment_size; 7935 rsm_cattr.attr_tot_export_segment_size = 7936 adapter->rsm_attr.attr_tot_export_segment_size; 7937 rsm_cattr.attr_max_export_segments = 7938 adapter->rsm_attr.attr_max_export_segments; 7939 rsm_cattr.attr_max_import_map_size = 7940 adapter->rsm_attr.attr_max_import_map_size; 7941 rsm_cattr.attr_tot_import_map_size = 7942 adapter->rsm_attr.attr_tot_import_map_size; 7943 rsm_cattr.attr_max_import_segments = 7944 adapter->rsm_attr.attr_max_import_segments; 7945 rsm_cattr.attr_controller_addr = 7946 adapter->rsm_attr.attr_controller_addr; 7947 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7948 "rsmattr_ddi_copyout done\n")); 7949 if (ddi_copyout((caddr_t)&rsm_cattr, arg, 7950 sizeof (rsmka_int_controller_attr_t), mode)) { 7951 return (RSMERR_BAD_ADDR); 7952 } 7953 else 7954 return (RSM_SUCCESS); 7955 } 7956 7957 /*ARGSUSED*/ 7958 static int 7959 rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, 7960 int *rvalp) 7961 { 7962 rsmseg_t *seg; 7963 rsmresource_t *res; 7964 minor_t rnum; 7965 rsm_ioctlmsg_t msg = {0}; 7966 int error; 7967 adapter_t *adapter; 7968 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL); 7969 7970 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl enter\n")); 7971 7972 if (cmd == RSM_IOCTL_CONSUMEEVENT) { 7973 error = rsm_consumeevent_ioctl((caddr_t)arg, mode); 7974 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7975 "rsm_ioctl RSM_IOCTL_CONSUMEEVENT done: %d\n", error)); 7976 return (error); 7977 } 7978 7979 /* topology cmd does not use the arg common to other cmds */ 7980 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_TOPOLOGY) { 7981 error = rsmka_topology_ioctl((caddr_t)arg, cmd, mode); 7982 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7983 "rsm_ioctl done: %d\n", error)); 7984 return (error); 7985 } 7986 7987 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_IOVEC) { 7988 error = rsm_iovec_ioctl(dev, (caddr_t)arg, cmd, mode, credp); 7989 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 7990 "rsm_ioctl done: %d\n", error)); 7991 return (error); 7992 } 7993 7994 /* 7995 * try to load arguments 7996 */ 7997 if (cmd != RSM_IOCTL_RING_BELL && 7998 rsm_ddi_copyin((caddr_t)arg, &msg, mode)) { 7999 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8000 "rsm_ioctl done: EFAULT\n")); 8001 return (RSMERR_BAD_ADDR); 8002 } 8003 8004 if (cmd == RSM_IOCTL_ATTR) { 8005 adapter = rsm_getadapter(&msg, mode); 8006 if (adapter == NULL) { 8007 DBG_PRINTF((category, RSM_DEBUG, 8008 "rsm_ioctl done: ENODEV\n")); 8009 return (RSMERR_CTLR_NOT_PRESENT); 8010 } 8011 error = rsmattr_ddi_copyout(adapter, msg.arg, mode); 8012 rsmka_release_adapter(adapter); 8013 DBG_PRINTF((category, RSM_DEBUG, 8014 "rsm_ioctl:after copyout %d\n", error)); 8015 return (error); 8016 } 8017 8018 if (cmd == RSM_IOCTL_BAR_INFO) { 8019 /* Return library off,len of barrier page */ 8020 msg.off = barrier_offset; 8021 msg.len = (int)barrier_size; 8022 #ifdef _MULTI_DATAMODEL 8023 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 8024 rsm_ioctlmsg32_t msg32; 8025 8026 if (msg.len > UINT_MAX) 8027 msg.len = RSM_MAXSZ_PAGE_ALIGNED; 8028 else 8029 msg32.len = (int32_t)msg.len; 8030 msg32.off = (int32_t)msg.off; 8031 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8032 "rsm_ioctl done\n")); 8033 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg, 8034 sizeof (msg32), mode)) 8035 return (RSMERR_BAD_ADDR); 8036 else 8037 return (RSM_SUCCESS); 8038 } 8039 #endif 8040 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8041 "rsm_ioctl done\n")); 8042 if (ddi_copyout((caddr_t)&msg, (caddr_t)arg, 8043 sizeof (msg), mode)) 8044 return (RSMERR_BAD_ADDR); 8045 else 8046 return (RSM_SUCCESS); 8047 } 8048 8049 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_MAP_ADDR) { 8050 /* map the nodeid or hwaddr */ 8051 error = rsmaddr_ioctl(cmd, &msg, mode); 8052 if (error == RSM_SUCCESS) { 8053 #ifdef _MULTI_DATAMODEL 8054 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) { 8055 rsm_ioctlmsg32_t msg32; 8056 8057 msg32.hwaddr = (uint64_t)msg.hwaddr; 8058 msg32.nodeid = (uint32_t)msg.nodeid; 8059 8060 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8061 "rsm_ioctl done\n")); 8062 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg, 8063 sizeof (msg32), mode)) 8064 return (RSMERR_BAD_ADDR); 8065 else 8066 return (RSM_SUCCESS); 8067 } 8068 #endif 8069 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8070 "rsm_ioctl done\n")); 8071 if (ddi_copyout((caddr_t)&msg, (caddr_t)arg, 8072 sizeof (msg), mode)) 8073 return (RSMERR_BAD_ADDR); 8074 else 8075 return (RSM_SUCCESS); 8076 } 8077 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8078 "rsm_ioctl done: %d\n", error)); 8079 return (error); 8080 } 8081 8082 /* Find resource and look it in read mode */ 8083 rnum = getminor(dev); 8084 res = rsmresource_lookup(rnum, RSM_NOLOCK); 8085 ASSERT(res != NULL); 8086 8087 /* 8088 * Find command group 8089 */ 8090 switch (RSM_IOCTL_CMDGRP(cmd)) { 8091 case RSM_IOCTL_EXPORT_SEG: 8092 /* 8093 * Export list is searched during publish, loopback and 8094 * remote lookup call. 8095 */ 8096 seg = rsmresource_seg(res, rnum, credp, 8097 RSM_RESOURCE_EXPORT_SEGMENT); 8098 if (seg->s_type == RSM_RESOURCE_EXPORT_SEGMENT) { 8099 error = rsmexport_ioctl(seg, &msg, cmd, arg, mode, 8100 credp); 8101 } else { /* export ioctl on an import/barrier resource */ 8102 error = RSMERR_BAD_SEG_HNDL; 8103 } 8104 break; 8105 case RSM_IOCTL_IMPORT_SEG: 8106 /* Import list is searched during remote unmap call. */ 8107 seg = rsmresource_seg(res, rnum, credp, 8108 RSM_RESOURCE_IMPORT_SEGMENT); 8109 if (seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT) { 8110 error = rsmimport_ioctl(seg, &msg, cmd, arg, mode, 8111 credp); 8112 } else { /* import ioctl on an export/barrier resource */ 8113 error = RSMERR_BAD_SEG_HNDL; 8114 } 8115 break; 8116 case RSM_IOCTL_BAR: 8117 if (res != RSMRC_RESERVED && 8118 res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT) { 8119 error = rsmbar_ioctl((rsmseg_t *)res, &msg, cmd, arg, 8120 mode); 8121 } else { /* invalid res value */ 8122 error = RSMERR_BAD_SEG_HNDL; 8123 } 8124 break; 8125 case RSM_IOCTL_BELL: 8126 if (res != RSMRC_RESERVED) { 8127 if (res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT) 8128 error = exportbell_ioctl((rsmseg_t *)res, cmd); 8129 else if (res->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT) 8130 error = importbell_ioctl((rsmseg_t *)res, cmd); 8131 else /* RSM_RESOURCE_BAR */ 8132 error = RSMERR_BAD_SEG_HNDL; 8133 } else { /* invalid res value */ 8134 error = RSMERR_BAD_SEG_HNDL; 8135 } 8136 break; 8137 default: 8138 error = EINVAL; 8139 } 8140 8141 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl done: %d\n", 8142 error)); 8143 return (error); 8144 } 8145 8146 8147 /* **************************** Segment Mapping Operations ********* */ 8148 static rsm_mapinfo_t * 8149 rsm_get_mapinfo(rsmseg_t *seg, off_t off, size_t len, off_t *dev_offset, 8150 size_t *map_len) 8151 { 8152 rsm_mapinfo_t *p; 8153 /* 8154 * Find the correct mapinfo structure to use during the mapping 8155 * from the seg->s_mapinfo list. 8156 * The seg->s_mapinfo list contains in reverse order the mappings 8157 * as returned by the RSMPI rsm_map. In rsm_devmap, we need to 8158 * access the correct entry within this list for the mapping 8159 * requested. 8160 * 8161 * The algorithm for selecting a list entry is as follows: 8162 * 8163 * When start_offset of an entry <= off we have found the entry 8164 * we were looking for. Adjust the dev_offset and map_len (needs 8165 * to be PAGESIZE aligned). 8166 */ 8167 p = seg->s_mapinfo; 8168 for (; p; p = p->next) { 8169 if (p->start_offset <= off) { 8170 *dev_offset = p->dev_offset + off - p->start_offset; 8171 *map_len = (len > p->individual_len) ? 8172 p->individual_len : ptob(btopr(len)); 8173 return (p); 8174 } 8175 p = p->next; 8176 } 8177 8178 return (NULL); 8179 } 8180 8181 static void 8182 rsm_free_mapinfo(rsm_mapinfo_t *mapinfo) 8183 { 8184 rsm_mapinfo_t *p; 8185 8186 while (mapinfo != NULL) { 8187 p = mapinfo; 8188 mapinfo = mapinfo->next; 8189 kmem_free(p, sizeof (*p)); 8190 } 8191 } 8192 8193 static int 8194 rsmmap_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, offset_t off, 8195 size_t len, void **pvtp) 8196 { 8197 rsmcookie_t *p; 8198 rsmresource_t *res; 8199 rsmseg_t *seg; 8200 minor_t rnum; 8201 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8202 8203 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map enter\n")); 8204 8205 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8206 "rsmmap_map: dhp = %x\n", dhp)); 8207 8208 flags = flags; 8209 8210 rnum = getminor(dev); 8211 res = (rsmresource_t *)rsmresource_lookup(rnum, RSM_NOLOCK); 8212 ASSERT(res != NULL); 8213 8214 seg = (rsmseg_t *)res; 8215 8216 rsmseglock_acquire(seg); 8217 8218 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 8219 8220 /* 8221 * Allocate structure and add cookie to segment list 8222 */ 8223 p = kmem_alloc(sizeof (*p), KM_SLEEP); 8224 8225 p->c_dhp = dhp; 8226 p->c_off = off; 8227 p->c_len = len; 8228 p->c_next = seg->s_ckl; 8229 seg->s_ckl = p; 8230 8231 *pvtp = (void *)seg; 8232 8233 rsmseglock_release(seg); 8234 8235 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map done\n")); 8236 return (DDI_SUCCESS); 8237 } 8238 8239 /* 8240 * Page fault handling is done here. The prerequisite mapping setup 8241 * has been done in rsm_devmap with calls to ddi_devmem_setup or 8242 * ddi_umem_setup 8243 */ 8244 static int 8245 rsmmap_access(devmap_cookie_t dhp, void *pvt, offset_t offset, size_t len, 8246 uint_t type, uint_t rw) 8247 { 8248 int e; 8249 rsmseg_t *seg = (rsmseg_t *)pvt; 8250 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8251 8252 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access enter\n")); 8253 8254 rsmseglock_acquire(seg); 8255 8256 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 8257 8258 while (seg->s_state == RSM_STATE_MAP_QUIESCE) { 8259 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 8260 DBG_PRINTF((category, RSM_DEBUG, 8261 "rsmmap_access done: cv_wait INTR")); 8262 rsmseglock_release(seg); 8263 return (RSMERR_INTERRUPTED); 8264 } 8265 } 8266 8267 ASSERT(seg->s_state == RSM_STATE_DISCONNECT || 8268 seg->s_state == RSM_STATE_ACTIVE); 8269 8270 if (seg->s_state == RSM_STATE_DISCONNECT) 8271 seg->s_flags |= RSM_IMPORT_DUMMY; 8272 8273 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8274 "rsmmap_access: dhp = %x\n", dhp)); 8275 8276 rsmseglock_release(seg); 8277 8278 if (e = devmap_load(dhp, offset, len, type, rw)) { 8279 DBG_PRINTF((category, RSM_ERR, "devmap_load failed\n")); 8280 } 8281 8282 8283 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access done\n")); 8284 8285 return (e); 8286 } 8287 8288 static int 8289 rsmmap_dup(devmap_cookie_t dhp, void *oldpvt, devmap_cookie_t new_dhp, 8290 void **newpvt) 8291 { 8292 rsmseg_t *seg = (rsmseg_t *)oldpvt; 8293 rsmcookie_t *p, *old; 8294 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8295 8296 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup enter\n")); 8297 8298 /* 8299 * Same as map, create an entry to hold cookie and add it to 8300 * connect segment list. The oldpvt is a pointer to segment. 8301 * Return segment pointer in newpvt. 8302 */ 8303 rsmseglock_acquire(seg); 8304 8305 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 8306 8307 /* 8308 * Find old cookie 8309 */ 8310 for (old = seg->s_ckl; old != NULL; old = old->c_next) { 8311 if (old->c_dhp == dhp) { 8312 break; 8313 } 8314 } 8315 if (old == NULL) { 8316 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8317 "rsmmap_dup done: EINVAL\n")); 8318 rsmseglock_release(seg); 8319 return (EINVAL); 8320 } 8321 8322 p = kmem_alloc(sizeof (*p), KM_SLEEP); 8323 8324 p->c_dhp = new_dhp; 8325 p->c_off = old->c_off; 8326 p->c_len = old->c_len; 8327 p->c_next = seg->s_ckl; 8328 seg->s_ckl = p; 8329 8330 *newpvt = (void *)seg; 8331 8332 rsmseglock_release(seg); 8333 8334 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup done\n")); 8335 8336 return (DDI_SUCCESS); 8337 } 8338 8339 static void 8340 rsmmap_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, size_t len, 8341 devmap_cookie_t new_dhp1, void **pvtp1, 8342 devmap_cookie_t new_dhp2, void **pvtp2) 8343 { 8344 /* 8345 * Remove pvtp structure from segment list. 8346 */ 8347 rsmseg_t *seg = (rsmseg_t *)pvtp; 8348 int freeflag; 8349 8350 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8351 8352 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap enter\n")); 8353 8354 off = off; len = len; 8355 pvtp1 = pvtp1; pvtp2 = pvtp2; 8356 8357 rsmseglock_acquire(seg); 8358 8359 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 8360 8361 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8362 "rsmmap_unmap: dhp = %x\n", dhp)); 8363 /* 8364 * We can go ahead and remove the dhps even if we are in 8365 * the MAPPING state because the dhps being removed here 8366 * belong to a different mmap and we are holding the segment 8367 * lock. 8368 */ 8369 if (new_dhp1 == NULL && new_dhp2 == NULL) { 8370 /* find and remove dhp handle */ 8371 rsmcookie_t *tmp, **back = &seg->s_ckl; 8372 8373 while (*back != NULL) { 8374 tmp = *back; 8375 if (tmp->c_dhp == dhp) { 8376 *back = tmp->c_next; 8377 kmem_free(tmp, sizeof (*tmp)); 8378 break; 8379 } 8380 back = &tmp->c_next; 8381 } 8382 } else { 8383 DBG_PRINTF((category, RSM_DEBUG_LVL2, 8384 "rsmmap_unmap:parital unmap" 8385 "new_dhp1 %lx, new_dhp2 %lx\n", 8386 (size_t)new_dhp1, (size_t)new_dhp2)); 8387 } 8388 8389 /* 8390 * rsmmap_unmap is called for each mapping cookie on the list. 8391 * When the list becomes empty and we are not in the MAPPING 8392 * state then unmap in the rsmpi driver. 8393 */ 8394 if ((seg->s_ckl == NULL) && (seg->s_state != RSM_STATE_MAPPING)) 8395 (void) rsm_unmap(seg); 8396 8397 if (seg->s_state == RSM_STATE_END && seg->s_ckl == NULL) { 8398 freeflag = 1; 8399 } else { 8400 freeflag = 0; 8401 } 8402 8403 rsmseglock_release(seg); 8404 8405 if (freeflag) { 8406 /* Free the segment structure */ 8407 rsmseg_free(seg); 8408 } 8409 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap done\n")); 8410 8411 } 8412 8413 static struct devmap_callback_ctl rsmmap_ops = { 8414 DEVMAP_OPS_REV, /* devmap_ops version number */ 8415 rsmmap_map, /* devmap_ops map routine */ 8416 rsmmap_access, /* devmap_ops access routine */ 8417 rsmmap_dup, /* devmap_ops dup routine */ 8418 rsmmap_unmap, /* devmap_ops unmap routine */ 8419 }; 8420 8421 static int 8422 rsm_devmap(dev_t dev, devmap_cookie_t dhc, offset_t off, size_t len, 8423 size_t *maplen, uint_t model /*ARGSUSED*/) 8424 { 8425 struct devmap_callback_ctl *callbackops = &rsmmap_ops; 8426 int err; 8427 uint_t maxprot; 8428 minor_t rnum; 8429 rsmseg_t *seg; 8430 off_t dev_offset; 8431 size_t cur_len; 8432 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8433 8434 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_devmap enter\n")); 8435 8436 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8437 "rsm_devmap: off = %lx, len = %lx\n", off, len)); 8438 rnum = getminor(dev); 8439 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_NOLOCK); 8440 ASSERT(seg != NULL); 8441 8442 if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) { 8443 if ((off == barrier_offset) && 8444 (len == barrier_size)) { 8445 8446 ASSERT(bar_va != NULL && bar_cookie != NULL); 8447 8448 /* 8449 * The offset argument in devmap_umem_setup represents 8450 * the offset within the kernel memory defined by the 8451 * cookie. We use this offset as barrier_offset. 8452 */ 8453 err = devmap_umem_setup(dhc, rsm_dip, NULL, bar_cookie, 8454 barrier_offset, len, PROT_USER|PROT_READ, 8455 DEVMAP_DEFAULTS, 0); 8456 8457 if (err != 0) { 8458 DBG_PRINTF((category, RSM_ERR, 8459 "rsm_devmap done: %d\n", err)); 8460 return (RSMERR_MAP_FAILED); 8461 } 8462 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8463 "rsm_devmap done: %d\n", err)); 8464 8465 *maplen = barrier_size; 8466 8467 return (err); 8468 } else { 8469 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8470 "rsm_devmap done: %d\n", err)); 8471 return (RSMERR_MAP_FAILED); 8472 } 8473 } 8474 8475 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT); 8476 ASSERT(seg->s_state == RSM_STATE_MAPPING); 8477 8478 /* 8479 * Make sure we still have permission for the map operation. 8480 */ 8481 maxprot = PROT_USER; 8482 if (seg->s_mode & RSM_PERM_READ) { 8483 maxprot |= PROT_READ; 8484 } 8485 8486 if (seg->s_mode & RSM_PERM_WRITE) { 8487 maxprot |= PROT_WRITE; 8488 } 8489 8490 /* 8491 * For each devmap call, rsmmap_map is called. This maintains driver 8492 * private information for the mapping. Thus, if there are multiple 8493 * devmap calls there will be multiple rsmmap_map calls and for each 8494 * call, the mapping information will be stored. 8495 * In case of an error during the processing of the devmap call, error 8496 * will be returned. This error return causes the caller of rsm_devmap 8497 * to undo all the mappings by calling rsmmap_unmap for each one. 8498 * rsmmap_unmap will free up the private information for the requested 8499 * mapping. 8500 */ 8501 if (seg->s_node != my_nodeid) { 8502 rsm_mapinfo_t *p; 8503 8504 p = rsm_get_mapinfo(seg, off, len, &dev_offset, &cur_len); 8505 if (p == NULL) { 8506 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8507 "rsm_devmap: incorrect mapping info\n")); 8508 return (RSMERR_MAP_FAILED); 8509 } 8510 err = devmap_devmem_setup(dhc, p->dip, 8511 callbackops, p->dev_register, 8512 dev_offset, cur_len, maxprot, 8513 DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS, 0); 8514 8515 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8516 "rsm_devmap: dip=%lx,dreg=%lu,doff=%lx," 8517 "off=%lx,len=%lx\n", 8518 p->dip, p->dev_register, dev_offset, off, cur_len)); 8519 8520 if (err != 0) { 8521 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8522 "rsm_devmap: devmap_devmem_setup failed %d\n", 8523 err)); 8524 return (RSMERR_MAP_FAILED); 8525 } 8526 /* cur_len is always an integral multiple pagesize */ 8527 ASSERT((cur_len & (PAGESIZE-1)) == 0); 8528 *maplen = cur_len; 8529 return (err); 8530 8531 } else { 8532 err = devmap_umem_setup(dhc, rsm_dip, callbackops, 8533 seg->s_cookie, off, len, maxprot, 8534 DEVMAP_ALLOW_REMAP|DEVMAP_DEFAULTS, 0); 8535 if (err != 0) { 8536 DBG_PRINTF((category, RSM_DEBUG, 8537 "rsm_devmap: devmap_umem_setup failed %d\n", 8538 err)); 8539 return (RSMERR_MAP_FAILED); 8540 } 8541 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8542 "rsm_devmap: loopback done\n")); 8543 8544 *maplen = ptob(btopr(len)); 8545 8546 return (err); 8547 } 8548 } 8549 8550 /* 8551 * We can use the devmap framework for mapping device memory to user space by 8552 * specifying this routine in the rsm_cb_ops structure. The kernel mmap 8553 * processing calls this entry point and devmap_setup is called within this 8554 * function, which eventually calls rsm_devmap 8555 */ 8556 static int 8557 rsm_segmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len, 8558 uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) 8559 { 8560 int error = 0; 8561 int old_state; 8562 minor_t rnum; 8563 rsmseg_t *seg, *eseg; 8564 adapter_t *adapter; 8565 rsm_import_share_t *sharedp; 8566 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI); 8567 8568 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_segmap enter\n")); 8569 8570 /* 8571 * find segment 8572 */ 8573 rnum = getminor(dev); 8574 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK); 8575 8576 if (seg == NULL) { 8577 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8578 "rsm_segmap done: invalid segment\n")); 8579 return (EINVAL); 8580 } 8581 8582 /* 8583 * the user is trying to map a resource that has not been 8584 * defined yet. The library uses this to map in the 8585 * barrier page. 8586 */ 8587 if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) { 8588 rsmseglock_release(seg); 8589 8590 /* 8591 * The mapping for the barrier page is identified 8592 * by the special offset barrier_offset 8593 */ 8594 8595 if (off == (off_t)barrier_offset || 8596 len == (off_t)barrier_size) { 8597 if (bar_cookie == NULL || bar_va == NULL) { 8598 DBG_PRINTF((category, RSM_DEBUG, 8599 "rsm_segmap: bar cookie/va is NULL\n")); 8600 return (EINVAL); 8601 } 8602 8603 error = devmap_setup(dev, (offset_t)off, as, addrp, 8604 (size_t)len, prot, maxprot, flags, cred); 8605 8606 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8607 "rsm_segmap done: %d\n", error)); 8608 return (error); 8609 } else { 8610 DBG_PRINTF((category, RSM_DEBUG, 8611 "rsm_segmap: bad offset/length\n")); 8612 return (EINVAL); 8613 } 8614 } 8615 8616 /* Make sure you can only map imported segments */ 8617 if (seg->s_hdr.rsmrc_type != RSM_RESOURCE_IMPORT_SEGMENT) { 8618 rsmseglock_release(seg); 8619 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8620 "rsm_segmap done: not an import segment\n")); 8621 return (EINVAL); 8622 } 8623 /* check means library is broken */ 8624 ASSERT(seg->s_hdr.rsmrc_num == rnum); 8625 8626 /* wait for the segment to become unquiesced */ 8627 while (seg->s_state == RSM_STATE_CONN_QUIESCE) { 8628 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) { 8629 rsmseglock_release(seg); 8630 DBG_PRINTF((category, RSM_DEBUG, 8631 "rsm_segmap done: cv_wait INTR")); 8632 return (ENODEV); 8633 } 8634 } 8635 8636 /* wait until segment leaves the mapping state */ 8637 while (seg->s_state == RSM_STATE_MAPPING) 8638 cv_wait(&seg->s_cv, &seg->s_lock); 8639 8640 /* 8641 * we allow multiple maps of the same segment in the KA 8642 * and it works because we do an rsmpi map of the whole 8643 * segment during the first map and all the device mapping 8644 * information needed in rsm_devmap is in the mapinfo list. 8645 */ 8646 if ((seg->s_state != RSM_STATE_CONNECT) && 8647 (seg->s_state != RSM_STATE_ACTIVE)) { 8648 rsmseglock_release(seg); 8649 DBG_PRINTF((category, RSM_DEBUG, 8650 "rsm_segmap done: segment not connected\n")); 8651 return (ENODEV); 8652 } 8653 8654 /* 8655 * Make sure we are not mapping a larger segment than what's 8656 * exported 8657 */ 8658 if ((size_t)off + ptob(btopr(len)) > seg->s_len) { 8659 rsmseglock_release(seg); 8660 DBG_PRINTF((category, RSM_DEBUG, 8661 "rsm_segmap done: off+len>seg size\n")); 8662 return (ENXIO); 8663 } 8664 8665 /* 8666 * Make sure we still have permission for the map operation. 8667 */ 8668 maxprot = PROT_USER; 8669 if (seg->s_mode & RSM_PERM_READ) { 8670 maxprot |= PROT_READ; 8671 } 8672 8673 if (seg->s_mode & RSM_PERM_WRITE) { 8674 maxprot |= PROT_WRITE; 8675 } 8676 8677 if ((prot & maxprot) != prot) { 8678 /* No permission */ 8679 rsmseglock_release(seg); 8680 DBG_PRINTF((category, RSM_DEBUG, 8681 "rsm_segmap done: no permission\n")); 8682 return (EACCES); 8683 } 8684 8685 old_state = seg->s_state; 8686 8687 ASSERT(seg->s_share != NULL); 8688 8689 rsmsharelock_acquire(seg); 8690 8691 sharedp = seg->s_share; 8692 8693 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 8694 "rsm_segmap:RSMSI_STATE=%d\n", sharedp->rsmsi_state)); 8695 8696 if ((sharedp->rsmsi_state != RSMSI_STATE_CONNECTED) && 8697 (sharedp->rsmsi_state != RSMSI_STATE_MAPPED)) { 8698 rsmsharelock_release(seg); 8699 rsmseglock_release(seg); 8700 DBG_PRINTF((category, RSM_DEBUG, 8701 "rsm_segmap done:RSMSI_STATE %d invalid\n", 8702 sharedp->rsmsi_state)); 8703 return (ENODEV); 8704 } 8705 8706 /* 8707 * Do the map - since we want importers to share mappings 8708 * we do the rsmpi map for the whole segment 8709 */ 8710 if (seg->s_node != my_nodeid) { 8711 uint_t dev_register; 8712 off_t dev_offset; 8713 dev_info_t *dip; 8714 size_t tmp_len; 8715 size_t total_length_mapped = 0; 8716 size_t length_to_map = seg->s_len; 8717 off_t tmp_off = 0; 8718 rsm_mapinfo_t *p; 8719 8720 /* 8721 * length_to_map = seg->s_len is always an integral 8722 * multiple of PAGESIZE. Length mapped in each entry in mapinfo 8723 * list is a multiple of PAGESIZE - RSMPI map ensures this 8724 */ 8725 8726 adapter = seg->s_adapter; 8727 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED || 8728 sharedp->rsmsi_state == RSMSI_STATE_MAPPED); 8729 8730 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) { 8731 error = 0; 8732 /* map the whole segment */ 8733 while (total_length_mapped < seg->s_len) { 8734 tmp_len = 0; 8735 8736 error = adapter->rsmpi_ops->rsm_map( 8737 seg->s_handle.in, tmp_off, 8738 length_to_map, &tmp_len, 8739 &dip, &dev_register, &dev_offset, 8740 NULL, NULL); 8741 8742 if (error != 0) 8743 break; 8744 8745 /* 8746 * Store the mapping info obtained from rsm_map 8747 */ 8748 p = kmem_alloc(sizeof (*p), KM_SLEEP); 8749 p->dev_register = dev_register; 8750 p->dev_offset = dev_offset; 8751 p->dip = dip; 8752 p->individual_len = tmp_len; 8753 p->start_offset = tmp_off; 8754 p->next = sharedp->rsmsi_mapinfo; 8755 sharedp->rsmsi_mapinfo = p; 8756 8757 total_length_mapped += tmp_len; 8758 length_to_map -= tmp_len; 8759 tmp_off += tmp_len; 8760 } 8761 seg->s_mapinfo = sharedp->rsmsi_mapinfo; 8762 8763 if (error != RSM_SUCCESS) { 8764 /* Check if this is the the first rsm_map */ 8765 if (sharedp->rsmsi_mapinfo != NULL) { 8766 /* 8767 * A single rsm_unmap undoes 8768 * multiple rsm_maps. 8769 */ 8770 (void) seg->s_adapter->rsmpi_ops-> 8771 rsm_unmap(sharedp->rsmsi_handle); 8772 rsm_free_mapinfo(sharedp-> 8773 rsmsi_mapinfo); 8774 } 8775 sharedp->rsmsi_mapinfo = NULL; 8776 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED; 8777 rsmsharelock_release(seg); 8778 rsmseglock_release(seg); 8779 DBG_PRINTF((category, RSM_DEBUG, 8780 "rsm_segmap done: rsmpi map err %d\n", 8781 error)); 8782 ASSERT(error != RSMERR_BAD_LENGTH && 8783 error != RSMERR_BAD_MEM_ALIGNMENT && 8784 error != RSMERR_BAD_SEG_HNDL); 8785 if (error == RSMERR_UNSUPPORTED_OPERATION) 8786 return (ENOTSUP); 8787 else if (error == RSMERR_INSUFFICIENT_RESOURCES) 8788 return (EAGAIN); 8789 else if (error == RSMERR_CONN_ABORTED) 8790 return (ENODEV); 8791 else 8792 return (error); 8793 } else { 8794 sharedp->rsmsi_state = RSMSI_STATE_MAPPED; 8795 } 8796 } else { 8797 seg->s_mapinfo = sharedp->rsmsi_mapinfo; 8798 } 8799 8800 sharedp->rsmsi_mapcnt++; 8801 8802 rsmsharelock_release(seg); 8803 8804 /* move to an intermediate mapping state */ 8805 seg->s_state = RSM_STATE_MAPPING; 8806 rsmseglock_release(seg); 8807 8808 error = devmap_setup(dev, (offset_t)off, as, addrp, 8809 len, prot, maxprot, flags, cred); 8810 8811 rsmseglock_acquire(seg); 8812 ASSERT(seg->s_state == RSM_STATE_MAPPING); 8813 8814 if (error == DDI_SUCCESS) { 8815 seg->s_state = RSM_STATE_ACTIVE; 8816 } else { 8817 rsmsharelock_acquire(seg); 8818 8819 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED); 8820 8821 sharedp->rsmsi_mapcnt--; 8822 if (sharedp->rsmsi_mapcnt == 0) { 8823 /* unmap the shared RSMPI mapping */ 8824 ASSERT(sharedp->rsmsi_handle != NULL); 8825 (void) adapter->rsmpi_ops-> 8826 rsm_unmap(sharedp->rsmsi_handle); 8827 rsm_free_mapinfo(sharedp->rsmsi_mapinfo); 8828 sharedp->rsmsi_mapinfo = NULL; 8829 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED; 8830 } 8831 8832 rsmsharelock_release(seg); 8833 seg->s_state = old_state; 8834 DBG_PRINTF((category, RSM_ERR, 8835 "rsm: devmap_setup failed %d\n", error)); 8836 } 8837 cv_broadcast(&seg->s_cv); 8838 rsmseglock_release(seg); 8839 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsm_segmap done: %d\n", 8840 error)); 8841 return (error); 8842 } else { 8843 /* 8844 * For loopback, the export segment mapping cookie (s_cookie) 8845 * is also used as the s_cookie value for its import segments 8846 * during mapping. 8847 * Note that reference counting for s_cookie of the export 8848 * segment is not required due to the following: 8849 * We never have a case of the export segment being destroyed, 8850 * leaving the import segments with a stale value for the 8851 * s_cookie field, since a force disconnect is done prior to a 8852 * destroy of an export segment. The force disconnect causes 8853 * the s_cookie value to be reset to NULL. Also for the 8854 * rsm_rebind operation, we change the s_cookie value of the 8855 * export segment as well as of all its local (loopback) 8856 * importers. 8857 */ 8858 DBG_ADDCATEGORY(category, RSM_LOOPBACK); 8859 8860 rsmsharelock_release(seg); 8861 /* 8862 * In order to maintain the lock ordering between the export 8863 * and import segment locks, we need to acquire the export 8864 * segment lock first and only then acquire the import 8865 * segment lock. 8866 * The above is necessary to avoid any deadlock scenarios 8867 * with rsm_rebind which also acquires both the export 8868 * and import segment locks in the above mentioned order. 8869 * Based on code inspection, there seem to be no other 8870 * situations in which both the export and import segment 8871 * locks are acquired either in the same or opposite order 8872 * as mentioned above. 8873 * Thus in order to conform to the above lock order, we 8874 * need to change the state of the import segment to 8875 * RSM_STATE_MAPPING, release the lock. Once this is done we 8876 * can now safely acquire the export segment lock first 8877 * followed by the import segment lock which is as per 8878 * the lock order mentioned above. 8879 */ 8880 /* move to an intermediate mapping state */ 8881 seg->s_state = RSM_STATE_MAPPING; 8882 rsmseglock_release(seg); 8883 8884 eseg = rsmexport_lookup(seg->s_key); 8885 8886 if (eseg == NULL) { 8887 rsmseglock_acquire(seg); 8888 /* 8889 * Revert to old_state and signal any waiters 8890 * The shared state is not changed 8891 */ 8892 8893 seg->s_state = old_state; 8894 cv_broadcast(&seg->s_cv); 8895 rsmseglock_release(seg); 8896 DBG_PRINTF((category, RSM_DEBUG, 8897 "rsm_segmap done: key %d not found\n", seg->s_key)); 8898 return (ENODEV); 8899 } 8900 8901 rsmsharelock_acquire(seg); 8902 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED || 8903 sharedp->rsmsi_state == RSMSI_STATE_MAPPED); 8904 8905 sharedp->rsmsi_mapcnt++; 8906 sharedp->rsmsi_state = RSMSI_STATE_MAPPED; 8907 rsmsharelock_release(seg); 8908 8909 ASSERT(eseg->s_cookie != NULL); 8910 8911 /* 8912 * It is not required or necessary to acquire the import 8913 * segment lock here to change the value of s_cookie since 8914 * no one will touch the import segment as long as it is 8915 * in the RSM_STATE_MAPPING state. 8916 */ 8917 seg->s_cookie = eseg->s_cookie; 8918 8919 rsmseglock_release(eseg); 8920 8921 error = devmap_setup(dev, (offset_t)off, as, addrp, (size_t)len, 8922 prot, maxprot, flags, cred); 8923 8924 rsmseglock_acquire(seg); 8925 ASSERT(seg->s_state == RSM_STATE_MAPPING); 8926 if (error == 0) { 8927 seg->s_state = RSM_STATE_ACTIVE; 8928 } else { 8929 rsmsharelock_acquire(seg); 8930 8931 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED); 8932 8933 sharedp->rsmsi_mapcnt--; 8934 if (sharedp->rsmsi_mapcnt == 0) { 8935 sharedp->rsmsi_mapinfo = NULL; 8936 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED; 8937 } 8938 rsmsharelock_release(seg); 8939 seg->s_state = old_state; 8940 seg->s_cookie = NULL; 8941 } 8942 cv_broadcast(&seg->s_cv); 8943 rsmseglock_release(seg); 8944 DBG_PRINTF((category, RSM_DEBUG_LVL2, 8945 "rsm_segmap done: %d\n", error)); 8946 return (error); 8947 } 8948 } 8949 8950 int 8951 rsmka_null_seg_create( 8952 rsm_controller_handle_t argcp, 8953 rsm_memseg_export_handle_t *handle, 8954 size_t size, 8955 uint_t flags, 8956 rsm_memory_local_t *memory, 8957 rsm_resource_callback_t callback, 8958 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/) 8959 { 8960 return (RSM_SUCCESS); 8961 } 8962 8963 8964 int 8965 rsmka_null_seg_destroy( 8966 rsm_memseg_export_handle_t argmemseg /*ARGSUSED*/) 8967 { 8968 return (RSM_SUCCESS); 8969 } 8970 8971 8972 int 8973 rsmka_null_bind( 8974 rsm_memseg_export_handle_t argmemseg, 8975 off_t offset, 8976 rsm_memory_local_t *argmemory, 8977 rsm_resource_callback_t callback, 8978 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/) 8979 { 8980 return (RSM_SUCCESS); 8981 } 8982 8983 8984 int 8985 rsmka_null_unbind( 8986 rsm_memseg_export_handle_t argmemseg, 8987 off_t offset, 8988 size_t length /*ARGSUSED*/) 8989 { 8990 return (DDI_SUCCESS); 8991 } 8992 8993 int 8994 rsmka_null_rebind( 8995 rsm_memseg_export_handle_t argmemseg, 8996 off_t offset, 8997 rsm_memory_local_t *memory, 8998 rsm_resource_callback_t callback, 8999 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/) 9000 { 9001 return (RSM_SUCCESS); 9002 } 9003 9004 int 9005 rsmka_null_publish( 9006 rsm_memseg_export_handle_t argmemseg, 9007 rsm_access_entry_t access_list[], 9008 uint_t access_list_length, 9009 rsm_memseg_id_t segment_id, 9010 rsm_resource_callback_t callback, 9011 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/) 9012 { 9013 return (RSM_SUCCESS); 9014 } 9015 9016 9017 int 9018 rsmka_null_republish( 9019 rsm_memseg_export_handle_t memseg, 9020 rsm_access_entry_t access_list[], 9021 uint_t access_list_length, 9022 rsm_resource_callback_t callback, 9023 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/) 9024 { 9025 return (RSM_SUCCESS); 9026 } 9027 9028 int 9029 rsmka_null_unpublish( 9030 rsm_memseg_export_handle_t argmemseg /*ARGSUSED*/) 9031 { 9032 return (RSM_SUCCESS); 9033 } 9034 9035 9036 void 9037 rsmka_init_loopback() 9038 { 9039 rsm_ops_t *ops = &null_rsmpi_ops; 9040 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_LOOPBACK); 9041 9042 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9043 "rsmka_init_loopback enter\n")); 9044 9045 /* initialize null ops vector */ 9046 ops->rsm_seg_create = rsmka_null_seg_create; 9047 ops->rsm_seg_destroy = rsmka_null_seg_destroy; 9048 ops->rsm_bind = rsmka_null_bind; 9049 ops->rsm_unbind = rsmka_null_unbind; 9050 ops->rsm_rebind = rsmka_null_rebind; 9051 ops->rsm_publish = rsmka_null_publish; 9052 ops->rsm_unpublish = rsmka_null_unpublish; 9053 ops->rsm_republish = rsmka_null_republish; 9054 9055 /* initialize attributes for loopback adapter */ 9056 loopback_attr.attr_name = loopback_str; 9057 loopback_attr.attr_page_size = 0x8; /* 8K */ 9058 9059 /* initialize loopback adapter */ 9060 loopback_adapter.rsm_attr = loopback_attr; 9061 loopback_adapter.rsmpi_ops = &null_rsmpi_ops; 9062 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9063 "rsmka_init_loopback done\n")); 9064 } 9065 9066 /* ************** DR functions ********************************** */ 9067 static void 9068 rsm_quiesce_exp_seg(rsmresource_t *resp) 9069 { 9070 int recheck_state; 9071 rsmseg_t *segp = (rsmseg_t *)resp; 9072 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9073 DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg"); 9074 9075 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9076 "%s enter: key=%u\n", function, segp->s_key)); 9077 9078 rsmseglock_acquire(segp); 9079 do { 9080 recheck_state = 0; 9081 if ((segp->s_state == RSM_STATE_NEW_QUIESCED) || 9082 (segp->s_state == RSM_STATE_BIND_QUIESCED) || 9083 (segp->s_state == RSM_STATE_EXPORT_QUIESCING) || 9084 (segp->s_state == RSM_STATE_EXPORT_QUIESCED)) { 9085 rsmseglock_release(segp); 9086 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9087 "%s done:state =%d\n", function, 9088 segp->s_state)); 9089 return; 9090 } 9091 9092 if (segp->s_state == RSM_STATE_NEW) { 9093 segp->s_state = RSM_STATE_NEW_QUIESCED; 9094 rsmseglock_release(segp); 9095 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9096 "%s done:state =%d\n", function, 9097 segp->s_state)); 9098 return; 9099 } 9100 9101 if (segp->s_state == RSM_STATE_BIND) { 9102 /* unbind */ 9103 (void) rsm_unbind_pages(segp); 9104 segp->s_state = RSM_STATE_BIND_QUIESCED; 9105 rsmseglock_release(segp); 9106 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9107 "%s done:state =%d\n", function, 9108 segp->s_state)); 9109 return; 9110 } 9111 9112 if (segp->s_state == RSM_STATE_EXPORT) { 9113 /* 9114 * wait for putv/getv to complete if the segp is 9115 * a local memory handle 9116 */ 9117 while ((segp->s_state == RSM_STATE_EXPORT) && 9118 (segp->s_rdmacnt != 0)) { 9119 cv_wait(&segp->s_cv, &segp->s_lock); 9120 } 9121 9122 if (segp->s_state != RSM_STATE_EXPORT) { 9123 /* 9124 * state changed need to see what it 9125 * should be changed to. 9126 */ 9127 recheck_state = 1; 9128 continue; 9129 } 9130 9131 segp->s_state = RSM_STATE_EXPORT_QUIESCING; 9132 rsmseglock_release(segp); 9133 /* 9134 * send SUSPEND messages - currently it will be 9135 * done at the end 9136 */ 9137 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9138 "%s done:state =%d\n", function, 9139 segp->s_state)); 9140 return; 9141 } 9142 } while (recheck_state); 9143 9144 rsmseglock_release(segp); 9145 } 9146 9147 static void 9148 rsm_unquiesce_exp_seg(rsmresource_t *resp) 9149 { 9150 int ret; 9151 rsmseg_t *segp = (rsmseg_t *)resp; 9152 rsmapi_access_entry_t *acl; 9153 rsm_access_entry_t *rsmpi_acl; 9154 int acl_len; 9155 int create_flags = 0; 9156 struct buf *xbuf; 9157 rsm_memory_local_t mem; 9158 adapter_t *adapter; 9159 dev_t sdev = 0; 9160 rsm_resource_callback_t callback_flag; 9161 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9162 DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg"); 9163 9164 rsmseglock_acquire(segp); 9165 9166 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9167 "%s enter: key=%u, state=%d\n", function, segp->s_key, 9168 segp->s_state)); 9169 9170 if ((segp->s_state == RSM_STATE_NEW) || 9171 (segp->s_state == RSM_STATE_BIND) || 9172 (segp->s_state == RSM_STATE_EXPORT)) { 9173 rsmseglock_release(segp); 9174 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n", 9175 function, segp->s_state)); 9176 return; 9177 } 9178 9179 if (segp->s_state == RSM_STATE_NEW_QUIESCED) { 9180 segp->s_state = RSM_STATE_NEW; 9181 cv_broadcast(&segp->s_cv); 9182 rsmseglock_release(segp); 9183 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n", 9184 function, segp->s_state)); 9185 return; 9186 } 9187 9188 if (segp->s_state == RSM_STATE_BIND_QUIESCED) { 9189 /* bind the segment */ 9190 ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr, 9191 segp->s_len, segp->s_proc); 9192 if (ret == RSM_SUCCESS) { /* bind successful */ 9193 segp->s_state = RSM_STATE_BIND; 9194 } else { /* bind failed - resource unavailable */ 9195 segp->s_state = RSM_STATE_NEW; 9196 } 9197 cv_broadcast(&segp->s_cv); 9198 rsmseglock_release(segp); 9199 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9200 "%s done: bind_qscd bind = %d\n", function, ret)); 9201 return; 9202 } 9203 9204 while (segp->s_state == RSM_STATE_EXPORT_QUIESCING) { 9205 /* wait for the segment to move to EXPORT_QUIESCED state */ 9206 cv_wait(&segp->s_cv, &segp->s_lock); 9207 } 9208 9209 if (segp->s_state == RSM_STATE_EXPORT_QUIESCED) { 9210 /* bind the segment */ 9211 ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr, 9212 segp->s_len, segp->s_proc); 9213 9214 if (ret != RSM_SUCCESS) { 9215 /* bind failed - resource unavailable */ 9216 acl_len = segp->s_acl_len; 9217 acl = segp->s_acl; 9218 rsmpi_acl = segp->s_acl_in; 9219 segp->s_acl_len = 0; 9220 segp->s_acl = NULL; 9221 segp->s_acl_in = NULL; 9222 rsmseglock_release(segp); 9223 9224 rsmexport_rm(segp); 9225 rsmacl_free(acl, acl_len); 9226 rsmpiacl_free(rsmpi_acl, acl_len); 9227 9228 rsmseglock_acquire(segp); 9229 segp->s_state = RSM_STATE_NEW; 9230 cv_broadcast(&segp->s_cv); 9231 rsmseglock_release(segp); 9232 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9233 "%s done: exp_qscd bind failed = %d\n", 9234 function, ret)); 9235 return; 9236 } 9237 /* 9238 * publish the segment 9239 * if successful 9240 * segp->s_state = RSM_STATE_EXPORT; 9241 * else failed 9242 * segp->s_state = RSM_STATE_BIND; 9243 */ 9244 9245 /* check whether it is a local_memory_handle */ 9246 if (segp->s_acl != (rsmapi_access_entry_t *)NULL) { 9247 if ((segp->s_acl[0].ae_node == my_nodeid) && 9248 (segp->s_acl[0].ae_permission == 0)) { 9249 segp->s_state = RSM_STATE_EXPORT; 9250 cv_broadcast(&segp->s_cv); 9251 rsmseglock_release(segp); 9252 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9253 "%s done:exp_qscd\n", function)); 9254 return; 9255 } 9256 } 9257 xbuf = ddi_umem_iosetup(segp->s_cookie, 0, segp->s_len, B_WRITE, 9258 sdev, 0, NULL, DDI_UMEM_SLEEP); 9259 ASSERT(xbuf != NULL); 9260 9261 mem.ms_type = RSM_MEM_BUF; 9262 mem.ms_bp = xbuf; 9263 9264 adapter = segp->s_adapter; 9265 9266 if (segp->s_flags & RSMKA_ALLOW_UNBIND_REBIND) { 9267 create_flags = RSM_ALLOW_UNBIND_REBIND; 9268 } 9269 9270 if (segp->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) { 9271 callback_flag = RSM_RESOURCE_DONTWAIT; 9272 } else { 9273 callback_flag = RSM_RESOURCE_SLEEP; 9274 } 9275 9276 ret = adapter->rsmpi_ops->rsm_seg_create( 9277 adapter->rsmpi_handle, &segp->s_handle.out, 9278 segp->s_len, create_flags, &mem, 9279 callback_flag, NULL); 9280 9281 if (ret != RSM_SUCCESS) { 9282 acl_len = segp->s_acl_len; 9283 acl = segp->s_acl; 9284 rsmpi_acl = segp->s_acl_in; 9285 segp->s_acl_len = 0; 9286 segp->s_acl = NULL; 9287 segp->s_acl_in = NULL; 9288 rsmseglock_release(segp); 9289 9290 rsmexport_rm(segp); 9291 rsmacl_free(acl, acl_len); 9292 rsmpiacl_free(rsmpi_acl, acl_len); 9293 9294 rsmseglock_acquire(segp); 9295 segp->s_state = RSM_STATE_BIND; 9296 cv_broadcast(&segp->s_cv); 9297 rsmseglock_release(segp); 9298 DBG_PRINTF((category, RSM_ERR, 9299 "%s done: exp_qscd create failed = %d\n", 9300 function, ret)); 9301 return; 9302 } 9303 9304 ret = adapter->rsmpi_ops->rsm_publish( 9305 segp->s_handle.out, segp->s_acl_in, segp->s_acl_len, 9306 segp->s_segid, RSM_RESOURCE_DONTWAIT, NULL); 9307 9308 if (ret != RSM_SUCCESS) { 9309 acl_len = segp->s_acl_len; 9310 acl = segp->s_acl; 9311 rsmpi_acl = segp->s_acl_in; 9312 segp->s_acl_len = 0; 9313 segp->s_acl = NULL; 9314 segp->s_acl_in = NULL; 9315 adapter->rsmpi_ops->rsm_seg_destroy(segp->s_handle.out); 9316 rsmseglock_release(segp); 9317 9318 rsmexport_rm(segp); 9319 rsmacl_free(acl, acl_len); 9320 rsmpiacl_free(rsmpi_acl, acl_len); 9321 9322 rsmseglock_acquire(segp); 9323 segp->s_state = RSM_STATE_BIND; 9324 cv_broadcast(&segp->s_cv); 9325 rsmseglock_release(segp); 9326 DBG_PRINTF((category, RSM_ERR, 9327 "%s done: exp_qscd publish failed = %d\n", 9328 function, ret)); 9329 return; 9330 } 9331 9332 segp->s_state = RSM_STATE_EXPORT; 9333 cv_broadcast(&segp->s_cv); 9334 rsmseglock_release(segp); 9335 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done: exp_qscd\n", 9336 function)); 9337 return; 9338 } 9339 9340 rsmseglock_release(segp); 9341 9342 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function)); 9343 } 9344 9345 static void 9346 rsm_quiesce_imp_seg(rsmresource_t *resp) 9347 { 9348 rsmseg_t *segp = (rsmseg_t *)resp; 9349 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9350 DBG_DEFINE_STR(function, "rsm_quiesce_imp_seg"); 9351 9352 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9353 "%s enter: key=%u\n", function, segp->s_key)); 9354 9355 rsmseglock_acquire(segp); 9356 segp->s_flags |= RSM_DR_INPROGRESS; 9357 9358 while (segp->s_rdmacnt != 0) { 9359 /* wait for the RDMA to complete */ 9360 cv_wait(&segp->s_cv, &segp->s_lock); 9361 } 9362 9363 rsmseglock_release(segp); 9364 9365 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function)); 9366 9367 } 9368 9369 static void 9370 rsm_unquiesce_imp_seg(rsmresource_t *resp) 9371 { 9372 rsmseg_t *segp = (rsmseg_t *)resp; 9373 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9374 DBG_DEFINE_STR(function, "rsm_unquiesce_imp_seg"); 9375 9376 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9377 "%s enter: key=%u\n", function, segp->s_key)); 9378 9379 rsmseglock_acquire(segp); 9380 9381 segp->s_flags &= ~RSM_DR_INPROGRESS; 9382 /* wake up any waiting putv/getv ops */ 9383 cv_broadcast(&segp->s_cv); 9384 9385 rsmseglock_release(segp); 9386 9387 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function)); 9388 9389 9390 } 9391 9392 static void 9393 rsm_process_exp_seg(rsmresource_t *resp, int event) 9394 { 9395 if (event == RSM_DR_QUIESCE) 9396 rsm_quiesce_exp_seg(resp); 9397 else /* UNQUIESCE */ 9398 rsm_unquiesce_exp_seg(resp); 9399 } 9400 9401 static void 9402 rsm_process_imp_seg(rsmresource_t *resp, int event) 9403 { 9404 if (event == RSM_DR_QUIESCE) 9405 rsm_quiesce_imp_seg(resp); 9406 else /* UNQUIESCE */ 9407 rsm_unquiesce_imp_seg(resp); 9408 } 9409 9410 static void 9411 rsm_dr_process_local_segments(int event) 9412 { 9413 9414 int i, j; 9415 rsmresource_blk_t *blk; 9416 rsmresource_t *p; 9417 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9418 9419 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9420 "rsm_dr_process_local_segments enter\n")); 9421 9422 /* iterate through the resource structure */ 9423 9424 rw_enter(&rsm_resource.rsmrc_lock, RW_READER); 9425 9426 for (i = 0; i < rsm_resource.rsmrc_len; i++) { 9427 blk = rsm_resource.rsmrc_root[i]; 9428 if (blk != NULL) { 9429 for (j = 0; j < RSMRC_BLKSZ; j++) { 9430 p = blk->rsmrcblk_blks[j]; 9431 if ((p != NULL) && (p != RSMRC_RESERVED)) { 9432 /* valid resource */ 9433 if (p->rsmrc_type == 9434 RSM_RESOURCE_EXPORT_SEGMENT) 9435 rsm_process_exp_seg(p, event); 9436 else if (p->rsmrc_type == 9437 RSM_RESOURCE_IMPORT_SEGMENT) 9438 rsm_process_imp_seg(p, event); 9439 } 9440 } 9441 } 9442 } 9443 9444 rw_exit(&rsm_resource.rsmrc_lock); 9445 9446 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9447 "rsm_dr_process_local_segments done\n")); 9448 } 9449 9450 /* *************** DR callback functions ************ */ 9451 static void 9452 rsm_dr_callback_post_add(void *arg, pgcnt_t delta /* ARGSUSED */) 9453 { 9454 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9455 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9456 "rsm_dr_callback_post_add is a no-op\n")); 9457 /* Noop */ 9458 } 9459 9460 static int 9461 rsm_dr_callback_pre_del(void *arg, pgcnt_t delta /* ARGSUSED */) 9462 { 9463 int recheck_state = 0; 9464 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9465 9466 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9467 "rsm_dr_callback_pre_del enter\n")); 9468 9469 mutex_enter(&rsm_drv_data.drv_lock); 9470 9471 do { 9472 recheck_state = 0; 9473 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9474 "rsm_dr_callback_pre_del:state=%d\n", 9475 rsm_drv_data.drv_state)); 9476 9477 switch (rsm_drv_data.drv_state) { 9478 case RSM_DRV_NEW: 9479 /* 9480 * The state should usually never be RSM_DRV_NEW 9481 * since in this state the callbacks have not yet 9482 * been registered. So, ASSERT. 9483 */ 9484 ASSERT(0); 9485 return (0); 9486 case RSM_DRV_REG_PROCESSING: 9487 /* 9488 * The driver is in the process of registering 9489 * with the DR framework. So, wait till the 9490 * registration process is complete. 9491 */ 9492 recheck_state = 1; 9493 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 9494 break; 9495 case RSM_DRV_UNREG_PROCESSING: 9496 /* 9497 * If the state is RSM_DRV_UNREG_PROCESSING, the 9498 * module is in the process of detaching and 9499 * unregistering the callbacks from the DR 9500 * framework. So, simply return. 9501 */ 9502 mutex_exit(&rsm_drv_data.drv_lock); 9503 DBG_PRINTF((category, RSM_DEBUG, 9504 "rsm_dr_callback_pre_del:" 9505 "pre-del on NEW/UNREG\n")); 9506 return (0); 9507 case RSM_DRV_OK: 9508 rsm_drv_data.drv_state = RSM_DRV_PREDEL_STARTED; 9509 break; 9510 case RSM_DRV_PREDEL_STARTED: 9511 /* FALLTHRU */ 9512 case RSM_DRV_PREDEL_COMPLETED: 9513 /* FALLTHRU */ 9514 case RSM_DRV_POSTDEL_IN_PROGRESS: 9515 recheck_state = 1; 9516 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 9517 break; 9518 case RSM_DRV_DR_IN_PROGRESS: 9519 rsm_drv_data.drv_memdel_cnt++; 9520 mutex_exit(&rsm_drv_data.drv_lock); 9521 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9522 "rsm_dr_callback_pre_del done\n")); 9523 return (0); 9524 /* break; */ 9525 default: 9526 ASSERT(0); 9527 break; 9528 } 9529 9530 } while (recheck_state); 9531 9532 rsm_drv_data.drv_memdel_cnt++; 9533 9534 mutex_exit(&rsm_drv_data.drv_lock); 9535 9536 /* Do all the quiescing stuff here */ 9537 DBG_PRINTF((category, RSM_DEBUG, 9538 "rsm_dr_callback_pre_del: quiesce things now\n")); 9539 9540 rsm_dr_process_local_segments(RSM_DR_QUIESCE); 9541 9542 /* 9543 * now that all local segments have been quiesced lets inform 9544 * the importers 9545 */ 9546 rsm_send_suspend(); 9547 9548 /* 9549 * In response to the suspend message the remote node(s) will process 9550 * the segments and send a suspend_complete message. Till all 9551 * the nodes send the suspend_complete message we wait in the 9552 * RSM_DRV_PREDEL_STARTED state. In the exporter_quiesce 9553 * function we transition to the RSM_DRV_PREDEL_COMPLETED state. 9554 */ 9555 mutex_enter(&rsm_drv_data.drv_lock); 9556 9557 while (rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED) { 9558 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 9559 } 9560 9561 ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_COMPLETED); 9562 9563 rsm_drv_data.drv_state = RSM_DRV_DR_IN_PROGRESS; 9564 cv_broadcast(&rsm_drv_data.drv_cv); 9565 9566 mutex_exit(&rsm_drv_data.drv_lock); 9567 9568 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9569 "rsm_dr_callback_pre_del done\n")); 9570 9571 return (0); 9572 } 9573 9574 static void 9575 rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled /* ARGSUSED */) 9576 { 9577 int recheck_state = 0; 9578 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL); 9579 9580 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9581 "rsm_dr_callback_post_del enter\n")); 9582 9583 mutex_enter(&rsm_drv_data.drv_lock); 9584 9585 do { 9586 recheck_state = 0; 9587 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9588 "rsm_dr_callback_post_del:state=%d\n", 9589 rsm_drv_data.drv_state)); 9590 9591 switch (rsm_drv_data.drv_state) { 9592 case RSM_DRV_NEW: 9593 /* 9594 * The driver state cannot not be RSM_DRV_NEW 9595 * since in this state the callbacks have not 9596 * yet been registered. 9597 */ 9598 ASSERT(0); 9599 return; 9600 case RSM_DRV_REG_PROCESSING: 9601 /* 9602 * The driver is in the process of registering with 9603 * the DR framework. Wait till the registration is 9604 * complete. 9605 */ 9606 recheck_state = 1; 9607 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 9608 break; 9609 case RSM_DRV_UNREG_PROCESSING: 9610 /* 9611 * RSM_DRV_UNREG_PROCESSING state means the module 9612 * is detaching and unregistering the callbacks 9613 * from the DR framework. So simply return. 9614 */ 9615 /* FALLTHRU */ 9616 case RSM_DRV_OK: 9617 /* 9618 * RSM_DRV_OK means we missed the pre-del 9619 * corresponding to this post-del coz we had not 9620 * registered yet, so simply return. 9621 */ 9622 mutex_exit(&rsm_drv_data.drv_lock); 9623 DBG_PRINTF((category, RSM_DEBUG, 9624 "rsm_dr_callback_post_del:" 9625 "post-del on OK/UNREG\n")); 9626 return; 9627 /* break; */ 9628 case RSM_DRV_PREDEL_STARTED: 9629 /* FALLTHRU */ 9630 case RSM_DRV_PREDEL_COMPLETED: 9631 /* FALLTHRU */ 9632 case RSM_DRV_POSTDEL_IN_PROGRESS: 9633 recheck_state = 1; 9634 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock); 9635 break; 9636 case RSM_DRV_DR_IN_PROGRESS: 9637 rsm_drv_data.drv_memdel_cnt--; 9638 if (rsm_drv_data.drv_memdel_cnt > 0) { 9639 mutex_exit(&rsm_drv_data.drv_lock); 9640 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9641 "rsm_dr_callback_post_del done:\n")); 9642 return; 9643 } 9644 rsm_drv_data.drv_state = RSM_DRV_POSTDEL_IN_PROGRESS; 9645 break; 9646 default: 9647 ASSERT(0); 9648 return; 9649 /* break; */ 9650 } 9651 } while (recheck_state); 9652 9653 mutex_exit(&rsm_drv_data.drv_lock); 9654 9655 /* Do all the unquiescing stuff here */ 9656 DBG_PRINTF((category, RSM_DEBUG, 9657 "rsm_dr_callback_post_del: unquiesce things now\n")); 9658 9659 rsm_dr_process_local_segments(RSM_DR_UNQUIESCE); 9660 9661 /* 9662 * now that all local segments have been unquiesced lets inform 9663 * the importers 9664 */ 9665 rsm_send_resume(); 9666 9667 mutex_enter(&rsm_drv_data.drv_lock); 9668 9669 rsm_drv_data.drv_state = RSM_DRV_OK; 9670 9671 cv_broadcast(&rsm_drv_data.drv_cv); 9672 9673 mutex_exit(&rsm_drv_data.drv_lock); 9674 9675 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, 9676 "rsm_dr_callback_post_del done\n")); 9677 9678 return; 9679 9680 } 9681