1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 1999-2003 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _RSM_IN_H 28 #define _RSM_IN_H 29 30 #ifdef __cplusplus 31 extern "C" { 32 #endif 33 34 #include <sys/ddi.h> 35 #include <sys/sunddi.h> 36 #include <sys/rsm/rsm.h> 37 #include <sys/rsm/rsmpi.h> 38 39 #define DRIVER_NAME "rsm" 40 41 42 #define RSM_DRIVER_MINOR 0 43 44 #define RSM_CNUM 8 45 46 #define RSMIPC_SZ 10 /* number of outstanding requests, max: 256 */ 47 48 #define RSMIPC_MAX_MESSAGES 64 /* max msgs that receiver can buffer */ 49 #define RSMIPC_LOTSFREE_MSGBUFS 16 /* chunks of credits sent to sender */ 50 51 /* 52 * The base for Sun RSMAPI Kernel Agent service idenitifiers is RSM_INTR_T_KA 53 * as defined below. This is as per the RSMPI specification. Thus, 54 * in the kernel agent, we need to use this value as the service identifier 55 * while registering the service handlers. 56 */ 57 #define RSM_INTR_T_KA 0x88 58 #define RSM_SERVICE RSM_INTR_T_KA 59 60 #define RSM_PRI 2 61 #define RSM_QUEUE_SZ 256 62 63 #define RSM_LOCK 0 64 #define RSM_NOLOCK 1 65 66 #define RSM_MAX_NUM_SEG 4095 /* default value for max imp and exp segs */ 67 68 #define RSM_MAX_NODE 64 /* maximum number of nodes in the cluster */ 69 70 #define RSM_MAX_CTRL 32 /* maximum number of controllers per node */ 71 72 /* 73 * The following defines UINT_MAX rounded down to a page aligned value. 74 */ 75 #define RSM_MAXSZ_PAGE_ALIGNED (UINT_MAX & PAGEMASK) 76 /* 77 * Define TRASHSIZE as the maximum possible size which is page aligned 78 * This value cannot be 0xffffffffffffe000 since this is taken as a 79 * negative value in the devmap_umem_remap call, thus causing the call 80 * to fail. 81 */ 82 #define TRASHSIZE 0x7fffffffffffe000 83 84 #define RSM_ACCESS_READ 0444 85 #define RSM_ACCESS_WRITE 0222 86 #define RSM_ACCESS_TRUSTED 0666 87 88 /* flag values for rsmseg_unload */ 89 #define DISCONNECT 1 90 #define NO_DISCONNECT 0 91 92 struct rsm_driver_data { 93 kmutex_t drv_lock; 94 kcondvar_t drv_cv; 95 int drv_state; /* RSM_DRV_YYYY states */ 96 int drv_memdel_cnt; /* number of memdel callbacks */ 97 }; 98 99 /* rsm driver state */ 100 #define RSM_DRV_NEW 0 101 #define RSM_DRV_OK 1 102 #define RSM_DRV_PREDEL_STARTED 2 103 #define RSM_DRV_PREDEL_COMPLETED 3 104 #define RSM_DRV_POSTDEL_IN_PROGRESS 4 105 #define RSM_DRV_DR_IN_PROGRESS 5 106 #define RSM_DRV_REG_PROCESSING 6 107 #define RSM_DRV_UNREG_PROCESSING 7 108 109 /* internal flags */ 110 #define RSM_DR_QUIESCE 0 111 #define RSM_DR_UNQUIESCE 1 112 113 typedef enum { 114 RSM_STATE_NEW = 0, 115 RSM_STATE_NEW_QUIESCED, 116 RSM_STATE_BIND, 117 RSM_STATE_BIND_QUIESCED, 118 RSM_STATE_EXPORT, 119 RSM_STATE_EXPORT_QUIESCING, 120 RSM_STATE_EXPORT_QUIESCED, 121 RSM_STATE_ZOMBIE, 122 RSM_STATE_CONNECTING, 123 RSM_STATE_ABORT_CONNECT, 124 RSM_STATE_CONNECT, 125 RSM_STATE_CONN_QUIESCE, 126 RSM_STATE_MAPPING, 127 RSM_STATE_ACTIVE, 128 RSM_STATE_MAP_QUIESCE, 129 RSM_STATE_DISCONNECT, 130 RSM_STATE_END 131 } rsm_resource_state_t; 132 133 typedef enum { 134 RSM_RESOURCE_EXPORT_SEGMENT, 135 RSM_RESOURCE_IMPORT_SEGMENT, 136 RSM_RESOURCE_BAR 137 }rsm_resource_type_t; 138 139 /* 140 * All resources have the only common info. whether it is a segment or 141 * a notification queue. 142 */ 143 typedef struct rsm_resource { 144 kmutex_t rsmrc_lock; /* sync on resource */ 145 minor_t rsmrc_num; /* (minor) number */ 146 rsm_memseg_id_t rsmrc_key; /* user key */ 147 mode_t rsmrc_mode; /* access permission */ 148 struct adapter *rsmrc_adapter; /* controller number */ 149 rsm_node_id_t rsmrc_node; /* nodeid */ 150 rsm_resource_type_t rsmrc_type; /* type of this resource */ 151 rsm_resource_state_t rsmrc_state; /* segment state */ 152 struct rsm_resource *rsmrc_next; 153 } rsmresource_t; 154 155 #define RSMRC_BLKSZ 16 156 #define RSMRC_RESERVED ((rsmresource_t *)0x1) 157 158 #define RSM_HASHSZ 128 159 160 #define RSM_USER_MEMORY 0x1 161 #define RSM_KERNEL_MEMORY 0x2 162 #define RSM_EXPORT_WAIT 0x4 163 #define RSM_SEGMENT_POLL 0x8 164 #define RSM_FORCE_DISCONNECT 0x10 165 #define RSM_IMPORT_DUMMY 0x20 166 /* 167 * The following macro is used within the kernel agent to indicate that 168 * rebind/unbind is allowed for an exported segment. It is a part of the 169 * segment's s_flags field. 170 */ 171 #define RSMKA_ALLOW_UNBIND_REBIND 0x40 172 #define RSM_REPUBLISH_WAIT 0x80 173 #define RSM_DR_INPROGRESS 0x100 174 #define RSM_FORCE_DESTROY_WAIT 0x200 175 #define RSMKA_SET_RESOURCE_DONTWAIT 0x400 176 177 #define RSMRC_LOCK(p) mutex_enter(&(p)->rsmrc_lock) 178 #define RSMRC_UNLOCK(p) mutex_exit(&(p)->rsmrc_lock) 179 #define RSMRC_HELD(p) MUTEX_HELD(&(p)->rsmrc_lock) 180 #define RSMRC_TRY(p) mutex_tryenter(&(p)->rsmrc_lock) 181 182 typedef struct rsm_region { 183 caddr_t r_vaddr; /* exported virtual address */ 184 size_t r_len; /* length of export region */ 185 offset_t r_off; /* offset of this region in segment */ 186 struct as *r_asp; 187 struct rsm_region *r_next; /* next region of segment */ 188 }rsm_region; 189 190 typedef struct rsm_cookie { 191 devmap_cookie_t c_dhp; /* devmap cookie handle */ 192 offset_t c_off; /* offset of mapping */ 193 size_t c_len; /* len of mapping */ 194 struct rsm_cookie *c_next; /* next handle */ 195 }rsmcookie_t; 196 197 typedef struct rsm_mapinfo { 198 dev_info_t *dip; 199 uint_t dev_register; 200 off_t dev_offset; 201 off_t start_offset; 202 size_t individual_len; 203 struct rsm_mapinfo *next; 204 } rsm_mapinfo_t; 205 206 207 208 /* 209 * Shared Importer data structure 210 * 211 */ 212 typedef struct rsm_import_share { 213 kmutex_t rsmsi_lock; /* lock for shared importers */ 214 kcondvar_t rsmsi_cv; /* condvar to wait at */ 215 rsm_node_id_t rsmsi_node; 216 rsm_memseg_id_t rsmsi_segid; 217 size_t rsmsi_seglen; 218 rsm_memseg_import_handle_t rsmsi_handle; /* RSMPI handle */ 219 uint_t rsmsi_state; 220 #define RSMSI_STATE_NEW 0x0001 221 #define RSMSI_STATE_CONNECTING 0x0002 222 #define RSMSI_STATE_ABORT_CONNECT 0x0004 223 #define RSMSI_STATE_CONNECTED 0x0008 224 #define RSMSI_STATE_CONN_QUIESCE 0x0010 225 #define RSMSI_STATE_MAPPED 0x0020 226 #define RSMSI_STATE_MAP_QUIESCE 0x0040 227 #define RSMSI_STATE_DISCONNECTED 0x0080 228 229 uint_t rsmsi_refcnt; /* ref count of importers */ 230 uint_t rsmsi_mapcnt; /* count of mapped importers */ 231 mode_t rsmsi_mode; /* mode of last (re)publish */ 232 uid_t rsmsi_uid; 233 gid_t rsmsi_gid; 234 rsm_mapinfo_t *rsmsi_mapinfo; /* register, offset, len values */ 235 uint_t rsmsi_flags; /* flags */ 236 #define RSMSI_FLAGS_ABORTDONE 0x0001 /* NOT_IMPORTING msg for abort conn */ 237 /* has been sent */ 238 void *rsmsi_cookie; /* cookie of the first seg connect */ 239 } rsm_import_share_t; 240 241 #define RSMSI_LOCK(sharep) mutex_enter(&(sharep)->rsmsi_lock) 242 #define RSMSI_UNLOCK(sharep) mutex_exit(&(sharep)->rsmsi_lock) 243 #define RSMSI_HELD(sharep) MUTEX_HELD(&(sharep)->rsmsi_lock) 244 #define RSMSI_TRY(sharep) mutex_tryenter(&(sharep)->rsmsi_lock) 245 246 typedef struct rsm_seginfo { 247 rsmresource_t s_hdr; /* resource hdr */ 248 #define s_state s_hdr.rsmrc_state /* segment state */ 249 #define s_adapter s_hdr.rsmrc_adapter 250 #define s_node s_hdr.rsmrc_node 251 #define s_lock s_hdr.rsmrc_lock 252 #define s_minor s_hdr.rsmrc_num /* minor # of segment */ 253 #define s_key s_hdr.rsmrc_key /* user segment key */ 254 #define s_mode s_hdr.rsmrc_mode /* user segment mode */ 255 #define s_type s_hdr.rsmrc_type /* segment type */ 256 uid_t s_uid; /* owner id */ 257 gid_t s_gid; /* owner id */ 258 259 size_t s_len; /* total segment size */ 260 rsm_region s_region; /* regions of segment */ 261 262 int s_flags; 263 int s_pollflag; /* indicates poll status */ 264 265 kcondvar_t s_cv; /* condition to wait on */ 266 267 rsm_memseg_id_t s_segid; /* NIC segment id */ 268 269 int s_acl_len; /* length of access list */ 270 rsmapi_access_entry_t *s_acl; /* access list */ 271 rsm_access_entry_t *s_acl_in; /* access list with hwaddr */ 272 273 struct pollhead s_poll; 274 uint32_t s_pollevent; 275 pid_t s_pid; 276 277 rsmcookie_t *s_ckl; /* list of devmap cookie */ 278 279 size_t s_total_maplen; 280 rsm_mapinfo_t *s_mapinfo; /* register, offset, len */ 281 282 union { 283 rsm_memseg_import_handle_t in; 284 rsm_memseg_export_handle_t out; 285 } s_handle; /* NIC handle for segment */ 286 287 /* 288 * This field is used to indicate the cookie returned by the 289 * ddi_umem_lock when binding pages for an export segment. 290 * Also, for importers on the same node as the export segment, 291 * this field indicates the cookie used during import mapping. 292 */ 293 ddi_umem_cookie_t s_cookie; 294 rsm_import_share_t *s_share; /* shared importer data */ 295 /* 296 * This field in an import segments indicates the number of 297 * putv/getv operations in progress and in an export segment 298 * it is the number of putv/getv ops currently using it as 299 * a handle in the iovec. 300 */ 301 uint_t s_rdmacnt; 302 struct proc *s_proc; 303 } rsmseg_t; 304 305 #define rsmseglock_acquire(p) RSMRC_LOCK((rsmresource_t *)(p)) 306 #define rsmseglock_release(p) RSMRC_UNLOCK((rsmresource_t *)(p)) 307 #define rsmseglock_held(p) RSMRC_HELD((rsmresource_t *)(p)) 308 #define rsmseglock_try(p) RSMRC_TRY((rsmresource_t *)(p)) 309 310 #define rsmsharelock_acquire(p) RSMSI_LOCK(p->s_share) 311 #define rsmsharelock_release(p) RSMSI_UNLOCK(p->s_share) 312 #define rsmsharelock_held(p) RSMSI_HELD(p->s_share) 313 #define rsmsharelock_try(p) RSMSI_TRY(p->s_share) 314 315 /* 316 * Resource elements structure 317 */ 318 typedef struct { 319 int rsmrcblk_avail; 320 rsmresource_t *rsmrcblk_blks[RSMRC_BLKSZ]; 321 }rsmresource_blk_t; 322 323 struct rsmresource_table { 324 krwlock_t rsmrc_lock; 325 int rsmrc_len; 326 int rsmrc_sz; 327 rsmresource_blk_t **rsmrc_root; 328 }; 329 330 /* 331 * Struct for advertised resource list 332 */ 333 /* 334 * Hashtable structs 335 * bucket points to an array of pointers, each entry in the bucket array 336 * points to a linked list of resource items. 337 * bucket index = bucket_address%RSM_HASHSZ 338 */ 339 typedef struct rsmhash_table { 340 krwlock_t rsmhash_rw; 341 rsmresource_t **bucket; 342 } rsmhash_table_t; 343 344 /* 345 * Remote messaging related structure 346 */ 347 348 /* 349 * Flags for ipc slot 350 */ 351 #define RSMIPC_FREE 0x1 /* slot is free */ 352 #define RSMIPC_PENDING 0x2 /* slot has pending request */ 353 354 #define RSMIPC_SET(x, v) ((x)->rsmipc_flags |= (v)) 355 #define RSMIPC_GET(x, v) ((x)->rsmipc_flags & (v)) 356 #define RSMIPC_CLEAR(x, v) ((x)->rsmipc_flags &= ~(v)) 357 358 typedef struct rsmipc_slot { 359 kmutex_t rsmipc_lock; /* lock for remote msgs */ 360 kcondvar_t rsmipc_cv; /* condition var to wait on */ 361 int rsmipc_flags; 362 rsmipc_cookie_t rsmipc_cookie; /* cookie of request in wire */ 363 void *rsmipc_data; /* ptr to data to copy */ 364 }rsmipc_slot_t; 365 366 /* 367 * Messaging struc 368 */ 369 typedef struct { 370 kmutex_t lock; 371 kcondvar_t cv; 372 int count; 373 int wanted; 374 int sequence; 375 rsmipc_slot_t slots[RSMIPC_SZ]; 376 }rsm_ipc_t; 377 378 /* 379 * These tokens are used for building the list of remote node importers 380 * of a segment exported from the local node 381 */ 382 typedef struct importing_token { 383 struct importing_token *next; 384 rsm_memseg_id_t key; 385 rsm_node_id_t importing_node; 386 void *import_segment_cookie; 387 rsm_addr_t importing_adapter_hwaddr; 388 } importing_token_t; 389 390 typedef struct { 391 kmutex_t lock; 392 importing_token_t **bucket; 393 } importers_table_t; 394 395 /* 396 * Used by the rsm_send_republish() fn 397 */ 398 typedef struct republish_token { 399 struct republish_token *next; 400 rsm_memseg_id_t key; 401 rsm_node_id_t importing_node; 402 rsm_permission_t permission; 403 } republish_token_t; 404 405 /* 406 * data strucuture for list manipulation 407 */ 408 typedef struct list_element { 409 struct list_element *next; 410 rsm_node_id_t nodeid; 411 uint32_t flags; 412 #define RSM_SUSPEND_ACKPENDING 0x01 413 #define RSM_SUSPEND_NODEDEAD 0x02 414 } list_element_t; 415 416 typedef struct list_head { 417 struct list_element *list_head; 418 kmutex_t list_lock; 419 } list_head_t; 420 421 #ifdef __cplusplus 422 } 423 #endif 424 425 #endif /* _RSM_IN_H */ 426