1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/systm.h> 29 #include <sys/kmem.h> 30 #include <sys/cmn_err.h> 31 #include <sys/atomic.h> 32 #include <sys/clconf.h> 33 #include <sys/cladm.h> 34 #include <sys/flock.h> 35 #include <nfs/export.h> 36 #include <nfs/nfs.h> 37 #include <nfs/nfs4.h> 38 #include <nfs/nfssys.h> 39 #include <nfs/lm.h> 40 #include <sys/pathname.h> 41 #include <sys/sdt.h> 42 #include <sys/nvpair.h> 43 44 45 extern time_t rfs4_start_time; 46 extern uint_t nfs4_srv_vkey; 47 48 stateid4 special0 = { 49 0, 50 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } 51 }; 52 53 stateid4 special1 = { 54 0xffffffff, 55 { 56 (char)0xff, (char)0xff, (char)0xff, (char)0xff, 57 (char)0xff, (char)0xff, (char)0xff, (char)0xff, 58 (char)0xff, (char)0xff, (char)0xff, (char)0xff 59 } 60 }; 61 62 63 #define ISSPECIAL(id) (stateid4_cmp(id, &special0) || \ 64 stateid4_cmp(id, &special1)) 65 66 /* For embedding the cluster nodeid into our clientid */ 67 #define CLUSTER_NODEID_SHIFT 24 68 #define CLUSTER_MAX_NODEID 255 69 70 #ifdef DEBUG 71 int rfs4_debug; 72 #endif 73 74 static uint32_t rfs4_database_debug = 0x00; 75 76 static void rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf); 77 static void rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dir, char *leaf); 78 static void rfs4_dss_clear_oldstate(rfs4_servinst_t *sip); 79 static void rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip); 80 81 /* 82 * Couple of simple init/destroy functions for a general waiter 83 */ 84 void 85 rfs4_sw_init(rfs4_state_wait_t *swp) 86 { 87 mutex_init(swp->sw_cv_lock, NULL, MUTEX_DEFAULT, NULL); 88 cv_init(swp->sw_cv, NULL, CV_DEFAULT, NULL); 89 swp->sw_active = FALSE; 90 swp->sw_wait_count = 0; 91 } 92 93 void 94 rfs4_sw_destroy(rfs4_state_wait_t *swp) 95 { 96 mutex_destroy(swp->sw_cv_lock); 97 cv_destroy(swp->sw_cv); 98 } 99 100 void 101 rfs4_sw_enter(rfs4_state_wait_t *swp) 102 { 103 mutex_enter(swp->sw_cv_lock); 104 while (swp->sw_active) { 105 swp->sw_wait_count++; 106 cv_wait(swp->sw_cv, swp->sw_cv_lock); 107 swp->sw_wait_count--; 108 } 109 ASSERT(swp->sw_active == FALSE); 110 swp->sw_active = TRUE; 111 mutex_exit(swp->sw_cv_lock); 112 } 113 114 void 115 rfs4_sw_exit(rfs4_state_wait_t *swp) 116 { 117 mutex_enter(swp->sw_cv_lock); 118 ASSERT(swp->sw_active == TRUE); 119 swp->sw_active = FALSE; 120 if (swp->sw_wait_count != 0) 121 cv_broadcast(swp->sw_cv); 122 mutex_exit(swp->sw_cv_lock); 123 } 124 125 /* 126 * CPR callback id -- not related to v4 callbacks 127 */ 128 static callb_id_t cpr_id = 0; 129 130 static void 131 deep_lock_copy(LOCK4res *dres, LOCK4res *sres) 132 { 133 lock_owner4 *slo = &sres->LOCK4res_u.denied.owner; 134 lock_owner4 *dlo = &dres->LOCK4res_u.denied.owner; 135 136 if (sres->status == NFS4ERR_DENIED) { 137 dlo->owner_val = kmem_alloc(slo->owner_len, KM_SLEEP); 138 bcopy(slo->owner_val, dlo->owner_val, slo->owner_len); 139 } 140 } 141 142 static void 143 deep_lock_free(LOCK4res *res) 144 { 145 lock_owner4 *lo = &res->LOCK4res_u.denied.owner; 146 147 if (res->status == NFS4ERR_DENIED) 148 kmem_free(lo->owner_val, lo->owner_len); 149 } 150 151 static void 152 deep_open_copy(OPEN4res *dres, OPEN4res *sres) 153 { 154 nfsace4 *sacep, *dacep; 155 156 if (sres->status != NFS4_OK) { 157 return; 158 } 159 160 dres->attrset = sres->attrset; 161 162 switch (sres->delegation.delegation_type) { 163 case OPEN_DELEGATE_NONE: 164 return; 165 case OPEN_DELEGATE_READ: 166 sacep = &sres->delegation.open_delegation4_u.read.permissions; 167 dacep = &dres->delegation.open_delegation4_u.read.permissions; 168 break; 169 case OPEN_DELEGATE_WRITE: 170 sacep = &sres->delegation.open_delegation4_u.write.permissions; 171 dacep = &dres->delegation.open_delegation4_u.write.permissions; 172 break; 173 } 174 dacep->who.utf8string_val = 175 kmem_alloc(sacep->who.utf8string_len, KM_SLEEP); 176 bcopy(sacep->who.utf8string_val, dacep->who.utf8string_val, 177 sacep->who.utf8string_len); 178 } 179 180 static void 181 deep_open_free(OPEN4res *res) 182 { 183 nfsace4 *acep; 184 if (res->status != NFS4_OK) 185 return; 186 187 switch (res->delegation.delegation_type) { 188 case OPEN_DELEGATE_NONE: 189 return; 190 case OPEN_DELEGATE_READ: 191 acep = &res->delegation.open_delegation4_u.read.permissions; 192 break; 193 case OPEN_DELEGATE_WRITE: 194 acep = &res->delegation.open_delegation4_u.write.permissions; 195 break; 196 } 197 198 if (acep->who.utf8string_val) { 199 kmem_free(acep->who.utf8string_val, acep->who.utf8string_len); 200 acep->who.utf8string_val = NULL; 201 } 202 } 203 204 void 205 rfs4_free_reply(nfs_resop4 *rp) 206 { 207 switch (rp->resop) { 208 case OP_LOCK: 209 deep_lock_free(&rp->nfs_resop4_u.oplock); 210 break; 211 case OP_OPEN: 212 deep_open_free(&rp->nfs_resop4_u.opopen); 213 default: 214 break; 215 } 216 } 217 218 void 219 rfs4_copy_reply(nfs_resop4 *dst, nfs_resop4 *src) 220 { 221 *dst = *src; 222 223 /* Handle responses that need deep copy */ 224 switch (src->resop) { 225 case OP_LOCK: 226 deep_lock_copy(&dst->nfs_resop4_u.oplock, 227 &src->nfs_resop4_u.oplock); 228 break; 229 case OP_OPEN: 230 deep_open_copy(&dst->nfs_resop4_u.opopen, 231 &src->nfs_resop4_u.opopen); 232 break; 233 default: 234 break; 235 }; 236 } 237 238 /* 239 * This is the implementation of the underlying state engine. The 240 * public interface to this engine is described by 241 * nfs4_state.h. Callers to the engine should hold no state engine 242 * locks when they call in to it. If the protocol needs to lock data 243 * structures it should do so after acquiring all references to them 244 * first and then follow the following lock order: 245 * 246 * client > openowner > state > lo_state > lockowner > file. 247 * 248 * Internally we only allow a thread to hold one hash bucket lock at a 249 * time and the lock is higher in the lock order (must be acquired 250 * first) than the data structure that is on that hash list. 251 * 252 * If a new reference was acquired by the caller, that reference needs 253 * to be released after releasing all acquired locks with the 254 * corresponding rfs4_*_rele routine. 255 */ 256 257 /* 258 * This code is some what prototypical for now. Its purpose currently is to 259 * implement the interfaces sufficiently to finish the higher protocol 260 * elements. This will be replaced by a dynamically resizeable tables 261 * backed by kmem_cache allocator. However synchronization is handled 262 * correctly (I hope) and will not change by much. The mutexes for 263 * the hash buckets that can be used to create new instances of data 264 * structures might be good candidates to evolve into reader writer 265 * locks. If it has to do a creation, it would be holding the 266 * mutex across a kmem_alloc with KM_SLEEP specified. 267 */ 268 269 #ifdef DEBUG 270 #define TABSIZE 17 271 #else 272 #define TABSIZE 2047 273 #endif 274 275 #define ADDRHASH(key) ((unsigned long)(key) >> 3) 276 277 /* Used to serialize create/destroy of rfs4_server_state database */ 278 kmutex_t rfs4_state_lock; 279 static rfs4_database_t *rfs4_server_state = NULL; 280 281 /* Used to serialize lookups of clientids */ 282 static krwlock_t rfs4_findclient_lock; 283 284 /* 285 * For now this "table" is exposed so that the CPR callback 286 * function can tromp through it.. 287 */ 288 rfs4_table_t *rfs4_client_tab; 289 290 static rfs4_index_t *rfs4_clientid_idx; 291 static rfs4_index_t *rfs4_nfsclnt_idx; 292 static rfs4_table_t *rfs4_openowner_tab; 293 static rfs4_index_t *rfs4_openowner_idx; 294 static rfs4_table_t *rfs4_state_tab; 295 static rfs4_index_t *rfs4_state_idx; 296 static rfs4_index_t *rfs4_state_owner_file_idx; 297 static rfs4_index_t *rfs4_state_file_idx; 298 static rfs4_table_t *rfs4_lo_state_tab; 299 static rfs4_index_t *rfs4_lo_state_idx; 300 static rfs4_index_t *rfs4_lo_state_owner_idx; 301 static rfs4_table_t *rfs4_lockowner_tab; 302 static rfs4_index_t *rfs4_lockowner_idx; 303 static rfs4_index_t *rfs4_lockowner_pid_idx; 304 static rfs4_table_t *rfs4_file_tab; 305 static rfs4_index_t *rfs4_file_idx; 306 static rfs4_table_t *rfs4_deleg_state_tab; 307 static rfs4_index_t *rfs4_deleg_idx; 308 static rfs4_index_t *rfs4_deleg_state_idx; 309 310 #define MAXTABSZ 1024*1024 311 312 /* The values below are rfs4_lease_time units */ 313 314 #ifdef DEBUG 315 #define CLIENT_CACHE_TIME 1 316 #define OPENOWNER_CACHE_TIME 1 317 #define STATE_CACHE_TIME 1 318 #define LO_STATE_CACHE_TIME 1 319 #define LOCKOWNER_CACHE_TIME 1 320 #define FILE_CACHE_TIME 3 321 #define DELEG_STATE_CACHE_TIME 1 322 #else 323 #define CLIENT_CACHE_TIME 10 324 #define OPENOWNER_CACHE_TIME 5 325 #define STATE_CACHE_TIME 1 326 #define LO_STATE_CACHE_TIME 1 327 #define LOCKOWNER_CACHE_TIME 3 328 #define FILE_CACHE_TIME 40 329 #define DELEG_STATE_CACHE_TIME 1 330 #endif 331 332 333 static time_t rfs4_client_cache_time = 0; 334 static time_t rfs4_openowner_cache_time = 0; 335 static time_t rfs4_state_cache_time = 0; 336 static time_t rfs4_lo_state_cache_time = 0; 337 static time_t rfs4_lockowner_cache_time = 0; 338 static time_t rfs4_file_cache_time = 0; 339 static time_t rfs4_deleg_state_cache_time = 0; 340 341 static bool_t rfs4_client_create(rfs4_entry_t, void *); 342 static void rfs4_dss_remove_cpleaf(rfs4_client_t *); 343 static void rfs4_dss_remove_leaf(rfs4_servinst_t *, char *, char *); 344 static void rfs4_client_destroy(rfs4_entry_t); 345 static bool_t rfs4_client_expiry(rfs4_entry_t); 346 static uint32_t clientid_hash(void *); 347 static bool_t clientid_compare(rfs4_entry_t, void *); 348 static void *clientid_mkkey(rfs4_entry_t); 349 static uint32_t nfsclnt_hash(void *); 350 static bool_t nfsclnt_compare(rfs4_entry_t, void *); 351 static void *nfsclnt_mkkey(rfs4_entry_t); 352 static bool_t rfs4_openowner_create(rfs4_entry_t, void *); 353 static void rfs4_openowner_destroy(rfs4_entry_t); 354 static bool_t rfs4_openowner_expiry(rfs4_entry_t); 355 static uint32_t openowner_hash(void *); 356 static bool_t openowner_compare(rfs4_entry_t, void *); 357 static void *openowner_mkkey(rfs4_entry_t); 358 static bool_t rfs4_state_create(rfs4_entry_t, void *); 359 static void rfs4_state_destroy(rfs4_entry_t); 360 static bool_t rfs4_state_expiry(rfs4_entry_t); 361 static uint32_t state_hash(void *); 362 static bool_t state_compare(rfs4_entry_t, void *); 363 static void *state_mkkey(rfs4_entry_t); 364 static uint32_t state_owner_file_hash(void *); 365 static bool_t state_owner_file_compare(rfs4_entry_t, void *); 366 static void *state_owner_file_mkkey(rfs4_entry_t); 367 static uint32_t state_file_hash(void *); 368 static bool_t state_file_compare(rfs4_entry_t, void *); 369 static void *state_file_mkkey(rfs4_entry_t); 370 static bool_t rfs4_lo_state_create(rfs4_entry_t, void *); 371 static void rfs4_lo_state_destroy(rfs4_entry_t); 372 static bool_t rfs4_lo_state_expiry(rfs4_entry_t); 373 static uint32_t lo_state_hash(void *); 374 static bool_t lo_state_compare(rfs4_entry_t, void *); 375 static void *lo_state_mkkey(rfs4_entry_t); 376 static uint32_t lo_state_lo_hash(void *); 377 static bool_t lo_state_lo_compare(rfs4_entry_t, void *); 378 static void *lo_state_lo_mkkey(rfs4_entry_t); 379 static bool_t rfs4_lockowner_create(rfs4_entry_t, void *); 380 static void rfs4_lockowner_destroy(rfs4_entry_t); 381 static bool_t rfs4_lockowner_expiry(rfs4_entry_t); 382 static uint32_t lockowner_hash(void *); 383 static bool_t lockowner_compare(rfs4_entry_t, void *); 384 static void *lockowner_mkkey(rfs4_entry_t); 385 static uint32_t pid_hash(void *); 386 static bool_t pid_compare(rfs4_entry_t, void *); 387 static void *pid_mkkey(rfs4_entry_t); 388 static bool_t rfs4_file_create(rfs4_entry_t, void *); 389 static void rfs4_file_destroy(rfs4_entry_t); 390 static uint32_t file_hash(void *); 391 static bool_t file_compare(rfs4_entry_t, void *); 392 static void *file_mkkey(rfs4_entry_t); 393 static bool_t rfs4_deleg_state_create(rfs4_entry_t, void *); 394 static void rfs4_deleg_state_destroy(rfs4_entry_t); 395 static bool_t rfs4_deleg_state_expiry(rfs4_entry_t); 396 static uint32_t deleg_hash(void *); 397 static bool_t deleg_compare(rfs4_entry_t, void *); 398 static void *deleg_mkkey(rfs4_entry_t); 399 static uint32_t deleg_state_hash(void *); 400 static bool_t deleg_state_compare(rfs4_entry_t, void *); 401 static void *deleg_state_mkkey(rfs4_entry_t); 402 403 static void rfs4_state_rele_nounlock(rfs4_state_t *); 404 405 static int rfs4_ss_enabled = 0; 406 407 extern void (*rfs4_client_clrst)(struct nfs4clrst_args *); 408 409 void 410 rfs4_ss_pnfree(rfs4_ss_pn_t *ss_pn) 411 { 412 kmem_free(ss_pn, sizeof (rfs4_ss_pn_t)); 413 } 414 415 static rfs4_ss_pn_t * 416 rfs4_ss_pnalloc(char *dir, char *leaf) 417 { 418 rfs4_ss_pn_t *ss_pn; 419 int dir_len, leaf_len; 420 421 /* 422 * validate we have a resonable path 423 * (account for the '/' and trailing null) 424 */ 425 if ((dir_len = strlen(dir)) > MAXPATHLEN || 426 (leaf_len = strlen(leaf)) > MAXNAMELEN || 427 (dir_len + leaf_len + 2) > MAXPATHLEN) { 428 return (NULL); 429 } 430 431 ss_pn = kmem_alloc(sizeof (rfs4_ss_pn_t), KM_SLEEP); 432 433 (void) snprintf(ss_pn->pn, MAXPATHLEN, "%s/%s", dir, leaf); 434 /* Handy pointer to just the leaf name */ 435 ss_pn->leaf = ss_pn->pn + dir_len + 1; 436 return (ss_pn); 437 } 438 439 440 /* 441 * Move the "leaf" filename from "sdir" directory 442 * to the "ddir" directory. Return the pathname of 443 * the destination unless the rename fails in which 444 * case we need to return the source pathname. 445 */ 446 static rfs4_ss_pn_t * 447 rfs4_ss_movestate(char *sdir, char *ddir, char *leaf) 448 { 449 rfs4_ss_pn_t *src, *dst; 450 451 if ((src = rfs4_ss_pnalloc(sdir, leaf)) == NULL) 452 return (NULL); 453 454 if ((dst = rfs4_ss_pnalloc(ddir, leaf)) == NULL) { 455 rfs4_ss_pnfree(src); 456 return (NULL); 457 } 458 459 /* 460 * If the rename fails we shall return the src 461 * pathname and free the dst. Otherwise we need 462 * to free the src and return the dst pathanme. 463 */ 464 if (vn_rename(src->pn, dst->pn, UIO_SYSSPACE)) { 465 rfs4_ss_pnfree(dst); 466 return (src); 467 } 468 rfs4_ss_pnfree(src); 469 return (dst); 470 } 471 472 473 static rfs4_oldstate_t * 474 rfs4_ss_getstate(vnode_t *dvp, rfs4_ss_pn_t *ss_pn) 475 { 476 struct uio uio; 477 struct iovec iov[3]; 478 479 rfs4_oldstate_t *cl_ss = NULL; 480 vnode_t *vp; 481 vattr_t va; 482 uint_t id_len; 483 int err, kill_file, file_vers; 484 485 if (ss_pn == NULL) 486 return (NULL); 487 488 /* 489 * open the state file. 490 */ 491 if (vn_open(ss_pn->pn, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0) != 0) { 492 return (NULL); 493 } 494 495 if (vp->v_type != VREG) { 496 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 497 VN_RELE(vp); 498 return (NULL); 499 } 500 501 err = VOP_ACCESS(vp, VREAD, 0, CRED()); 502 if (err) { 503 /* 504 * We don't have read access? better get the heck out. 505 */ 506 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 507 VN_RELE(vp); 508 return (NULL); 509 } 510 511 (void) VOP_RWLOCK(vp, V_WRITELOCK_FALSE, NULL); 512 /* 513 * get the file size to do some basic validation 514 */ 515 va.va_mask = AT_SIZE; 516 err = VOP_GETATTR(vp, &va, 0, CRED()); 517 518 kill_file = (va.va_size == 0 || va.va_size < 519 (NFS4_VERIFIER_SIZE + sizeof (uint_t)+1)); 520 521 if (err || kill_file) { 522 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL); 523 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 524 VN_RELE(vp); 525 if (kill_file) { 526 (void) VOP_REMOVE(dvp, ss_pn->leaf, CRED()); 527 } 528 return (NULL); 529 } 530 531 cl_ss = kmem_alloc(sizeof (rfs4_oldstate_t), KM_SLEEP); 532 533 /* 534 * build iovecs to read in the file_version, verifier and id_len 535 */ 536 iov[0].iov_base = (caddr_t)&file_vers; 537 iov[0].iov_len = sizeof (int); 538 iov[1].iov_base = (caddr_t)&cl_ss->cl_id4.verifier; 539 iov[1].iov_len = NFS4_VERIFIER_SIZE; 540 iov[2].iov_base = (caddr_t)&id_len; 541 iov[2].iov_len = sizeof (uint_t); 542 543 uio.uio_iov = iov; 544 uio.uio_iovcnt = 3; 545 uio.uio_segflg = UIO_SYSSPACE; 546 uio.uio_loffset = 0; 547 uio.uio_resid = sizeof (int) + NFS4_VERIFIER_SIZE + sizeof (uint_t); 548 549 if (err = VOP_READ(vp, &uio, FREAD, CRED(), NULL)) { 550 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL); 551 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 552 VN_RELE(vp); 553 kmem_free(cl_ss, sizeof (rfs4_oldstate_t)); 554 return (NULL); 555 } 556 557 /* 558 * if the file_version doesn't match or if the 559 * id_len is zero or the combination of the verifier, 560 * id_len and id_val is bigger than the file we have 561 * a problem. If so ditch the file. 562 */ 563 kill_file = (file_vers != NFS4_SS_VERSION || id_len == 0 || 564 (id_len + NFS4_VERIFIER_SIZE + sizeof (uint_t)) > va.va_size); 565 566 if (err || kill_file) { 567 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL); 568 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 569 VN_RELE(vp); 570 kmem_free(cl_ss, sizeof (rfs4_oldstate_t)); 571 if (kill_file) { 572 (void) VOP_REMOVE(dvp, ss_pn->leaf, CRED()); 573 } 574 return (NULL); 575 } 576 577 /* 578 * now get the client id value 579 */ 580 cl_ss->cl_id4.id_val = kmem_alloc(id_len, KM_SLEEP); 581 iov[0].iov_base = cl_ss->cl_id4.id_val; 582 iov[0].iov_len = id_len; 583 584 uio.uio_iov = iov; 585 uio.uio_iovcnt = 1; 586 uio.uio_segflg = UIO_SYSSPACE; 587 uio.uio_resid = cl_ss->cl_id4.id_len = id_len; 588 589 if (err = VOP_READ(vp, &uio, FREAD, CRED(), NULL)) { 590 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL); 591 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 592 VN_RELE(vp); 593 kmem_free(cl_ss->cl_id4.id_val, id_len); 594 kmem_free(cl_ss, sizeof (rfs4_oldstate_t)); 595 return (NULL); 596 } 597 598 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL); 599 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED()); 600 VN_RELE(vp); 601 return (cl_ss); 602 } 603 604 #ifdef nextdp 605 #undef nextdp 606 #endif 607 #define nextdp(dp) ((struct dirent64 *)((char *)(dp) + (dp)->d_reclen)) 608 609 /* 610 * Add entries from statedir to supplied oldstate list. 611 * Optionally, move all entries from statedir -> destdir. 612 */ 613 void 614 rfs4_ss_oldstate(rfs4_oldstate_t *oldstate, char *statedir, char *destdir) 615 { 616 rfs4_ss_pn_t *ss_pn; 617 rfs4_oldstate_t *cl_ss = NULL; 618 char *dirt = NULL; 619 int err, dir_eof = 0, size = 0; 620 vnode_t *dvp; 621 struct iovec iov; 622 struct uio uio; 623 struct dirent64 *dep; 624 offset_t dirchunk_offset = 0; 625 626 /* 627 * open the state directory 628 */ 629 if (vn_open(statedir, UIO_SYSSPACE, FREAD, 0, &dvp, 0, 0)) 630 return; 631 632 if (dvp->v_type != VDIR || VOP_ACCESS(dvp, VREAD, 0, CRED())) 633 goto out; 634 635 dirt = kmem_alloc(RFS4_SS_DIRSIZE, KM_SLEEP); 636 637 /* 638 * Get and process the directory entries 639 */ 640 while (!dir_eof) { 641 (void) VOP_RWLOCK(dvp, V_WRITELOCK_FALSE, NULL); 642 iov.iov_base = dirt; 643 iov.iov_len = RFS4_SS_DIRSIZE; 644 uio.uio_iov = &iov; 645 uio.uio_iovcnt = 1; 646 uio.uio_segflg = UIO_SYSSPACE; 647 uio.uio_loffset = dirchunk_offset; 648 uio.uio_resid = RFS4_SS_DIRSIZE; 649 650 err = VOP_READDIR(dvp, &uio, CRED(), &dir_eof); 651 VOP_RWUNLOCK(dvp, V_WRITELOCK_FALSE, NULL); 652 if (err) 653 goto out; 654 655 size = RFS4_SS_DIRSIZE - uio.uio_resid; 656 657 /* 658 * Process all the directory entries in this 659 * readdir chunk 660 */ 661 for (dep = (struct dirent64 *)dirt; size > 0; 662 dep = nextdp(dep)) { 663 664 size -= dep->d_reclen; 665 dirchunk_offset = dep->d_off; 666 667 /* 668 * Skip '.' and '..' 669 */ 670 if (NFS_IS_DOTNAME(dep->d_name)) 671 continue; 672 673 ss_pn = rfs4_ss_pnalloc(statedir, dep->d_name); 674 if (ss_pn == NULL) 675 continue; 676 677 if (cl_ss = rfs4_ss_getstate(dvp, ss_pn)) { 678 if (destdir != NULL) { 679 rfs4_ss_pnfree(ss_pn); 680 cl_ss->ss_pn = rfs4_ss_movestate( 681 statedir, destdir, dep->d_name); 682 } else { 683 cl_ss->ss_pn = ss_pn; 684 } 685 insque(cl_ss, oldstate); 686 } else { 687 rfs4_ss_pnfree(ss_pn); 688 } 689 } 690 } 691 692 out: 693 (void) VOP_CLOSE(dvp, FREAD, 1, (offset_t)0, CRED()); 694 VN_RELE(dvp); 695 if (dirt) 696 kmem_free((caddr_t)dirt, RFS4_SS_DIRSIZE); 697 } 698 699 static void 700 rfs4_ss_init(void) 701 { 702 int npaths = 1; 703 char *default_dss_path = NFS4_DSS_VAR_DIR; 704 705 /* read the default stable storage state */ 706 rfs4_dss_readstate(npaths, &default_dss_path); 707 708 rfs4_ss_enabled = 1; 709 } 710 711 static void 712 rfs4_ss_fini(void) 713 { 714 rfs4_servinst_t *sip; 715 716 mutex_enter(&rfs4_servinst_lock); 717 sip = rfs4_cur_servinst; 718 while (sip != NULL) { 719 rfs4_dss_clear_oldstate(sip); 720 sip = sip->next; 721 } 722 mutex_exit(&rfs4_servinst_lock); 723 } 724 725 /* 726 * Remove all oldstate files referenced by this servinst. 727 */ 728 static void 729 rfs4_dss_clear_oldstate(rfs4_servinst_t *sip) 730 { 731 rfs4_oldstate_t *os_head, *osp; 732 733 rw_enter(&sip->oldstate_lock, RW_WRITER); 734 os_head = sip->oldstate; 735 736 if (os_head == NULL) 737 return; 738 739 /* skip dummy entry */ 740 osp = os_head->next; 741 while (osp != os_head) { 742 char *leaf = osp->ss_pn->leaf; 743 rfs4_oldstate_t *os_next; 744 745 rfs4_dss_remove_leaf(sip, NFS4_DSS_OLDSTATE_LEAF, leaf); 746 747 if (osp->cl_id4.id_val) 748 kmem_free(osp->cl_id4.id_val, osp->cl_id4.id_len); 749 if (osp->ss_pn) 750 kmem_free(osp->ss_pn, sizeof (rfs4_ss_pn_t)); 751 752 os_next = osp->next; 753 remque(osp); 754 kmem_free(osp, sizeof (rfs4_oldstate_t)); 755 osp = os_next; 756 } 757 758 /* free dummy entry */ 759 kmem_free(osp, sizeof (rfs4_oldstate_t)); 760 761 sip->oldstate = NULL; 762 763 rw_exit(&sip->oldstate_lock); 764 } 765 766 /* 767 * Form the state and oldstate paths, and read in the stable storage files. 768 */ 769 void 770 rfs4_dss_readstate(int npaths, char **paths) 771 { 772 int i; 773 char *state, *oldstate; 774 775 state = kmem_alloc(MAXPATHLEN, KM_SLEEP); 776 oldstate = kmem_alloc(MAXPATHLEN, KM_SLEEP); 777 778 for (i = 0; i < npaths; i++) { 779 char *path = paths[i]; 780 781 (void) sprintf(state, "%s/%s", path, NFS4_DSS_STATE_LEAF); 782 (void) sprintf(oldstate, "%s/%s", path, NFS4_DSS_OLDSTATE_LEAF); 783 784 /* 785 * Populate the current server instance's oldstate list. 786 * 787 * 1. Read stable storage data from old state directory, 788 * leaving its contents alone. 789 * 790 * 2. Read stable storage data from state directory, 791 * and move the latter's contents to old state 792 * directory. 793 */ 794 rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, oldstate, NULL); 795 rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, state, oldstate); 796 } 797 798 kmem_free(state, MAXPATHLEN); 799 kmem_free(oldstate, MAXPATHLEN); 800 } 801 802 803 /* 804 * Check if we are still in grace and if the client can be 805 * granted permission to perform reclaims. 806 */ 807 void 808 rfs4_ss_chkclid(rfs4_client_t *cp) 809 { 810 rfs4_servinst_t *sip; 811 812 /* 813 * It should be sufficient to check the oldstate data for just 814 * this client's instance. However, since our per-instance 815 * client grouping is solely temporal, HA-NFSv4 RG failover 816 * might result in clients of the same RG being partitioned into 817 * separate instances. 818 * 819 * Until the client grouping is improved, we must check the 820 * oldstate data for all instances with an active grace period. 821 * 822 * This also serves as the mechanism to remove stale oldstate data. 823 * The first time we check an instance after its grace period has 824 * expired, the oldstate data should be cleared. 825 * 826 * Start at the current instance, and walk the list backwards 827 * to the first. 828 */ 829 mutex_enter(&rfs4_servinst_lock); 830 for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) { 831 rfs4_ss_chkclid_sip(cp, sip); 832 833 /* if the above check found this client, we're done */ 834 if (cp->can_reclaim) 835 break; 836 } 837 mutex_exit(&rfs4_servinst_lock); 838 } 839 840 static void 841 rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip) 842 { 843 rfs4_oldstate_t *osp, *os_head; 844 845 /* short circuit everything if this server instance has no oldstate */ 846 rw_enter(&sip->oldstate_lock, RW_READER); 847 os_head = sip->oldstate; 848 rw_exit(&sip->oldstate_lock); 849 if (os_head == NULL) 850 return; 851 852 /* 853 * If this server instance is no longer in a grace period then 854 * the client won't be able to reclaim. No further need for this 855 * instance's oldstate data, so it can be cleared. 856 */ 857 if (!rfs4_servinst_in_grace(sip)) 858 return; 859 860 /* this instance is still in grace; search for the clientid */ 861 862 rw_enter(&sip->oldstate_lock, RW_READER); 863 864 os_head = sip->oldstate; 865 /* skip dummy entry */ 866 osp = os_head->next; 867 while (osp != os_head) { 868 if (osp->cl_id4.id_len == cp->nfs_client.id_len) { 869 if (bcmp(osp->cl_id4.id_val, cp->nfs_client.id_val, 870 osp->cl_id4.id_len) == 0) { 871 cp->can_reclaim = 1; 872 break; 873 } 874 } 875 osp = osp->next; 876 } 877 878 rw_exit(&sip->oldstate_lock); 879 } 880 881 /* 882 * Place client information into stable storage: 1/3. 883 * First, generate the leaf filename, from the client's IP address and 884 * the server-generated short-hand clientid. 885 */ 886 void 887 rfs4_ss_clid(rfs4_client_t *cp, struct svc_req *req) 888 { 889 const char *kinet_ntop6(uchar_t *, char *, size_t); 890 char leaf[MAXNAMELEN], buf[INET6_ADDRSTRLEN]; 891 struct sockaddr *ca; 892 uchar_t *b; 893 894 if (rfs4_ss_enabled == 0) { 895 return; 896 } 897 898 buf[0] = 0; 899 900 901 ca = (struct sockaddr *)svc_getrpccaller(req->rq_xprt)->buf; 902 if (ca == NULL) { 903 return; 904 } 905 906 /* 907 * Convert the caller's IP address to a dotted string 908 */ 909 if (ca->sa_family == AF_INET) { 910 911 bcopy(svc_getrpccaller(req->rq_xprt)->buf, &cp->cl_addr, 912 sizeof (struct sockaddr_in)); 913 b = (uchar_t *)&((struct sockaddr_in *)ca)->sin_addr; 914 (void) sprintf(buf, "%03d.%03d.%03d.%03d", b[0] & 0xFF, 915 b[1] & 0xFF, b[2] & 0xFF, b[3] & 0xFF); 916 } else if (ca->sa_family == AF_INET6) { 917 struct sockaddr_in6 *sin6; 918 919 sin6 = (struct sockaddr_in6 *)ca; 920 bcopy(svc_getrpccaller(req->rq_xprt)->buf, &cp->cl_addr, 921 sizeof (struct sockaddr_in6)); 922 (void) kinet_ntop6((uchar_t *)&sin6->sin6_addr, 923 buf, INET6_ADDRSTRLEN); 924 } 925 926 (void) snprintf(leaf, MAXNAMELEN, "%s-%llx", buf, 927 (longlong_t)cp->clientid); 928 rfs4_ss_clid_write(cp, leaf); 929 } 930 931 /* 932 * Place client information into stable storage: 2/3. 933 * DSS: distributed stable storage: the file may need to be written to 934 * multiple directories. 935 */ 936 static void 937 rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf) 938 { 939 rfs4_servinst_t *sip; 940 941 /* 942 * It should be sufficient to write the leaf file to (all) DSS paths 943 * associated with just this client's instance. However, since our 944 * per-instance client grouping is solely temporal, HA-NFSv4 RG 945 * failover might result in us losing DSS data. 946 * 947 * Until the client grouping is improved, we must write the DSS data 948 * to all instances' paths. Start at the current instance, and 949 * walk the list backwards to the first. 950 */ 951 mutex_enter(&rfs4_servinst_lock); 952 for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) { 953 int i, npaths = sip->dss_npaths; 954 955 /* write the leaf file to all DSS paths */ 956 for (i = 0; i < npaths; i++) { 957 rfs4_dss_path_t *dss_path = sip->dss_paths[i]; 958 959 /* HA-NFSv4 path might have been failed-away from us */ 960 if (dss_path == NULL) 961 continue; 962 963 rfs4_ss_clid_write_one(cp, dss_path->path, leaf); 964 } 965 } 966 mutex_exit(&rfs4_servinst_lock); 967 } 968 969 /* 970 * Place client information into stable storage: 3/3. 971 * Write the stable storage data to the requested file. 972 */ 973 static void 974 rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dss_path, char *leaf) 975 { 976 int ioflag; 977 int file_vers = NFS4_SS_VERSION; 978 size_t dirlen; 979 struct uio uio; 980 struct iovec iov[4]; 981 char *dir; 982 rfs4_ss_pn_t *ss_pn; 983 vnode_t *vp; 984 nfs_client_id4 *cl_id4 = &(cp->nfs_client); 985 986 /* allow 2 extra bytes for '/' & NUL */ 987 dirlen = strlen(dss_path) + strlen(NFS4_DSS_STATE_LEAF) + 2; 988 dir = kmem_alloc(dirlen, KM_SLEEP); 989 (void) sprintf(dir, "%s/%s", dss_path, NFS4_DSS_STATE_LEAF); 990 991 ss_pn = rfs4_ss_pnalloc(dir, leaf); 992 /* rfs4_ss_pnalloc takes its own copy */ 993 kmem_free(dir, dirlen); 994 if (ss_pn == NULL) 995 return; 996 997 if (vn_open(ss_pn->pn, UIO_SYSSPACE, FCREAT|FWRITE, 0600, &vp, 998 CRCREAT, 0)) { 999 rfs4_ss_pnfree(ss_pn); 1000 return; 1001 } 1002 1003 /* 1004 * We need to record leaf - i.e. the filename - so that we know 1005 * what to remove, in the future. However, the dir part of cp->ss_pn 1006 * should never be referenced directly, since it's potentially only 1007 * one of several paths with this leaf in it. 1008 */ 1009 if (cp->ss_pn != NULL) { 1010 if (strcmp(cp->ss_pn->leaf, leaf) == 0) { 1011 /* we've already recorded *this* leaf */ 1012 rfs4_ss_pnfree(ss_pn); 1013 } else { 1014 /* replace with this leaf */ 1015 rfs4_ss_pnfree(cp->ss_pn); 1016 cp->ss_pn = ss_pn; 1017 } 1018 } else { 1019 cp->ss_pn = ss_pn; 1020 } 1021 1022 /* 1023 * Build a scatter list that points to the nfs_client_id4 1024 */ 1025 iov[0].iov_base = (caddr_t)&file_vers; 1026 iov[0].iov_len = sizeof (int); 1027 iov[1].iov_base = (caddr_t)&(cl_id4->verifier); 1028 iov[1].iov_len = NFS4_VERIFIER_SIZE; 1029 iov[2].iov_base = (caddr_t)&(cl_id4->id_len); 1030 iov[2].iov_len = sizeof (uint_t); 1031 iov[3].iov_base = (caddr_t)cl_id4->id_val; 1032 iov[3].iov_len = cl_id4->id_len; 1033 1034 uio.uio_iov = iov; 1035 uio.uio_iovcnt = 4; 1036 uio.uio_loffset = 0; 1037 uio.uio_segflg = UIO_SYSSPACE; 1038 uio.uio_llimit = (rlim64_t)MAXOFFSET_T; 1039 uio.uio_resid = cl_id4->id_len + sizeof (int) + 1040 NFS4_VERIFIER_SIZE + sizeof (uint_t); 1041 1042 ioflag = uio.uio_fmode = (FWRITE|FSYNC); 1043 uio.uio_extflg = UIO_COPY_DEFAULT; 1044 1045 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 1046 /* write the full client id to the file. */ 1047 (void) VOP_WRITE(vp, &uio, ioflag, CRED(), NULL); 1048 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 1049 1050 (void) VOP_CLOSE(vp, FWRITE, 1, (offset_t)0, CRED()); 1051 VN_RELE(vp); 1052 } 1053 1054 /* 1055 * DSS: distributed stable storage. 1056 * Unpack the list of paths passed by nfsd. 1057 * Use nvlist_alloc(9F) to manage the data. 1058 * The caller is responsible for allocating and freeing the buffer. 1059 */ 1060 int 1061 rfs4_dss_setpaths(char *buf, size_t buflen) 1062 { 1063 int error; 1064 1065 /* 1066 * If this is a "warm start", i.e. we previously had DSS paths, 1067 * preserve the old paths. 1068 */ 1069 if (rfs4_dss_paths != NULL) { 1070 /* 1071 * Before we lose the ptr, destroy the nvlist and pathnames 1072 * array from the warm start before this one. 1073 */ 1074 if (rfs4_dss_oldpaths) 1075 nvlist_free(rfs4_dss_oldpaths); 1076 rfs4_dss_oldpaths = rfs4_dss_paths; 1077 } 1078 1079 /* unpack the buffer into a searchable nvlist */ 1080 error = nvlist_unpack(buf, buflen, &rfs4_dss_paths, KM_SLEEP); 1081 if (error) 1082 return (error); 1083 1084 /* 1085 * Search the nvlist for the pathnames nvpair (which is the only nvpair 1086 * in the list, and record its location. 1087 */ 1088 error = nvlist_lookup_string_array(rfs4_dss_paths, NFS4_DSS_NVPAIR_NAME, 1089 &rfs4_dss_newpaths, &rfs4_dss_numnewpaths); 1090 return (error); 1091 } 1092 1093 /* 1094 * Ultimately the nfssys() call NFS4_CLR_STATE endsup here 1095 * to find and mark the client for forced expire. 1096 */ 1097 static void 1098 rfs4_client_scrub(rfs4_entry_t ent, void *arg) 1099 { 1100 rfs4_client_t *cp = (rfs4_client_t *)ent; 1101 struct nfs4clrst_args *clr = arg; 1102 struct sockaddr_in6 *ent_sin6; 1103 struct in6_addr clr_in6; 1104 struct sockaddr_in *ent_sin; 1105 struct in_addr clr_in; 1106 1107 if (clr->addr_type != cp->cl_addr.ss_family) { 1108 return; 1109 } 1110 1111 switch (clr->addr_type) { 1112 1113 case AF_INET6: 1114 /* copyin the address from user space */ 1115 if (copyin(clr->ap, &clr_in6, sizeof (clr_in6))) { 1116 break; 1117 } 1118 1119 ent_sin6 = (struct sockaddr_in6 *)&cp->cl_addr; 1120 1121 /* 1122 * now compare, and if equivalent mark entry 1123 * for forced expiration 1124 */ 1125 if (IN6_ARE_ADDR_EQUAL(&ent_sin6->sin6_addr, &clr_in6)) { 1126 cp->forced_expire = 1; 1127 } 1128 break; 1129 1130 case AF_INET: 1131 /* copyin the address from user space */ 1132 if (copyin(clr->ap, &clr_in, sizeof (clr_in))) { 1133 break; 1134 } 1135 1136 ent_sin = (struct sockaddr_in *)&cp->cl_addr; 1137 1138 /* 1139 * now compare, and if equivalent mark entry 1140 * for forced expiration 1141 */ 1142 if (ent_sin->sin_addr.s_addr == clr_in.s_addr) { 1143 cp->forced_expire = 1; 1144 } 1145 break; 1146 1147 default: 1148 /* force this assert to fail */ 1149 ASSERT(clr->addr_type != clr->addr_type); 1150 } 1151 } 1152 1153 /* 1154 * This is called from nfssys() in order to clear server state 1155 * for the specified client IP Address. 1156 */ 1157 void 1158 rfs4_clear_client_state(struct nfs4clrst_args *clr) 1159 { 1160 (void) rfs4_dbe_walk(rfs4_client_tab, rfs4_client_scrub, clr); 1161 } 1162 1163 /* 1164 * Used to initialize the NFSv4 server's state or database. All of 1165 * the tables are created and timers are set. Only called when NFSv4 1166 * service is provided. 1167 */ 1168 void 1169 rfs4_state_init() 1170 { 1171 int start_grace; 1172 extern boolean_t rfs4_cpr_callb(void *, int); 1173 char *dss_path = NFS4_DSS_VAR_DIR; 1174 1175 mutex_enter(&rfs4_state_lock); 1176 1177 /* 1178 * If the server state database has already been initialized, 1179 * skip it 1180 */ 1181 if (rfs4_server_state != NULL) { 1182 mutex_exit(&rfs4_state_lock); 1183 return; 1184 } 1185 1186 rw_init(&rfs4_findclient_lock, NULL, RW_DEFAULT, NULL); 1187 1188 /* 1189 * Set the boot time. If the server 1190 * has been restarted quickly and has had the opportunity to 1191 * service clients, then the start_time needs to be bumped 1192 * regardless. A small window but it exists... 1193 */ 1194 if (rfs4_start_time != gethrestime_sec()) 1195 rfs4_start_time = gethrestime_sec(); 1196 else 1197 rfs4_start_time++; 1198 1199 /* DSS: distributed stable storage: initialise served paths list */ 1200 rfs4_dss_pathlist = NULL; 1201 1202 /* 1203 * Create the first server instance, or a new one if the server has 1204 * been restarted; see above comments on rfs4_start_time. Don't 1205 * start its grace period; that will be done later, to maximise the 1206 * clients' recovery window. 1207 */ 1208 start_grace = 0; 1209 rfs4_servinst_create(start_grace, 1, &dss_path); 1210 1211 /* reset the "first NFSv4 request" status */ 1212 rfs4_seen_first_compound = 0; 1213 1214 /* 1215 * Add a CPR callback so that we can update client 1216 * access times to extend the lease after a suspend 1217 * and resume (using the same class as rpcmod/connmgr) 1218 */ 1219 cpr_id = callb_add(rfs4_cpr_callb, 0, CB_CL_CPR_RPC, "rfs4"); 1220 1221 /* set the various cache timers for table creation */ 1222 if (rfs4_client_cache_time == 0) 1223 rfs4_client_cache_time = CLIENT_CACHE_TIME; 1224 if (rfs4_openowner_cache_time == 0) 1225 rfs4_openowner_cache_time = OPENOWNER_CACHE_TIME; 1226 if (rfs4_state_cache_time == 0) 1227 rfs4_state_cache_time = STATE_CACHE_TIME; 1228 if (rfs4_lo_state_cache_time == 0) 1229 rfs4_lo_state_cache_time = LO_STATE_CACHE_TIME; 1230 if (rfs4_lockowner_cache_time == 0) 1231 rfs4_lockowner_cache_time = LOCKOWNER_CACHE_TIME; 1232 if (rfs4_file_cache_time == 0) 1233 rfs4_file_cache_time = FILE_CACHE_TIME; 1234 if (rfs4_deleg_state_cache_time == 0) 1235 rfs4_deleg_state_cache_time = DELEG_STATE_CACHE_TIME; 1236 1237 /* Create the overall database to hold all server state */ 1238 rfs4_server_state = rfs4_database_create(rfs4_database_debug); 1239 1240 /* Now create the individual tables */ 1241 rfs4_client_cache_time *= rfs4_lease_time; 1242 rfs4_client_tab = rfs4_table_create(rfs4_server_state, 1243 "Client", 1244 rfs4_client_cache_time, 1245 2, 1246 rfs4_client_create, 1247 rfs4_client_destroy, 1248 rfs4_client_expiry, 1249 sizeof (rfs4_client_t), 1250 TABSIZE, 1251 MAXTABSZ/8, 100); 1252 rfs4_nfsclnt_idx = rfs4_index_create(rfs4_client_tab, 1253 "nfs_client_id4", nfsclnt_hash, 1254 nfsclnt_compare, nfsclnt_mkkey, 1255 TRUE); 1256 rfs4_clientid_idx = rfs4_index_create(rfs4_client_tab, 1257 "client_id", clientid_hash, 1258 clientid_compare, clientid_mkkey, 1259 FALSE); 1260 1261 rfs4_openowner_cache_time *= rfs4_lease_time; 1262 rfs4_openowner_tab = rfs4_table_create(rfs4_server_state, 1263 "OpenOwner", 1264 rfs4_openowner_cache_time, 1265 1, 1266 rfs4_openowner_create, 1267 rfs4_openowner_destroy, 1268 rfs4_openowner_expiry, 1269 sizeof (rfs4_openowner_t), 1270 TABSIZE, 1271 MAXTABSZ, 100); 1272 rfs4_openowner_idx = rfs4_index_create(rfs4_openowner_tab, 1273 "open_owner4", openowner_hash, 1274 openowner_compare, 1275 openowner_mkkey, TRUE); 1276 1277 rfs4_state_cache_time *= rfs4_lease_time; 1278 rfs4_state_tab = rfs4_table_create(rfs4_server_state, 1279 "OpenStateID", 1280 rfs4_state_cache_time, 1281 3, 1282 rfs4_state_create, 1283 rfs4_state_destroy, 1284 rfs4_state_expiry, 1285 sizeof (rfs4_state_t), 1286 TABSIZE, 1287 MAXTABSZ, 100); 1288 1289 rfs4_state_owner_file_idx = rfs4_index_create(rfs4_state_tab, 1290 "Openowner-File", 1291 state_owner_file_hash, 1292 state_owner_file_compare, 1293 state_owner_file_mkkey, TRUE); 1294 1295 rfs4_state_idx = rfs4_index_create(rfs4_state_tab, 1296 "State-id", state_hash, 1297 state_compare, state_mkkey, FALSE); 1298 1299 rfs4_state_file_idx = rfs4_index_create(rfs4_state_tab, 1300 "File", state_file_hash, 1301 state_file_compare, state_file_mkkey, 1302 FALSE); 1303 1304 rfs4_lo_state_cache_time *= rfs4_lease_time; 1305 rfs4_lo_state_tab = rfs4_table_create(rfs4_server_state, 1306 "LockStateID", 1307 rfs4_lo_state_cache_time, 1308 2, 1309 rfs4_lo_state_create, 1310 rfs4_lo_state_destroy, 1311 rfs4_lo_state_expiry, 1312 sizeof (rfs4_lo_state_t), 1313 TABSIZE, 1314 MAXTABSZ, 100); 1315 1316 rfs4_lo_state_owner_idx = rfs4_index_create(rfs4_lo_state_tab, 1317 "lockownerxstate", 1318 lo_state_lo_hash, 1319 lo_state_lo_compare, 1320 lo_state_lo_mkkey, TRUE); 1321 1322 rfs4_lo_state_idx = rfs4_index_create(rfs4_lo_state_tab, 1323 "State-id", 1324 lo_state_hash, lo_state_compare, 1325 lo_state_mkkey, FALSE); 1326 1327 rfs4_lockowner_cache_time *= rfs4_lease_time; 1328 1329 rfs4_lockowner_tab = rfs4_table_create(rfs4_server_state, 1330 "Lockowner", 1331 rfs4_lockowner_cache_time, 1332 2, 1333 rfs4_lockowner_create, 1334 rfs4_lockowner_destroy, 1335 rfs4_lockowner_expiry, 1336 sizeof (rfs4_lockowner_t), 1337 TABSIZE, 1338 MAXTABSZ, 100); 1339 1340 rfs4_lockowner_idx = rfs4_index_create(rfs4_lockowner_tab, 1341 "lock_owner4", lockowner_hash, 1342 lockowner_compare, 1343 lockowner_mkkey, TRUE); 1344 1345 rfs4_lockowner_pid_idx = rfs4_index_create(rfs4_lockowner_tab, 1346 "pid", pid_hash, 1347 pid_compare, pid_mkkey, 1348 FALSE); 1349 1350 rfs4_file_cache_time *= rfs4_lease_time; 1351 rfs4_file_tab = rfs4_table_create(rfs4_server_state, 1352 "File", 1353 rfs4_file_cache_time, 1354 1, 1355 rfs4_file_create, 1356 rfs4_file_destroy, 1357 NULL, 1358 sizeof (rfs4_file_t), 1359 TABSIZE, 1360 MAXTABSZ, -1); 1361 1362 rfs4_file_idx = rfs4_index_create(rfs4_file_tab, 1363 "Filehandle", file_hash, 1364 file_compare, file_mkkey, TRUE); 1365 1366 rfs4_deleg_state_cache_time *= rfs4_lease_time; 1367 rfs4_deleg_state_tab = rfs4_table_create(rfs4_server_state, 1368 "DelegStateID", 1369 rfs4_deleg_state_cache_time, 1370 2, 1371 rfs4_deleg_state_create, 1372 rfs4_deleg_state_destroy, 1373 rfs4_deleg_state_expiry, 1374 sizeof (rfs4_deleg_state_t), 1375 TABSIZE, 1376 MAXTABSZ, 100); 1377 rfs4_deleg_idx = rfs4_index_create(rfs4_deleg_state_tab, 1378 "DelegByFileClient", 1379 deleg_hash, 1380 deleg_compare, 1381 deleg_mkkey, TRUE); 1382 1383 rfs4_deleg_state_idx = rfs4_index_create(rfs4_deleg_state_tab, 1384 "DelegState", 1385 deleg_state_hash, 1386 deleg_state_compare, 1387 deleg_state_mkkey, FALSE); 1388 1389 /* 1390 * Init the stable storage. 1391 */ 1392 rfs4_ss_init(); 1393 1394 rfs4_client_clrst = rfs4_clear_client_state; 1395 1396 mutex_exit(&rfs4_state_lock); 1397 } 1398 1399 1400 /* 1401 * Used at server shutdown to cleanup all of the NFSv4 server's structures 1402 * and other state. 1403 */ 1404 void 1405 rfs4_state_fini() 1406 { 1407 rfs4_database_t *dbp; 1408 1409 mutex_enter(&rfs4_state_lock); 1410 1411 if (rfs4_server_state == NULL) { 1412 mutex_exit(&rfs4_state_lock); 1413 return; 1414 } 1415 1416 rfs4_client_clrst = NULL; 1417 1418 rfs4_set_deleg_policy(SRV_NEVER_DELEGATE); 1419 dbp = rfs4_server_state; 1420 rfs4_server_state = NULL; 1421 1422 /* 1423 * Cleanup the CPR callback. 1424 */ 1425 if (cpr_id) 1426 (void) callb_delete(cpr_id); 1427 1428 rw_destroy(&rfs4_findclient_lock); 1429 1430 /* First stop all of the reaper threads in the database */ 1431 rfs4_database_shutdown(dbp); 1432 /* clean up any dangling stable storage structures */ 1433 rfs4_ss_fini(); 1434 /* Now actually destroy/release the database and its tables */ 1435 rfs4_database_destroy(dbp); 1436 1437 /* Reset the cache timers for next time */ 1438 rfs4_client_cache_time = 0; 1439 rfs4_openowner_cache_time = 0; 1440 rfs4_state_cache_time = 0; 1441 rfs4_lo_state_cache_time = 0; 1442 rfs4_lockowner_cache_time = 0; 1443 rfs4_file_cache_time = 0; 1444 rfs4_deleg_state_cache_time = 0; 1445 1446 mutex_exit(&rfs4_state_lock); 1447 1448 /* destroy server instances and current instance ptr */ 1449 rfs4_servinst_destroy_all(); 1450 1451 /* reset the "first NFSv4 request" status */ 1452 rfs4_seen_first_compound = 0; 1453 1454 /* DSS: distributed stable storage */ 1455 if (rfs4_dss_oldpaths) 1456 nvlist_free(rfs4_dss_oldpaths); 1457 if (rfs4_dss_paths) 1458 nvlist_free(rfs4_dss_paths); 1459 rfs4_dss_paths = rfs4_dss_oldpaths = NULL; 1460 } 1461 1462 typedef union { 1463 struct { 1464 uint32_t start_time; 1465 uint32_t c_id; 1466 } impl_id; 1467 clientid4 id4; 1468 } cid; 1469 1470 static int foreign_stateid(stateid_t *id); 1471 static int foreign_clientid(cid *cidp); 1472 static void embed_nodeid(cid *cidp); 1473 1474 typedef union { 1475 struct { 1476 uint32_t c_id; 1477 uint32_t gen_num; 1478 } cv_impl; 1479 verifier4 confirm_verf; 1480 } scid_confirm_verf; 1481 1482 static uint32_t 1483 clientid_hash(void *key) 1484 { 1485 cid *idp = key; 1486 1487 return (idp->impl_id.c_id); 1488 } 1489 1490 static bool_t 1491 clientid_compare(rfs4_entry_t entry, void *key) 1492 { 1493 rfs4_client_t *client = (rfs4_client_t *)entry; 1494 clientid4 *idp = key; 1495 1496 return (*idp == client->clientid); 1497 } 1498 1499 static void * 1500 clientid_mkkey(rfs4_entry_t entry) 1501 { 1502 rfs4_client_t *client = (rfs4_client_t *)entry; 1503 1504 return (&client->clientid); 1505 } 1506 1507 static uint32_t 1508 nfsclnt_hash(void *key) 1509 { 1510 nfs_client_id4 *client = key; 1511 int i; 1512 uint32_t hash = 0; 1513 1514 for (i = 0; i < client->id_len; i++) { 1515 hash <<= 1; 1516 hash += (uint_t)client->id_val[i]; 1517 } 1518 return (hash); 1519 } 1520 1521 1522 static bool_t 1523 nfsclnt_compare(rfs4_entry_t entry, void *key) 1524 { 1525 rfs4_client_t *client = (rfs4_client_t *)entry; 1526 nfs_client_id4 *nfs_client = key; 1527 1528 if (client->nfs_client.id_len != nfs_client->id_len) 1529 return (FALSE); 1530 1531 return (bcmp(client->nfs_client.id_val, nfs_client->id_val, 1532 nfs_client->id_len) == 0); 1533 } 1534 1535 static void * 1536 nfsclnt_mkkey(rfs4_entry_t entry) 1537 { 1538 rfs4_client_t *client = (rfs4_client_t *)entry; 1539 1540 return (&client->nfs_client); 1541 } 1542 1543 static bool_t 1544 rfs4_client_expiry(rfs4_entry_t u_entry) 1545 { 1546 rfs4_client_t *cp = (rfs4_client_t *)u_entry; 1547 bool_t cp_expired; 1548 1549 if (rfs4_dbe_is_invalid(cp->dbe)) 1550 return (TRUE); 1551 /* 1552 * If the sysadmin has used clear_locks for this 1553 * entry then forced_expire will be set and we 1554 * want this entry to be reaped. Or the entry 1555 * has exceeded its lease period. 1556 */ 1557 cp_expired = (cp->forced_expire || 1558 (gethrestime_sec() - cp->last_access 1559 > rfs4_lease_time)); 1560 1561 if (!cp->ss_remove && cp_expired) 1562 cp->ss_remove = 1; 1563 return (cp_expired); 1564 } 1565 1566 /* 1567 * Remove the leaf file from all distributed stable storage paths. 1568 */ 1569 static void 1570 rfs4_dss_remove_cpleaf(rfs4_client_t *cp) 1571 { 1572 char *leaf = cp->ss_pn->leaf; 1573 1574 rfs4_dss_remove_leaf(cp->server_instance, NFS4_DSS_STATE_LEAF, leaf); 1575 } 1576 1577 static void 1578 rfs4_dss_remove_leaf(rfs4_servinst_t *sip, char *dir_leaf, char *leaf) 1579 { 1580 int i, npaths = sip->dss_npaths; 1581 1582 for (i = 0; i < npaths; i++) { 1583 rfs4_dss_path_t *dss_path = sip->dss_paths[i]; 1584 char *path, *dir; 1585 size_t pathlen; 1586 1587 /* the HA-NFSv4 path might have been failed-over away from us */ 1588 if (dss_path == NULL) 1589 continue; 1590 1591 dir = dss_path->path; 1592 1593 /* allow 3 extra bytes for two '/' & a NUL */ 1594 pathlen = strlen(dir) + strlen(dir_leaf) + strlen(leaf) + 3; 1595 path = kmem_alloc(pathlen, KM_SLEEP); 1596 (void) sprintf(path, "%s/%s/%s", dir, dir_leaf, leaf); 1597 1598 (void) vn_remove(path, UIO_SYSSPACE, RMFILE); 1599 1600 kmem_free(path, pathlen); 1601 } 1602 } 1603 1604 static void 1605 rfs4_client_destroy(rfs4_entry_t u_entry) 1606 { 1607 rfs4_client_t *cp = (rfs4_client_t *)u_entry; 1608 1609 mutex_destroy(cp->cbinfo.cb_lock); 1610 cv_destroy(cp->cbinfo.cb_cv); 1611 cv_destroy(cp->cbinfo.cb_cv_nullcaller); 1612 1613 /* free callback info */ 1614 rfs4_cbinfo_free(&cp->cbinfo); 1615 1616 if (cp->cp_confirmed) 1617 rfs4_client_rele(cp->cp_confirmed); 1618 1619 if (cp->ss_pn) { 1620 /* check if the stable storage files need to be removed */ 1621 if (cp->ss_remove) 1622 rfs4_dss_remove_cpleaf(cp); 1623 rfs4_ss_pnfree(cp->ss_pn); 1624 } 1625 1626 /* Free the client supplied client id */ 1627 kmem_free(cp->nfs_client.id_val, cp->nfs_client.id_len); 1628 1629 if (cp->sysidt != LM_NOSYSID) 1630 lm_free_sysidt(cp->sysidt); 1631 } 1632 1633 static bool_t 1634 rfs4_client_create(rfs4_entry_t u_entry, void *arg) 1635 { 1636 rfs4_client_t *cp = (rfs4_client_t *)u_entry; 1637 nfs_client_id4 *client = (nfs_client_id4 *)arg; 1638 cid *cidp; 1639 scid_confirm_verf *scvp; 1640 1641 /* Get a clientid to give to the client */ 1642 cidp = (cid *)&cp->clientid; 1643 cidp->impl_id.start_time = rfs4_start_time; 1644 cidp->impl_id.c_id = (uint32_t)rfs4_dbe_getid(cp->dbe); 1645 1646 /* If we are booted as a cluster node, embed our nodeid */ 1647 if (cluster_bootflags & CLUSTER_BOOTED) 1648 embed_nodeid(cidp); 1649 1650 /* Allocate and copy client's client id value */ 1651 cp->nfs_client.id_val = kmem_alloc(client->id_len, KM_SLEEP); 1652 cp->nfs_client.id_len = client->id_len; 1653 bcopy(client->id_val, cp->nfs_client.id_val, client->id_len); 1654 cp->nfs_client.verifier = client->verifier; 1655 1656 /* Init the value for the SETCLIENTID_CONFIRM verifier */ 1657 scvp = (scid_confirm_verf *)&cp->confirm_verf; 1658 scvp->cv_impl.c_id = cidp->impl_id.c_id; 1659 scvp->cv_impl.gen_num = 0; 1660 1661 /* An F_UNLKSYS has been done for this client */ 1662 cp->unlksys_completed = FALSE; 1663 1664 /* We need the client to ack us */ 1665 cp->need_confirm = TRUE; 1666 cp->cp_confirmed = NULL; 1667 1668 /* TRUE all the time until the callback path actually fails */ 1669 cp->cbinfo.cb_notified_of_cb_path_down = TRUE; 1670 1671 /* Initialize the access time to now */ 1672 cp->last_access = gethrestime_sec(); 1673 1674 cp->cr_set = NULL; 1675 /* Initialize list for insque/remque */ 1676 cp->openownerlist.next = cp->openownerlist.prev = &cp->openownerlist; 1677 cp->openownerlist.oop = NULL; /* This is not an openowner */ 1678 1679 cp->sysidt = LM_NOSYSID; 1680 1681 cp->clientdeleglist.next = cp->clientdeleglist.prev = 1682 &cp->clientdeleglist; 1683 cp->clientdeleglist.dsp = NULL; 1684 1685 /* set up the callback control structure */ 1686 cp->cbinfo.cb_state = CB_UNINIT; 1687 mutex_init(cp->cbinfo.cb_lock, NULL, MUTEX_DEFAULT, NULL); 1688 cv_init(cp->cbinfo.cb_cv, NULL, CV_DEFAULT, NULL); 1689 cv_init(cp->cbinfo.cb_cv_nullcaller, NULL, CV_DEFAULT, NULL); 1690 1691 /* 1692 * Associate the client_t with the current server instance. 1693 * The hold is solely to satisfy the calling requirement of 1694 * rfs4_servinst_assign(). In this case it's not strictly necessary. 1695 */ 1696 rfs4_dbe_hold(cp->dbe); 1697 rfs4_servinst_assign(cp, rfs4_cur_servinst); 1698 rfs4_dbe_rele(cp->dbe); 1699 1700 return (TRUE); 1701 } 1702 1703 /* 1704 * Caller wants to generate/update the setclientid_confirm verifier 1705 * associated with a client. This is done during the SETCLIENTID 1706 * processing. 1707 */ 1708 void 1709 rfs4_client_scv_next(rfs4_client_t *cp) 1710 { 1711 scid_confirm_verf *scvp; 1712 1713 /* Init the value for the SETCLIENTID_CONFIRM verifier */ 1714 scvp = (scid_confirm_verf *)&cp->confirm_verf; 1715 scvp->cv_impl.gen_num++; 1716 } 1717 1718 void 1719 rfs4_client_rele(rfs4_client_t *cp) 1720 { 1721 rfs4_dbe_rele(cp->dbe); 1722 } 1723 1724 rfs4_client_t * 1725 rfs4_findclient(nfs_client_id4 *client, bool_t *create, rfs4_client_t *oldcp) 1726 { 1727 rfs4_client_t *cp; 1728 1729 1730 if (oldcp) { 1731 rw_enter(&rfs4_findclient_lock, RW_WRITER); 1732 rfs4_dbe_hide(oldcp->dbe); 1733 } else { 1734 rw_enter(&rfs4_findclient_lock, RW_READER); 1735 } 1736 1737 cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_nfsclnt_idx, client, 1738 create, (void *)client, RFS4_DBS_VALID); 1739 1740 if (oldcp) 1741 rfs4_dbe_unhide(oldcp->dbe); 1742 1743 rw_exit(&rfs4_findclient_lock); 1744 1745 return (cp); 1746 } 1747 1748 rfs4_client_t * 1749 rfs4_findclient_by_id(clientid4 clientid, bool_t find_unconfirmed) 1750 { 1751 rfs4_client_t *cp; 1752 bool_t create = FALSE; 1753 cid *cidp = (cid *)&clientid; 1754 1755 /* If we're a cluster and the nodeid isn't right, short-circuit */ 1756 if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp)) 1757 return (NULL); 1758 1759 rw_enter(&rfs4_findclient_lock, RW_READER); 1760 1761 cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx, &clientid, 1762 &create, NULL, RFS4_DBS_VALID); 1763 1764 rw_exit(&rfs4_findclient_lock); 1765 1766 if (cp && cp->need_confirm && find_unconfirmed == FALSE) { 1767 rfs4_client_rele(cp); 1768 return (NULL); 1769 } else { 1770 return (cp); 1771 } 1772 } 1773 1774 bool_t 1775 rfs4_lease_expired(rfs4_client_t *cp) 1776 { 1777 bool_t rc; 1778 1779 rfs4_dbe_lock(cp->dbe); 1780 1781 /* 1782 * If the admin has executed clear_locks for this 1783 * client id, force expire will be set, so no need 1784 * to calculate anything because it's "outa here". 1785 */ 1786 if (cp->forced_expire) { 1787 rc = TRUE; 1788 } else { 1789 rc = (gethrestime_sec() - cp->last_access > rfs4_lease_time); 1790 } 1791 1792 /* 1793 * If the lease has expired we will also want 1794 * to remove any stable storage state data. So 1795 * mark the client id accordingly. 1796 */ 1797 if (!cp->ss_remove) 1798 cp->ss_remove = (rc == TRUE); 1799 1800 rfs4_dbe_unlock(cp->dbe); 1801 1802 return (rc); 1803 } 1804 1805 void 1806 rfs4_update_lease(rfs4_client_t *cp) 1807 { 1808 rfs4_dbe_lock(cp->dbe); 1809 if (!cp->forced_expire) 1810 cp->last_access = gethrestime_sec(); 1811 rfs4_dbe_unlock(cp->dbe); 1812 } 1813 1814 1815 static bool_t 1816 EQOPENOWNER(open_owner4 *a, open_owner4 *b) 1817 { 1818 bool_t rc; 1819 1820 if (a->clientid != b->clientid) 1821 return (FALSE); 1822 1823 if (a->owner_len != b->owner_len) 1824 return (FALSE); 1825 1826 rc = (bcmp(a->owner_val, b->owner_val, a->owner_len) == 0); 1827 1828 return (rc); 1829 } 1830 1831 static uint_t 1832 openowner_hash(void *key) 1833 { 1834 int i; 1835 open_owner4 *openowner = key; 1836 uint_t hash = 0; 1837 1838 for (i = 0; i < openowner->owner_len; i++) { 1839 hash <<= 4; 1840 hash += (uint_t)openowner->owner_val[i]; 1841 } 1842 hash += (uint_t)openowner->clientid; 1843 hash |= (openowner->clientid >> 32); 1844 1845 return (hash); 1846 } 1847 1848 static bool_t 1849 openowner_compare(rfs4_entry_t u_entry, void *key) 1850 { 1851 rfs4_openowner_t *op = (rfs4_openowner_t *)u_entry; 1852 open_owner4 *arg = key; 1853 1854 return (EQOPENOWNER(&op->owner, arg)); 1855 } 1856 1857 void * 1858 openowner_mkkey(rfs4_entry_t u_entry) 1859 { 1860 rfs4_openowner_t *op = (rfs4_openowner_t *)u_entry; 1861 1862 return (&op->owner); 1863 } 1864 1865 static bool_t 1866 rfs4_openowner_expiry(rfs4_entry_t u_entry) 1867 { 1868 rfs4_openowner_t *op = (rfs4_openowner_t *)u_entry; 1869 1870 if (rfs4_dbe_is_invalid(op->dbe)) 1871 return (TRUE); 1872 return ((gethrestime_sec() - op->client->last_access 1873 > rfs4_lease_time)); 1874 } 1875 1876 static void 1877 rfs4_openowner_destroy(rfs4_entry_t u_entry) 1878 { 1879 rfs4_openowner_t *op = (rfs4_openowner_t *)u_entry; 1880 1881 rfs4_sw_destroy(&op->oo_sw); 1882 1883 /* Remove open owner from client's lists of open owners */ 1884 rfs4_dbe_lock(op->client->dbe); 1885 1886 remque(&op->openownerlist); 1887 op->openownerlist.next = op->openownerlist.prev = &op->openownerlist; 1888 1889 rfs4_dbe_unlock(op->client->dbe); 1890 1891 /* One less reference to the client */ 1892 rfs4_client_rele(op->client); 1893 op->client = NULL; 1894 1895 /* Free the last reply for this lock owner */ 1896 rfs4_free_reply(op->reply); 1897 1898 if (op->reply_fh.nfs_fh4_val) { 1899 kmem_free(op->reply_fh.nfs_fh4_val, op->reply_fh.nfs_fh4_len); 1900 op->reply_fh.nfs_fh4_val = NULL; 1901 op->reply_fh.nfs_fh4_len = 0; 1902 } 1903 1904 /* Free the lock owner id */ 1905 kmem_free(op->owner.owner_val, op->owner.owner_len); 1906 } 1907 1908 void 1909 rfs4_openowner_rele(rfs4_openowner_t *op) 1910 { 1911 rfs4_dbe_rele(op->dbe); 1912 } 1913 1914 static bool_t 1915 rfs4_openowner_create(rfs4_entry_t u_entry, void *arg) 1916 { 1917 rfs4_openowner_t *op = (rfs4_openowner_t *)u_entry; 1918 rfs4_openowner_t *argp = (rfs4_openowner_t *)arg; 1919 open_owner4 *openowner = &argp->owner; 1920 seqid4 seqid = argp->open_seqid; 1921 rfs4_client_t *cp; 1922 bool_t create = FALSE; 1923 1924 rw_enter(&rfs4_findclient_lock, RW_READER); 1925 1926 cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx, 1927 &openowner->clientid, 1928 &create, NULL, RFS4_DBS_VALID); 1929 1930 rw_exit(&rfs4_findclient_lock); 1931 1932 if (cp == NULL) 1933 return (FALSE); 1934 1935 op->reply_fh.nfs_fh4_len = 0; 1936 op->reply_fh.nfs_fh4_val = NULL; 1937 1938 op->owner.clientid = openowner->clientid; 1939 op->owner.owner_val = 1940 kmem_alloc(openowner->owner_len, KM_SLEEP); 1941 1942 bcopy(openowner->owner_val, 1943 op->owner.owner_val, openowner->owner_len); 1944 1945 op->owner.owner_len = openowner->owner_len; 1946 1947 op->need_confirm = TRUE; 1948 1949 rfs4_sw_init(&op->oo_sw); 1950 1951 op->open_seqid = seqid; 1952 bzero(op->reply, sizeof (nfs_resop4)); 1953 op->client = cp; 1954 op->cr_set = NULL; 1955 /* Init lists for remque/insque */ 1956 op->ownerstateids.next = op->ownerstateids.prev = &op->ownerstateids; 1957 op->ownerstateids.sp = NULL; /* NULL since this is the state list */ 1958 op->openownerlist.next = op->openownerlist.prev = &op->openownerlist; 1959 op->openownerlist.oop = op; /* ourselves */ 1960 1961 /* Insert openowner into client's open owner list */ 1962 rfs4_dbe_lock(cp->dbe); 1963 1964 insque(&op->openownerlist, cp->openownerlist.prev); 1965 1966 rfs4_dbe_unlock(cp->dbe); 1967 1968 return (TRUE); 1969 } 1970 1971 rfs4_openowner_t * 1972 rfs4_findopenowner(open_owner4 *openowner, bool_t *create, seqid4 seqid) 1973 { 1974 rfs4_openowner_t *op; 1975 rfs4_openowner_t arg; 1976 1977 arg.owner = *openowner; 1978 arg.open_seqid = seqid; 1979 op = (rfs4_openowner_t *)rfs4_dbsearch(rfs4_openowner_idx, openowner, 1980 create, &arg, RFS4_DBS_VALID); 1981 1982 return (op); 1983 } 1984 1985 void 1986 rfs4_update_open_sequence(rfs4_openowner_t *op) 1987 { 1988 1989 rfs4_dbe_lock(op->dbe); 1990 1991 op->open_seqid++; 1992 1993 rfs4_dbe_unlock(op->dbe); 1994 } 1995 1996 void 1997 rfs4_update_open_resp(rfs4_openowner_t *op, nfs_resop4 *resp, nfs_fh4 *fh) 1998 { 1999 2000 rfs4_dbe_lock(op->dbe); 2001 2002 rfs4_free_reply(op->reply); 2003 2004 rfs4_copy_reply(op->reply, resp); 2005 2006 /* Save the filehandle if provided and free if not used */ 2007 if (resp->nfs_resop4_u.opopen.status == NFS4_OK && 2008 fh && fh->nfs_fh4_len) { 2009 if (op->reply_fh.nfs_fh4_val == NULL) 2010 op->reply_fh.nfs_fh4_val = 2011 kmem_alloc(fh->nfs_fh4_len, KM_SLEEP); 2012 nfs_fh4_copy(fh, &op->reply_fh); 2013 } else { 2014 if (op->reply_fh.nfs_fh4_val) { 2015 kmem_free(op->reply_fh.nfs_fh4_val, 2016 op->reply_fh.nfs_fh4_len); 2017 op->reply_fh.nfs_fh4_val = NULL; 2018 op->reply_fh.nfs_fh4_len = 0; 2019 } 2020 } 2021 2022 rfs4_dbe_unlock(op->dbe); 2023 } 2024 2025 static bool_t 2026 lockowner_compare(rfs4_entry_t u_entry, void *key) 2027 { 2028 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2029 lock_owner4 *b = (lock_owner4 *)key; 2030 2031 if (lo->owner.clientid != b->clientid) 2032 return (FALSE); 2033 2034 if (lo->owner.owner_len != b->owner_len) 2035 return (FALSE); 2036 2037 return (bcmp(lo->owner.owner_val, b->owner_val, 2038 lo->owner.owner_len) == 0); 2039 } 2040 2041 void * 2042 lockowner_mkkey(rfs4_entry_t u_entry) 2043 { 2044 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2045 2046 return (&lo->owner); 2047 } 2048 2049 static uint32_t 2050 lockowner_hash(void *key) 2051 { 2052 int i; 2053 lock_owner4 *lockowner = key; 2054 uint_t hash = 0; 2055 2056 for (i = 0; i < lockowner->owner_len; i++) { 2057 hash <<= 4; 2058 hash += (uint_t)lockowner->owner_val[i]; 2059 } 2060 hash += (uint_t)lockowner->clientid; 2061 hash |= (lockowner->clientid >> 32); 2062 2063 return (hash); 2064 } 2065 2066 static uint32_t 2067 pid_hash(void *key) 2068 { 2069 return ((uint32_t)(uintptr_t)key); 2070 } 2071 2072 static void * 2073 pid_mkkey(rfs4_entry_t u_entry) 2074 { 2075 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2076 2077 return ((void *)(uintptr_t)lo->pid); 2078 } 2079 2080 static bool_t 2081 pid_compare(rfs4_entry_t u_entry, void *key) 2082 { 2083 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2084 2085 return (lo->pid == (pid_t)(uintptr_t)key); 2086 } 2087 2088 static void 2089 rfs4_lockowner_destroy(rfs4_entry_t u_entry) 2090 { 2091 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2092 2093 /* Free the lock owner id */ 2094 kmem_free(lo->owner.owner_val, lo->owner.owner_len); 2095 rfs4_client_rele(lo->client); 2096 } 2097 2098 void 2099 rfs4_lockowner_rele(rfs4_lockowner_t *lo) 2100 { 2101 rfs4_dbe_rele(lo->dbe); 2102 } 2103 2104 /* ARGSUSED */ 2105 static bool_t 2106 rfs4_lockowner_expiry(rfs4_entry_t u_entry) 2107 { 2108 /* 2109 * Since expiry is called with no other references on 2110 * this struct, go ahead and have it removed. 2111 */ 2112 return (TRUE); 2113 } 2114 2115 static bool_t 2116 rfs4_lockowner_create(rfs4_entry_t u_entry, void *arg) 2117 { 2118 rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry; 2119 lock_owner4 *lockowner = (lock_owner4 *)arg; 2120 rfs4_client_t *cp; 2121 bool_t create = FALSE; 2122 2123 rw_enter(&rfs4_findclient_lock, RW_READER); 2124 2125 cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx, 2126 &lockowner->clientid, 2127 &create, NULL, RFS4_DBS_VALID); 2128 2129 rw_exit(&rfs4_findclient_lock); 2130 2131 if (cp == NULL) 2132 return (FALSE); 2133 2134 /* Reference client */ 2135 lo->client = cp; 2136 lo->owner.clientid = lockowner->clientid; 2137 lo->owner.owner_val = kmem_alloc(lockowner->owner_len, KM_SLEEP); 2138 bcopy(lockowner->owner_val, lo->owner.owner_val, lockowner->owner_len); 2139 lo->owner.owner_len = lockowner->owner_len; 2140 lo->pid = rfs4_dbe_getid(lo->dbe); 2141 2142 return (TRUE); 2143 } 2144 2145 rfs4_lockowner_t * 2146 rfs4_findlockowner(lock_owner4 *lockowner, bool_t *create) 2147 { 2148 rfs4_lockowner_t *lo; 2149 2150 lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_idx, lockowner, 2151 create, lockowner, RFS4_DBS_VALID); 2152 2153 return (lo); 2154 } 2155 2156 rfs4_lockowner_t * 2157 rfs4_findlockowner_by_pid(pid_t pid) 2158 { 2159 rfs4_lockowner_t *lo; 2160 bool_t create = FALSE; 2161 2162 lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_pid_idx, 2163 (void *)(uintptr_t)pid, &create, NULL, RFS4_DBS_VALID); 2164 2165 return (lo); 2166 } 2167 2168 2169 static uint32_t 2170 file_hash(void *key) 2171 { 2172 return (ADDRHASH(key)); 2173 } 2174 2175 static void * 2176 file_mkkey(rfs4_entry_t u_entry) 2177 { 2178 rfs4_file_t *fp = (rfs4_file_t *)u_entry; 2179 2180 return (fp->vp); 2181 } 2182 2183 static bool_t 2184 file_compare(rfs4_entry_t u_entry, void *key) 2185 { 2186 rfs4_file_t *fp = (rfs4_file_t *)u_entry; 2187 2188 return (fp->vp == (vnode_t *)key); 2189 } 2190 2191 static void 2192 rfs4_file_destroy(rfs4_entry_t u_entry) 2193 { 2194 rfs4_file_t *fp = (rfs4_file_t *)u_entry; 2195 2196 ASSERT(fp->delegationlist.next == &fp->delegationlist); 2197 if (fp->filehandle.nfs_fh4_val) 2198 kmem_free(fp->filehandle.nfs_fh4_val, 2199 fp->filehandle.nfs_fh4_len); 2200 cv_destroy(fp->dinfo->recall_cv); 2201 if (fp->vp) { 2202 vnode_t *vp = fp->vp; 2203 2204 mutex_enter(&vp->v_lock); 2205 (void) vsd_set(vp, nfs4_srv_vkey, NULL); 2206 mutex_exit(&vp->v_lock); 2207 VN_RELE(vp); 2208 fp->vp = NULL; 2209 } 2210 rw_destroy(&fp->file_rwlock); 2211 } 2212 2213 /* 2214 * Used to unlock the underlying dbe struct only 2215 */ 2216 void 2217 rfs4_file_rele(rfs4_file_t *fp) 2218 { 2219 rfs4_dbe_rele(fp->dbe); 2220 } 2221 2222 /* 2223 * Used to unlock the file rw lock and the file's dbe entry 2224 * Only used to pair with rfs4_findfile_withlock() 2225 */ 2226 void 2227 rfs4_file_rele_withunlock(rfs4_file_t *fp) 2228 { 2229 rw_exit(&fp->file_rwlock); 2230 rfs4_dbe_rele(fp->dbe); 2231 } 2232 2233 typedef struct { 2234 vnode_t *vp; 2235 nfs_fh4 *fh; 2236 } rfs4_fcreate_arg; 2237 2238 static bool_t 2239 rfs4_file_create(rfs4_entry_t u_entry, void *arg) 2240 { 2241 rfs4_file_t *fp = (rfs4_file_t *)u_entry; 2242 rfs4_fcreate_arg *ap = (rfs4_fcreate_arg *)arg; 2243 vnode_t *vp = ap->vp; 2244 nfs_fh4 *fh = ap->fh; 2245 2246 VN_HOLD(vp); 2247 2248 fp->filehandle.nfs_fh4_len = 0; 2249 fp->filehandle.nfs_fh4_val = NULL; 2250 ASSERT(fh && fh->nfs_fh4_len); 2251 if (fh && fh->nfs_fh4_len) { 2252 fp->filehandle.nfs_fh4_val = 2253 kmem_alloc(fh->nfs_fh4_len, KM_SLEEP); 2254 nfs_fh4_copy(fh, &fp->filehandle); 2255 } 2256 fp->vp = vp; 2257 2258 /* Init list for remque/insque */ 2259 fp->delegationlist.next = fp->delegationlist.prev = 2260 &fp->delegationlist; 2261 fp->delegationlist.dsp = NULL; /* NULL since this is state list */ 2262 2263 fp->share_deny = fp->share_access = fp->access_read = 0; 2264 fp->access_write = fp->deny_read = fp->deny_write = 0; 2265 2266 mutex_init(fp->dinfo->recall_lock, NULL, MUTEX_DEFAULT, NULL); 2267 cv_init(fp->dinfo->recall_cv, NULL, CV_DEFAULT, NULL); 2268 2269 fp->dinfo->dtype = OPEN_DELEGATE_NONE; 2270 2271 rw_init(&fp->file_rwlock, NULL, RW_DEFAULT, NULL); 2272 2273 mutex_enter(&vp->v_lock); 2274 if (vsd_set(vp, nfs4_srv_vkey, (void *)fp)) { 2275 ASSERT(FALSE); 2276 cmn_err(CE_WARN, "rfs4_file_create: vsd_set failed."); 2277 } 2278 mutex_exit(&vp->v_lock); 2279 2280 return (TRUE); 2281 } 2282 2283 rfs4_file_t * 2284 rfs4_findfile(vnode_t *vp, nfs_fh4 *fh, bool_t *create) 2285 { 2286 rfs4_file_t *fp; 2287 rfs4_fcreate_arg arg; 2288 2289 arg.vp = vp; 2290 arg.fh = fh; 2291 2292 if (*create == TRUE) 2293 fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create, 2294 &arg, RFS4_DBS_VALID); 2295 else { 2296 mutex_enter(&vp->v_lock); 2297 fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey); 2298 mutex_exit(&vp->v_lock); 2299 if (fp) { 2300 rfs4_dbe_lock(fp->dbe); 2301 if (rfs4_dbe_is_invalid(fp->dbe) || 2302 (rfs4_dbe_refcnt(fp->dbe) == 0)) { 2303 rfs4_dbe_unlock(fp->dbe); 2304 fp = NULL; 2305 } else { 2306 rfs4_dbe_hold(fp->dbe); 2307 rfs4_dbe_unlock(fp->dbe); 2308 } 2309 } 2310 } 2311 return (fp); 2312 } 2313 2314 /* 2315 * Find a file in the db and once it is located, take the rw lock. 2316 * Need to check the vnode pointer and if it does not exist (it was 2317 * removed between the db location and check) redo the find. This 2318 * assumes that a file struct that has a NULL vnode pointer is marked 2319 * at 'invalid' and will not be found in the db the second time 2320 * around. 2321 */ 2322 rfs4_file_t * 2323 rfs4_findfile_withlock(vnode_t *vp, nfs_fh4 *fh, bool_t *create) 2324 { 2325 rfs4_file_t *fp; 2326 rfs4_fcreate_arg arg; 2327 bool_t screate = *create; 2328 2329 if (screate == FALSE) { 2330 mutex_enter(&vp->v_lock); 2331 fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey); 2332 mutex_exit(&vp->v_lock); 2333 if (fp) { 2334 rfs4_dbe_lock(fp->dbe); 2335 if (rfs4_dbe_is_invalid(fp->dbe) || 2336 (rfs4_dbe_refcnt(fp->dbe) == 0)) { 2337 rfs4_dbe_unlock(fp->dbe); 2338 fp = NULL; 2339 } else { 2340 rfs4_dbe_hold(fp->dbe); 2341 rfs4_dbe_unlock(fp->dbe); 2342 rw_enter(&fp->file_rwlock, RW_WRITER); 2343 } 2344 } 2345 } else { 2346 retry: 2347 arg.vp = vp; 2348 arg.fh = fh; 2349 2350 fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create, 2351 &arg, RFS4_DBS_VALID); 2352 if (fp != NULL) { 2353 rw_enter(&fp->file_rwlock, RW_WRITER); 2354 if (fp->vp == NULL) { 2355 rw_exit(&fp->file_rwlock); 2356 rfs4_file_rele(fp); 2357 *create = screate; 2358 goto retry; 2359 } 2360 } 2361 } 2362 2363 return (fp); 2364 } 2365 2366 static uint32_t 2367 lo_state_hash(void *key) 2368 { 2369 stateid_t *id = key; 2370 2371 return (id->bits.ident+id->bits.pid); 2372 } 2373 2374 static bool_t 2375 lo_state_compare(rfs4_entry_t u_entry, void *key) 2376 { 2377 rfs4_lo_state_t *lop = (rfs4_lo_state_t *)u_entry; 2378 stateid_t *id = key; 2379 bool_t rc; 2380 2381 rc = (lop->lockid.bits.boottime == id->bits.boottime && 2382 lop->lockid.bits.type == id->bits.type && 2383 lop->lockid.bits.ident == id->bits.ident && 2384 lop->lockid.bits.pid == id->bits.pid); 2385 2386 return (rc); 2387 } 2388 2389 static void * 2390 lo_state_mkkey(rfs4_entry_t u_entry) 2391 { 2392 rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry; 2393 2394 return (&lsp->lockid); 2395 } 2396 2397 static bool_t 2398 rfs4_lo_state_expiry(rfs4_entry_t u_entry) 2399 { 2400 rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry; 2401 2402 if (rfs4_dbe_is_invalid(lsp->dbe)) 2403 return (TRUE); 2404 if (lsp->state->closed) 2405 return (TRUE); 2406 return ((gethrestime_sec() - lsp->state->owner->client->last_access 2407 > rfs4_lease_time)); 2408 } 2409 2410 static void 2411 rfs4_lo_state_destroy(rfs4_entry_t u_entry) 2412 { 2413 rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry; 2414 2415 rfs4_sw_destroy(&lsp->ls_sw); 2416 2417 /* Make sure to release the file locks */ 2418 if (lsp->locks_cleaned == FALSE) { 2419 lsp->locks_cleaned = TRUE; 2420 if (lsp->locker->client->sysidt != LM_NOSYSID) { 2421 /* Is the PxFS kernel module loaded? */ 2422 if (lm_remove_file_locks != NULL) { 2423 int new_sysid; 2424 2425 /* Encode the cluster nodeid in new sysid */ 2426 new_sysid = lsp->locker->client->sysidt; 2427 lm_set_nlmid_flk(&new_sysid); 2428 2429 /* 2430 * This PxFS routine removes file locks for a 2431 * client over all nodes of a cluster. 2432 */ 2433 DTRACE_PROBE1(nfss_i_clust_rm_lck, 2434 int, new_sysid); 2435 (*lm_remove_file_locks)(new_sysid); 2436 } else { 2437 (void) cleanlocks(lsp->state->finfo->vp, 2438 lsp->locker->pid, 2439 lsp->locker->client->sysidt); 2440 } 2441 } 2442 } 2443 2444 rfs4_dbe_lock(lsp->state->dbe); 2445 2446 remque(&lsp->lockownerlist); 2447 lsp->lockownerlist.next = lsp->lockownerlist.prev = 2448 &lsp->lockownerlist; 2449 2450 rfs4_dbe_unlock(lsp->state->dbe); 2451 2452 /* Free the last reply for this state */ 2453 rfs4_free_reply(lsp->reply); 2454 2455 rfs4_lockowner_rele(lsp->locker); 2456 lsp->locker = NULL; 2457 2458 rfs4_state_rele_nounlock(lsp->state); 2459 lsp->state = NULL; 2460 } 2461 2462 static bool_t 2463 rfs4_lo_state_create(rfs4_entry_t u_entry, void *arg) 2464 { 2465 rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry; 2466 rfs4_lo_state_t *argp = (rfs4_lo_state_t *)arg; 2467 rfs4_lockowner_t *lo = argp->locker; 2468 rfs4_state_t *sp = argp->state; 2469 2470 lsp->state = sp; 2471 2472 lsp->lockid = sp->stateid; 2473 lsp->lockid.bits.type = LOCKID; 2474 lsp->lockid.bits.chgseq = 0; 2475 lsp->lockid.bits.pid = lo->pid; 2476 2477 lsp->locks_cleaned = FALSE; 2478 lsp->lock_completed = FALSE; 2479 2480 rfs4_sw_init(&lsp->ls_sw); 2481 2482 /* Attached the supplied lock owner */ 2483 rfs4_dbe_hold(lo->dbe); 2484 lsp->locker = lo; 2485 2486 lsp->lockownerlist.next = lsp->lockownerlist.prev = 2487 &lsp->lockownerlist; 2488 lsp->lockownerlist.lsp = lsp; 2489 2490 rfs4_dbe_lock(sp->dbe); 2491 2492 insque(&lsp->lockownerlist, sp->lockownerlist.prev); 2493 2494 rfs4_dbe_hold(sp->dbe); 2495 2496 rfs4_dbe_unlock(sp->dbe); 2497 2498 return (TRUE); 2499 } 2500 2501 void 2502 rfs4_lo_state_rele(rfs4_lo_state_t *lsp, bool_t unlock_fp) 2503 { 2504 if (unlock_fp == TRUE) 2505 rw_exit(&lsp->state->finfo->file_rwlock); 2506 rfs4_dbe_rele(lsp->dbe); 2507 } 2508 2509 static rfs4_lo_state_t * 2510 rfs4_findlo_state(stateid_t *id, bool_t lock_fp) 2511 { 2512 rfs4_lo_state_t *lsp; 2513 bool_t create = FALSE; 2514 2515 lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_idx, id, 2516 &create, NULL, RFS4_DBS_VALID); 2517 if (lock_fp == TRUE && lsp != NULL) 2518 rw_enter(&lsp->state->finfo->file_rwlock, RW_READER); 2519 2520 return (lsp); 2521 } 2522 2523 2524 static uint32_t 2525 lo_state_lo_hash(void *key) 2526 { 2527 rfs4_lo_state_t *lop = key; 2528 2529 return (ADDRHASH(lop->locker) ^ ADDRHASH(lop->state)); 2530 } 2531 2532 static bool_t 2533 lo_state_lo_compare(rfs4_entry_t u_entry, void *key) 2534 { 2535 rfs4_lo_state_t *lop = (rfs4_lo_state_t *)u_entry; 2536 rfs4_lo_state_t *keyp = key; 2537 2538 return (keyp->locker == lop->locker && keyp->state == lop->state); 2539 } 2540 2541 static void * 2542 lo_state_lo_mkkey(rfs4_entry_t u_entry) 2543 { 2544 return (u_entry); 2545 } 2546 2547 rfs4_lo_state_t * 2548 rfs4_findlo_state_by_owner(rfs4_lockowner_t *lo, 2549 rfs4_state_t *sp, bool_t *create) 2550 { 2551 rfs4_lo_state_t *lsp; 2552 rfs4_lo_state_t arg; 2553 2554 arg.locker = lo; 2555 arg.state = sp; 2556 2557 lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_owner_idx, &arg, 2558 create, &arg, RFS4_DBS_VALID); 2559 2560 return (lsp); 2561 } 2562 2563 static stateid_t 2564 get_stateid(id_t eid) 2565 { 2566 stateid_t id; 2567 2568 id.bits.boottime = rfs4_start_time; 2569 id.bits.ident = eid; 2570 id.bits.chgseq = 0; 2571 id.bits.type = 0; 2572 id.bits.pid = 0; 2573 2574 /* 2575 * If we are booted as a cluster node, embed our nodeid. 2576 * We've already done sanity checks in rfs4_client_create() so no 2577 * need to repeat them here. 2578 */ 2579 id.bits.clnodeid = (cluster_bootflags & CLUSTER_BOOTED) ? 2580 clconf_get_nodeid() : 0; 2581 2582 return (id); 2583 } 2584 2585 /* 2586 * For use only when booted as a cluster node. 2587 * Returns TRUE if the embedded nodeid indicates that this stateid was 2588 * generated on another node. 2589 */ 2590 static int 2591 foreign_stateid(stateid_t *id) 2592 { 2593 ASSERT(cluster_bootflags & CLUSTER_BOOTED); 2594 return (id->bits.clnodeid != (uint32_t)clconf_get_nodeid()); 2595 } 2596 2597 /* 2598 * For use only when booted as a cluster node. 2599 * Returns TRUE if the embedded nodeid indicates that this clientid was 2600 * generated on another node. 2601 */ 2602 static int 2603 foreign_clientid(cid *cidp) 2604 { 2605 ASSERT(cluster_bootflags & CLUSTER_BOOTED); 2606 return (cidp->impl_id.c_id >> CLUSTER_NODEID_SHIFT != 2607 (uint32_t)clconf_get_nodeid()); 2608 } 2609 2610 /* 2611 * For use only when booted as a cluster node. 2612 * Embed our cluster nodeid into the clientid. 2613 */ 2614 static void 2615 embed_nodeid(cid *cidp) 2616 { 2617 int clnodeid; 2618 /* 2619 * Currently, our state tables are small enough that their 2620 * ids will leave enough bits free for the nodeid. If the 2621 * tables become larger, we mustn't overwrite the id. 2622 * Equally, we only have room for so many bits of nodeid, so 2623 * must check that too. 2624 */ 2625 ASSERT(cluster_bootflags & CLUSTER_BOOTED); 2626 ASSERT(cidp->impl_id.c_id >> CLUSTER_NODEID_SHIFT == 0); 2627 clnodeid = clconf_get_nodeid(); 2628 ASSERT(clnodeid <= CLUSTER_MAX_NODEID); 2629 ASSERT(clnodeid != NODEID_UNKNOWN); 2630 cidp->impl_id.c_id |= (clnodeid << CLUSTER_NODEID_SHIFT); 2631 } 2632 2633 static uint32_t 2634 state_hash(void *key) 2635 { 2636 stateid_t *ip = (stateid_t *)key; 2637 2638 return (ip->bits.ident); 2639 } 2640 2641 static bool_t 2642 state_compare(rfs4_entry_t u_entry, void *key) 2643 { 2644 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2645 stateid_t *id = (stateid_t *)key; 2646 bool_t rc; 2647 2648 rc = (sp->stateid.bits.boottime == id->bits.boottime && 2649 sp->stateid.bits.ident == id->bits.ident); 2650 2651 return (rc); 2652 } 2653 2654 static void * 2655 state_mkkey(rfs4_entry_t u_entry) 2656 { 2657 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2658 2659 return (&sp->stateid); 2660 } 2661 2662 static void 2663 rfs4_state_destroy(rfs4_entry_t u_entry) 2664 { 2665 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2666 2667 ASSERT(&sp->lockownerlist == sp->lockownerlist.next); 2668 2669 /* release any share locks for this stateid if it's still open */ 2670 if (!sp->closed) 2671 rfs4_unshare(sp); 2672 2673 /* Were done with the file */ 2674 rfs4_file_rele(sp->finfo); 2675 sp->finfo = NULL; 2676 2677 /* And now with the openowner */ 2678 rfs4_dbe_lock(sp->owner->dbe); 2679 2680 remque(&sp->ownerstateids); 2681 sp->ownerstateids.next = sp->ownerstateids.prev = &sp->ownerstateids; 2682 2683 rfs4_dbe_unlock(sp->owner->dbe); 2684 2685 rfs4_openowner_rele(sp->owner); 2686 sp->owner = NULL; 2687 } 2688 2689 static void 2690 rfs4_state_rele_nounlock(rfs4_state_t *sp) 2691 { 2692 rfs4_dbe_rele(sp->dbe); 2693 } 2694 2695 void 2696 rfs4_state_rele(rfs4_state_t *sp) 2697 { 2698 rw_exit(&sp->finfo->file_rwlock); 2699 rfs4_dbe_rele(sp->dbe); 2700 } 2701 2702 static uint32_t 2703 deleg_hash(void *key) 2704 { 2705 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)key; 2706 2707 return (ADDRHASH(dsp->client) ^ ADDRHASH(dsp->finfo)); 2708 } 2709 2710 static bool_t 2711 deleg_compare(rfs4_entry_t u_entry, void *key) 2712 { 2713 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2714 rfs4_deleg_state_t *kdsp = (rfs4_deleg_state_t *)key; 2715 2716 return (dsp->client == kdsp->client && dsp->finfo == kdsp->finfo); 2717 } 2718 2719 static void * 2720 deleg_mkkey(rfs4_entry_t u_entry) 2721 { 2722 return (u_entry); 2723 } 2724 2725 static uint32_t 2726 deleg_state_hash(void *key) 2727 { 2728 stateid_t *ip = (stateid_t *)key; 2729 2730 return (ip->bits.ident); 2731 } 2732 2733 static bool_t 2734 deleg_state_compare(rfs4_entry_t u_entry, void *key) 2735 { 2736 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2737 stateid_t *id = (stateid_t *)key; 2738 bool_t rc; 2739 2740 if (id->bits.type != DELEGID) 2741 return (FALSE); 2742 2743 rc = (dsp->delegid.bits.boottime == id->bits.boottime && 2744 dsp->delegid.bits.ident == id->bits.ident); 2745 2746 return (rc); 2747 } 2748 2749 static void * 2750 deleg_state_mkkey(rfs4_entry_t u_entry) 2751 { 2752 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2753 2754 return (&dsp->delegid); 2755 } 2756 2757 static bool_t 2758 rfs4_deleg_state_expiry(rfs4_entry_t u_entry) 2759 { 2760 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2761 2762 if (rfs4_dbe_is_invalid(dsp->dbe)) 2763 return (TRUE); 2764 2765 if ((gethrestime_sec() - dsp->client->last_access 2766 > rfs4_lease_time)) { 2767 rfs4_dbe_invalidate(dsp->dbe); 2768 return (TRUE); 2769 } 2770 2771 return (FALSE); 2772 } 2773 2774 static bool_t 2775 rfs4_deleg_state_create(rfs4_entry_t u_entry, void *argp) 2776 { 2777 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2778 rfs4_file_t *fp = ((rfs4_deleg_state_t *)argp)->finfo; 2779 rfs4_client_t *cp = ((rfs4_deleg_state_t *)argp)->client; 2780 2781 rfs4_dbe_hold(fp->dbe); 2782 rfs4_dbe_hold(cp->dbe); 2783 2784 dsp->delegid = get_stateid(rfs4_dbe_getid(dsp->dbe)); 2785 dsp->delegid.bits.type = DELEGID; 2786 dsp->finfo = fp; 2787 dsp->client = cp; 2788 dsp->dtype = OPEN_DELEGATE_NONE; 2789 2790 dsp->time_granted = gethrestime_sec(); /* observability */ 2791 dsp->time_revoked = 0; 2792 2793 /* Init lists for remque/insque */ 2794 dsp->delegationlist.next = dsp->delegationlist.prev = 2795 &dsp->delegationlist; 2796 dsp->delegationlist.dsp = dsp; 2797 2798 dsp->clientdeleglist.next = dsp->clientdeleglist.prev = 2799 &dsp->clientdeleglist; 2800 dsp->clientdeleglist.dsp = dsp; 2801 2802 /* Insert state on per open owner's list */ 2803 rfs4_dbe_lock(cp->dbe); 2804 2805 insque(&dsp->clientdeleglist, cp->clientdeleglist.prev); 2806 2807 rfs4_dbe_unlock(cp->dbe); 2808 2809 return (TRUE); 2810 } 2811 2812 static void 2813 rfs4_deleg_state_destroy(rfs4_entry_t u_entry) 2814 { 2815 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 2816 2817 if (&dsp->delegationlist != dsp->delegationlist.next) 2818 rfs4_return_deleg(dsp, FALSE); 2819 2820 /* Were done with the file */ 2821 rfs4_file_rele(dsp->finfo); 2822 dsp->finfo = NULL; 2823 2824 /* And now with the openowner */ 2825 rfs4_dbe_lock(dsp->client->dbe); 2826 2827 remque(&dsp->clientdeleglist); 2828 dsp->clientdeleglist.next = dsp->clientdeleglist.prev = 2829 &dsp->clientdeleglist; 2830 2831 rfs4_dbe_unlock(dsp->client->dbe); 2832 2833 rfs4_client_rele(dsp->client); 2834 dsp->client = NULL; 2835 } 2836 2837 rfs4_deleg_state_t * 2838 rfs4_finddeleg(rfs4_state_t *sp, bool_t *create) 2839 { 2840 rfs4_deleg_state_t ds, *dsp; 2841 2842 ds.client = sp->owner->client; 2843 ds.finfo = sp->finfo; 2844 2845 dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_idx, &ds, 2846 create, &ds, RFS4_DBS_VALID); 2847 2848 return (dsp); 2849 } 2850 2851 rfs4_deleg_state_t * 2852 rfs4_finddelegstate(stateid_t *id) 2853 { 2854 rfs4_deleg_state_t *dsp; 2855 bool_t create = FALSE; 2856 2857 dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_state_idx, id, 2858 &create, NULL, RFS4_DBS_VALID); 2859 2860 return (dsp); 2861 } 2862 2863 void 2864 rfs4_deleg_state_rele(rfs4_deleg_state_t *dsp) 2865 { 2866 rfs4_dbe_rele(dsp->dbe); 2867 } 2868 2869 void 2870 rfs4_update_lock_sequence(rfs4_lo_state_t *lsp) 2871 { 2872 2873 rfs4_dbe_lock(lsp->dbe); 2874 2875 /* 2876 * If we are skipping sequence id checking, this means that 2877 * this is the first lock request and therefore the sequence 2878 * id does not need to be updated. This only happens on the 2879 * first lock request for a lockowner 2880 */ 2881 if (!lsp->skip_seqid_check) 2882 lsp->seqid++; 2883 2884 rfs4_dbe_unlock(lsp->dbe); 2885 } 2886 2887 void 2888 rfs4_update_lock_resp(rfs4_lo_state_t *lsp, nfs_resop4 *resp) 2889 { 2890 2891 rfs4_dbe_lock(lsp->dbe); 2892 2893 rfs4_free_reply(lsp->reply); 2894 2895 rfs4_copy_reply(lsp->reply, resp); 2896 2897 rfs4_dbe_unlock(lsp->dbe); 2898 } 2899 2900 void 2901 rfs4_free_opens(rfs4_openowner_t *op, bool_t invalidate, 2902 bool_t close_of_client) 2903 { 2904 rfs4_state_t *sp; 2905 2906 rfs4_dbe_lock(op->dbe); 2907 2908 for (sp = op->ownerstateids.next->sp; sp != NULL; 2909 sp = sp->ownerstateids.next->sp) { 2910 rfs4_state_close(sp, FALSE, close_of_client, CRED()); 2911 if (invalidate == TRUE) 2912 rfs4_dbe_invalidate(sp->dbe); 2913 } 2914 2915 rfs4_dbe_unlock(op->dbe); 2916 rfs4_dbe_invalidate(op->dbe); 2917 } 2918 2919 static uint32_t 2920 state_owner_file_hash(void *key) 2921 { 2922 rfs4_state_t *sp = key; 2923 2924 return (ADDRHASH(sp->owner) ^ ADDRHASH(sp->finfo)); 2925 } 2926 2927 static bool_t 2928 state_owner_file_compare(rfs4_entry_t u_entry, void *key) 2929 { 2930 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2931 rfs4_state_t *arg = key; 2932 2933 if (sp->closed == TRUE) 2934 return (FALSE); 2935 2936 return (arg->owner == sp->owner && arg->finfo == sp->finfo); 2937 } 2938 2939 static void * 2940 state_owner_file_mkkey(rfs4_entry_t u_entry) 2941 { 2942 return (u_entry); 2943 } 2944 2945 static uint32_t 2946 state_file_hash(void *key) 2947 { 2948 return (ADDRHASH(key)); 2949 } 2950 2951 static bool_t 2952 state_file_compare(rfs4_entry_t u_entry, void *key) 2953 { 2954 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2955 rfs4_file_t *fp = key; 2956 2957 if (sp->closed == TRUE) 2958 return (FALSE); 2959 2960 return (fp == sp->finfo); 2961 } 2962 2963 static void * 2964 state_file_mkkey(rfs4_entry_t u_entry) 2965 { 2966 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 2967 2968 return (sp->finfo); 2969 } 2970 2971 rfs4_state_t * 2972 rfs4_findstate_by_owner_file(rfs4_openowner_t *op, rfs4_file_t *file, 2973 bool_t *create) 2974 { 2975 rfs4_state_t *sp; 2976 rfs4_state_t key; 2977 2978 key.owner = op; 2979 key.finfo = file; 2980 2981 sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_owner_file_idx, &key, 2982 create, &key, RFS4_DBS_VALID); 2983 2984 return (sp); 2985 } 2986 2987 /* This returns ANY state struct that refers to this file */ 2988 static rfs4_state_t * 2989 rfs4_findstate_by_file(rfs4_file_t *fp) 2990 { 2991 bool_t create = FALSE; 2992 2993 return ((rfs4_state_t *)rfs4_dbsearch(rfs4_state_file_idx, fp, 2994 &create, fp, RFS4_DBS_VALID)); 2995 } 2996 2997 static bool_t 2998 rfs4_state_expiry(rfs4_entry_t u_entry) 2999 { 3000 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 3001 3002 if (rfs4_dbe_is_invalid(sp->dbe)) 3003 return (TRUE); 3004 3005 if (sp->closed == TRUE && 3006 ((gethrestime_sec() - rfs4_dbe_get_timerele(sp->dbe)) 3007 > rfs4_lease_time)) 3008 return (TRUE); 3009 3010 return ((gethrestime_sec() - sp->owner->client->last_access 3011 > rfs4_lease_time)); 3012 } 3013 3014 static bool_t 3015 rfs4_state_create(rfs4_entry_t u_entry, void *argp) 3016 { 3017 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 3018 rfs4_file_t *fp = ((rfs4_state_t *)argp)->finfo; 3019 rfs4_openowner_t *op = ((rfs4_state_t *)argp)->owner; 3020 3021 rfs4_dbe_hold(fp->dbe); 3022 rfs4_dbe_hold(op->dbe); 3023 sp->stateid = get_stateid(rfs4_dbe_getid(sp->dbe)); 3024 sp->stateid.bits.type = OPENID; 3025 sp->owner = op; 3026 sp->finfo = fp; 3027 3028 /* Init lists for remque/insque */ 3029 sp->ownerstateids.next = sp->ownerstateids.prev = &sp->ownerstateids; 3030 sp->ownerstateids.sp = sp; 3031 sp->lockownerlist.next = sp->lockownerlist.prev = &sp->lockownerlist; 3032 sp->lockownerlist.lsp = NULL; 3033 3034 /* Insert state on per open owner's list */ 3035 rfs4_dbe_lock(op->dbe); 3036 3037 insque(&sp->ownerstateids, op->ownerstateids.prev); 3038 3039 rfs4_dbe_unlock(op->dbe); 3040 3041 return (TRUE); 3042 } 3043 3044 static rfs4_state_t * 3045 rfs4_findstate(stateid_t *id, rfs4_dbsearch_type_t find_invalid, 3046 bool_t lock_fp) 3047 { 3048 rfs4_state_t *sp; 3049 bool_t create = FALSE; 3050 3051 sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_idx, id, 3052 &create, NULL, find_invalid); 3053 if (lock_fp == TRUE && sp != NULL) 3054 rw_enter(&sp->finfo->file_rwlock, RW_READER); 3055 3056 return (sp); 3057 } 3058 3059 void 3060 rfs4_state_close(rfs4_state_t *sp, bool_t lock_held, 3061 bool_t close_of_client, cred_t *cr) 3062 { 3063 /* Remove the associated lo_state owners */ 3064 if (!lock_held) 3065 rfs4_dbe_lock(sp->dbe); 3066 3067 /* 3068 * If refcnt == 0, the dbe is about to be destroyed. 3069 * lock state will be released by the reaper thread. 3070 */ 3071 3072 if (rfs4_dbe_refcnt(sp->dbe) > 0) { 3073 if (sp->closed == FALSE) { 3074 sp->closed = TRUE; 3075 3076 rfs4_release_share_lock_state(sp, cr, close_of_client); 3077 } 3078 } 3079 3080 if (!lock_held) 3081 rfs4_dbe_unlock(sp->dbe); 3082 } 3083 3084 /* 3085 * Remove all state associated with the given client. 3086 */ 3087 void 3088 rfs4_client_state_remove(rfs4_client_t *cp) 3089 { 3090 rfs4_openowner_t *oop; 3091 3092 rfs4_dbe_lock(cp->dbe); 3093 3094 for (oop = cp->openownerlist.next->oop; oop != NULL; 3095 oop = oop->openownerlist.next->oop) { 3096 rfs4_free_opens(oop, TRUE, TRUE); 3097 } 3098 3099 rfs4_dbe_unlock(cp->dbe); 3100 } 3101 3102 void 3103 rfs4_client_close(rfs4_client_t *cp) 3104 { 3105 /* Mark client as going away. */ 3106 rfs4_dbe_lock(cp->dbe); 3107 rfs4_dbe_invalidate(cp->dbe); 3108 rfs4_dbe_unlock(cp->dbe); 3109 3110 rfs4_client_state_remove(cp); 3111 3112 /* Release the client */ 3113 rfs4_client_rele(cp); 3114 } 3115 3116 nfsstat4 3117 rfs4_check_clientid(clientid4 *cp, int setclid_confirm) 3118 { 3119 cid *cidp = (cid *) cp; 3120 3121 /* 3122 * If we are booted as a cluster node, check the embedded nodeid. 3123 * If it indicates that this clientid was generated on another node, 3124 * inform the client accordingly. 3125 */ 3126 if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp)) 3127 return (NFS4ERR_STALE_CLIENTID); 3128 3129 /* 3130 * If the server start time matches the time provided 3131 * by the client (via the clientid) and this is NOT a 3132 * setclientid_confirm then return EXPIRED. 3133 */ 3134 if (!setclid_confirm && cidp->impl_id.start_time == rfs4_start_time) 3135 return (NFS4ERR_EXPIRED); 3136 3137 return (NFS4ERR_STALE_CLIENTID); 3138 } 3139 3140 /* 3141 * This is used when a stateid has not been found amongst the 3142 * current server's state. Check the stateid to see if it 3143 * was from this server instantiation or not. 3144 */ 3145 static nfsstat4 3146 what_stateid_error(stateid_t *id, stateid_type_t type) 3147 { 3148 /* If we are booted as a cluster node, was stateid locally generated? */ 3149 if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id)) 3150 return (NFS4ERR_STALE_STATEID); 3151 3152 /* If types don't match then no use checking further */ 3153 if (type != id->bits.type) 3154 return (NFS4ERR_BAD_STATEID); 3155 3156 /* From a previous server instantiation, return STALE */ 3157 if (id->bits.boottime < rfs4_start_time) 3158 return (NFS4ERR_STALE_STATEID); 3159 3160 /* 3161 * From this server but the state is most likely beyond lease 3162 * timeout: return NFS4ERR_EXPIRED. However, there is the 3163 * case of a delegation stateid. For delegations, there is a 3164 * case where the state can be removed without the client's 3165 * knowledge/consent: revocation. In the case of delegation 3166 * revocation, the delegation state will be removed and will 3167 * not be found. If the client does something like a 3168 * DELEGRETURN or even a READ/WRITE with a delegatoin stateid 3169 * that has been revoked, the server should return BAD_STATEID 3170 * instead of the more common EXPIRED error. 3171 */ 3172 if (id->bits.boottime == rfs4_start_time) { 3173 if (type == DELEGID) 3174 return (NFS4ERR_BAD_STATEID); 3175 else 3176 return (NFS4ERR_EXPIRED); 3177 } 3178 3179 return (NFS4ERR_BAD_STATEID); 3180 } 3181 3182 /* 3183 * Used later on to find the various state structs. When called from 3184 * rfs4_check_stateid()->rfs4_get_all_state(), no file struct lock is 3185 * taken (it is not needed) and helps on the read/write path with 3186 * respect to performance. 3187 */ 3188 static nfsstat4 3189 rfs4_get_state_lockit(stateid4 *stateid, rfs4_state_t **spp, 3190 rfs4_dbsearch_type_t find_invalid, bool_t lock_fp) 3191 { 3192 stateid_t *id = (stateid_t *)stateid; 3193 rfs4_state_t *sp; 3194 3195 *spp = NULL; 3196 3197 /* If we are booted as a cluster node, was stateid locally generated? */ 3198 if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id)) 3199 return (NFS4ERR_STALE_STATEID); 3200 3201 sp = rfs4_findstate(id, find_invalid, lock_fp); 3202 if (sp == NULL) { 3203 return (what_stateid_error(id, OPENID)); 3204 } 3205 3206 if (rfs4_lease_expired(sp->owner->client)) { 3207 if (lock_fp == TRUE) 3208 rfs4_state_rele(sp); 3209 else 3210 rfs4_state_rele_nounlock(sp); 3211 return (NFS4ERR_EXPIRED); 3212 } 3213 3214 *spp = sp; 3215 3216 return (NFS4_OK); 3217 } 3218 3219 nfsstat4 3220 rfs4_get_state(stateid4 *stateid, rfs4_state_t **spp, 3221 rfs4_dbsearch_type_t find_invalid) 3222 { 3223 return (rfs4_get_state_lockit(stateid, spp, find_invalid, TRUE)); 3224 } 3225 3226 int 3227 rfs4_check_stateid_seqid(rfs4_state_t *sp, stateid4 *stateid) 3228 { 3229 stateid_t *id = (stateid_t *)stateid; 3230 3231 if (rfs4_lease_expired(sp->owner->client)) 3232 return (NFS4_CHECK_STATEID_EXPIRED); 3233 3234 /* Stateid is some time in the future - that's bad */ 3235 if (sp->stateid.bits.chgseq < id->bits.chgseq) 3236 return (NFS4_CHECK_STATEID_BAD); 3237 3238 if (sp->stateid.bits.chgseq == id->bits.chgseq + 1) 3239 return (NFS4_CHECK_STATEID_REPLAY); 3240 3241 /* Stateid is some time in the past - that's old */ 3242 if (sp->stateid.bits.chgseq > id->bits.chgseq) 3243 return (NFS4_CHECK_STATEID_OLD); 3244 3245 /* Caller needs to know about confirmation before closure */ 3246 if (sp->owner->need_confirm) 3247 return (NFS4_CHECK_STATEID_UNCONFIRMED); 3248 3249 if (sp->closed == TRUE) 3250 return (NFS4_CHECK_STATEID_CLOSED); 3251 3252 return (NFS4_CHECK_STATEID_OKAY); 3253 } 3254 3255 int 3256 rfs4_check_lo_stateid_seqid(rfs4_lo_state_t *lsp, stateid4 *stateid) 3257 { 3258 stateid_t *id = (stateid_t *)stateid; 3259 3260 if (rfs4_lease_expired(lsp->state->owner->client)) 3261 return (NFS4_CHECK_STATEID_EXPIRED); 3262 3263 /* Stateid is some time in the future - that's bad */ 3264 if (lsp->lockid.bits.chgseq < id->bits.chgseq) 3265 return (NFS4_CHECK_STATEID_BAD); 3266 3267 if (lsp->lockid.bits.chgseq == id->bits.chgseq + 1) 3268 return (NFS4_CHECK_STATEID_REPLAY); 3269 3270 /* Stateid is some time in the past - that's old */ 3271 if (lsp->lockid.bits.chgseq > id->bits.chgseq) 3272 return (NFS4_CHECK_STATEID_OLD); 3273 3274 return (NFS4_CHECK_STATEID_OKAY); 3275 } 3276 3277 nfsstat4 3278 rfs4_get_deleg_state(stateid4 *stateid, rfs4_deleg_state_t **dspp) 3279 { 3280 stateid_t *id = (stateid_t *)stateid; 3281 rfs4_deleg_state_t *dsp; 3282 3283 *dspp = NULL; 3284 3285 /* If we are booted as a cluster node, was stateid locally generated? */ 3286 if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id)) 3287 return (NFS4ERR_STALE_STATEID); 3288 3289 dsp = rfs4_finddelegstate(id); 3290 if (dsp == NULL) { 3291 return (what_stateid_error(id, DELEGID)); 3292 } 3293 3294 if (rfs4_lease_expired(dsp->client)) { 3295 rfs4_deleg_state_rele(dsp); 3296 return (NFS4ERR_EXPIRED); 3297 } 3298 3299 *dspp = dsp; 3300 3301 return (NFS4_OK); 3302 } 3303 3304 nfsstat4 3305 rfs4_get_lo_state(stateid4 *stateid, rfs4_lo_state_t **lspp, bool_t lock_fp) 3306 { 3307 stateid_t *id = (stateid_t *)stateid; 3308 rfs4_lo_state_t *lsp; 3309 3310 *lspp = NULL; 3311 3312 /* If we are booted as a cluster node, was stateid locally generated? */ 3313 if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id)) 3314 return (NFS4ERR_STALE_STATEID); 3315 3316 lsp = rfs4_findlo_state(id, lock_fp); 3317 if (lsp == NULL) { 3318 return (what_stateid_error(id, LOCKID)); 3319 } 3320 3321 if (rfs4_lease_expired(lsp->state->owner->client)) { 3322 rfs4_lo_state_rele(lsp, lock_fp); 3323 return (NFS4ERR_EXPIRED); 3324 } 3325 3326 *lspp = lsp; 3327 3328 return (NFS4_OK); 3329 } 3330 3331 static nfsstat4 3332 rfs4_get_all_state(stateid4 *sid, rfs4_state_t **spp, 3333 rfs4_deleg_state_t **dspp, rfs4_lo_state_t **lospp) 3334 { 3335 rfs4_state_t *sp = NULL; 3336 rfs4_deleg_state_t *dsp = NULL; 3337 rfs4_lo_state_t *losp = NULL; 3338 stateid_t *id; 3339 nfsstat4 status; 3340 3341 *spp = NULL; *dspp = NULL; *lospp = NULL; 3342 3343 id = (stateid_t *)sid; 3344 switch (id->bits.type) { 3345 case OPENID: 3346 status = rfs4_get_state_lockit(sid, &sp, FALSE, FALSE); 3347 break; 3348 case DELEGID: 3349 status = rfs4_get_deleg_state(sid, &dsp); 3350 break; 3351 case LOCKID: 3352 status = rfs4_get_lo_state(sid, &losp, FALSE); 3353 if (status == NFS4_OK) { 3354 sp = losp->state; 3355 rfs4_dbe_hold(sp->dbe); 3356 } 3357 break; 3358 default: 3359 status = NFS4ERR_BAD_STATEID; 3360 } 3361 3362 if (status == NFS4_OK) { 3363 *spp = sp; 3364 *dspp = dsp; 3365 *lospp = losp; 3366 } 3367 3368 return (status); 3369 } 3370 3371 /* 3372 * Given the I/O mode (FREAD or FWRITE), this checks whether the 3373 * rfs4_state_t struct has access to do this operation and if so 3374 * return NFS4_OK; otherwise the proper NFSv4 error is returned. 3375 */ 3376 nfsstat4 3377 rfs4_state_has_access(rfs4_state_t *sp, int mode, vnode_t *vp) 3378 { 3379 nfsstat4 stat = NFS4_OK; 3380 rfs4_file_t *fp; 3381 bool_t create = FALSE; 3382 3383 rfs4_dbe_lock(sp->dbe); 3384 if (mode == FWRITE) { 3385 if (!(sp->share_access & OPEN4_SHARE_ACCESS_WRITE)) { 3386 stat = NFS4ERR_OPENMODE; 3387 } 3388 } else if (mode == FREAD) { 3389 if (!(sp->share_access & OPEN4_SHARE_ACCESS_READ)) { 3390 /* 3391 * If we have OPENed the file with DENYing access 3392 * to both READ and WRITE then no one else could 3393 * have OPENed the file, hence no conflicting READ 3394 * deny. This check is merely an optimization. 3395 */ 3396 if (sp->share_deny == OPEN4_SHARE_DENY_BOTH) 3397 goto out; 3398 3399 /* Check against file struct's DENY mode */ 3400 fp = rfs4_findfile(vp, NULL, &create); 3401 if (fp != NULL) { 3402 int deny_read = 0; 3403 rfs4_dbe_lock(fp->dbe); 3404 /* 3405 * Check if any other open owner has the file 3406 * OPENed with deny READ. 3407 */ 3408 if (sp->share_deny & OPEN4_SHARE_DENY_READ) 3409 deny_read = 1; 3410 ASSERT(fp->deny_read - deny_read >= 0); 3411 if (fp->deny_read - deny_read > 0) 3412 stat = NFS4ERR_OPENMODE; 3413 rfs4_dbe_unlock(fp->dbe); 3414 rfs4_file_rele(fp); 3415 } 3416 } 3417 } else { 3418 /* Illegal I/O mode */ 3419 stat = NFS4ERR_INVAL; 3420 } 3421 out: 3422 rfs4_dbe_unlock(sp->dbe); 3423 return (stat); 3424 } 3425 3426 /* 3427 * Given the I/O mode (FREAD or FWRITE), the vnode, the stateid and whether 3428 * the file is being truncated, return NFS4_OK if allowed or approriate 3429 * V4 error if not. Note NFS4ERR_DELAY will be returned and a recall on 3430 * the associated file will be done if the I/O is not consistent with any 3431 * delegation in effect on the file. Should be holding VOP_RWLOCK, either 3432 * as reader or writer as appropriate. rfs4_op_open will accquire the 3433 * VOP_RWLOCK as writer when setting up delegation. If the stateid is bad 3434 * this routine will return NFS4ERR_BAD_STATEID. In addition, through the 3435 * deleg parameter, we will return whether a write delegation is held by 3436 * the client associated with this stateid. 3437 * If the server instance associated with the relevant client is in its 3438 * grace period, return NFS4ERR_GRACE. 3439 */ 3440 3441 nfsstat4 3442 rfs4_check_stateid(int mode, vnode_t *vp, 3443 stateid4 *stateid, bool_t trunc, bool_t *deleg, 3444 bool_t do_access) 3445 { 3446 rfs4_file_t *fp; 3447 bool_t create = FALSE; 3448 rfs4_state_t *sp; 3449 rfs4_deleg_state_t *dsp; 3450 rfs4_lo_state_t *lsp; 3451 stateid_t *id = (stateid_t *)stateid; 3452 nfsstat4 stat = NFS4_OK; 3453 3454 if (ISSPECIAL(stateid)) { 3455 fp = rfs4_findfile(vp, NULL, &create); 3456 if (fp == NULL) 3457 return (NFS4_OK); 3458 if (fp->dinfo->dtype == OPEN_DELEGATE_NONE) { 3459 rfs4_file_rele(fp); 3460 return (NFS4_OK); 3461 } 3462 if (mode == FWRITE || 3463 fp->dinfo->dtype == OPEN_DELEGATE_WRITE) { 3464 rfs4_recall_deleg(fp, trunc, NULL); 3465 rfs4_file_rele(fp); 3466 return (NFS4ERR_DELAY); 3467 } 3468 rfs4_file_rele(fp); 3469 return (NFS4_OK); 3470 } else { 3471 stat = rfs4_get_all_state(stateid, &sp, &dsp, &lsp); 3472 if (stat != NFS4_OK) 3473 return (stat); 3474 if (lsp != NULL) { 3475 /* Is associated server instance in its grace period? */ 3476 if (rfs4_clnt_in_grace(lsp->locker->client)) { 3477 rfs4_lo_state_rele(lsp, FALSE); 3478 if (sp != NULL) 3479 rfs4_state_rele_nounlock(sp); 3480 return (NFS4ERR_GRACE); 3481 } 3482 if (id->bits.type == LOCKID) { 3483 /* Seqid in the future? - that's bad */ 3484 if (lsp->lockid.bits.chgseq < 3485 id->bits.chgseq) { 3486 rfs4_lo_state_rele(lsp, FALSE); 3487 if (sp != NULL) 3488 rfs4_state_rele_nounlock(sp); 3489 return (NFS4ERR_BAD_STATEID); 3490 } 3491 /* Seqid in the past? - that's old */ 3492 if (lsp->lockid.bits.chgseq > 3493 id->bits.chgseq) { 3494 rfs4_lo_state_rele(lsp, FALSE); 3495 if (sp != NULL) 3496 rfs4_state_rele_nounlock(sp); 3497 return (NFS4ERR_OLD_STATEID); 3498 } 3499 /* Ensure specified filehandle matches */ 3500 if (lsp->state->finfo->vp != vp) { 3501 rfs4_lo_state_rele(lsp, FALSE); 3502 if (sp != NULL) 3503 rfs4_state_rele_nounlock(sp); 3504 return (NFS4ERR_BAD_STATEID); 3505 } 3506 } 3507 rfs4_lo_state_rele(lsp, FALSE); 3508 } 3509 3510 /* Stateid provided was an "open" stateid */ 3511 if (sp != NULL) { 3512 /* Is associated server instance in its grace period? */ 3513 if (rfs4_clnt_in_grace(sp->owner->client)) { 3514 rfs4_state_rele_nounlock(sp); 3515 return (NFS4ERR_GRACE); 3516 } 3517 if (id->bits.type == OPENID) { 3518 /* Seqid in the future? - that's bad */ 3519 if (sp->stateid.bits.chgseq < 3520 id->bits.chgseq) { 3521 rfs4_state_rele_nounlock(sp); 3522 return (NFS4ERR_BAD_STATEID); 3523 } 3524 /* Seqid in the past - that's old */ 3525 if (sp->stateid.bits.chgseq > 3526 id->bits.chgseq) { 3527 rfs4_state_rele_nounlock(sp); 3528 return (NFS4ERR_OLD_STATEID); 3529 } 3530 } 3531 /* Ensure specified filehandle matches */ 3532 if (sp->finfo->vp != vp) { 3533 rfs4_state_rele_nounlock(sp); 3534 return (NFS4ERR_BAD_STATEID); 3535 } 3536 3537 if (sp->owner->need_confirm) { 3538 rfs4_state_rele_nounlock(sp); 3539 return (NFS4ERR_BAD_STATEID); 3540 } 3541 3542 if (sp->closed == TRUE) { 3543 rfs4_state_rele_nounlock(sp); 3544 return (NFS4ERR_OLD_STATEID); 3545 } 3546 3547 if (do_access) 3548 stat = rfs4_state_has_access(sp, mode, vp); 3549 else 3550 stat = NFS4_OK; 3551 3552 /* 3553 * Return whether this state has write 3554 * delegation if desired 3555 */ 3556 if (deleg && 3557 (sp->finfo->dinfo->dtype == OPEN_DELEGATE_WRITE)) 3558 *deleg = TRUE; 3559 3560 /* 3561 * We got a valid stateid, so we update the 3562 * lease on the client. Ideally we would like 3563 * to do this after the calling op succeeds, 3564 * but for now this will be good 3565 * enough. Callers of this routine are 3566 * currently insulated from the state stuff. 3567 */ 3568 rfs4_update_lease(sp->owner->client); 3569 3570 /* 3571 * If a delegation is present on this file and 3572 * this is a WRITE, then update the lastwrite 3573 * time to indicate that activity is present. 3574 */ 3575 if (sp->finfo->dinfo->dtype == OPEN_DELEGATE_WRITE && 3576 mode == FWRITE) { 3577 sp->finfo->dinfo->time_lastwrite = 3578 gethrestime_sec(); 3579 } 3580 3581 rfs4_state_rele_nounlock(sp); 3582 3583 return (stat); 3584 } 3585 3586 if (dsp != NULL) { 3587 /* Is associated server instance in its grace period? */ 3588 if (rfs4_clnt_in_grace(dsp->client)) { 3589 rfs4_deleg_state_rele(dsp); 3590 return (NFS4ERR_GRACE); 3591 } 3592 if (dsp->delegid.bits.chgseq != id->bits.chgseq) { 3593 rfs4_deleg_state_rele(dsp); 3594 return (NFS4ERR_BAD_STATEID); 3595 } 3596 3597 /* Ensure specified filehandle matches */ 3598 if (dsp->finfo->vp != vp) { 3599 rfs4_deleg_state_rele(dsp); 3600 return (NFS4ERR_BAD_STATEID); 3601 } 3602 /* 3603 * Return whether this state has write 3604 * delegation if desired 3605 */ 3606 if (deleg && 3607 (dsp->finfo->dinfo->dtype == OPEN_DELEGATE_WRITE)) 3608 *deleg = TRUE; 3609 3610 rfs4_update_lease(dsp->client); 3611 3612 /* 3613 * If a delegation is present on this file and 3614 * this is a WRITE, then update the lastwrite 3615 * time to indicate that activity is present. 3616 */ 3617 if (dsp->finfo->dinfo->dtype == OPEN_DELEGATE_WRITE && 3618 mode == FWRITE) { 3619 dsp->finfo->dinfo->time_lastwrite = 3620 gethrestime_sec(); 3621 } 3622 3623 /* 3624 * XXX - what happens if this is a WRITE and the 3625 * delegation type of for READ. 3626 */ 3627 rfs4_deleg_state_rele(dsp); 3628 3629 return (stat); 3630 } 3631 /* 3632 * If we got this far, something bad happened 3633 */ 3634 return (NFS4ERR_BAD_STATEID); 3635 } 3636 } 3637 3638 3639 /* 3640 * This is a special function in that for the file struct provided the 3641 * server wants to remove/close all current state associated with the 3642 * file. The prime use of this would be with OP_REMOVE to force the 3643 * release of state and particularly of file locks. 3644 * 3645 * There is an assumption that there is no delegations outstanding on 3646 * this file at this point. The caller should have waited for those 3647 * to be returned or revoked. 3648 */ 3649 void 3650 rfs4_close_all_state(rfs4_file_t *fp) 3651 { 3652 rfs4_state_t *sp; 3653 3654 rfs4_dbe_lock(fp->dbe); 3655 3656 #ifdef DEBUG 3657 /* only applies when server is handing out delegations */ 3658 if (rfs4_deleg_policy != SRV_NEVER_DELEGATE) 3659 ASSERT(fp->dinfo->hold_grant > 0); 3660 #endif 3661 3662 /* No delegations for this file */ 3663 ASSERT(fp->delegationlist.next == &fp->delegationlist); 3664 3665 /* Make sure that it can not be found */ 3666 rfs4_dbe_invalidate(fp->dbe); 3667 3668 if (fp->vp == NULL) { 3669 rfs4_dbe_unlock(fp->dbe); 3670 return; 3671 } 3672 rfs4_dbe_unlock(fp->dbe); 3673 3674 /* 3675 * Hold as writer to prevent other server threads from 3676 * processing requests related to the file while all state is 3677 * being removed. 3678 */ 3679 rw_enter(&fp->file_rwlock, RW_WRITER); 3680 3681 /* Remove ALL state from the file */ 3682 while (sp = rfs4_findstate_by_file(fp)) { 3683 rfs4_state_close(sp, FALSE, FALSE, CRED()); 3684 rfs4_state_rele_nounlock(sp); 3685 } 3686 3687 /* 3688 * This is only safe since there are no further references to 3689 * the file. 3690 */ 3691 rfs4_dbe_lock(fp->dbe); 3692 if (fp->vp) { 3693 vnode_t *vp = fp->vp; 3694 3695 mutex_enter(&vp->v_lock); 3696 (void) vsd_set(vp, nfs4_srv_vkey, NULL); 3697 mutex_exit(&vp->v_lock); 3698 VN_RELE(vp); 3699 fp->vp = NULL; 3700 } 3701 rfs4_dbe_unlock(fp->dbe); 3702 3703 /* Finally let other references to proceed */ 3704 rw_exit(&fp->file_rwlock); 3705 } 3706 3707 /* 3708 * This function is used as a target for the rfs4_dbe_walk() call 3709 * below. The purpose of this function is to see if the 3710 * lockowner_state refers to a file that resides within the exportinfo 3711 * export. If so, then remove the lock_owner state (file locks and 3712 * share "locks") for this object since the intent is the server is 3713 * unexporting the specified directory. Be sure to invalidate the 3714 * object after the state has been released 3715 */ 3716 static void 3717 rfs4_lo_state_walk_callout(rfs4_entry_t u_entry, void *e) 3718 { 3719 rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry; 3720 struct exportinfo *exi = (struct exportinfo *)e; 3721 nfs_fh4_fmt_t fhfmt4, *exi_fhp, *finfo_fhp; 3722 fhandle_t *efhp; 3723 3724 efhp = (fhandle_t *)&exi->exi_fh; 3725 exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4; 3726 3727 FH_TO_FMT4(efhp, exi_fhp); 3728 3729 finfo_fhp = 3730 (nfs_fh4_fmt_t *)lsp->state->finfo->filehandle.nfs_fh4_val; 3731 3732 if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) && 3733 bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata, 3734 exi_fhp->fh4_xlen) == 0) { 3735 rfs4_state_close(lsp->state, FALSE, FALSE, CRED()); 3736 rfs4_dbe_invalidate(lsp->dbe); 3737 rfs4_dbe_invalidate(lsp->state->dbe); 3738 } 3739 } 3740 3741 /* 3742 * This function is used as a target for the rfs4_dbe_walk() call 3743 * below. The purpose of this function is to see if the state refers 3744 * to a file that resides within the exportinfo export. If so, then 3745 * remove the open state for this object since the intent is the 3746 * server is unexporting the specified directory. The main result for 3747 * this type of entry is to invalidate it such it will not be found in 3748 * the future. 3749 */ 3750 static void 3751 rfs4_state_walk_callout(rfs4_entry_t u_entry, void *e) 3752 { 3753 rfs4_state_t *sp = (rfs4_state_t *)u_entry; 3754 struct exportinfo *exi = (struct exportinfo *)e; 3755 nfs_fh4_fmt_t fhfmt4, *exi_fhp, *finfo_fhp; 3756 fhandle_t *efhp; 3757 3758 efhp = (fhandle_t *)&exi->exi_fh; 3759 exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4; 3760 3761 FH_TO_FMT4(efhp, exi_fhp); 3762 3763 finfo_fhp = 3764 (nfs_fh4_fmt_t *)sp->finfo->filehandle.nfs_fh4_val; 3765 3766 if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) && 3767 bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata, 3768 exi_fhp->fh4_xlen) == 0) { 3769 rfs4_state_close(sp, TRUE, FALSE, CRED()); 3770 rfs4_dbe_invalidate(sp->dbe); 3771 } 3772 } 3773 3774 /* 3775 * This function is used as a target for the rfs4_dbe_walk() call 3776 * below. The purpose of this function is to see if the state refers 3777 * to a file that resides within the exportinfo export. If so, then 3778 * remove the deleg state for this object since the intent is the 3779 * server is unexporting the specified directory. The main result for 3780 * this type of entry is to invalidate it such it will not be found in 3781 * the future. 3782 */ 3783 static void 3784 rfs4_deleg_state_walk_callout(rfs4_entry_t u_entry, void *e) 3785 { 3786 rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry; 3787 struct exportinfo *exi = (struct exportinfo *)e; 3788 nfs_fh4_fmt_t fhfmt4, *exi_fhp, *finfo_fhp; 3789 fhandle_t *efhp; 3790 3791 efhp = (fhandle_t *)&exi->exi_fh; 3792 exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4; 3793 3794 FH_TO_FMT4(efhp, exi_fhp); 3795 3796 finfo_fhp = 3797 (nfs_fh4_fmt_t *)dsp->finfo->filehandle.nfs_fh4_val; 3798 3799 if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) && 3800 bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata, 3801 exi_fhp->fh4_xlen) == 0) { 3802 rfs4_dbe_invalidate(dsp->dbe); 3803 } 3804 } 3805 3806 /* 3807 * This function is used as a target for the rfs4_dbe_walk() call 3808 * below. The purpose of this function is to see if the state refers 3809 * to a file that resides within the exportinfo export. If so, then 3810 * release vnode hold for this object since the intent is the server 3811 * is unexporting the specified directory. Invalidation will prevent 3812 * this struct from being found in the future. 3813 */ 3814 static void 3815 rfs4_file_walk_callout(rfs4_entry_t u_entry, void *e) 3816 { 3817 rfs4_file_t *fp = (rfs4_file_t *)u_entry; 3818 struct exportinfo *exi = (struct exportinfo *)e; 3819 nfs_fh4_fmt_t fhfmt4, *exi_fhp, *finfo_fhp; 3820 fhandle_t *efhp; 3821 3822 efhp = (fhandle_t *)&exi->exi_fh; 3823 exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4; 3824 3825 FH_TO_FMT4(efhp, exi_fhp); 3826 3827 finfo_fhp = (nfs_fh4_fmt_t *)fp->filehandle.nfs_fh4_val; 3828 3829 if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) && 3830 bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata, 3831 exi_fhp->fh4_xlen) == 0) { 3832 if (fp->vp) { 3833 vnode_t *vp = fp->vp; 3834 3835 /* don't leak monitors */ 3836 if (fp->dinfo->dtype == OPEN_DELEGATE_READ) 3837 (void) fem_uninstall(vp, deleg_rdops, 3838 (void *)fp); 3839 else if (fp->dinfo->dtype == OPEN_DELEGATE_WRITE) 3840 (void) fem_uninstall(vp, deleg_wrops, 3841 (void *)fp); 3842 mutex_enter(&vp->v_lock); 3843 (void) vsd_set(vp, nfs4_srv_vkey, NULL); 3844 mutex_exit(&vp->v_lock); 3845 VN_RELE(vp); 3846 fp->vp = NULL; 3847 } 3848 rfs4_dbe_invalidate(fp->dbe); 3849 } 3850 } 3851 3852 /* 3853 * Given a directory that is being unexported, cleanup/release all 3854 * state in the server that refers to objects residing underneath this 3855 * particular export. The ordering of the release is important. 3856 * Lock_owner, then state and then file. 3857 */ 3858 void 3859 rfs4_clean_state_exi(struct exportinfo *exi) 3860 { 3861 mutex_enter(&rfs4_state_lock); 3862 3863 if (rfs4_server_state == NULL) { 3864 mutex_exit(&rfs4_state_lock); 3865 return; 3866 } 3867 3868 rfs4_dbe_walk(rfs4_lo_state_tab, rfs4_lo_state_walk_callout, exi); 3869 rfs4_dbe_walk(rfs4_state_tab, rfs4_state_walk_callout, exi); 3870 rfs4_dbe_walk(rfs4_deleg_state_tab, rfs4_deleg_state_walk_callout, exi); 3871 rfs4_dbe_walk(rfs4_file_tab, rfs4_file_walk_callout, exi); 3872 3873 mutex_exit(&rfs4_state_lock); 3874 } 3875