39    Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is
46    given rsb and lkb and queues callbacks.
173 	printk(KERN_ERR "rsb: nodeid %d master %d dir %d flags %lx first %x "  in dlm_print_rsb()
186 	printk(KERN_ERR "rsb: root_list empty %d recover_list empty %d\n",  in dlm_dump_rsb()
188 	printk(KERN_ERR "rsb lookup list\n");  in dlm_dump_rsb()
191 	printk(KERN_ERR "rsb grant queue:\n");  in dlm_dump_rsb()
194 	printk(KERN_ERR "rsb convert queue:\n");  in dlm_dump_rsb()
197 	printk(KERN_ERR "rsb wait queue:\n");  in dlm_dump_rsb()
320  * Basic operations on rsb's and lkb's
329    a valid reference to the rsb, so there's no need for locking. */
399 /* This function tries to resume the timer callback if a rsb
429 	/* if the rsb is not queued do nothing */  in del_scan()
437 	/* check if the first element was the rsb we deleted */  in del_scan()
463 	/* A dir record for a remote master rsb should never be on the scan list. */  in add_scan()
468 	/* An active rsb should never be on the scan list. */  in add_scan()
471 	/* An rsb should not already be on the scan list. */  in add_scan()
475 	/* set the new rsb absolute expire time in the rsb */  in add_scan()
485 		 * to this rsb with the oldest expire time to the end  in add_scan()
487 		 * rsb expire time is our next expiration if it wasn't  in add_scan()
508 /* Called by lockspace scan_timer to free unused rsb's. */
541 		 * If the first rsb is not yet expired, then stop because the  in dlm_rsb_scan()
545 			/* rearm with the next rsb to expire in the future */  in dlm_rsb_scan()
574 		/* An rsb that is a dir record for a remote master rsb  in dlm_rsb_scan()
581 		/* We're the master of this rsb but we're not  in dlm_rsb_scan()
596    Otherwise, take an rsb off the list and return it. */
638 static int rsb_insert(struct dlm_rsb *rsb, struct rhashtable *rhash)  in rsb_insert()  argument
642 	rv = rhashtable_insert_fast(rhash, &rsb->res_node,  in rsb_insert()
645 		rsb_set_flag(rsb, RSB_HASHED);  in rsb_insert()
651  * Find rsb in rsbtbl and potentially create/add one
653  * Delaying the release of rsb's has a similar benefit to applications keeping
654  * NL locks on an rsb, but without the guarantee that the cached master value
655  * will still be valid when the rsb is reused.  Apps aren't always smart enough
656  * to keep NL locks on an rsb that they may lock again shortly; this can lead
659  * Searching for an rsb means looking through both the normal list and toss
660  * list.  When found on the toss list the rsb is moved to the normal list with
663  * rsb's on the keep list are being used locally and refcounted.
664  * rsb's on the toss list are not being used locally, and are not refcounted.
666  * The toss list rsb's were either
673  * The purpose of find_rsb() is to return a refcounted rsb for local use.
674  * So, if the given rsb is on the toss list, it is moved to the keep list
677  * deactivate_rsb() happens when all local usage of the rsb is done, i.e. no
678  * more refcounts exist, so the rsb is moved from the keep list to the
681  * rsb's on both keep and toss lists are used for doing a name to master
682  * lookups.  rsb's that are in use locally (and being refcounted) are on
683  * the keep list, rsb's that are not in use locally (not refcounted) and
686  * rsb's on the toss list who's dir_nodeid is not local can have stale
687  * name/master mappings.  So, remote requests on such rsb's can potentially
690  * first_lkid is to keep only a single outstanding request on an rsb
691  * while that rsb has a potentially stale master.)
720 	 * create the rsb; dlm_recover_process_copy() will handle EBADR  in find_rsb_dir()
724 	 * not find the rsb anywhere, then recreate it.  This happens if  in find_rsb_dir()
725 	 * someone sends us a request after we have removed/freed an rsb.  in find_rsb_dir()
727 	 * an rsb taken from their scan list.)  in find_rsb_dir()
740 	/* check if the rsb is active under read lock - likely path */  in find_rsb_dir()
748 	 * rsb is active, so we can't check master_nodeid without lock_rsb.  in find_rsb_dir()
765 	 * The expectation here is that the rsb will have HASHED and  in find_rsb_dir()
766 	 * INACTIVE flags set, and that the rsb can be moved from  in find_rsb_dir()
768 	 * the read lock and acquiring the write lock, this rsb could  in find_rsb_dir()
773 	 * the rcu read lock means the rsb will not be freed yet.  in find_rsb_dir()
774 	 * If the HASHED flag is not set, then the rsb is being freed,  in find_rsb_dir()
775 	 * so we add a new rsb struct.  If the HASHED flag is set,  in find_rsb_dir()
777 	 * made the rsb active, as we're expecting to do here, and  in find_rsb_dir()
791 	 * rsb found inactive (master_nodeid may be out of date unless  in find_rsb_dir()
793 	 * is using this rsb because it's inactive, so we can  in find_rsb_dir()
798 		/* our rsb was not master, and another node (not the dir node)  in find_rsb_dir()
821 		/* Because we have held no locks on this rsb,  in find_rsb_dir()
840 	 * rsb not found  in find_rsb_dir()
895 		 * rsb exists now, we do a whole relookup  in find_rsb_dir()
927 	/* check if the rsb is in active state under read lock - likely path */  in find_rsb_nodir()
940 	 * rsb is active, so we can't check master_nodeid without lock_rsb.  in find_rsb_nodir()
965 	 * rsb found inactive. No other thread is using this rsb because  in find_rsb_nodir()
971 		/* our rsb is not master, and another node has sent us a  in find_rsb_nodir()
983 		/* our rsb is not master, and we are dir; may as well fix it;  in find_rsb_nodir()
1003 	 * rsb not found  in find_rsb_nodir()
1020 		 * rsb exists now, we do a whole relookup  in find_rsb_nodir()
1036  * rsb rcu usage
1038  * While rcu read lock is held, the rsb cannot be freed,
1041  * Two threads are accessing the same rsb concurrently,
1042  * the first (A) is trying to use the rsb, the second (B)
1043  * is trying to free the rsb.
1046  * (trying to use rsb)      (trying to free rsb)
1050  * A3. look up rsb in rsbtbl
1053  *                          B2. look up rsb in rsbtbl
1054  *                          B3. remove rsb from rsbtbl
1055  *                          B4. clear rsb HASHED flag
1057  *                          B6. begin freeing rsb using rcu...
1059  * (rsb is inactive, so try to make it active again)
1060  * A5. read rsb HASHED flag (safe because rsb is not freed yet)
1061  * A6. the rsb HASHED flag is not set, which it means the rsb
1065  *                          B7. ...finish freeing rsb using rcu
1066  * A8. create a new rsb
1071  * A6. look up rsb in rsbtbl, not found
1073  * A8. create a new rsb
1116 		/* our rsb is not master, and another node (not the dir node)  in validate_master_nodeid()
1128 		/* our rsb is not master, but the dir nodeid has sent us a  in validate_master_nodeid()
1162 		 * rsb even though the res_nodeid is no longer removed.  in __dlm_master_lookup()
1199 		 * up the master for this rsb  in __dlm_master_lookup()
1211 		 * finds the rsb on the active list and ignores the remove,  in __dlm_master_lookup()
1242  *   remote node sends back the rsb names it is master of and we are dir of
1244  *   we either create new rsb setting remote node as master, or find existing
1245  *   rsb and set master to be the remote node.
1287 	/* check if the rsb is active under read lock - likely path */  in _dlm_master_lookup()
1299 	/* because the rsb is active, we need to lock_rsb before  in _dlm_master_lookup()
1310 	/* the rsb was active */  in _dlm_master_lookup()
1334 	/* because the rsb is inactive, it's not refcounted and lock_rsb  in _dlm_master_lookup()
1340 	/* A dir record rsb should never be on scan list. */  in _dlm_master_lookup()
1363 		 * rsb exists now, we do a whole relookup  in _dlm_master_lookup()
1433 	 * When the rsb becomes unused:  in deactivate_rsb()
1434 	 * - If it's not a dir record for a remote master rsb,  in deactivate_rsb()
1436 	 * - If it's a dir record for a remote master rsb,  in deactivate_rsb()
1467 /* Attaching/detaching lkb's from rsb's is for rsb reference counting.
1468    The rsb must exist as long as any lkb's for it do. */
1560 /* __put_lkb() is used when an lkb may not have an rsb attached to
1637 /* add/remove lkb to rsb's grant/convert/wait queue */
1891 	   b=0 lvb written to rsb or invalidated  in set_lvb_lock()
1987 /* Manipulate lkb's on rsb's convert/granted/waiting queues
2282 	 * locks for a recovered rsb, on which lkb's have been rebuilt.  in _can_be_granted()
2632    lkb using the nodeid field in the given rsb.  If the rsb's nodeid is
2634    0.  If the rsb's nodeid is _not_ known, it needs to be looked up
2637    When the rsb nodeid is being looked up remotely, the initial lkb
2639    lookup reply.  Other lkb's waiting for the same rsb lookup are kept
2640    on the rsb's res_lookup list until the master is verified.
2643    0: nodeid is set in rsb/lkb and the caller should go ahead and use it
2644    1: the rsb master is not available and the lkb has been placed on
2705 /* confirm_master -- confirm (or deny) an rsb's master nodeid */
2795 	   an active lkb cannot be modified before locking the rsb */  in set_lock_args()
2897 	/* an lkb may be waiting for an rsb lookup to complete where the  in validate_unlock_args()
3179 /* add a new lkb to a possibly new rsb, called by requesting process */
3655 /* FIXME: if this lkb is the only lock we hold on the rsb, then set
3656    MASTER_UNCERTAIN to force the next request on the rsb to confirm
3934 /* This is called after the rsb is locked so that we can safely inspect
4011 	   for this rsb or not, so if the master sends us a request, we should  in receive_request()
4012 	   recreate the rsb if we've destroyed it.   This race happens when we  in receive_request()
4014 	   node sends us a request for the rsb. */  in receive_request()
4315 	 * Look for inactive rsb, if it's there, free it.  in receive_remove()
4316 	 * If the rsb is active, it's being used, and we should ignore this  in receive_remove()
4321 	 * recreate the master rsb when it gets a request from the dir node for  in receive_remove()
4322 	 * an rsb it doesn't have.  in receive_remove()
4347 	/* at this stage the rsb can only being freed here */  in receive_remove()
4353 			log_error(ls, "receive_remove on active rsb from %d master %d",  in receive_remove()
4458 		/* find_rsb failed to find rsb or rsb wasn't master */  in receive_request_reply()
4727 	   rsb before we've received the dir node's loookup_reply for it.  in receive_lookup_reply()
5468  * previously blocked locks to now be granted.  The subset of rsb's
5472  * Simplest would be to go through each master rsb and check for non-empty
5476  * rsb's very inefficient.  So, we rely on earlier recovery routines
5477  * to set RECOVER_GRANT on any rsb's that we should attempt to grant
5573 	   until all locks have been rebuilt on the rsb (recover_conversion) */  in receive_rcom_lock_args()
5586    to check if the rsb already has an lkb with the given remote nodeid/lkid.
5612 	/* In general we expect the rsb returned to be R_MASTER, but we don't  in dlm_recover_master_copy()
5615 	   locks before we've made ourselves master of this rsb.  We can still  in dlm_recover_master_copy()
6107    granted.  Regardless of what rsb queue the lock is on, it's removed and