176 static noinline void __process_finished_items(struct rcu_pending *pending,  in __process_finished_items()  argument
185 	    __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) {  in __process_finished_items()
197 	switch ((ulong) pending->process) {  in __process_finished_items()
251 			pending->process(pending, *genradix_ptr(&objs.objs, i));  in __process_finished_items()
261 			pending->process(pending, obj);  in __process_finished_items()
267 static bool process_finished_items(struct rcu_pending *pending,  in process_finished_items()  argument
276 	if ((p->objs.nr && __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) ||  in process_finished_items()
277 	    (p->lists[0].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[0].seq)) ||  in process_finished_items()
278 	    (p->lists[1].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[1].seq)) ||  in process_finished_items()
280 		__process_finished_items(pending, p, flags);  in process_finished_items()
291 	struct rcu_pending *pending = p->parent;  in rcu_pending_work()  local
296 	} while (process_finished_items(pending, p, flags));  in rcu_pending_work()
387  * __rcu_pending_enqueue: enqueue a pending RCU item, to be processed (via
388  * pending->pracess) once grace period elapses.
393  * - If @ptr is NULL, we're enqueuing an item for a generic @pending with a
404 __rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *head,  in __rcu_pending_enqueue()  argument
414 	BUG_ON((ptr != NULL) != (pending->process == RCU_PENDING_KVFREE_FN));  in __rcu_pending_enqueue()
417 	p = this_cpu_ptr(pending->p);  in __rcu_pending_enqueue()
419 	seq = __get_state_synchronize_rcu(pending->srcu);  in __rcu_pending_enqueue()
422 	    unlikely(process_finished_items(pending, p, flags)))  in __rcu_pending_enqueue()
481 			__call_rcu(pending->srcu, &p->cb, rcu_pending_rcu_cb);  in __rcu_pending_enqueue()
483 			__start_poll_synchronize_rcu(pending->srcu);  in __rcu_pending_enqueue()
492 	if (unlikely(__poll_state_synchronize_rcu(pending->srcu, seq))) {  in __rcu_pending_enqueue()
493 		switch ((ulong) pending->process) {  in __rcu_pending_enqueue()
501 			pending->process(pending, head);  in __rcu_pending_enqueue()
508 	p = this_cpu_ptr(pending->p);  in __rcu_pending_enqueue()
513 void rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *obj)  in rcu_pending_enqueue()  argument
515 	__rcu_pending_enqueue(pending, obj, NULL, true);  in rcu_pending_enqueue()
550 struct rcu_head *rcu_pending_dequeue(struct rcu_pending *pending)  in rcu_pending_dequeue()  argument
552 	return rcu_pending_pcpu_dequeue(raw_cpu_ptr(pending->p));  in rcu_pending_dequeue()
555 struct rcu_head *rcu_pending_dequeue_from_all(struct rcu_pending *pending)  in rcu_pending_dequeue_from_all()  argument
557 	struct rcu_head *ret = rcu_pending_dequeue(pending);  in rcu_pending_dequeue_from_all()
564 		ret = rcu_pending_pcpu_dequeue(per_cpu_ptr(pending->p, cpu));  in rcu_pending_dequeue_from_all()
571 static bool rcu_pending_has_pending_or_armed(struct rcu_pending *pending)  in rcu_pending_has_pending_or_armed()  argument
575 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_has_pending_or_armed()
587 void rcu_pending_exit(struct rcu_pending *pending)  in rcu_pending_exit()  argument
591 	if (!pending->p)  in rcu_pending_exit()
594 	while (rcu_pending_has_pending_or_armed(pending)) {  in rcu_pending_exit()
595 		__rcu_barrier(pending->srcu);  in rcu_pending_exit()
598 			struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
604 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
609 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
616 	free_percpu(pending->p);  in rcu_pending_exit()
622  * @pending:	Object to init
628 int rcu_pending_init(struct rcu_pending *pending,  in rcu_pending_init()  argument
632 	pending->p = alloc_percpu(struct rcu_pending_pcpu);  in rcu_pending_init()
633 	if (!pending->p)  in rcu_pending_init()
638 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_init()
639 		p->parent	= pending;  in rcu_pending_init()
646 	pending->srcu = srcu;  in rcu_pending_init()
647 	pending->process = process;  in rcu_pending_init()