Lines Matching refs:objs
87 GENRADIX(struct rcu_head *) objs;
109 DARRAY_PREALLOCATED(struct rcu_pending_seq, 4) objs;
121 if (p->objs.nr) in __rcu_pending_has_pending()
197 struct rcu_pending_seq objs = {}; in __process_finished_items() local
200 if (p->objs.nr && in __process_finished_items()
201 __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) { in __process_finished_items()
202 objs = p->objs.data[0]; in __process_finished_items()
203 darray_remove_item(&p->objs, p->objs.data); in __process_finished_items()
215 for (size_t i = 0; i < objs.nr; ) { in __process_finished_items()
216 size_t nr_this_node = min(GENRADIX_NODE_SIZE / sizeof(void *), objs.nr - i); in __process_finished_items()
218 kfree_bulk(nr_this_node, (void **) genradix_ptr(&objs.objs, i)); in __process_finished_items()
221 genradix_free(&objs.objs); in __process_finished_items()
248 for (size_t i = 0; i < objs.nr; i++) { in __process_finished_items()
249 struct rcu_head *obj = *genradix_ptr(&objs.objs, i); in __process_finished_items()
252 genradix_free(&objs.objs); in __process_finished_items()
266 for (size_t i = 0; i < objs.nr; i++) in __process_finished_items()
267 pending->process(pending, *genradix_ptr(&objs.objs, i)); in __process_finished_items()
268 genradix_free(&objs.objs); in __process_finished_items()
292 if ((p->objs.nr && __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) || in process_finished_items()
337 darray_for_each_reverse(p->objs, objs) in get_object_radix()
338 if (rcu_gp_poll_cookie_eq(objs->seq, seq)) in get_object_radix()
339 return objs; in get_object_radix()
341 if (darray_push_gfp(&p->objs, ((struct rcu_pending_seq) { .seq = seq }), GFP_ATOMIC)) in get_object_radix()
344 return &darray_last(p->objs); in get_object_radix()
425 struct rcu_pending_seq *objs; in __rcu_pending_enqueue() local
449 objs = get_object_radix(p, seq); in __rcu_pending_enqueue()
450 if (unlikely(!objs)) in __rcu_pending_enqueue()
453 if (unlikely(!objs->cursor)) { in __rcu_pending_enqueue()
460 objs->cursor = genradix_ptr_alloc_preallocated_inlined(&objs->objs, in __rcu_pending_enqueue()
461 objs->nr, &new_node, GFP_ATOMIC|__GFP_NOWARN); in __rcu_pending_enqueue()
462 if (unlikely(!objs->cursor)) { in __rcu_pending_enqueue()
481 *objs->cursor++ = ptr ?: head; in __rcu_pending_enqueue()
483 if (!(((ulong) objs->cursor) & (GENRADIX_NODE_SIZE - 1))) in __rcu_pending_enqueue()
484 objs->cursor = NULL; in __rcu_pending_enqueue()
485 start_gp = !objs->nr; in __rcu_pending_enqueue()
486 objs->nr++; in __rcu_pending_enqueue()
539 darray_for_each(p->objs, objs) in rcu_pending_pcpu_dequeue()
540 if (objs->nr) { in rcu_pending_pcpu_dequeue()
541 ret = *genradix_ptr(&objs->objs, --objs->nr); in rcu_pending_pcpu_dequeue()
542 objs->cursor = NULL; in rcu_pending_pcpu_dequeue()
543 if (!objs->nr) in rcu_pending_pcpu_dequeue()
544 genradix_free(&objs->objs); in rcu_pending_pcpu_dequeue()
629 WARN_ON(p->objs.nr); in rcu_pending_exit()
630 darray_exit(&p->objs); in rcu_pending_exit()
658 darray_init(&p->objs); in rcu_pending_init()