180 int padata_do_parallel(struct padata_shell *ps,  in padata_do_parallel()  argument
183 	struct padata_instance *pinst = ps->pinst;  in padata_do_parallel()
190 	pd = rcu_dereference_bh(ps->pd);  in padata_do_parallel()
283 	struct padata_instance *pinst = pd->ps->pinst;  in padata_reorder()
407 	struct padata_mt_job_state *ps = pw->pw_data;  in padata_mt_helper()  local
408 	struct padata_mt_job *job = ps->job;  in padata_mt_helper()
411 	spin_lock(&ps->lock);  in padata_mt_helper()
418 		size = roundup(start + 1, ps->chunk_size) - start;  in padata_mt_helper()
425 		spin_unlock(&ps->lock);  in padata_mt_helper()
427 		spin_lock(&ps->lock);  in padata_mt_helper()
430 	++ps->nworks_fini;  in padata_mt_helper()
431 	done = (ps->nworks_fini == ps->nworks);  in padata_mt_helper()
432 	spin_unlock(&ps->lock);  in padata_mt_helper()
435 		complete(&ps->completion);  in padata_mt_helper()
449 	struct padata_mt_job_state ps;  in padata_do_multithreaded()  local
467 	spin_lock_init(&ps.lock);  in padata_do_multithreaded()
468 	init_completion(&ps.completion);  in padata_do_multithreaded()
469 	ps.job	       = job;  in padata_do_multithreaded()
470 	ps.nworks      = padata_work_alloc_mt(nworks, &ps, &works);  in padata_do_multithreaded()
471 	ps.nworks_fini = 0;  in padata_do_multithreaded()
481 	ps.chunk_size = job->size / (ps.nworks * load_balance_factor);  in padata_do_multithreaded()
482 	ps.chunk_size = max(ps.chunk_size, job->min_chunk);  in padata_do_multithreaded()
483 	ps.chunk_size = max(ps.chunk_size, 1ul);  in padata_do_multithreaded()
484 	ps.chunk_size = roundup(ps.chunk_size, job->align);  in padata_do_multithreaded()
499 	padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK);  in padata_do_multithreaded()
503 	wait_for_completion(&ps.completion);  in padata_do_multithreaded()
538 static struct parallel_data *padata_alloc_pd(struct padata_shell *ps)  in padata_alloc_pd()  argument
540 	struct padata_instance *pinst = ps->pinst;  in padata_alloc_pd()
555 	pd->ps = ps;  in padata_alloc_pd()
610 static int padata_replace_one(struct padata_shell *ps)  in padata_replace_one()  argument
614 	pd_new = padata_alloc_pd(ps);  in padata_replace_one()
618 	ps->opd = rcu_dereference_protected(ps->pd, 1);  in padata_replace_one()
619 	rcu_assign_pointer(ps->pd, pd_new);  in padata_replace_one()
626 	struct padata_shell *ps;  in padata_replace()  local
631 	list_for_each_entry(ps, &pinst->pslist, list) {  in padata_replace()
632 		err = padata_replace_one(ps);  in padata_replace()
639 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)  in padata_replace()
640 		padata_put_pd(ps->opd);  in padata_replace()
1039 	struct padata_shell *ps;  in padata_alloc_shell()  local
1041 	ps = kzalloc(sizeof(*ps), GFP_KERNEL);  in padata_alloc_shell()
1042 	if (!ps)  in padata_alloc_shell()
1045 	ps->pinst = pinst;  in padata_alloc_shell()
1048 	pd = padata_alloc_pd(ps);  in padata_alloc_shell()
1055 	RCU_INIT_POINTER(ps->pd, pd);  in padata_alloc_shell()
1056 	list_add(&ps->list, &pinst->pslist);  in padata_alloc_shell()
1059 	return ps;  in padata_alloc_shell()
1062 	kfree(ps);  in padata_alloc_shell()
1073 void padata_free_shell(struct padata_shell *ps)  in padata_free_shell()  argument
1077 	if (!ps)  in padata_free_shell()
1080 	mutex_lock(&ps->pinst->lock);  in padata_free_shell()
1081 	list_del(&ps->list);  in padata_free_shell()
1082 	pd = rcu_dereference_protected(ps->pd, 1);  in padata_free_shell()
1084 	mutex_unlock(&ps->pinst->lock);  in padata_free_shell()
1086 	kfree(ps);  in padata_free_shell()