183 	struct padata_instance *pinst = ps->pinst;  in padata_do_parallel()  local
193 	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)  in padata_do_parallel()
206 	if ((pinst->flags & PADATA_RESET))  in padata_do_parallel()
227 		queue_work(pinst->parallel_wq, &pw->pw_work);  in padata_do_parallel()
283 	struct padata_instance *pinst = pd->ps->pinst;  in padata_reorder()  local
306 		queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);  in padata_reorder()
387 static int padata_setup_cpumasks(struct padata_instance *pinst)  in padata_setup_cpumasks()  argument
397 	cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu);  in padata_setup_cpumasks()
398 	err = apply_workqueue_attrs(pinst->parallel_wq, attrs);  in padata_setup_cpumasks()
540 	struct padata_instance *pinst = ps->pinst;  in padata_alloc_pd()  local
562 	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);  in padata_alloc_pd()
563 	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);  in padata_alloc_pd()
594 static void __padata_start(struct padata_instance *pinst)  in __padata_start()  argument
596 	pinst->flags |= PADATA_INIT;  in __padata_start()
599 static void __padata_stop(struct padata_instance *pinst)  in __padata_stop()  argument
601 	if (!(pinst->flags & PADATA_INIT))  in __padata_stop()
604 	pinst->flags &= ~PADATA_INIT;  in __padata_stop()
624 static int padata_replace(struct padata_instance *pinst)  in padata_replace()  argument
629 	pinst->flags |= PADATA_RESET;  in padata_replace()
631 	list_for_each_entry(ps, &pinst->pslist, list) {  in padata_replace()
639 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)  in padata_replace()
642 	pinst->flags &= ~PADATA_RESET;  in padata_replace()
648 static bool padata_validate_cpumask(struct padata_instance *pinst,  in padata_validate_cpumask()  argument
652 		pinst->flags |= PADATA_INVALID;  in padata_validate_cpumask()
656 	pinst->flags &= ~PADATA_INVALID;  in padata_validate_cpumask()
660 static int __padata_set_cpumasks(struct padata_instance *pinst,  in __padata_set_cpumasks()  argument
667 	valid = padata_validate_cpumask(pinst, pcpumask);  in __padata_set_cpumasks()
669 		__padata_stop(pinst);  in __padata_set_cpumasks()
673 	valid = padata_validate_cpumask(pinst, cbcpumask);  in __padata_set_cpumasks()
675 		__padata_stop(pinst);  in __padata_set_cpumasks()
678 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in __padata_set_cpumasks()
679 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in __padata_set_cpumasks()
681 	err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst);  in __padata_set_cpumasks()
684 		__padata_start(pinst);  in __padata_set_cpumasks()
699 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,  in padata_set_cpumask()  argument
706 	mutex_lock(&pinst->lock);  in padata_set_cpumask()
710 		serial_mask = pinst->cpumask.cbcpu;  in padata_set_cpumask()
714 		parallel_mask = pinst->cpumask.pcpu;  in padata_set_cpumask()
721 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);  in padata_set_cpumask()
724 	mutex_unlock(&pinst->lock);  in padata_set_cpumask()
733 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)  in __padata_add_cpu()  argument
738 		err = padata_replace(pinst);  in __padata_add_cpu()
740 		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&  in __padata_add_cpu()
741 		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_add_cpu()
742 			__padata_start(pinst);  in __padata_add_cpu()
748 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)  in __padata_remove_cpu()  argument
753 		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||  in __padata_remove_cpu()
754 		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_remove_cpu()
755 			__padata_stop(pinst);  in __padata_remove_cpu()
757 		err = padata_replace(pinst);  in __padata_remove_cpu()
763 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)  in pinst_has_cpu()  argument
765 	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||  in pinst_has_cpu()
766 		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);  in pinst_has_cpu()
771 	struct padata_instance *pinst;  in padata_cpu_online()  local
774 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node);  in padata_cpu_online()
775 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_online()
778 	mutex_lock(&pinst->lock);  in padata_cpu_online()
779 	ret = __padata_add_cpu(pinst, cpu);  in padata_cpu_online()
780 	mutex_unlock(&pinst->lock);  in padata_cpu_online()
786 	struct padata_instance *pinst;  in padata_cpu_dead()  local
789 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node);  in padata_cpu_dead()
790 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_dead()
793 	mutex_lock(&pinst->lock);  in padata_cpu_dead()
794 	ret = __padata_remove_cpu(pinst, cpu);  in padata_cpu_dead()
795 	mutex_unlock(&pinst->lock);  in padata_cpu_dead()
802 static void __padata_free(struct padata_instance *pinst)  in __padata_free()  argument
806 					    &pinst->cpu_dead_node);  in __padata_free()
807 	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node);  in __padata_free()
810 	WARN_ON(!list_empty(&pinst->pslist));  in __padata_free()
812 	free_cpumask_var(pinst->cpumask.pcpu);  in __padata_free()
813 	free_cpumask_var(pinst->cpumask.cbcpu);  in __padata_free()
814 	destroy_workqueue(pinst->serial_wq);  in __padata_free()
815 	destroy_workqueue(pinst->parallel_wq);  in __padata_free()
816 	kfree(pinst);  in __padata_free()
826 	struct padata_instance *pinst = kobj2pinst(kobj);  in padata_sysfs_release()  local
827 	__padata_free(pinst);  in padata_sysfs_release()
837 static ssize_t show_cpumask(struct padata_instance *pinst,  in show_cpumask()  argument
843 	mutex_lock(&pinst->lock);  in show_cpumask()
845 		cpumask = pinst->cpumask.cbcpu;  in show_cpumask()
847 		cpumask = pinst->cpumask.pcpu;  in show_cpumask()
851 	mutex_unlock(&pinst->lock);  in show_cpumask()
855 static ssize_t store_cpumask(struct padata_instance *pinst,  in store_cpumask()  argument
873 	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);  in store_cpumask()
908 	struct padata_instance *pinst;  in padata_sysfs_show()  local
911 	pinst = kobj2pinst(kobj);  in padata_sysfs_show()
914 		ret = pentry->show(pinst, attr, buf);  in padata_sysfs_show()
923 	struct padata_instance *pinst;  in padata_sysfs_store()  local
926 	pinst = kobj2pinst(kobj);  in padata_sysfs_store()
929 		ret = pentry->store(pinst, attr, buf, count);  in padata_sysfs_store()
953 	struct padata_instance *pinst;  in padata_alloc()  local
955 	pinst = kzalloc_obj(struct padata_instance);  in padata_alloc()
956 	if (!pinst)  in padata_alloc()
959 	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,  in padata_alloc()
961 	if (!pinst->parallel_wq)  in padata_alloc()
966 	pinst->serial_wq = alloc_workqueue("%s_serial",  in padata_alloc()
969 	if (!pinst->serial_wq)  in padata_alloc()
972 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))  in padata_alloc()
974 	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {  in padata_alloc()
975 		free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
979 	INIT_LIST_HEAD(&pinst->pslist);  in padata_alloc()
981 	cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);  in padata_alloc()
982 	cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask);  in padata_alloc()
984 	if (padata_setup_cpumasks(pinst))  in padata_alloc()
987 	__padata_start(pinst);  in padata_alloc()
989 	kobject_init(&pinst->kobj, &padata_attr_type);  in padata_alloc()
990 	mutex_init(&pinst->lock);  in padata_alloc()
994 						    &pinst->cpu_online_node);  in padata_alloc()
996 						    &pinst->cpu_dead_node);  in padata_alloc()
1001 	return pinst;  in padata_alloc()
1004 	free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
1005 	free_cpumask_var(pinst->cpumask.cbcpu);  in padata_alloc()
1007 	destroy_workqueue(pinst->serial_wq);  in padata_alloc()
1010 	destroy_workqueue(pinst->parallel_wq);  in padata_alloc()
1012 	kfree(pinst);  in padata_alloc()
1023 void padata_free(struct padata_instance *pinst)  in padata_free()  argument
1025 	kobject_put(&pinst->kobj);  in padata_free()
1036 struct padata_shell *padata_alloc_shell(struct padata_instance *pinst)  in padata_alloc_shell()  argument
1045 	ps->pinst = pinst;  in padata_alloc_shell()
1054 	mutex_lock(&pinst->lock);  in padata_alloc_shell()
1056 	list_add(&ps->list, &pinst->pslist);  in padata_alloc_shell()
1057 	mutex_unlock(&pinst->lock);  in padata_alloc_shell()
1080 	mutex_lock(&ps->pinst->lock);  in padata_free_shell()
1084 	mutex_unlock(&ps->pinst->lock);  in padata_free_shell()