258 	struct llbitmap_page_ctl **pctl;  member
361 	return llbitmap->pctl[idx]->state[offset];  in llbitmap_read()
366 				       struct llbitmap_page_ctl *pctl,  in llbitmap_infect_dirty_bits()  argument
374 		switch (pctl->state[pos]) {  in llbitmap_infect_dirty_bits()
376 			pctl->state[pos] = level_456 ? BitNeedSync : BitDirty;  in llbitmap_infect_dirty_bits()
379 			pctl->state[pos] = BitDirty;  in llbitmap_infect_dirty_bits()
388 	struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];  in llbitmap_set_page_dirty()  local
393 	if (!test_bit(LLPageDirty, &pctl->flags))  in llbitmap_set_page_dirty()
394 		set_bit(LLPageDirty, &pctl->flags);  in llbitmap_set_page_dirty()
402 		set_bit(block, pctl->dirty);  in llbitmap_set_page_dirty()
413 	if (test_and_set_bit(block, pctl->dirty)) {  in llbitmap_set_page_dirty()
414 		llbitmap_infect_dirty_bits(llbitmap, pctl, block);  in llbitmap_set_page_dirty()
421 		if (pctl->state[pos] == BitDirty ||  in llbitmap_set_page_dirty()
422 		    pctl->state[pos] == BitNeedSync) {  in llbitmap_set_page_dirty()
423 			llbitmap_infect_dirty_bits(llbitmap, pctl, block);  in llbitmap_set_page_dirty()
439 	llbitmap->pctl[idx]->state[bit] = state;  in llbitmap_write()
450 	if (llbitmap->pctl && llbitmap->pctl[idx])  in llbitmap_read_page()
451 		page = llbitmap->pctl[idx]->page;  in llbitmap_read_page()
481 	struct page *page = llbitmap->pctl[idx]->page;  in llbitmap_write_page()
487 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];  in llbitmap_write_page()  local
489 		if (!test_and_clear_bit(block, pctl->dirty))  in llbitmap_write_page()
511 	struct llbitmap_page_ctl *pctl =  in active_release()  local
514 	wake_up(&pctl->wait);  in active_release()
521 	if (!llbitmap->pctl)  in llbitmap_free_pages()
525 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];  in llbitmap_free_pages()  local
527 		if (!pctl || !pctl->page)  in llbitmap_free_pages()
530 		__free_page(pctl->page);  in llbitmap_free_pages()
531 		percpu_ref_exit(&pctl->active);  in llbitmap_free_pages()
534 	kfree(llbitmap->pctl[0]);  in llbitmap_free_pages()
535 	kfree(llbitmap->pctl);  in llbitmap_free_pages()
536 	llbitmap->pctl = NULL;  in llbitmap_free_pages()
541 	struct llbitmap_page_ctl *pctl;  in llbitmap_cache_pages()  local
544 	unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS(  in llbitmap_cache_pages()
548 	llbitmap->pctl = kmalloc_array(nr_pages, sizeof(void *),  in llbitmap_cache_pages()
550 	if (!llbitmap->pctl)  in llbitmap_cache_pages()
554 	pctl = kmalloc_array(nr_pages, size, GFP_KERNEL | __GFP_ZERO);  in llbitmap_cache_pages()
555 	if (!pctl) {  in llbitmap_cache_pages()
556 		kfree(llbitmap->pctl);  in llbitmap_cache_pages()
562 	for (i = 0; i < nr_pages; i++, pctl = (void *)pctl + size) {  in llbitmap_cache_pages()
565 		llbitmap->pctl[i] = pctl;  in llbitmap_cache_pages()
572 		if (percpu_ref_init(&pctl->active, active_release,  in llbitmap_cache_pages()
579 		pctl->page = page;  in llbitmap_cache_pages()
580 		pctl->state = page_address(page);  in llbitmap_cache_pages()
581 		init_waitqueue_head(&pctl->wait);  in llbitmap_cache_pages()
689 	struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];  in llbitmap_raise_barrier()  local
692 	if (likely(percpu_ref_tryget_live(&pctl->active))) {  in llbitmap_raise_barrier()
693 		WRITE_ONCE(pctl->expire, jiffies + llbitmap->barrier_idle * HZ);  in llbitmap_raise_barrier()
697 	wait_event(pctl->wait, !percpu_ref_is_dying(&pctl->active));  in llbitmap_raise_barrier()
703 	struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];  in llbitmap_release_barrier()  local
705 	percpu_ref_put(&pctl->active);  in llbitmap_release_barrier()
710 	struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];  in llbitmap_suspend_timeout()  local
712 	percpu_ref_kill(&pctl->active);  in llbitmap_suspend_timeout()
714 	if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active),  in llbitmap_suspend_timeout()
716 		percpu_ref_resurrect(&pctl->active);  in llbitmap_suspend_timeout()
725 	struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];  in llbitmap_resume()  local
727 	pctl->expire = LONG_MAX;  in llbitmap_resume()
728 	percpu_ref_resurrect(&pctl->active);  in llbitmap_resume()
729 	wake_up(&pctl->wait);  in llbitmap_resume()
940 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];  in md_llbitmap_daemon_fn()  local
947 		if (!test_bit(LLPageFlush, &pctl->flags) &&  in md_llbitmap_daemon_fn()
948 		    time_before(jiffies, pctl->expire)) {  in md_llbitmap_daemon_fn()
1138 		if (!test_bit(LLPageDirty, &llbitmap->pctl[i]->flags) ||  in llbitmap_unplug_fn()
1139 		    !test_and_clear_bit(LLPageDirty, &llbitmap->pctl[i]->flags))  in llbitmap_unplug_fn()
1155 		if (test_bit(LLPageDirty, &llbitmap->pctl[i]->flags))  in llbitmap_dirty()
1198 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];  in __llbitmap_flush()  local
1201 		set_bit(LLPageDirty, &pctl->flags);  in __llbitmap_flush()
1202 		bitmap_fill(pctl->dirty, llbitmap->blocks_per_page);  in __llbitmap_flush()
1215 		set_bit(LLPageFlush, &llbitmap->pctl[i]->flags);  in llbitmap_flush()
1292 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];  in llbitmap_close_sync()  local
1295 		WRITE_ONCE(pctl->expire, jiffies);  in llbitmap_close_sync()
1349 	bitmap_fill(llbitmap->pctl[0]->dirty, nr_blocks);  in llbitmap_write_sb()
1412 		struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];  in llbitmap_write_all()  local
1414 		set_bit(LLPageDirty, &pctl->flags);  in llbitmap_write_all()
1415 		bitmap_fill(pctl->dirty, llbitmap->blocks_per_page);  in llbitmap_write_all()
1454 	if (!llbitmap || !llbitmap->pctl) {  in bits_show()