353 	return !folio->mapping && folio->share;  in dax_folio_is_shared()
372 	folio->mapping = NULL;  in dax_folio_make_shared()
394 	folio->mapping = NULL;  in dax_folio_put()
408 		new_folio->mapping = NULL;  in dax_folio_put()
441 static void dax_associate_entry(void *entry, struct address_space *mapping,  in dax_associate_entry()  argument
452 	if (shared && (folio->mapping || dax_folio_is_shared(folio))) {  in dax_associate_entry()
453 		if (folio->mapping)  in dax_associate_entry()
460 		WARN_ON_ONCE(folio->mapping);  in dax_associate_entry()
463 		folio->mapping = mapping;  in dax_associate_entry()
468 static void dax_disassociate_entry(void *entry, struct address_space *mapping,  in dax_disassociate_entry()  argument
508 		struct address_space *mapping = READ_ONCE(folio->mapping);  in dax_lock_folio()  local
511 		if (!mapping || !dax_mapping(mapping))  in dax_lock_folio()
522 		if (S_ISCHR(mapping->host->i_mode))  in dax_lock_folio()
525 		xas.xa = &mapping->i_pages;  in dax_lock_folio()
527 		if (mapping != folio->mapping) {  in dax_lock_folio()
549 	struct address_space *mapping = folio->mapping;  in dax_unlock_folio()  local
550 	XA_STATE(xas, &mapping->i_pages, folio->index);  in dax_unlock_folio()
552 	if (S_ISCHR(mapping->host->i_mode))  in dax_unlock_folio()
567 dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, pgoff_t index,  in dax_lock_mapping_entry()  argument
576 		if (!dax_mapping(mapping))  in dax_lock_mapping_entry()
579 		xas.xa = &mapping->i_pages;  in dax_lock_mapping_entry()
610 void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index,  in dax_unlock_mapping_entry()  argument
613 	XA_STATE(xas, &mapping->i_pages, index);  in dax_unlock_mapping_entry()
651 		struct address_space *mapping, unsigned int order)  in grab_mapping_entry()  argument
693 			unmap_mapping_pages(mapping,  in grab_mapping_entry()
700 		dax_disassociate_entry(entry, mapping, false);  in grab_mapping_entry()
703 		mapping->nrpages -= PG_PMD_NR;  in grab_mapping_entry()
719 		mapping->nrpages += 1UL << order;  in grab_mapping_entry()
724 	if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))  in grab_mapping_entry()
754 struct page *dax_layout_busy_page_range(struct address_space *mapping,  in dax_layout_busy_page_range()  argument
762 	XA_STATE(xas, &mapping->i_pages, start_idx);  in dax_layout_busy_page_range()
764 	if (!dax_mapping(mapping))  in dax_layout_busy_page_range()
784 	unmap_mapping_pages(mapping, start_idx, end_idx - start_idx + 1, 0);  in dax_layout_busy_page_range()
809 struct page *dax_layout_busy_page(struct address_space *mapping)  in dax_layout_busy_page()  argument
811 	return dax_layout_busy_page_range(mapping, 0, LLONG_MAX);  in dax_layout_busy_page()
815 static int __dax_invalidate_entry(struct address_space *mapping,  in __dax_invalidate_entry()  argument
818 	XA_STATE(xas, &mapping->i_pages, index);  in __dax_invalidate_entry()
830 	dax_disassociate_entry(entry, mapping, trunc);  in __dax_invalidate_entry()
832 	mapping->nrpages -= 1UL << dax_entry_order(entry);  in __dax_invalidate_entry()
840 static int __dax_clear_dirty_range(struct address_space *mapping,  in __dax_clear_dirty_range()  argument
843 	XA_STATE(xas, &mapping->i_pages, start);  in __dax_clear_dirty_range()
873 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)  in dax_delete_mapping_entry()  argument
875 	int ret = __dax_invalidate_entry(mapping, index, true);  in dax_delete_mapping_entry()
888 void dax_delete_mapping_range(struct address_space *mapping,  in dax_delete_mapping_range()  argument
894 	XA_STATE(xas, &mapping->i_pages, start_idx);  in dax_delete_mapping_range()
909 		dax_disassociate_entry(entry, mapping, true);  in dax_delete_mapping_range()
911 		mapping->nrpages -= 1UL << dax_entry_order(entry);  in dax_delete_mapping_range()
992 int dax_invalidate_mapping_entry_sync(struct address_space *mapping,  in dax_invalidate_mapping_entry_sync()  argument
995 	return __dax_invalidate_entry(mapping, index, false);  in dax_invalidate_mapping_entry_sync()
1046 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_insert_entry()  local
1053 		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);  in dax_insert_entry()
1059 			unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,  in dax_insert_entry()
1062 			unmap_mapping_pages(mapping, index, 1, false);  in dax_insert_entry()
1070 		dax_disassociate_entry(entry, mapping, false);  in dax_insert_entry()
1071 		dax_associate_entry(new_entry, mapping, vmf->vma,  in dax_insert_entry()
1101 		struct address_space *mapping, void *entry)  in dax_writeback_one()  argument
1166 	i_mmap_lock_read(mapping);  in dax_writeback_one()
1167 	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) {  in dax_writeback_one()
1171 	i_mmap_unlock_read(mapping);  in dax_writeback_one()
1186 	trace_dax_writeback_one(mapping->host, index, count);  in dax_writeback_one()
1199 int dax_writeback_mapping_range(struct address_space *mapping,  in dax_writeback_mapping_range()  argument
1202 	XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);  in dax_writeback_mapping_range()
1203 	struct inode *inode = mapping->host;  in dax_writeback_mapping_range()
1212 	if (mapping_empty(mapping) || wbc->sync_mode != WB_SYNC_ALL)  in dax_writeback_mapping_range()
1217 	tag_pages_for_writeback(mapping, xas.xa_index, end_index);  in dax_writeback_mapping_range()
1221 		ret = dax_writeback_one(&xas, dax_dev, mapping, entry);  in dax_writeback_mapping_range()
1223 			mapping_set_error(mapping, ret);  in dax_writeback_mapping_range()
1377 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_pmd_load_hole()  local
1378 	struct inode *inode = mapping->host;  in dax_pmd_load_hole()
1865 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_iomap_pte_fault()  local
1866 	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);  in dax_iomap_pte_fault()
1868 		.inode		= mapping->host,  in dax_iomap_pte_fault()
1891 	entry = grab_mapping_entry(&xas, mapping, 0);  in dax_iomap_pte_fault()
1975 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_iomap_pmd_fault()  local
1976 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);  in dax_iomap_pmd_fault()
1978 		.inode		= mapping->host,  in dax_iomap_pmd_fault()
2012 	entry = grab_mapping_entry(&xas, mapping, PMD_ORDER);  in dax_iomap_pmd_fault()
2096 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in dax_insert_pfn_mkwrite()  local
2097 	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);  in dax_insert_pfn_mkwrite()
2109 		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,  in dax_insert_pfn_mkwrite()
2128 	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);  in dax_insert_pfn_mkwrite()