25  * @mark is an xa_mark_t; a small number indicating one of the mark bits.
68 static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)  in xa_mark_set()  argument
70 	if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))  in xa_mark_set()
71 		xa->xa_flags |= XA_FLAGS_MARK(mark);  in xa_mark_set()
74 static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark)  in xa_mark_clear()  argument
76 	if (xa->xa_flags & XA_FLAGS_MARK(mark))  in xa_mark_clear()
77 		xa->xa_flags &= ~(XA_FLAGS_MARK(mark));  in xa_mark_clear()
80 static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark)  in node_marks()  argument
82 	return node->marks[(__force unsigned)mark];  in node_marks()
86 		unsigned int offset, xa_mark_t mark)  in node_get_mark()  argument
88 	return test_bit(offset, node_marks(node, mark));  in node_get_mark()
93 				xa_mark_t mark)  in node_set_mark()  argument
95 	return __test_and_set_bit(offset, node_marks(node, mark));  in node_set_mark()
100 				xa_mark_t mark)  in node_clear_mark()  argument
102 	return __test_and_clear_bit(offset, node_marks(node, mark));  in node_clear_mark()
105 static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark)  in node_any_mark()  argument
107 	return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);  in node_any_mark()
110 static inline void node_mark_all(struct xa_node *node, xa_mark_t mark)  in node_mark_all()  argument
112 	bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE);  in node_mark_all()
115 #define mark_inc(mark) do { \  argument
116 	mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \
123  * Set a mark on the first entry if any entry has it set.  Clear marks on
128 	xa_mark_t mark = 0;  in xas_squash_marks()  local
132 		unsigned long *marks = node_marks(xas->xa_node, mark);  in xas_squash_marks()
138 		if (mark == XA_MARK_MAX)  in xas_squash_marks()
140 		mark_inc(mark);  in xas_squash_marks()
587 		xa_mark_t mark = 0;  in xas_expand()  local
599 		/* Propagate the aggregated mark info to the new child */  in xas_expand()
601 			if (xa_track_free(xa) && mark == XA_FREE_MARK) {  in xas_expand()
607 			} else if (xa_marked(xa, mark)) {  in xas_expand()
608 				node_set_mark(node, 0, mark);  in xas_expand()
610 			if (mark == XA_MARK_MAX)  in xas_expand()
612 			mark_inc(mark);  in xas_expand()
824 		 * so the mark clearing will appear to happen before the  in xas_store()
858  * xas_get_mark() - Returns the state of this mark.
860  * @mark: Mark number.
862  * Return: true if the mark is set, false if the mark is clear or @xas
865 bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_get_mark()  argument
870 		return xa_marked(xas->xa, mark);  in xas_get_mark()
871 	return node_get_mark(xas->xa_node, xas->xa_offset, mark);  in xas_get_mark()
876  * xas_set_mark() - Sets the mark on this entry and its parents.
878  * @mark: Mark number.
880  * Sets the specified mark on this entry, and walks up the tree setting it
884 void xas_set_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_set_mark()  argument
893 		if (node_set_mark(node, offset, mark))  in xas_set_mark()
899 	if (!xa_marked(xas->xa, mark))  in xas_set_mark()
900 		xa_mark_set(xas->xa, mark);  in xas_set_mark()
905  * xas_clear_mark() - Clears the mark on this entry and its parents.
907  * @mark: Mark number.
909  * Clears the specified mark on this entry, and walks back to the head
913 void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_clear_mark()  argument
922 		if (!node_clear_mark(node, offset, mark))  in xas_clear_mark()
924 		if (node_any_mark(node, mark))  in xas_clear_mark()
931 	if (xa_marked(xas->xa, mark))  in xas_clear_mark()
932 		xa_mark_clear(xas->xa, mark);  in xas_clear_mark()
941  * free entries with a mark, we need to set it on all entries.  All other
949 	xa_mark_t mark = 0;  in xas_init_marks()  local
952 		if (xa_track_free(xas->xa) && mark == XA_FREE_MARK)  in xas_init_marks()
953 			xas_set_mark(xas, mark);  in xas_init_marks()
955 			xas_clear_mark(xas, mark);  in xas_init_marks()
956 		if (mark == XA_MARK_MAX)  in xas_init_marks()
958 		mark_inc(mark);  in xas_init_marks()
967 	xa_mark_t mark = XA_MARK_0;  in node_get_marks()  local
970 		if (node_get_mark(node, offset, mark))  in node_get_marks()
971 			marks |= 1 << (__force unsigned int)mark;  in node_get_marks()
972 		if (mark == XA_MARK_MAX)  in node_get_marks()
974 		mark_inc(mark);  in node_get_marks()
981 		xa_mark_t mark)  in node_mark_slots()  argument
986 		node_mark_all(node, mark);  in node_mark_slots()
989 			node_set_mark(node, i, mark);  in node_mark_slots()
997 	xa_mark_t mark = XA_MARK_0;  in node_set_marks()  local
1000 		if (marks & (1 << (__force unsigned int)mark)) {  in node_set_marks()
1001 			node_set_mark(node, offset, mark);  in node_set_marks()
1003 				node_mark_slots(child, sibs, mark);  in node_set_marks()
1005 		if (mark == XA_MARK_MAX)  in node_set_marks()
1007 		mark_inc(mark);  in node_set_marks()
1445  * @mark: Mark number to search for.
1462 void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark)  in xas_find_marked()  argument
1483 			if (xa_marked(xas->xa, mark))  in xas_find_marked()
1510 		offset = xas_find_chunk(xas, advance, mark);  in xas_find_marked()
1523 		if (!entry && !(xa_track_free(xas->xa) && mark == XA_FREE_MARK))  in xas_find_marked()
2064  * __xa_set_mark() - Set this mark on this entry while locked.
2067  * @mark: Mark number.
2069  * Attempting to set a mark on a %NULL entry does not succeed.
2073 void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in __xa_set_mark()  argument
2079 		xas_set_mark(&xas, mark);  in __xa_set_mark()
2084  * __xa_clear_mark() - Clear this mark on this entry while locked.
2087  * @mark: Mark number.
2091 void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in __xa_clear_mark()  argument
2097 		xas_clear_mark(&xas, mark);  in __xa_clear_mark()
2102  * xa_get_mark() - Inquire whether this mark is set on this entry.
2105  * @mark: Mark number.
2111  * Return: True if the entry at @index has this mark set, false if it doesn't.
2113 bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_get_mark()  argument
2120 	while (xas_get_mark(&xas, mark)) {  in xa_get_mark()
2134  * xa_set_mark() - Set this mark on this entry.
2137  * @mark: Mark number.
2139  * Attempting to set a mark on a %NULL entry does not succeed.
2143 void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_set_mark()  argument
2146 	__xa_set_mark(xa, index, mark);  in xa_set_mark()
2152  * xa_clear_mark() - Clear this mark on this entry.
2155  * @mark: Mark number.
2157  * Clearing a mark always succeeds.
2161 void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_clear_mark()  argument
2164 	__xa_clear_mark(xa, index, mark);  in xa_clear_mark()
2287 			unsigned long max, unsigned int n, xa_mark_t mark)  in xas_extract_marked()  argument
2293 	xas_for_each_marked(xas, entry, max, mark) {  in xas_extract_marked()
2317  * The @filter may be an XArray mark value, in which case entries which are
2318  * marked with that mark will be copied.  It may also be %XA_PRESENT, in