21  * pt_check_range() - Validate the range can be iterated
22  * @range: Range to validate
24  * Check that VA and last_va fall within the permitted range of VAs. If the
28 static inline int pt_check_range(struct pt_range *range)  in pt_check_range()  argument
32 	PT_WARN_ON(!range->max_vasz_lg2);  in pt_check_range()
34 	if (pt_feature(range->common, PT_FEAT_SIGN_EXTEND)) {  in pt_check_range()
35 		PT_WARN_ON(range->common->max_vasz_lg2 != range->max_vasz_lg2);  in pt_check_range()
36 		prefix = fvalog2_div(range->va, range->max_vasz_lg2 - 1) ?  in pt_check_range()
40 		prefix = pt_full_va_prefix(range->common);  in pt_check_range()
43 	if (!fvalog2_div_eq(range->va, prefix, range->max_vasz_lg2) ||  in pt_check_range()
44 	    !fvalog2_div_eq(range->last_va, prefix, range->max_vasz_lg2))  in pt_check_range()
50  * pt_index_to_va() - Update range->va to the current pts->index
53  * Adjust range->va to match the current index. This is done in a lazy manner
61 	pts->range->va = fvalog2_set_mod(pts->range->va, lower_va,  in pt_index_to_va()
78  *                            within pts->range
83  * Returns: true if the item is fully enclosed by the pts->range.
88 	struct pt_range *range = pts->range;  in pt_entry_fully_covered()  local
90 	/* Range begins at the start of the entry */  in pt_entry_fully_covered()
91 	if (log2_mod(pts->range->va, oasz_lg2))  in pt_entry_fully_covered()
94 	/* Range ends past the end of the entry */  in pt_entry_fully_covered()
95 	if (!log2_div_eq(range->va, range->last_va, oasz_lg2))  in pt_entry_fully_covered()
98 	/* Range ends at the end of the entry */  in pt_entry_fully_covered()
99 	return log2_mod_eq_max(range->last_va, oasz_lg2);  in pt_entry_fully_covered()
112 	PT_WARN_ON(pts->level > pts->range->top_level);  in pt_range_to_index()
113 	if (pts->range->top_level == pts->level)  in pt_range_to_index()
114 		return log2_div(fvalog2_mod(pts->range->va,  in pt_range_to_index()
115 					    pts->range->max_vasz_lg2),  in pt_range_to_index()
117 	return log2_mod(log2_div(pts->range->va, isz_lg2),  in pt_range_to_index()
130 	struct pt_range *range = pts->range;  in pt_range_to_end_index()  local
133 	if (range->va == range->last_va)  in pt_range_to_end_index()
136 	if (pts->range->top_level == pts->level)  in pt_range_to_end_index()
137 		return log2_div(fvalog2_mod(pts->range->last_va,  in pt_range_to_end_index()
138 					    pts->range->max_vasz_lg2),  in pt_range_to_end_index()
145 	if (log2_div_eq(range->va, range->last_va, num_entries_lg2 + isz_lg2))  in pt_range_to_end_index()
146 		return log2_mod(log2_div(pts->range->last_va, isz_lg2),  in pt_range_to_end_index()
186  * for_each_pt_level_entry() - For loop wrapper over entries in the range
190  * pts->range that fall within the pts's current table level. Each step does
213 	struct pt_range range = {  in _pt_top_range()  local
220 	struct pt_state pts = { .range = &range, .level = range.top_level };  in _pt_top_range()
231 	 * The top range will default to the lower region only with sign extend.  in _pt_top_range()
233 	range.max_vasz_lg2 = max_vasz_lg2;  in _pt_top_range()
237 	range.va = fvalog2_set_mod(pt_full_va_prefix(common), 0, max_vasz_lg2);  in _pt_top_range()
238 	range.last_va =  in _pt_top_range()
240 	return range;  in _pt_top_range()
244  * pt_top_range() - Return a range that spans part of the top level
247  * For PT_FEAT_SIGN_EXTEND this will return the lower range, and cover half the
261  * pt_all_range() - Return a range that spans the entire page table
264  * The returned range spans the whole page table. Due to how PT_FEAT_SIGN_EXTEND
265  * is supported range->va and range->last_va will be incorrect during the
270 	struct pt_range range = pt_top_range(common);  in pt_all_range()  local
273 		return range;  in pt_all_range()
279 	range.last_va = fvalog2_set_mod_max(0, range.max_vasz_lg2);  in pt_all_range()
280 	return range;  in pt_all_range()
284  * pt_upper_range() - Return a range that spans part of the top level
287  * For PT_FEAT_SIGN_EXTEND this will return the upper range, and cover half the
292 	struct pt_range range = pt_top_range(common);  in pt_upper_range()  local
295 		return range;  in pt_upper_range()
297 	range.va = fvalog2_set_mod(PT_VADDR_MAX, 0, range.max_vasz_lg2 - 1);  in pt_upper_range()
298 	range.last_va = PT_VADDR_MAX;  in pt_upper_range()
299 	return range;  in pt_upper_range()
303  * pt_make_range() - Return a range that spans part of the table
308  * The caller must validate the range with pt_check_range() before using it.
313 	struct pt_range range =  in pt_make_range()  local
316 	range.va = va;  in pt_make_range()
317 	range.last_va = last_va;  in pt_make_range()
319 	return range;  in pt_make_range()
330 	struct pt_range range = *parent;  in pt_make_child_range()  local
332 	range.va = va;  in pt_make_child_range()
333 	range.last_va = last_va;  in pt_make_child_range()
336 	PT_WARN_ON(pt_check_range(&range));  in pt_make_child_range()
338 	return range;  in pt_make_child_range()
343  * @range: Range pointer to embed in the state
350 pt_init(struct pt_range *range, unsigned int level, struct pt_table_p *table)  in pt_init()  argument
353 		.range = range,  in pt_init()
362  * @range: Range pointer to embed in the state
366 static __always_inline struct pt_state pt_init_top(struct pt_range *range)  in pt_init_top()  argument
368 	return pt_init(range, range->top_level, range->top_table);  in pt_init_top()
371 typedef int (*pt_level_fn_t)(struct pt_range *range, void *arg,
391 	ret = (*fn)(pts->range, arg, pts->level - 1, pts->table_lower);  in pt_descend()
396  * pt_walk_range() - Walk over a VA range
397  * @range: Range pointer
401  * Walk over a VA range. The caller should have done a validity check, at
402  * least calling pt_check_range(), when building range. The walk will
405 static __always_inline int pt_walk_range(struct pt_range *range,  in pt_walk_range()  argument
408 	return fn(range, arg, range->top_level, range->top_table);  in pt_walk_range()
422  * item. This creates a new walk and does not alter pts or pts->range.
428 	struct pt_range range = pt_make_child_range(pts->range, va, last_va);  in pt_walk_descend()  local
434 	return fn(&range, arg, pts->level - 1, pts->table_lower);  in pt_walk_descend()
444  * table. This creates a new walk and does not alter pts or pts->range.
453 			       log2_set_mod(parent_pts->range->va, 0, isz_lg2),  in pt_walk_descend_all()
454 			       log2_set_mod_max(parent_pts->range->va, isz_lg2),  in pt_walk_descend_all()
459  * pt_range_slice() - Return a range that spans indexes
464  * Create a range than spans an index range of the current table level
475 	va = fvalog2_set_mod(pts->range->va,  in pt_range_slice()
479 		pts->range->va,  in pt_range_slice()
481 	return pt_make_child_range(pts->range, va, last_va);  in pt_range_slice()
495 	struct pt_range range = _pt_top_range(common, top_of_table);  in pt_top_memsize_lg2()  local
496 	struct pt_state pts = pt_init_top(&range);  in pt_top_memsize_lg2()
500 	if (range.top_level != PT_MAX_TOP_LEVEL &&  in pt_top_memsize_lg2()
573 	static __always_inline int fn(struct pt_range *range, void *arg, \
579 			return CONCATENATE(fn, 0)(range, arg, 0, table); \
581 			return CONCATENATE(fn, 1)(range, arg, 1, table); \
583 			return CONCATENATE(fn, 2)(range, arg, 2, table); \
585 			return CONCATENATE(fn, 3)(range, arg, 3, table); \
587 			return CONCATENATE(fn, 4)(range, arg, 4, table); \
588 		return CONCATENATE(fn, 5)(range, arg, 5, table);         \
591 static inline int __pt_make_level_fn_err(struct pt_range *range, void *arg,  in __pt_make_level_fn_err()  argument
600 	static inline int fn(struct pt_range *range, void *arg,     \
604 		return do_fn(range, arg, level, table, descend_fn); \
615  *  static __always_inline int do_fn(struct pt_range *range, void *arg,