49 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable);
73 	u32 page_cnt;  member
549 static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt, int node_id,  in arena_alloc_pages()  argument
569 	if (page_cnt > page_cnt_max)  in arena_alloc_pages()
576 		if (pgoff > page_cnt_max - page_cnt)  in arena_alloc_pages()
583 	alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *));  in arena_alloc_pages()
595 		ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);  in arena_alloc_pages()
598 		ret = range_tree_clear(&arena->rt, pgoff, page_cnt);  in arena_alloc_pages()
600 		ret = pgoff = range_tree_find(&arena->rt, page_cnt);  in arena_alloc_pages()
602 			ret = range_tree_clear(&arena->rt, pgoff, page_cnt);  in arena_alloc_pages()
607 	remaining = page_cnt;  in arena_alloc_pages()
649 	range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped);  in arena_alloc_pages()
669 static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)  in zap_pages()  argument
676 		zap_vma_range(vml->vma, uaddr, PAGE_SIZE * page_cnt);  in zap_pages()
679 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable)  in arena_free_pages()  argument
696 	uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT));  in arena_free_pages()
700 	page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT;  in arena_free_pages()
713 	range_tree_set(&arena->rt, pgoff, page_cnt);  in arena_free_pages()
717 	apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,  in arena_free_pages()
724 	flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE));  in arena_free_pages()
726 	if (page_cnt > 1)  in arena_free_pages()
728 		zap_pages(arena, full_uaddr, page_cnt);  in arena_free_pages()
732 		if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */  in arena_free_pages()
756 	s->page_cnt = page_cnt;  in arena_free_pages()
766 static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cnt)  in arena_reserve_pages()  argument
778 	if (pgoff + page_cnt > page_cnt_max)  in arena_reserve_pages()
785 	ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);  in arena_reserve_pages()
793 	ret = range_tree_clear(&arena->rt, pgoff, page_cnt);  in arena_reserve_pages()
810 	long kaddr, page_cnt, pgoff;  in arena_free_worker()  local
827 		page_cnt = s->page_cnt;  in arena_free_worker()
832 		apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,  in arena_free_worker()
835 		range_tree_set(&arena->rt, pgoff, page_cnt);  in arena_free_worker()
842 		page_cnt = s->page_cnt;  in arena_free_worker()
847 		flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE));  in arena_free_worker()
850 		zap_pages(arena, full_uaddr, page_cnt);  in arena_free_worker()
873 __bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt,  in bpf_arena_alloc_pages()  argument
879 	if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt)  in bpf_arena_alloc_pages()
882 	return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, true);  in bpf_arena_alloc_pages()
885 void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt,  in bpf_arena_alloc_pages_non_sleepable()  argument
891 	if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt)  in bpf_arena_alloc_pages_non_sleepable()
894 	return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, false);  in bpf_arena_alloc_pages_non_sleepable()
896 __bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt)  in bpf_arena_free_pages()  argument
901 	if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || !ptr__ign)  in bpf_arena_free_pages()
903 	arena_free_pages(arena, (long)ptr__ign, page_cnt, true);  in bpf_arena_free_pages()
906 void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt)  in bpf_arena_free_pages_non_sleepable()  argument
911 	if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || !ptr__ign)  in bpf_arena_free_pages_non_sleepable()
913 	arena_free_pages(arena, (long)ptr__ign, page_cnt, false);  in bpf_arena_free_pages_non_sleepable()
916 __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cnt)  in bpf_arena_reserve_pages()  argument
924 	if (!page_cnt)  in bpf_arena_reserve_pages()
927 	return arena_reserve_pages(arena, (long)ptr__ign, page_cnt);  in bpf_arena_reserve_pages()