96 	struct htab_elem *__percpu *extra_elems;
109 struct htab_elem {  struct
116 				struct htab_elem *batch_flink;  argument
186 static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)  in htab_elem_value()
191 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()
197 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()
202 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)  in fd_htab_map_get_ptr()
207 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)  in get_htab_elem()
209 	return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);  in get_htab_elem()
230 		struct htab_elem *elem;  in htab_free_prealloced_internal_structs()
253 		struct htab_elem *elem;  in htab_free_prealloced_fields()
303 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,  in prealloc_lru_pop()
307 	struct htab_elem *l;  in prealloc_lru_pop()
311 		l = container_of(node, struct htab_elem, lru_node);  in prealloc_lru_pop()
352 				   offsetof(struct htab_elem, hash) -  in prealloc_init()
353 				   offsetof(struct htab_elem, lru_node),  in prealloc_init()
364 				 offsetof(struct htab_elem, lru_node),  in prealloc_init()
368 				       htab->elems + offsetof(struct htab_elem, fnode),  in prealloc_init()
390 	struct htab_elem *__percpu *pptr, *l_new;  in alloc_extra_elems()
394 	pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8,  in alloc_extra_elems()
404 		l_new = container_of(l, struct htab_elem, fnode);  in alloc_extra_elems()
428 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=  in htab_map_alloc_check()
429 		     offsetof(struct htab_elem, hash_node.pprev));  in htab_map_alloc_check()
456 	   sizeof(struct htab_elem))  in htab_map_alloc_check()
473 	struct htab_elem *elem = obj;  in htab_mem_dtor()
586 	htab->elem_size = sizeof(struct htab_elem) +  in htab_map_alloc()
695 static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,  in lookup_elem_raw()
699 	struct htab_elem *l;  in lookup_elem_raw()
712 static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,  in lookup_nulls_elem_raw()
717 	struct htab_elem *l;  in lookup_nulls_elem_raw()
739 	struct htab_elem *l;  in __htab_map_lookup_elem()
757 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()
786 				offsetof(struct htab_elem, key) +  in htab_map_gen_lookup()
794 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()
827 			      offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
831 			     offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
835 				offsetof(struct htab_elem, key) +  in htab_lru_map_gen_lookup()
841 				    struct htab_elem *elem)  in check_and_cancel_fields()
865 	struct htab_elem *l = NULL, *tgt_l;  in htab_lru_map_delete_node()
872 	tgt_l = container_of(node, struct htab_elem, lru_node);  in htab_lru_map_delete_node()
899 	struct htab_elem *l, *next_l;  in htab_map_get_next_key()
922 				  struct htab_elem, hash_node);  in htab_map_get_next_key()
941 					  struct htab_elem, hash_node);  in htab_map_get_next_key()
953 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)  in htab_elem_free()
962 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)  in htab_put_fd_value()
1001 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()
1075 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,  in alloc_htab_elem()
1078 					 struct htab_elem *old_elem, u64 map_flags)  in alloc_htab_elem()
1082 	struct htab_elem *l_new, **pl_new;  in alloc_htab_elem()
1099 			l_new = container_of(l, struct htab_elem, fnode);  in alloc_htab_elem()
1158 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,  in check_flags()
1177 	struct htab_elem *l_new, *l_old;  in htab_map_update_elem()
1273 static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)  in htab_lru_push_free()
1284 	struct htab_elem *l_new, *l_old = NULL;  in htab_lru_map_update_elem()
1360 	struct htab_elem *l_new, *l_old;  in htab_map_update_elem_in_place()
1423 	struct htab_elem *l_new = NULL, *l_old;  in __htab_lru_percpu_map_update_elem()
1506 	struct htab_elem *l;  in htab_map_delete_elem()
1541 	struct htab_elem *l;  in htab_lru_map_delete_elem()
1581 		struct htab_elem *l;  in delete_all_elements()
1599 		struct htab_elem *l;  in htab_free_malloced_internal_structs()
1684 	struct htab_elem *l;  in __htab_map_lookup_and_delete_elem()
1786 	struct htab_elem *node_to_free = NULL;  in __htab_map_lookup_and_delete_batch()
1791 	struct htab_elem *l;  in __htab_map_lookup_and_delete_batch()
2089 static struct htab_elem *
2091 			   struct htab_elem *prev_elem)  in bpf_hash_map_seq_find_next()
2098 	struct htab_elem *elem;  in bpf_hash_map_seq_find_next()
2111 		elem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);  in bpf_hash_map_seq_find_next()
2148 	struct htab_elem *elem;  in bpf_hash_map_seq_start()
2168 static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)  in __bpf_hash_map_seq_show()
2271 	struct htab_elem *elem;  in bpf_for_each_hash_elem()
2340 			usage += sizeof(struct htab_elem *) * num_possible_cpus();  in htab_map_mem_usage()
2405 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()
2426 				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));  in htab_percpu_map_gen_lookup()
2435 	struct htab_elem *l;  in htab_percpu_map_lookup_percpu_elem()
2449 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()
2461 	struct htab_elem *l;  in htab_lru_percpu_map_lookup_percpu_elem()
2477 	struct htab_elem *l;  in bpf_percpu_hash_copy()
2534 	struct htab_elem *l;  in htab_percpu_map_seq_show_elem()
2615 	struct htab_elem *l;  in fd_htab_map_free()
2715 				offsetof(struct htab_elem, key) +  in htab_of_map_gen_lookup()