31 static bool queue_stack_map_is_empty(struct bpf_queue_stack *qs)  in queue_stack_map_is_empty()  argument
33 	return qs->head == qs->tail;  in queue_stack_map_is_empty()
36 static bool queue_stack_map_is_full(struct bpf_queue_stack *qs)  in queue_stack_map_is_full()  argument
38 	u32 head = qs->head + 1;  in queue_stack_map_is_full()
40 	if (unlikely(head >= qs->size))  in queue_stack_map_is_full()
43 	return head == qs->tail;  in queue_stack_map_is_full()
68 	struct bpf_queue_stack *qs;  in queue_stack_map_alloc()  local
72 	queue_size = sizeof(*qs) + size * attr->value_size;  in queue_stack_map_alloc()
74 	qs = bpf_map_area_alloc(queue_size, numa_node);  in queue_stack_map_alloc()
75 	if (!qs)  in queue_stack_map_alloc()
78 	bpf_map_init_from_attr(&qs->map, attr);  in queue_stack_map_alloc()
80 	qs->size = size;  in queue_stack_map_alloc()
82 	raw_res_spin_lock_init(&qs->lock);  in queue_stack_map_alloc()
84 	return &qs->map;  in queue_stack_map_alloc()
90 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in queue_stack_map_free()  local
92 	bpf_map_area_free(qs);  in queue_stack_map_free()
97 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in __queue_map_get()  local
102 	if (raw_res_spin_lock_irqsave(&qs->lock, flags))  in __queue_map_get()
105 	if (queue_stack_map_is_empty(qs)) {  in __queue_map_get()
106 		memset(value, 0, qs->map.value_size);  in __queue_map_get()
111 	ptr = &qs->elements[qs->tail * qs->map.value_size];  in __queue_map_get()
112 	memcpy(value, ptr, qs->map.value_size);  in __queue_map_get()
115 		if (unlikely(++qs->tail >= qs->size))  in __queue_map_get()
116 			qs->tail = 0;  in __queue_map_get()
120 	raw_res_spin_unlock_irqrestore(&qs->lock, flags);  in __queue_map_get()
127 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in __stack_map_get()  local
133 	if (raw_res_spin_lock_irqsave(&qs->lock, flags))  in __stack_map_get()
136 	if (queue_stack_map_is_empty(qs)) {  in __stack_map_get()
137 		memset(value, 0, qs->map.value_size);  in __stack_map_get()
142 	index = qs->head - 1;  in __stack_map_get()
143 	if (unlikely(index >= qs->size))  in __stack_map_get()
144 		index = qs->size - 1;  in __stack_map_get()
146 	ptr = &qs->elements[index * qs->map.value_size];  in __stack_map_get()
147 	memcpy(value, ptr, qs->map.value_size);  in __stack_map_get()
150 		qs->head = index;  in __stack_map_get()
153 	raw_res_spin_unlock_irqrestore(&qs->lock, flags);  in __stack_map_get()
185 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in queue_stack_map_push_elem()  local
199 	if (raw_res_spin_lock_irqsave(&qs->lock, irq_flags))  in queue_stack_map_push_elem()
202 	if (queue_stack_map_is_full(qs)) {  in queue_stack_map_push_elem()
208 		if (unlikely(++qs->tail >= qs->size))  in queue_stack_map_push_elem()
209 			qs->tail = 0;  in queue_stack_map_push_elem()
212 	dst = &qs->elements[qs->head * qs->map.value_size];  in queue_stack_map_push_elem()
213 	memcpy(dst, value, qs->map.value_size);  in queue_stack_map_push_elem()
215 	if (unlikely(++qs->head >= qs->size))  in queue_stack_map_push_elem()
216 		qs->head = 0;  in queue_stack_map_push_elem()
219 	raw_res_spin_unlock_irqrestore(&qs->lock, irq_flags);  in queue_stack_map_push_elem()