25 	 * Length denotes the amount of data to be written as part of stream element,  in bpf_stream_elem_alloc()
60 static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)  in bpf_stream_consume_capacity()  argument
62 	if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY)  in bpf_stream_consume_capacity()
64 	if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) {  in bpf_stream_consume_capacity()
65 		atomic_sub(len, &stream->capacity);  in bpf_stream_consume_capacity()
71 static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)  in bpf_stream_release_capacity()  argument
75 	atomic_sub(len, &stream->capacity);  in bpf_stream_release_capacity()
78 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)  in bpf_stream_push_str()  argument
80 	int ret = bpf_stream_consume_capacity(stream, len);  in bpf_stream_push_str()
82 	return ret ?: __bpf_stream_push_str(&stream->log, str, len);  in bpf_stream_push_str()
89 	return &aux->stream[stream_id - 1];  in bpf_stream_get()
105 static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream)  in bpf_stream_backlog_peek()  argument
107 	return stream->backlog_head;  in bpf_stream_backlog_peek()
110 static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream)  in bpf_stream_backlog_pop()  argument
114 	node = stream->backlog_head;  in bpf_stream_backlog_pop()
115 	if (stream->backlog_head == stream->backlog_tail)  in bpf_stream_backlog_pop()
116 		stream->backlog_head = stream->backlog_tail = NULL;  in bpf_stream_backlog_pop()
118 		stream->backlog_head = node->next;  in bpf_stream_backlog_pop()
122 static void bpf_stream_backlog_fill(struct bpf_stream *stream)  in bpf_stream_backlog_fill()  argument
126 	if (llist_empty(&stream->log))  in bpf_stream_backlog_fill()
128 	tail = llist_del_all(&stream->log);  in bpf_stream_backlog_fill()
133 	if (!stream->backlog_head) {  in bpf_stream_backlog_fill()
134 		stream->backlog_head = head;  in bpf_stream_backlog_fill()
135 		stream->backlog_tail = tail;  in bpf_stream_backlog_fill()
137 		stream->backlog_tail->next = head;  in bpf_stream_backlog_fill()
138 		stream->backlog_tail = tail;  in bpf_stream_backlog_fill()
155 static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)  in bpf_stream_read()  argument
161 	mutex_lock(&stream->lock);  in bpf_stream_read()
167 		node = bpf_stream_backlog_peek(stream);  in bpf_stream_read()
169 			bpf_stream_backlog_fill(stream);  in bpf_stream_read()
170 			node = bpf_stream_backlog_peek(stream);  in bpf_stream_read()
190 		bpf_stream_backlog_pop(stream);  in bpf_stream_read()
191 		bpf_stream_release_capacity(stream, elem);  in bpf_stream_read()
195 	mutex_unlock(&stream->lock);  in bpf_stream_read()
201 	struct bpf_stream *stream;  in bpf_prog_stream_read()  local
203 	stream = bpf_stream_get(stream_id, prog->aux);  in bpf_prog_stream_read()
204 	if (!stream)  in bpf_prog_stream_read()
206 	return bpf_stream_read(stream, buf, len);  in bpf_prog_stream_read()
224 	struct bpf_stream *stream;  in bpf_stream_vprintk_impl()  local
228 	stream = bpf_stream_get(stream_id, aux);  in bpf_stream_vprintk_impl()
229 	if (!stream)  in bpf_stream_vprintk_impl()
243 	ret = bpf_stream_push_str(stream, data.buf, ret);  in bpf_stream_vprintk_impl()
257 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {  in bpf_prog_stream_init()
258 		atomic_set(&prog->aux->stream[i].capacity, 0);  in bpf_prog_stream_init()
259 		init_llist_head(&prog->aux->stream[i].log);  in bpf_prog_stream_init()
260 		mutex_init(&prog->aux->stream[i].lock);  in bpf_prog_stream_init()
261 		prog->aux->stream[i].backlog_head = NULL;  in bpf_prog_stream_init()
262 		prog->aux->stream[i].backlog_tail = NULL;  in bpf_prog_stream_init()
271 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {  in bpf_prog_stream_free()
272 		list = llist_del_all(&prog->aux->stream[i].log);  in bpf_prog_stream_free()
274 		bpf_stream_free_list(prog->aux->stream[i].backlog_head);  in bpf_prog_stream_free()
315 	struct bpf_stream *stream;  in bpf_stream_stage_commit()  local
318 	stream = bpf_stream_get(stream_id, prog->aux);  in bpf_stream_stage_commit()
319 	if (!stream)  in bpf_stream_stage_commit()
322 	ret = bpf_stream_consume_capacity(stream, ss->len);  in bpf_stream_stage_commit()
335 	llist_add_batch(head, tail, &stream->log);  in bpf_stream_stage_commit()