1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/bpf.h> 5 #include <linux/filter.h> 6 #include <linux/bpf_mem_alloc.h> 7 #include <linux/gfp.h> 8 #include <linux/memory.h> 9 #include <linux/mutex.h> 10 11 static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) 12 { 13 init_llist_node(&elem->node); 14 elem->total_len = len; 15 elem->consumed_len = 0; 16 } 17 18 static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) 19 { 20 const int max_len = ARRAY_SIZE((struct bpf_bprintf_buffers){}.buf); 21 struct bpf_stream_elem *elem; 22 size_t alloc_size; 23 24 /* 25 * Length denotes the amount of data to be written as part of stream element, 26 * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can 27 * accomodate, therefore deny allocations that won't fit into them. 28 */ 29 if (len < 0 || len > max_len) 30 return NULL; 31 32 alloc_size = offsetof(struct bpf_stream_elem, str[len]); 33 elem = kmalloc_nolock(alloc_size, __GFP_ZERO, -1); 34 if (!elem) 35 return NULL; 36 37 bpf_stream_elem_init(elem, len); 38 39 return elem; 40 } 41 42 static int __bpf_stream_push_str(struct llist_head *log, const char *str, int len) 43 { 44 struct bpf_stream_elem *elem = NULL; 45 46 /* 47 * Allocate a bpf_prog_stream_elem and push it to the bpf_prog_stream 48 * log, elements will be popped at once and reversed to print the log. 49 */ 50 elem = bpf_stream_elem_alloc(len); 51 if (!elem) 52 return -ENOMEM; 53 54 memcpy(elem->str, str, len); 55 llist_add(&elem->node, log); 56 57 return 0; 58 } 59 60 static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) 61 { 62 if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY) 63 return -ENOSPC; 64 if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) { 65 atomic_sub(len, &stream->capacity); 66 return -ENOSPC; 67 } 68 return 0; 69 } 70 71 static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) 72 { 73 int len = elem->total_len; 74 75 atomic_sub(len, &stream->capacity); 76 } 77 78 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) 79 { 80 int ret = bpf_stream_consume_capacity(stream, len); 81 82 return ret ?: __bpf_stream_push_str(&stream->log, str, len); 83 } 84 85 static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) 86 { 87 if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) 88 return NULL; 89 return &aux->stream[stream_id - 1]; 90 } 91 92 static void bpf_stream_free_elem(struct bpf_stream_elem *elem) 93 { 94 kfree_nolock(elem); 95 } 96 97 static void bpf_stream_free_list(struct llist_node *list) 98 { 99 struct bpf_stream_elem *elem, *tmp; 100 101 llist_for_each_entry_safe(elem, tmp, list, node) 102 bpf_stream_free_elem(elem); 103 } 104 105 static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream) 106 { 107 return stream->backlog_head; 108 } 109 110 static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream) 111 { 112 struct llist_node *node; 113 114 node = stream->backlog_head; 115 if (stream->backlog_head == stream->backlog_tail) 116 stream->backlog_head = stream->backlog_tail = NULL; 117 else 118 stream->backlog_head = node->next; 119 return node; 120 } 121 122 static void bpf_stream_backlog_fill(struct bpf_stream *stream) 123 { 124 struct llist_node *head, *tail; 125 126 if (llist_empty(&stream->log)) 127 return; 128 tail = llist_del_all(&stream->log); 129 if (!tail) 130 return; 131 head = llist_reverse_order(tail); 132 133 if (!stream->backlog_head) { 134 stream->backlog_head = head; 135 stream->backlog_tail = tail; 136 } else { 137 stream->backlog_tail->next = head; 138 stream->backlog_tail = tail; 139 } 140 141 return; 142 } 143 144 static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) 145 { 146 int rem = elem->total_len - elem->consumed_len; 147 int used = min(rem, *len); 148 149 elem->consumed_len += used; 150 *len -= used; 151 152 return elem->consumed_len == elem->total_len; 153 } 154 155 static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) 156 { 157 int rem_len = len, cons_len, ret = 0; 158 struct bpf_stream_elem *elem = NULL; 159 struct llist_node *node; 160 161 mutex_lock(&stream->lock); 162 163 while (rem_len) { 164 int pos = len - rem_len; 165 bool cont; 166 167 node = bpf_stream_backlog_peek(stream); 168 if (!node) { 169 bpf_stream_backlog_fill(stream); 170 node = bpf_stream_backlog_peek(stream); 171 } 172 if (!node) 173 break; 174 elem = container_of(node, typeof(*elem), node); 175 176 cons_len = elem->consumed_len; 177 cont = bpf_stream_consume_elem(elem, &rem_len) == false; 178 179 ret = copy_to_user(buf + pos, elem->str + cons_len, 180 elem->consumed_len - cons_len); 181 /* Restore in case of error. */ 182 if (ret) { 183 ret = -EFAULT; 184 elem->consumed_len = cons_len; 185 break; 186 } 187 188 if (cont) 189 continue; 190 bpf_stream_backlog_pop(stream); 191 bpf_stream_release_capacity(stream, elem); 192 bpf_stream_free_elem(elem); 193 } 194 195 mutex_unlock(&stream->lock); 196 return ret ? ret : len - rem_len; 197 } 198 199 int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) 200 { 201 struct bpf_stream *stream; 202 203 stream = bpf_stream_get(stream_id, prog->aux); 204 if (!stream) 205 return -ENOENT; 206 return bpf_stream_read(stream, buf, len); 207 } 208 209 __bpf_kfunc_start_defs(); 210 211 /* 212 * Avoid using enum bpf_stream_id so that kfunc users don't have to pull in the 213 * enum in headers. 214 */ 215 __bpf_kfunc int bpf_stream_vprintk_impl(int stream_id, const char *fmt__str, const void *args, 216 u32 len__sz, void *aux__prog) 217 { 218 struct bpf_bprintf_data data = { 219 .get_bin_args = true, 220 .get_buf = true, 221 }; 222 struct bpf_prog_aux *aux = aux__prog; 223 u32 fmt_size = strlen(fmt__str) + 1; 224 struct bpf_stream *stream; 225 u32 data_len = len__sz; 226 int ret, num_args; 227 228 stream = bpf_stream_get(stream_id, aux); 229 if (!stream) 230 return -ENOENT; 231 232 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 233 (data_len && !args)) 234 return -EINVAL; 235 num_args = data_len / 8; 236 237 ret = bpf_bprintf_prepare(fmt__str, fmt_size, args, num_args, &data); 238 if (ret < 0) 239 return ret; 240 241 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); 242 /* Exclude NULL byte during push. */ 243 ret = bpf_stream_push_str(stream, data.buf, ret); 244 bpf_bprintf_cleanup(&data); 245 246 return ret; 247 } 248 249 __bpf_kfunc_end_defs(); 250 251 /* Added kfunc to common_btf_ids */ 252 253 void bpf_prog_stream_init(struct bpf_prog *prog) 254 { 255 int i; 256 257 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { 258 atomic_set(&prog->aux->stream[i].capacity, 0); 259 init_llist_head(&prog->aux->stream[i].log); 260 mutex_init(&prog->aux->stream[i].lock); 261 prog->aux->stream[i].backlog_head = NULL; 262 prog->aux->stream[i].backlog_tail = NULL; 263 } 264 } 265 266 void bpf_prog_stream_free(struct bpf_prog *prog) 267 { 268 struct llist_node *list; 269 int i; 270 271 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { 272 list = llist_del_all(&prog->aux->stream[i].log); 273 bpf_stream_free_list(list); 274 bpf_stream_free_list(prog->aux->stream[i].backlog_head); 275 } 276 } 277 278 void bpf_stream_stage_init(struct bpf_stream_stage *ss) 279 { 280 init_llist_head(&ss->log); 281 ss->len = 0; 282 } 283 284 void bpf_stream_stage_free(struct bpf_stream_stage *ss) 285 { 286 struct llist_node *node; 287 288 node = llist_del_all(&ss->log); 289 bpf_stream_free_list(node); 290 } 291 292 int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) 293 { 294 struct bpf_bprintf_buffers *buf; 295 va_list args; 296 int ret; 297 298 if (bpf_try_get_buffers(&buf)) 299 return -EBUSY; 300 301 va_start(args, fmt); 302 ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); 303 va_end(args); 304 ss->len += ret; 305 /* Exclude NULL byte during push. */ 306 ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); 307 bpf_put_buffers(); 308 return ret; 309 } 310 311 int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, 312 enum bpf_stream_id stream_id) 313 { 314 struct llist_node *list, *head, *tail; 315 struct bpf_stream *stream; 316 int ret; 317 318 stream = bpf_stream_get(stream_id, prog->aux); 319 if (!stream) 320 return -EINVAL; 321 322 ret = bpf_stream_consume_capacity(stream, ss->len); 323 if (ret) 324 return ret; 325 326 list = llist_del_all(&ss->log); 327 head = tail = list; 328 329 if (!list) 330 return 0; 331 while (llist_next(list)) { 332 tail = llist_next(list); 333 list = tail; 334 } 335 llist_add_batch(head, tail, &stream->log); 336 return 0; 337 } 338 339 struct dump_stack_ctx { 340 struct bpf_stream_stage *ss; 341 int err; 342 }; 343 344 static bool dump_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp) 345 { 346 struct dump_stack_ctx *ctxp = cookie; 347 const char *file = "", *line = ""; 348 struct bpf_prog *prog; 349 int num, ret; 350 351 rcu_read_lock(); 352 prog = bpf_prog_ksym_find(ip); 353 rcu_read_unlock(); 354 if (prog) { 355 ret = bpf_prog_get_file_line(prog, ip, &file, &line, &num); 356 if (ret < 0) 357 goto end; 358 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n %s @ %s:%d\n", 359 (void *)(long)ip, line, file, num); 360 return !ctxp->err; 361 } 362 end: 363 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n", (void *)(long)ip); 364 return !ctxp->err; 365 } 366 367 int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss) 368 { 369 struct dump_stack_ctx ctx = { .ss = ss }; 370 int ret; 371 372 ret = bpf_stream_stage_printk(ss, "CPU: %d UID: %d PID: %d Comm: %s\n", 373 raw_smp_processor_id(), __kuid_val(current_real_cred()->euid), 374 current->pid, current->comm); 375 if (ret) 376 return ret; 377 ret = bpf_stream_stage_printk(ss, "Call trace:\n"); 378 if (ret) 379 return ret; 380 arch_bpf_stack_walk(dump_stack_cb, &ctx); 381 if (ctx.err) 382 return ctx.err; 383 return bpf_stream_stage_printk(ss, "\n"); 384 } 385