1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/bpf.h> 5 #include <linux/filter.h> 6 #include <linux/bpf_mem_alloc.h> 7 #include <linux/gfp.h> 8 #include <linux/memory.h> 9 #include <linux/mutex.h> 10 11 static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) 12 { 13 init_llist_node(&elem->node); 14 elem->total_len = len; 15 elem->consumed_len = 0; 16 } 17 18 static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) 19 { 20 const int max_len = ARRAY_SIZE((struct bpf_bprintf_buffers){}.buf); 21 struct bpf_stream_elem *elem; 22 size_t alloc_size; 23 24 /* 25 * Length denotes the amount of data to be written as part of stream element, 26 * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can 27 * accomodate, therefore deny allocations that won't fit into them. 28 */ 29 if (len < 0 || len > max_len) 30 return NULL; 31 32 alloc_size = offsetof(struct bpf_stream_elem, str[len]); 33 elem = kmalloc_nolock(alloc_size, __GFP_ZERO, -1); 34 if (!elem) 35 return NULL; 36 37 bpf_stream_elem_init(elem, len); 38 39 return elem; 40 } 41 42 static int __bpf_stream_push_str(struct llist_head *log, const char *str, int len) 43 { 44 struct bpf_stream_elem *elem = NULL; 45 46 /* 47 * Allocate a bpf_prog_stream_elem and push it to the bpf_prog_stream 48 * log, elements will be popped at once and reversed to print the log. 49 */ 50 elem = bpf_stream_elem_alloc(len); 51 if (!elem) 52 return -ENOMEM; 53 54 memcpy(elem->str, str, len); 55 llist_add(&elem->node, log); 56 57 return 0; 58 } 59 60 static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) 61 { 62 if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY) 63 return -ENOSPC; 64 if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) { 65 atomic_sub(len, &stream->capacity); 66 return -ENOSPC; 67 } 68 return 0; 69 } 70 71 static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) 72 { 73 int len = elem->total_len; 74 75 atomic_sub(len, &stream->capacity); 76 } 77 78 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) 79 { 80 int ret = bpf_stream_consume_capacity(stream, len); 81 82 return ret ?: __bpf_stream_push_str(&stream->log, str, len); 83 } 84 85 static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) 86 { 87 if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) 88 return NULL; 89 return &aux->stream[stream_id - 1]; 90 } 91 92 static void bpf_stream_free_elem(struct bpf_stream_elem *elem) 93 { 94 kfree_nolock(elem); 95 } 96 97 static void bpf_stream_free_list(struct llist_node *list) 98 { 99 struct bpf_stream_elem *elem, *tmp; 100 101 llist_for_each_entry_safe(elem, tmp, list, node) 102 bpf_stream_free_elem(elem); 103 } 104 105 static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream) 106 { 107 return stream->backlog_head; 108 } 109 110 static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream) 111 { 112 struct llist_node *node; 113 114 node = stream->backlog_head; 115 if (stream->backlog_head == stream->backlog_tail) 116 stream->backlog_head = stream->backlog_tail = NULL; 117 else 118 stream->backlog_head = node->next; 119 return node; 120 } 121 122 static void bpf_stream_backlog_fill(struct bpf_stream *stream) 123 { 124 struct llist_node *head, *tail; 125 126 if (llist_empty(&stream->log)) 127 return; 128 tail = llist_del_all(&stream->log); 129 if (!tail) 130 return; 131 head = llist_reverse_order(tail); 132 133 if (!stream->backlog_head) { 134 stream->backlog_head = head; 135 stream->backlog_tail = tail; 136 } else { 137 stream->backlog_tail->next = head; 138 stream->backlog_tail = tail; 139 } 140 141 return; 142 } 143 144 static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) 145 { 146 int rem = elem->total_len - elem->consumed_len; 147 int used = min(rem, *len); 148 149 elem->consumed_len += used; 150 *len -= used; 151 152 return elem->consumed_len == elem->total_len; 153 } 154 155 static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) 156 { 157 int rem_len = len, cons_len, ret = 0; 158 struct bpf_stream_elem *elem = NULL; 159 struct llist_node *node; 160 161 mutex_lock(&stream->lock); 162 163 while (rem_len) { 164 int pos = len - rem_len; 165 bool cont; 166 167 node = bpf_stream_backlog_peek(stream); 168 if (!node) { 169 bpf_stream_backlog_fill(stream); 170 node = bpf_stream_backlog_peek(stream); 171 } 172 if (!node) 173 break; 174 elem = container_of(node, typeof(*elem), node); 175 176 cons_len = elem->consumed_len; 177 cont = bpf_stream_consume_elem(elem, &rem_len) == false; 178 179 ret = copy_to_user(buf + pos, elem->str + cons_len, 180 elem->consumed_len - cons_len); 181 /* Restore in case of error. */ 182 if (ret) { 183 ret = -EFAULT; 184 elem->consumed_len = cons_len; 185 break; 186 } 187 188 if (cont) 189 continue; 190 bpf_stream_backlog_pop(stream); 191 bpf_stream_release_capacity(stream, elem); 192 bpf_stream_free_elem(elem); 193 } 194 195 mutex_unlock(&stream->lock); 196 return ret ? ret : len - rem_len; 197 } 198 199 int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) 200 { 201 struct bpf_stream *stream; 202 203 stream = bpf_stream_get(stream_id, prog->aux); 204 if (!stream) 205 return -ENOENT; 206 return bpf_stream_read(stream, buf, len); 207 } 208 209 __bpf_kfunc_start_defs(); 210 211 /* 212 * Avoid using enum bpf_stream_id so that kfunc users don't have to pull in the 213 * enum in headers. 214 */ 215 __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const void *args, 216 u32 len__sz, struct bpf_prog_aux *aux) 217 { 218 struct bpf_bprintf_data data = { 219 .get_bin_args = true, 220 .get_buf = true, 221 }; 222 u32 fmt_size = strlen(fmt__str) + 1; 223 struct bpf_stream *stream; 224 u32 data_len = len__sz; 225 int ret, num_args; 226 227 stream = bpf_stream_get(stream_id, aux); 228 if (!stream) 229 return -ENOENT; 230 231 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 232 (data_len && !args)) 233 return -EINVAL; 234 num_args = data_len / 8; 235 236 ret = bpf_bprintf_prepare(fmt__str, fmt_size, args, num_args, &data); 237 if (ret < 0) 238 return ret; 239 240 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); 241 /* Exclude NULL byte during push. */ 242 ret = bpf_stream_push_str(stream, data.buf, ret); 243 bpf_bprintf_cleanup(&data); 244 245 return ret; 246 } 247 248 /* Directly trigger a stack dump from the program. */ 249 __bpf_kfunc int bpf_stream_print_stack(int stream_id, struct bpf_prog_aux *aux) 250 { 251 struct bpf_stream_stage ss; 252 struct bpf_prog *prog; 253 254 /* Make sure the stream ID is valid. */ 255 if (!bpf_stream_get(stream_id, aux)) 256 return -ENOENT; 257 258 prog = aux->main_prog_aux->prog; 259 260 bpf_stream_stage(ss, prog, stream_id, ({ 261 bpf_stream_dump_stack(ss); 262 })); 263 264 return 0; 265 } 266 267 __bpf_kfunc_end_defs(); 268 269 /* Added kfunc to common_btf_ids */ 270 271 void bpf_prog_stream_init(struct bpf_prog *prog) 272 { 273 int i; 274 275 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { 276 atomic_set(&prog->aux->stream[i].capacity, 0); 277 init_llist_head(&prog->aux->stream[i].log); 278 mutex_init(&prog->aux->stream[i].lock); 279 prog->aux->stream[i].backlog_head = NULL; 280 prog->aux->stream[i].backlog_tail = NULL; 281 } 282 } 283 284 void bpf_prog_stream_free(struct bpf_prog *prog) 285 { 286 struct llist_node *list; 287 int i; 288 289 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { 290 list = llist_del_all(&prog->aux->stream[i].log); 291 bpf_stream_free_list(list); 292 bpf_stream_free_list(prog->aux->stream[i].backlog_head); 293 } 294 } 295 296 void bpf_stream_stage_init(struct bpf_stream_stage *ss) 297 { 298 init_llist_head(&ss->log); 299 ss->len = 0; 300 } 301 302 void bpf_stream_stage_free(struct bpf_stream_stage *ss) 303 { 304 struct llist_node *node; 305 306 node = llist_del_all(&ss->log); 307 bpf_stream_free_list(node); 308 } 309 310 int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) 311 { 312 struct bpf_bprintf_buffers *buf; 313 va_list args; 314 int ret; 315 316 if (bpf_try_get_buffers(&buf)) 317 return -EBUSY; 318 319 va_start(args, fmt); 320 ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); 321 va_end(args); 322 ss->len += ret; 323 /* Exclude NULL byte during push. */ 324 ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); 325 bpf_put_buffers(); 326 return ret; 327 } 328 329 int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, 330 enum bpf_stream_id stream_id) 331 { 332 struct llist_node *list, *head, *tail; 333 struct bpf_stream *stream; 334 int ret; 335 336 stream = bpf_stream_get(stream_id, prog->aux); 337 if (!stream) 338 return -EINVAL; 339 340 ret = bpf_stream_consume_capacity(stream, ss->len); 341 if (ret) 342 return ret; 343 344 list = llist_del_all(&ss->log); 345 head = tail = list; 346 347 if (!list) 348 return 0; 349 while (llist_next(list)) { 350 tail = llist_next(list); 351 list = tail; 352 } 353 llist_add_batch(head, tail, &stream->log); 354 return 0; 355 } 356 357 struct dump_stack_ctx { 358 struct bpf_stream_stage *ss; 359 int err; 360 }; 361 362 static bool dump_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp) 363 { 364 struct dump_stack_ctx *ctxp = cookie; 365 const char *file = "", *line = ""; 366 struct bpf_prog *prog; 367 int num, ret; 368 369 rcu_read_lock(); 370 prog = bpf_prog_ksym_find(ip); 371 rcu_read_unlock(); 372 if (prog) { 373 ret = bpf_prog_get_file_line(prog, ip, &file, &line, &num); 374 if (ret < 0) 375 goto end; 376 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n %s @ %s:%d\n", 377 (void *)(long)ip, line, file, num); 378 return !ctxp->err; 379 } 380 end: 381 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n", (void *)(long)ip); 382 return !ctxp->err; 383 } 384 385 int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss) 386 { 387 struct dump_stack_ctx ctx = { .ss = ss }; 388 int ret; 389 390 ret = bpf_stream_stage_printk(ss, "CPU: %d UID: %d PID: %d Comm: %s\n", 391 raw_smp_processor_id(), __kuid_val(current_real_cred()->euid), 392 current->pid, current->comm); 393 if (ret) 394 return ret; 395 ret = bpf_stream_stage_printk(ss, "Call trace:\n"); 396 if (ret) 397 return ret; 398 arch_bpf_stack_walk(dump_stack_cb, &ctx); 399 if (ctx.err) 400 return ctx.err; 401 return bpf_stream_stage_printk(ss, "\n"); 402 } 403