1 // SPDX-License-Identifier: GPL-2.0 2 /* Watch queue and general notification mechanism, built on pipes 3 * 4 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * 7 * See Documentation/core-api/watch_queue.rst 8 */ 9 10 #define pr_fmt(fmt) "watchq: " fmt 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/sched.h> 14 #include <linux/slab.h> 15 #include <linux/printk.h> 16 #include <linux/miscdevice.h> 17 #include <linux/fs.h> 18 #include <linux/mm.h> 19 #include <linux/pagemap.h> 20 #include <linux/poll.h> 21 #include <linux/uaccess.h> 22 #include <linux/vmalloc.h> 23 #include <linux/file.h> 24 #include <linux/security.h> 25 #include <linux/cred.h> 26 #include <linux/sched/signal.h> 27 #include <linux/watch_queue.h> 28 #include <linux/pipe_fs_i.h> 29 30 MODULE_DESCRIPTION("Watch queue"); 31 MODULE_AUTHOR("Red Hat, Inc."); 32 33 #define WATCH_QUEUE_NOTE_SIZE 128 34 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE) 35 36 /* 37 * This must be called under the RCU read-lock, which makes 38 * sure that the wqueue still exists. It can then take the lock, 39 * and check that the wqueue hasn't been destroyed, which in 40 * turn makes sure that the notification pipe still exists. 41 */ 42 static inline bool lock_wqueue(struct watch_queue *wqueue) 43 { 44 spin_lock_bh(&wqueue->lock); 45 if (unlikely(!wqueue->pipe)) { 46 spin_unlock_bh(&wqueue->lock); 47 return false; 48 } 49 return true; 50 } 51 52 static inline void unlock_wqueue(struct watch_queue *wqueue) 53 { 54 spin_unlock_bh(&wqueue->lock); 55 } 56 57 static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe, 58 struct pipe_buffer *buf) 59 { 60 struct watch_queue *wqueue = (struct watch_queue *)buf->private; 61 struct page *page; 62 unsigned int bit; 63 64 /* We need to work out which note within the page this refers to, but 65 * the note might have been maximum size, so merely ANDing the offset 66 * off doesn't work. OTOH, the note must've been more than zero size. 67 */ 68 bit = buf->offset + buf->len; 69 if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0) 70 bit -= WATCH_QUEUE_NOTE_SIZE; 71 bit /= WATCH_QUEUE_NOTE_SIZE; 72 73 page = buf->page; 74 bit += page->private; 75 76 set_bit(bit, wqueue->notes_bitmap); 77 generic_pipe_buf_release(pipe, buf); 78 } 79 80 /* New data written to a pipe may be appended to a buffer with this type. */ 81 static const struct pipe_buf_operations watch_queue_pipe_buf_ops = { 82 .release = watch_queue_pipe_buf_release, 83 .get = generic_pipe_buf_get, 84 }; 85 86 /* 87 * Post a notification to a watch queue. 88 * 89 * Must be called with the RCU lock for reading, and the 90 * watch_queue lock held, which guarantees that the pipe 91 * hasn't been released. 92 */ 93 static bool post_one_notification(struct watch_queue *wqueue, 94 struct watch_notification *n) 95 { 96 void *p; 97 struct pipe_inode_info *pipe = wqueue->pipe; 98 struct pipe_buffer *buf; 99 struct page *page; 100 unsigned int head, tail, note, offset, len, nr_notes; 101 bool done = false; 102 103 spin_lock_irq(&pipe->rd_wait.lock); 104 105 head = pipe->head; 106 tail = pipe->tail; 107 if (pipe_full(head, tail, pipe->ring_size)) 108 goto lost; 109 110 /* Pairs with the smp_store_release() in watch_queue_set_size(). */ 111 nr_notes = smp_load_acquire(&wqueue->nr_notes); 112 note = find_first_bit(wqueue->notes_bitmap, nr_notes); 113 if (note >= nr_notes) 114 goto lost; 115 116 page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE]; 117 offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE; 118 get_page(page); 119 len = n->info & WATCH_INFO_LENGTH; 120 p = kmap_local_page(page); 121 memcpy(p + offset, n, len); 122 kunmap_local(p); 123 124 buf = pipe_buf(pipe, head); 125 buf->page = page; 126 buf->private = (unsigned long)wqueue; 127 buf->ops = &watch_queue_pipe_buf_ops; 128 buf->offset = offset; 129 buf->len = len; 130 buf->flags = PIPE_BUF_FLAG_WHOLE; 131 smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */ 132 133 if (!test_and_clear_bit(note, wqueue->notes_bitmap)) { 134 spin_unlock_irq(&pipe->rd_wait.lock); 135 BUG(); 136 } 137 wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); 138 done = true; 139 140 out: 141 spin_unlock_irq(&pipe->rd_wait.lock); 142 if (done) 143 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 144 return done; 145 146 lost: 147 buf = pipe_buf(pipe, head - 1); 148 buf->flags |= PIPE_BUF_FLAG_LOSS; 149 goto out; 150 } 151 152 /* 153 * Apply filter rules to a notification. 154 */ 155 static bool filter_watch_notification(const struct watch_filter *wf, 156 const struct watch_notification *n) 157 { 158 const struct watch_type_filter *wt; 159 unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8; 160 unsigned int st_index = n->subtype / st_bits; 161 unsigned int st_bit = 1U << (n->subtype % st_bits); 162 int i; 163 164 if (!test_bit(n->type, wf->type_filter)) 165 return false; 166 167 for (i = 0; i < wf->nr_filters; i++) { 168 wt = &wf->filters[i]; 169 if (n->type == wt->type && 170 (wt->subtype_filter[st_index] & st_bit) && 171 (n->info & wt->info_mask) == wt->info_filter) 172 return true; 173 } 174 175 return false; /* If there is a filter, the default is to reject. */ 176 } 177 178 /** 179 * __post_watch_notification - Post an event notification 180 * @wlist: The watch list to post the event to. 181 * @n: The notification record to post. 182 * @cred: The creds of the process that triggered the notification. 183 * @id: The ID to match on the watch. 184 * 185 * Post a notification of an event into a set of watch queues and let the users 186 * know. 187 * 188 * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and 189 * should be in units of sizeof(*n). 190 */ 191 void __post_watch_notification(struct watch_list *wlist, 192 struct watch_notification *n, 193 const struct cred *cred, 194 u64 id) 195 { 196 const struct watch_filter *wf; 197 struct watch_queue *wqueue; 198 struct watch *watch; 199 200 if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) { 201 WARN_ON(1); 202 return; 203 } 204 205 rcu_read_lock(); 206 207 hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) { 208 if (watch->id != id) 209 continue; 210 n->info &= ~WATCH_INFO_ID; 211 n->info |= watch->info_id & WATCH_INFO_ID; 212 213 wqueue = rcu_dereference(watch->queue); 214 wf = rcu_dereference(wqueue->filter); 215 if (wf && !filter_watch_notification(wf, n)) 216 continue; 217 218 if (security_post_notification(watch->cred, cred, n) < 0) 219 continue; 220 221 if (lock_wqueue(wqueue)) { 222 post_one_notification(wqueue, n); 223 unlock_wqueue(wqueue); 224 } 225 } 226 227 rcu_read_unlock(); 228 } 229 EXPORT_SYMBOL(__post_watch_notification); 230 231 /* 232 * Allocate sufficient pages to preallocation for the requested number of 233 * notifications. 234 */ 235 long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes) 236 { 237 struct watch_queue *wqueue = pipe->watch_queue; 238 struct page **pages; 239 unsigned long *bitmap; 240 unsigned long user_bufs; 241 int ret, i, nr_pages; 242 243 if (!wqueue) 244 return -ENODEV; 245 if (wqueue->notes) 246 return -EBUSY; 247 248 if (nr_notes < 1 || 249 nr_notes > 512) /* TODO: choose a better hard limit */ 250 return -EINVAL; 251 252 nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1); 253 nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE; 254 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages); 255 256 if (nr_pages > pipe->max_usage && 257 (too_many_pipe_buffers_hard(user_bufs) || 258 too_many_pipe_buffers_soft(user_bufs)) && 259 pipe_is_unprivileged_user()) { 260 ret = -EPERM; 261 goto error; 262 } 263 264 nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE; 265 ret = pipe_resize_ring(pipe, roundup_pow_of_two(nr_notes)); 266 if (ret < 0) 267 goto error; 268 269 /* 270 * pipe_resize_ring() does not update nr_accounted for watch_queue 271 * pipes, because the above vastly overprovisions. Set nr_accounted on 272 * and max_usage this pipe to the number that was actually charged to 273 * the user above via account_pipe_buffers. 274 */ 275 pipe->max_usage = nr_pages; 276 pipe->nr_accounted = nr_pages; 277 278 ret = -ENOMEM; 279 pages = kzalloc_objs(struct page *, nr_pages); 280 if (!pages) 281 goto error; 282 283 for (i = 0; i < nr_pages; i++) { 284 pages[i] = alloc_page(GFP_KERNEL); 285 if (!pages[i]) 286 goto error_p; 287 pages[i]->private = i * WATCH_QUEUE_NOTES_PER_PAGE; 288 } 289 290 bitmap = bitmap_alloc(nr_notes, GFP_KERNEL); 291 if (!bitmap) 292 goto error_p; 293 294 bitmap_fill(bitmap, nr_notes); 295 wqueue->notes = pages; 296 wqueue->notes_bitmap = bitmap; 297 wqueue->nr_pages = nr_pages; 298 /* Pairs with the smp_load_acquire() in post_one_notification(). */ 299 smp_store_release(&wqueue->nr_notes, nr_notes); 300 return 0; 301 302 error_p: 303 while (--i >= 0) 304 __free_page(pages[i]); 305 kfree(pages); 306 error: 307 (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted); 308 return ret; 309 } 310 311 /* 312 * Set the filter on a watch queue. 313 */ 314 long watch_queue_set_filter(struct pipe_inode_info *pipe, 315 struct watch_notification_filter __user *_filter) 316 { 317 struct watch_notification_type_filter *tf; 318 struct watch_notification_filter filter; 319 struct watch_type_filter *q; 320 struct watch_filter *wfilter; 321 struct watch_queue *wqueue = pipe->watch_queue; 322 int ret, nr_filter = 0, i; 323 324 if (!wqueue) 325 return -ENODEV; 326 327 if (!_filter) { 328 /* Remove the old filter */ 329 wfilter = NULL; 330 goto set; 331 } 332 333 /* Grab the user's filter specification */ 334 if (copy_from_user(&filter, _filter, sizeof(filter)) != 0) 335 return -EFAULT; 336 if (filter.nr_filters == 0 || 337 filter.nr_filters > 16 || 338 filter.__reserved != 0) 339 return -EINVAL; 340 341 tf = memdup_array_user(_filter->filters, filter.nr_filters, sizeof(*tf)); 342 if (IS_ERR(tf)) 343 return PTR_ERR(tf); 344 345 ret = -EINVAL; 346 for (i = 0; i < filter.nr_filters; i++) { 347 if ((tf[i].info_filter & ~tf[i].info_mask) || 348 tf[i].info_mask & WATCH_INFO_LENGTH) 349 goto err_filter; 350 /* Ignore any unknown types */ 351 if (tf[i].type >= WATCH_TYPE__NR) 352 continue; 353 nr_filter++; 354 } 355 356 /* Now we need to build the internal filter from only the relevant 357 * user-specified filters. 358 */ 359 ret = -ENOMEM; 360 wfilter = kzalloc_flex(*wfilter, filters, nr_filter); 361 if (!wfilter) 362 goto err_filter; 363 wfilter->nr_filters = nr_filter; 364 365 q = wfilter->filters; 366 for (i = 0; i < filter.nr_filters; i++) { 367 if (tf[i].type >= WATCH_TYPE__NR) 368 continue; 369 370 q->type = tf[i].type; 371 q->info_filter = tf[i].info_filter; 372 q->info_mask = tf[i].info_mask; 373 q->subtype_filter[0] = tf[i].subtype_filter[0]; 374 __set_bit(q->type, wfilter->type_filter); 375 q++; 376 } 377 378 kfree(tf); 379 set: 380 pipe_lock(pipe); 381 wfilter = rcu_replace_pointer(wqueue->filter, wfilter, 382 lockdep_is_held(&pipe->mutex)); 383 pipe_unlock(pipe); 384 if (wfilter) 385 kfree_rcu(wfilter, rcu); 386 return 0; 387 388 err_filter: 389 kfree(tf); 390 return ret; 391 } 392 393 static void __put_watch_queue(struct kref *kref) 394 { 395 struct watch_queue *wqueue = 396 container_of(kref, struct watch_queue, usage); 397 struct watch_filter *wfilter; 398 int i; 399 400 for (i = 0; i < wqueue->nr_pages; i++) 401 __free_page(wqueue->notes[i]); 402 kfree(wqueue->notes); 403 bitmap_free(wqueue->notes_bitmap); 404 405 wfilter = rcu_access_pointer(wqueue->filter); 406 if (wfilter) 407 kfree_rcu(wfilter, rcu); 408 kfree_rcu(wqueue, rcu); 409 } 410 411 /** 412 * put_watch_queue - Dispose of a ref on a watchqueue. 413 * @wqueue: The watch queue to unref. 414 */ 415 void put_watch_queue(struct watch_queue *wqueue) 416 { 417 kref_put(&wqueue->usage, __put_watch_queue); 418 } 419 EXPORT_SYMBOL(put_watch_queue); 420 421 static void free_watch(struct rcu_head *rcu) 422 { 423 struct watch *watch = container_of(rcu, struct watch, rcu); 424 425 put_watch_queue(rcu_access_pointer(watch->queue)); 426 atomic_dec(&watch->cred->user->nr_watches); 427 put_cred(watch->cred); 428 kfree(watch); 429 } 430 431 static void __put_watch(struct kref *kref) 432 { 433 struct watch *watch = container_of(kref, struct watch, usage); 434 435 call_rcu(&watch->rcu, free_watch); 436 } 437 438 /* 439 * Discard a watch. 440 */ 441 static void put_watch(struct watch *watch) 442 { 443 kref_put(&watch->usage, __put_watch); 444 } 445 446 /** 447 * init_watch - Initialise a watch 448 * @watch: The watch to initialise. 449 * @wqueue: The queue to assign. 450 * 451 * Initialise a watch and set the watch queue. 452 */ 453 void init_watch(struct watch *watch, struct watch_queue *wqueue) 454 { 455 kref_init(&watch->usage); 456 INIT_HLIST_NODE(&watch->list_node); 457 INIT_HLIST_NODE(&watch->queue_node); 458 rcu_assign_pointer(watch->queue, wqueue); 459 } 460 461 static int add_one_watch(struct watch *watch, struct watch_list *wlist, struct watch_queue *wqueue) 462 { 463 const struct cred *cred; 464 struct watch *w; 465 466 hlist_for_each_entry(w, &wlist->watchers, list_node) { 467 struct watch_queue *wq = rcu_access_pointer(w->queue); 468 if (wqueue == wq && watch->id == w->id) 469 return -EBUSY; 470 } 471 472 cred = current_cred(); 473 if (atomic_inc_return(&cred->user->nr_watches) > task_rlimit(current, RLIMIT_NOFILE)) { 474 atomic_dec(&cred->user->nr_watches); 475 return -EAGAIN; 476 } 477 478 watch->cred = get_cred(cred); 479 rcu_assign_pointer(watch->watch_list, wlist); 480 481 kref_get(&wqueue->usage); 482 kref_get(&watch->usage); 483 hlist_add_head(&watch->queue_node, &wqueue->watches); 484 hlist_add_head_rcu(&watch->list_node, &wlist->watchers); 485 return 0; 486 } 487 488 /** 489 * add_watch_to_object - Add a watch on an object to a watch list 490 * @watch: The watch to add 491 * @wlist: The watch list to add to 492 * 493 * @watch->queue must have been set to point to the queue to post notifications 494 * to and the watch list of the object to be watched. @watch->cred must also 495 * have been set to the appropriate credentials and a ref taken on them. 496 * 497 * The caller must pin the queue and the list both and must hold the list 498 * locked against racing watch additions/removals. 499 */ 500 int add_watch_to_object(struct watch *watch, struct watch_list *wlist) 501 { 502 struct watch_queue *wqueue; 503 int ret = -ENOENT; 504 505 rcu_read_lock(); 506 507 wqueue = rcu_access_pointer(watch->queue); 508 if (lock_wqueue(wqueue)) { 509 spin_lock(&wlist->lock); 510 ret = add_one_watch(watch, wlist, wqueue); 511 spin_unlock(&wlist->lock); 512 unlock_wqueue(wqueue); 513 } 514 515 rcu_read_unlock(); 516 return ret; 517 } 518 EXPORT_SYMBOL(add_watch_to_object); 519 520 /** 521 * remove_watch_from_object - Remove a watch or all watches from an object. 522 * @wlist: The watch list to remove from 523 * @wq: The watch queue of interest (ignored if @all is true) 524 * @id: The ID of the watch to remove (ignored if @all is true) 525 * @all: True to remove all objects 526 * 527 * Remove a specific watch or all watches from an object. A notification is 528 * sent to the watcher to tell them that this happened. 529 */ 530 int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq, 531 u64 id, bool all) 532 { 533 struct watch_notification_removal n; 534 struct watch_queue *wqueue; 535 struct watch *watch; 536 int ret = -EBADSLT; 537 538 rcu_read_lock(); 539 540 again: 541 spin_lock(&wlist->lock); 542 hlist_for_each_entry(watch, &wlist->watchers, list_node) { 543 if (all || 544 (watch->id == id && rcu_access_pointer(watch->queue) == wq)) 545 goto found; 546 } 547 spin_unlock(&wlist->lock); 548 goto out; 549 550 found: 551 ret = 0; 552 hlist_del_init_rcu(&watch->list_node); 553 rcu_assign_pointer(watch->watch_list, NULL); 554 spin_unlock(&wlist->lock); 555 556 /* We now own the reference on watch that used to belong to wlist. */ 557 558 n.watch.type = WATCH_TYPE_META; 559 n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION; 560 n.watch.info = watch->info_id | watch_sizeof(n.watch); 561 n.id = id; 562 if (id != 0) 563 n.watch.info = watch->info_id | watch_sizeof(n); 564 565 wqueue = rcu_dereference(watch->queue); 566 567 if (lock_wqueue(wqueue)) { 568 post_one_notification(wqueue, &n.watch); 569 570 if (!hlist_unhashed(&watch->queue_node)) { 571 hlist_del_init_rcu(&watch->queue_node); 572 put_watch(watch); 573 } 574 575 unlock_wqueue(wqueue); 576 } 577 578 if (wlist->release_watch) { 579 void (*release_watch)(struct watch *); 580 581 release_watch = wlist->release_watch; 582 rcu_read_unlock(); 583 (*release_watch)(watch); 584 rcu_read_lock(); 585 } 586 put_watch(watch); 587 588 if (all && !hlist_empty(&wlist->watchers)) 589 goto again; 590 out: 591 rcu_read_unlock(); 592 return ret; 593 } 594 EXPORT_SYMBOL(remove_watch_from_object); 595 596 /* 597 * Remove all the watches that are contributory to a queue. This has the 598 * potential to race with removal of the watches by the destruction of the 599 * objects being watched or with the distribution of notifications. 600 */ 601 void watch_queue_clear(struct watch_queue *wqueue) 602 { 603 struct watch_list *wlist; 604 struct watch *watch; 605 bool release; 606 607 rcu_read_lock(); 608 spin_lock_bh(&wqueue->lock); 609 610 /* 611 * This pipe can be freed by callers like free_pipe_info(). 612 * Removing this reference also prevents new notifications. 613 */ 614 wqueue->pipe = NULL; 615 616 while (!hlist_empty(&wqueue->watches)) { 617 watch = hlist_entry(wqueue->watches.first, struct watch, queue_node); 618 hlist_del_init_rcu(&watch->queue_node); 619 /* We now own a ref on the watch. */ 620 spin_unlock_bh(&wqueue->lock); 621 622 /* We can't do the next bit under the queue lock as we need to 623 * get the list lock - which would cause a deadlock if someone 624 * was removing from the opposite direction at the same time or 625 * posting a notification. 626 */ 627 wlist = rcu_dereference(watch->watch_list); 628 if (wlist) { 629 void (*release_watch)(struct watch *); 630 631 spin_lock(&wlist->lock); 632 633 release = !hlist_unhashed(&watch->list_node); 634 if (release) { 635 hlist_del_init_rcu(&watch->list_node); 636 rcu_assign_pointer(watch->watch_list, NULL); 637 638 /* We now own a second ref on the watch. */ 639 } 640 641 release_watch = wlist->release_watch; 642 spin_unlock(&wlist->lock); 643 644 if (release) { 645 if (release_watch) { 646 rcu_read_unlock(); 647 /* This might need to call dput(), so 648 * we have to drop all the locks. 649 */ 650 (*release_watch)(watch); 651 rcu_read_lock(); 652 } 653 put_watch(watch); 654 } 655 } 656 657 put_watch(watch); 658 spin_lock_bh(&wqueue->lock); 659 } 660 661 spin_unlock_bh(&wqueue->lock); 662 rcu_read_unlock(); 663 } 664 665 /** 666 * get_watch_queue - Get a watch queue from its file descriptor. 667 * @fd: The fd to query. 668 */ 669 struct watch_queue *get_watch_queue(int fd) 670 { 671 struct pipe_inode_info *pipe; 672 struct watch_queue *wqueue = ERR_PTR(-EINVAL); 673 CLASS(fd, f)(fd); 674 675 if (!fd_empty(f)) { 676 pipe = get_pipe_info(fd_file(f), false); 677 if (pipe && pipe->watch_queue) { 678 wqueue = pipe->watch_queue; 679 kref_get(&wqueue->usage); 680 } 681 } 682 683 return wqueue; 684 } 685 EXPORT_SYMBOL(get_watch_queue); 686 687 /* 688 * Initialise a watch queue 689 */ 690 int watch_queue_init(struct pipe_inode_info *pipe) 691 { 692 struct watch_queue *wqueue; 693 694 wqueue = kzalloc_obj(*wqueue); 695 if (!wqueue) 696 return -ENOMEM; 697 698 wqueue->pipe = pipe; 699 kref_init(&wqueue->usage); 700 spin_lock_init(&wqueue->lock); 701 INIT_HLIST_HEAD(&wqueue->watches); 702 703 pipe->watch_queue = wqueue; 704 return 0; 705 } 706