1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 #include <linux/reboot.h> 9 10 #define CREATE_TRACE_POINTS 11 #include <trace/events/notifier.h> 12 13 /* 14 * Notifier list for kernel code which wants to be called 15 * at shutdown. This is used to stop any idling DMA operations 16 * and the like. 17 */ 18 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 19 20 /* 21 * Notifier chain core routines. The exported routines below 22 * are layered on top of these, with appropriate locking added. 23 */ 24 25 static int notifier_chain_register(struct notifier_block **nl, 26 struct notifier_block *n, 27 bool unique_priority) 28 { 29 while ((*nl) != NULL) { 30 if (unlikely((*nl) == n)) { 31 WARN(1, "notifier callback %ps already registered", 32 n->notifier_call); 33 return -EEXIST; 34 } 35 if (n->priority > (*nl)->priority) 36 break; 37 if (n->priority == (*nl)->priority && unique_priority) 38 return -EBUSY; 39 nl = &((*nl)->next); 40 } 41 n->next = *nl; 42 rcu_assign_pointer(*nl, n); 43 trace_notifier_register((void *)n->notifier_call); 44 return 0; 45 } 46 47 static int notifier_chain_unregister(struct notifier_block **nl, 48 struct notifier_block *n) 49 { 50 while ((*nl) != NULL) { 51 if ((*nl) == n) { 52 rcu_assign_pointer(*nl, n->next); 53 trace_notifier_unregister((void *)n->notifier_call); 54 return 0; 55 } 56 nl = &((*nl)->next); 57 } 58 return -ENOENT; 59 } 60 61 /** 62 * notifier_call_chain - Informs the registered notifiers about an event. 63 * @nl: Pointer to head of the blocking notifier chain 64 * @val: Value passed unmodified to notifier function 65 * @v: Pointer passed unmodified to notifier function 66 * @nr_to_call: Number of notifier functions to be called. Don't care 67 * value of this parameter is -1. 68 * @nr_calls: Records the number of notifications sent. Don't care 69 * value of this field is NULL. 70 * Return: notifier_call_chain returns the value returned by the 71 * last notifier function called. 72 */ 73 static int notifier_call_chain(struct notifier_block **nl, 74 unsigned long val, void *v, 75 int nr_to_call, int *nr_calls) 76 { 77 int ret = NOTIFY_DONE; 78 struct notifier_block *nb, *next_nb; 79 80 nb = rcu_dereference_raw(*nl); 81 82 while (nb && nr_to_call) { 83 next_nb = rcu_dereference_raw(nb->next); 84 85 #ifdef CONFIG_DEBUG_NOTIFIERS 86 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 87 WARN(1, "Invalid notifier called!"); 88 nb = next_nb; 89 continue; 90 } 91 #endif 92 trace_notifier_run((void *)nb->notifier_call); 93 ret = nb->notifier_call(nb, val, v); 94 95 if (nr_calls) 96 (*nr_calls)++; 97 98 if (ret & NOTIFY_STOP_MASK) 99 break; 100 nb = next_nb; 101 nr_to_call--; 102 } 103 return ret; 104 } 105 NOKPROBE_SYMBOL(notifier_call_chain); 106 107 /** 108 * notifier_call_chain_robust - Inform the registered notifiers about an event 109 * and rollback on error. 110 * @nl: Pointer to head of the blocking notifier chain 111 * @val_up: Value passed unmodified to the notifier function 112 * @val_down: Value passed unmodified to the notifier function when recovering 113 * from an error on @val_up 114 * @v: Pointer passed unmodified to the notifier function 115 * 116 * NOTE: It is important the @nl chain doesn't change between the two 117 * invocations of notifier_call_chain() such that we visit the 118 * exact same notifier callbacks; this rules out any RCU usage. 119 * 120 * Return: the return value of the @val_up call. 121 */ 122 static int notifier_call_chain_robust(struct notifier_block **nl, 123 unsigned long val_up, unsigned long val_down, 124 void *v) 125 { 126 int ret, nr = 0; 127 128 ret = notifier_call_chain(nl, val_up, v, -1, &nr); 129 if (ret & NOTIFY_STOP_MASK) 130 notifier_call_chain(nl, val_down, v, nr-1, NULL); 131 132 return ret; 133 } 134 135 /* 136 * Atomic notifier chain routines. Registration and unregistration 137 * use a spinlock, and call_chain is synchronized by RCU (no locks). 138 */ 139 140 /** 141 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 142 * @nh: Pointer to head of the atomic notifier chain 143 * @n: New entry in notifier chain 144 * 145 * Adds a notifier to an atomic notifier chain. 146 * 147 * Returns 0 on success, %-EEXIST on error. 148 */ 149 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 150 struct notifier_block *n) 151 { 152 unsigned long flags; 153 int ret; 154 155 spin_lock_irqsave(&nh->lock, flags); 156 ret = notifier_chain_register(&nh->head, n, false); 157 spin_unlock_irqrestore(&nh->lock, flags); 158 return ret; 159 } 160 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 161 162 /** 163 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain 164 * @nh: Pointer to head of the atomic notifier chain 165 * @n: New entry in notifier chain 166 * 167 * Adds a notifier to an atomic notifier chain if there is no other 168 * notifier registered using the same priority. 169 * 170 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 171 */ 172 int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh, 173 struct notifier_block *n) 174 { 175 unsigned long flags; 176 int ret; 177 178 spin_lock_irqsave(&nh->lock, flags); 179 ret = notifier_chain_register(&nh->head, n, true); 180 spin_unlock_irqrestore(&nh->lock, flags); 181 return ret; 182 } 183 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio); 184 185 /** 186 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 187 * @nh: Pointer to head of the atomic notifier chain 188 * @n: Entry to remove from notifier chain 189 * 190 * Removes a notifier from an atomic notifier chain. 191 * 192 * Returns zero on success or %-ENOENT on failure. 193 */ 194 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 195 struct notifier_block *n) 196 { 197 unsigned long flags; 198 int ret; 199 200 spin_lock_irqsave(&nh->lock, flags); 201 ret = notifier_chain_unregister(&nh->head, n); 202 spin_unlock_irqrestore(&nh->lock, flags); 203 synchronize_rcu(); 204 return ret; 205 } 206 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 207 208 /** 209 * atomic_notifier_call_chain - Call functions in an atomic notifier chain 210 * @nh: Pointer to head of the atomic notifier chain 211 * @val: Value passed unmodified to notifier function 212 * @v: Pointer passed unmodified to notifier function 213 * 214 * Calls each function in a notifier chain in turn. The functions 215 * run in an atomic context, so they must not block. 216 * This routine uses RCU to synchronize with changes to the chain. 217 * 218 * If the return value of the notifier can be and'ed 219 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 220 * will return immediately, with the return value of 221 * the notifier function which halted execution. 222 * Otherwise the return value is the return value 223 * of the last notifier function called. 224 */ 225 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 226 unsigned long val, void *v) 227 { 228 int ret; 229 230 rcu_read_lock(); 231 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 232 rcu_read_unlock(); 233 234 return ret; 235 } 236 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 237 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 238 239 /** 240 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty 241 * @nh: Pointer to head of the atomic notifier chain 242 * 243 * Checks whether notifier chain is empty. 244 * 245 * Returns true is notifier chain is empty, false otherwise. 246 */ 247 bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh) 248 { 249 return !rcu_access_pointer(nh->head); 250 } 251 252 /* 253 * Blocking notifier chain routines. All access to the chain is 254 * synchronized by an rwsem. 255 */ 256 257 static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh, 258 struct notifier_block *n, 259 bool unique_priority) 260 { 261 int ret; 262 263 /* 264 * This code gets used during boot-up, when task switching is 265 * not yet working and interrupts must remain disabled. At 266 * such times we must not call down_write(). 267 */ 268 if (unlikely(system_state == SYSTEM_BOOTING)) 269 return notifier_chain_register(&nh->head, n, unique_priority); 270 271 down_write(&nh->rwsem); 272 ret = notifier_chain_register(&nh->head, n, unique_priority); 273 up_write(&nh->rwsem); 274 return ret; 275 } 276 277 /** 278 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 279 * @nh: Pointer to head of the blocking notifier chain 280 * @n: New entry in notifier chain 281 * 282 * Adds a notifier to a blocking notifier chain. 283 * Must be called in process context. 284 * 285 * Returns 0 on success, %-EEXIST on error. 286 */ 287 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 288 struct notifier_block *n) 289 { 290 return __blocking_notifier_chain_register(nh, n, false); 291 } 292 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 293 294 /** 295 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain 296 * @nh: Pointer to head of the blocking notifier chain 297 * @n: New entry in notifier chain 298 * 299 * Adds a notifier to an blocking notifier chain if there is no other 300 * notifier registered using the same priority. 301 * 302 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 303 */ 304 int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh, 305 struct notifier_block *n) 306 { 307 return __blocking_notifier_chain_register(nh, n, true); 308 } 309 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio); 310 311 /** 312 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 313 * @nh: Pointer to head of the blocking notifier chain 314 * @n: Entry to remove from notifier chain 315 * 316 * Removes a notifier from a blocking notifier chain. 317 * Must be called from process context. 318 * 319 * Returns zero on success or %-ENOENT on failure. 320 */ 321 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 322 struct notifier_block *n) 323 { 324 int ret; 325 326 /* 327 * This code gets used during boot-up, when task switching is 328 * not yet working and interrupts must remain disabled. At 329 * such times we must not call down_write(). 330 */ 331 if (unlikely(system_state == SYSTEM_BOOTING)) 332 return notifier_chain_unregister(&nh->head, n); 333 334 down_write(&nh->rwsem); 335 ret = notifier_chain_unregister(&nh->head, n); 336 up_write(&nh->rwsem); 337 return ret; 338 } 339 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 340 341 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, 342 unsigned long val_up, unsigned long val_down, void *v) 343 { 344 int ret = NOTIFY_DONE; 345 346 /* 347 * We check the head outside the lock, but if this access is 348 * racy then it does not matter what the result of the test 349 * is, we re-check the list after having taken the lock anyway: 350 */ 351 if (rcu_access_pointer(nh->head)) { 352 down_read(&nh->rwsem); 353 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 354 up_read(&nh->rwsem); 355 } 356 return ret; 357 } 358 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); 359 360 /** 361 * blocking_notifier_call_chain - Call functions in a blocking notifier chain 362 * @nh: Pointer to head of the blocking notifier chain 363 * @val: Value passed unmodified to notifier function 364 * @v: Pointer passed unmodified to notifier function 365 * 366 * Calls each function in a notifier chain in turn. The functions 367 * run in a process context, so they are allowed to block. 368 * 369 * If the return value of the notifier can be and'ed 370 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 371 * will return immediately, with the return value of 372 * the notifier function which halted execution. 373 * Otherwise the return value is the return value 374 * of the last notifier function called. 375 */ 376 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 377 unsigned long val, void *v) 378 { 379 int ret = NOTIFY_DONE; 380 381 /* 382 * We check the head outside the lock, but if this access is 383 * racy then it does not matter what the result of the test 384 * is, we re-check the list after having taken the lock anyway: 385 */ 386 if (rcu_access_pointer(nh->head)) { 387 down_read(&nh->rwsem); 388 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 389 up_read(&nh->rwsem); 390 } 391 return ret; 392 } 393 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 394 395 /* 396 * Raw notifier chain routines. There is no protection; 397 * the caller must provide it. Use at your own risk! 398 */ 399 400 /** 401 * raw_notifier_chain_register - Add notifier to a raw notifier chain 402 * @nh: Pointer to head of the raw notifier chain 403 * @n: New entry in notifier chain 404 * 405 * Adds a notifier to a raw notifier chain. 406 * All locking must be provided by the caller. 407 * 408 * Returns 0 on success, %-EEXIST on error. 409 */ 410 int raw_notifier_chain_register(struct raw_notifier_head *nh, 411 struct notifier_block *n) 412 { 413 return notifier_chain_register(&nh->head, n, false); 414 } 415 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 416 417 /** 418 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 419 * @nh: Pointer to head of the raw notifier chain 420 * @n: Entry to remove from notifier chain 421 * 422 * Removes a notifier from a raw notifier chain. 423 * All locking must be provided by the caller. 424 * 425 * Returns zero on success or %-ENOENT on failure. 426 */ 427 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 428 struct notifier_block *n) 429 { 430 return notifier_chain_unregister(&nh->head, n); 431 } 432 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 433 434 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, 435 unsigned long val_up, unsigned long val_down, void *v) 436 { 437 return notifier_call_chain_robust(&nh->head, val_up, val_down, v); 438 } 439 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); 440 441 /** 442 * raw_notifier_call_chain - Call functions in a raw notifier chain 443 * @nh: Pointer to head of the raw notifier chain 444 * @val: Value passed unmodified to notifier function 445 * @v: Pointer passed unmodified to notifier function 446 * 447 * Calls each function in a notifier chain in turn. The functions 448 * run in an undefined context. 449 * All locking must be provided by the caller. 450 * 451 * If the return value of the notifier can be and'ed 452 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 453 * will return immediately, with the return value of 454 * the notifier function which halted execution. 455 * Otherwise the return value is the return value 456 * of the last notifier function called. 457 */ 458 int raw_notifier_call_chain(struct raw_notifier_head *nh, 459 unsigned long val, void *v) 460 { 461 return notifier_call_chain(&nh->head, val, v, -1, NULL); 462 } 463 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 464 465 /* 466 * SRCU notifier chain routines. Registration and unregistration 467 * use a mutex, and call_chain is synchronized by SRCU (no locks). 468 */ 469 470 /** 471 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 472 * @nh: Pointer to head of the SRCU notifier chain 473 * @n: New entry in notifier chain 474 * 475 * Adds a notifier to an SRCU notifier chain. 476 * Must be called in process context. 477 * 478 * Returns 0 on success, %-EEXIST on error. 479 */ 480 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 481 struct notifier_block *n) 482 { 483 int ret; 484 485 /* 486 * This code gets used during boot-up, when task switching is 487 * not yet working and interrupts must remain disabled. At 488 * such times we must not call mutex_lock(). 489 */ 490 if (unlikely(system_state == SYSTEM_BOOTING)) 491 return notifier_chain_register(&nh->head, n, false); 492 493 mutex_lock(&nh->mutex); 494 ret = notifier_chain_register(&nh->head, n, false); 495 mutex_unlock(&nh->mutex); 496 return ret; 497 } 498 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 499 500 /** 501 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 502 * @nh: Pointer to head of the SRCU notifier chain 503 * @n: Entry to remove from notifier chain 504 * 505 * Removes a notifier from an SRCU notifier chain. 506 * Must be called from process context. 507 * 508 * Returns zero on success or %-ENOENT on failure. 509 */ 510 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 511 struct notifier_block *n) 512 { 513 int ret; 514 515 /* 516 * This code gets used during boot-up, when task switching is 517 * not yet working and interrupts must remain disabled. At 518 * such times we must not call mutex_lock(). 519 */ 520 if (unlikely(system_state == SYSTEM_BOOTING)) 521 return notifier_chain_unregister(&nh->head, n); 522 523 mutex_lock(&nh->mutex); 524 ret = notifier_chain_unregister(&nh->head, n); 525 mutex_unlock(&nh->mutex); 526 synchronize_srcu(&nh->srcu); 527 return ret; 528 } 529 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 530 531 /** 532 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain 533 * @nh: Pointer to head of the SRCU notifier chain 534 * @val: Value passed unmodified to notifier function 535 * @v: Pointer passed unmodified to notifier function 536 * 537 * Calls each function in a notifier chain in turn. The functions 538 * run in a process context, so they are allowed to block. 539 * 540 * If the return value of the notifier can be and'ed 541 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 542 * will return immediately, with the return value of 543 * the notifier function which halted execution. 544 * Otherwise the return value is the return value 545 * of the last notifier function called. 546 */ 547 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 548 unsigned long val, void *v) 549 { 550 int ret; 551 int idx; 552 553 idx = srcu_read_lock(&nh->srcu); 554 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 555 srcu_read_unlock(&nh->srcu, idx); 556 return ret; 557 } 558 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 559 560 /** 561 * srcu_init_notifier_head - Initialize an SRCU notifier head 562 * @nh: Pointer to head of the srcu notifier chain 563 * 564 * Unlike other sorts of notifier heads, SRCU notifier heads require 565 * dynamic initialization. Be sure to call this routine before 566 * calling any of the other SRCU notifier routines for this head. 567 * 568 * If an SRCU notifier head is deallocated, it must first be cleaned 569 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 570 * per-cpu data (used by the SRCU mechanism) will leak. 571 */ 572 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 573 { 574 mutex_init(&nh->mutex); 575 if (init_srcu_struct(&nh->srcu) < 0) 576 BUG(); 577 nh->head = NULL; 578 } 579 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 580 581 static ATOMIC_NOTIFIER_HEAD(die_chain); 582 583 int notrace notify_die(enum die_val val, const char *str, 584 struct pt_regs *regs, long err, int trap, int sig) 585 { 586 struct die_args args = { 587 .regs = regs, 588 .str = str, 589 .err = err, 590 .trapnr = trap, 591 .signr = sig, 592 593 }; 594 RCU_LOCKDEP_WARN(!rcu_is_watching(), 595 "notify_die called but RCU thinks we're quiescent"); 596 return atomic_notifier_call_chain(&die_chain, val, &args); 597 } 598 NOKPROBE_SYMBOL(notify_die); 599 600 int register_die_notifier(struct notifier_block *nb) 601 { 602 return atomic_notifier_chain_register(&die_chain, nb); 603 } 604 EXPORT_SYMBOL_GPL(register_die_notifier); 605 606 int unregister_die_notifier(struct notifier_block *nb) 607 { 608 return atomic_notifier_chain_unregister(&die_chain, nb); 609 } 610 EXPORT_SYMBOL_GPL(unregister_die_notifier); 611