1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 #include <linux/reboot.h> 9 10 /* 11 * Notifier list for kernel code which wants to be called 12 * at shutdown. This is used to stop any idling DMA operations 13 * and the like. 14 */ 15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 16 17 /* 18 * Notifier chain core routines. The exported routines below 19 * are layered on top of these, with appropriate locking added. 20 */ 21 22 static int notifier_chain_register(struct notifier_block **nl, 23 struct notifier_block *n, 24 bool unique_priority) 25 { 26 while ((*nl) != NULL) { 27 if (unlikely((*nl) == n)) { 28 WARN(1, "notifier callback %ps already registered", 29 n->notifier_call); 30 return -EEXIST; 31 } 32 if (n->priority > (*nl)->priority) 33 break; 34 if (n->priority == (*nl)->priority && unique_priority) 35 return -EBUSY; 36 nl = &((*nl)->next); 37 } 38 n->next = *nl; 39 rcu_assign_pointer(*nl, n); 40 return 0; 41 } 42 43 static int notifier_chain_unregister(struct notifier_block **nl, 44 struct notifier_block *n) 45 { 46 while ((*nl) != NULL) { 47 if ((*nl) == n) { 48 rcu_assign_pointer(*nl, n->next); 49 return 0; 50 } 51 nl = &((*nl)->next); 52 } 53 return -ENOENT; 54 } 55 56 /** 57 * notifier_call_chain - Informs the registered notifiers about an event. 58 * @nl: Pointer to head of the blocking notifier chain 59 * @val: Value passed unmodified to notifier function 60 * @v: Pointer passed unmodified to notifier function 61 * @nr_to_call: Number of notifier functions to be called. Don't care 62 * value of this parameter is -1. 63 * @nr_calls: Records the number of notifications sent. Don't care 64 * value of this field is NULL. 65 * Return: notifier_call_chain returns the value returned by the 66 * last notifier function called. 67 */ 68 static int notifier_call_chain(struct notifier_block **nl, 69 unsigned long val, void *v, 70 int nr_to_call, int *nr_calls) 71 { 72 int ret = NOTIFY_DONE; 73 struct notifier_block *nb, *next_nb; 74 75 nb = rcu_dereference_raw(*nl); 76 77 while (nb && nr_to_call) { 78 next_nb = rcu_dereference_raw(nb->next); 79 80 #ifdef CONFIG_DEBUG_NOTIFIERS 81 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 82 WARN(1, "Invalid notifier called!"); 83 nb = next_nb; 84 continue; 85 } 86 #endif 87 ret = nb->notifier_call(nb, val, v); 88 89 if (nr_calls) 90 (*nr_calls)++; 91 92 if (ret & NOTIFY_STOP_MASK) 93 break; 94 nb = next_nb; 95 nr_to_call--; 96 } 97 return ret; 98 } 99 NOKPROBE_SYMBOL(notifier_call_chain); 100 101 /** 102 * notifier_call_chain_robust - Inform the registered notifiers about an event 103 * and rollback on error. 104 * @nl: Pointer to head of the blocking notifier chain 105 * @val_up: Value passed unmodified to the notifier function 106 * @val_down: Value passed unmodified to the notifier function when recovering 107 * from an error on @val_up 108 * @v: Pointer passed unmodified to the notifier function 109 * 110 * NOTE: It is important the @nl chain doesn't change between the two 111 * invocations of notifier_call_chain() such that we visit the 112 * exact same notifier callbacks; this rules out any RCU usage. 113 * 114 * Return: the return value of the @val_up call. 115 */ 116 static int notifier_call_chain_robust(struct notifier_block **nl, 117 unsigned long val_up, unsigned long val_down, 118 void *v) 119 { 120 int ret, nr = 0; 121 122 ret = notifier_call_chain(nl, val_up, v, -1, &nr); 123 if (ret & NOTIFY_STOP_MASK) 124 notifier_call_chain(nl, val_down, v, nr-1, NULL); 125 126 return ret; 127 } 128 129 /* 130 * Atomic notifier chain routines. Registration and unregistration 131 * use a spinlock, and call_chain is synchronized by RCU (no locks). 132 */ 133 134 /** 135 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 136 * @nh: Pointer to head of the atomic notifier chain 137 * @n: New entry in notifier chain 138 * 139 * Adds a notifier to an atomic notifier chain. 140 * 141 * Returns 0 on success, %-EEXIST on error. 142 */ 143 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 144 struct notifier_block *n) 145 { 146 unsigned long flags; 147 int ret; 148 149 spin_lock_irqsave(&nh->lock, flags); 150 ret = notifier_chain_register(&nh->head, n, false); 151 spin_unlock_irqrestore(&nh->lock, flags); 152 return ret; 153 } 154 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 155 156 /** 157 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain 158 * @nh: Pointer to head of the atomic notifier chain 159 * @n: New entry in notifier chain 160 * 161 * Adds a notifier to an atomic notifier chain if there is no other 162 * notifier registered using the same priority. 163 * 164 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 165 */ 166 int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh, 167 struct notifier_block *n) 168 { 169 unsigned long flags; 170 int ret; 171 172 spin_lock_irqsave(&nh->lock, flags); 173 ret = notifier_chain_register(&nh->head, n, true); 174 spin_unlock_irqrestore(&nh->lock, flags); 175 return ret; 176 } 177 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio); 178 179 /** 180 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 181 * @nh: Pointer to head of the atomic notifier chain 182 * @n: Entry to remove from notifier chain 183 * 184 * Removes a notifier from an atomic notifier chain. 185 * 186 * Returns zero on success or %-ENOENT on failure. 187 */ 188 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 189 struct notifier_block *n) 190 { 191 unsigned long flags; 192 int ret; 193 194 spin_lock_irqsave(&nh->lock, flags); 195 ret = notifier_chain_unregister(&nh->head, n); 196 spin_unlock_irqrestore(&nh->lock, flags); 197 synchronize_rcu(); 198 return ret; 199 } 200 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 201 202 /** 203 * atomic_notifier_call_chain - Call functions in an atomic notifier chain 204 * @nh: Pointer to head of the atomic notifier chain 205 * @val: Value passed unmodified to notifier function 206 * @v: Pointer passed unmodified to notifier function 207 * 208 * Calls each function in a notifier chain in turn. The functions 209 * run in an atomic context, so they must not block. 210 * This routine uses RCU to synchronize with changes to the chain. 211 * 212 * If the return value of the notifier can be and'ed 213 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 214 * will return immediately, with the return value of 215 * the notifier function which halted execution. 216 * Otherwise the return value is the return value 217 * of the last notifier function called. 218 */ 219 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 220 unsigned long val, void *v) 221 { 222 int ret; 223 224 rcu_read_lock(); 225 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 226 rcu_read_unlock(); 227 228 return ret; 229 } 230 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 231 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 232 233 /** 234 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty 235 * @nh: Pointer to head of the atomic notifier chain 236 * 237 * Checks whether notifier chain is empty. 238 * 239 * Returns true is notifier chain is empty, false otherwise. 240 */ 241 bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh) 242 { 243 return !rcu_access_pointer(nh->head); 244 } 245 246 /* 247 * Blocking notifier chain routines. All access to the chain is 248 * synchronized by an rwsem. 249 */ 250 251 static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh, 252 struct notifier_block *n, 253 bool unique_priority) 254 { 255 int ret; 256 257 /* 258 * This code gets used during boot-up, when task switching is 259 * not yet working and interrupts must remain disabled. At 260 * such times we must not call down_write(). 261 */ 262 if (unlikely(system_state == SYSTEM_BOOTING)) 263 return notifier_chain_register(&nh->head, n, unique_priority); 264 265 down_write(&nh->rwsem); 266 ret = notifier_chain_register(&nh->head, n, unique_priority); 267 up_write(&nh->rwsem); 268 return ret; 269 } 270 271 /** 272 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 273 * @nh: Pointer to head of the blocking notifier chain 274 * @n: New entry in notifier chain 275 * 276 * Adds a notifier to a blocking notifier chain. 277 * Must be called in process context. 278 * 279 * Returns 0 on success, %-EEXIST on error. 280 */ 281 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 282 struct notifier_block *n) 283 { 284 return __blocking_notifier_chain_register(nh, n, false); 285 } 286 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 287 288 /** 289 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain 290 * @nh: Pointer to head of the blocking notifier chain 291 * @n: New entry in notifier chain 292 * 293 * Adds a notifier to an blocking notifier chain if there is no other 294 * notifier registered using the same priority. 295 * 296 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 297 */ 298 int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh, 299 struct notifier_block *n) 300 { 301 return __blocking_notifier_chain_register(nh, n, true); 302 } 303 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio); 304 305 /** 306 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 307 * @nh: Pointer to head of the blocking notifier chain 308 * @n: Entry to remove from notifier chain 309 * 310 * Removes a notifier from a blocking notifier chain. 311 * Must be called from process context. 312 * 313 * Returns zero on success or %-ENOENT on failure. 314 */ 315 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 316 struct notifier_block *n) 317 { 318 int ret; 319 320 /* 321 * This code gets used during boot-up, when task switching is 322 * not yet working and interrupts must remain disabled. At 323 * such times we must not call down_write(). 324 */ 325 if (unlikely(system_state == SYSTEM_BOOTING)) 326 return notifier_chain_unregister(&nh->head, n); 327 328 down_write(&nh->rwsem); 329 ret = notifier_chain_unregister(&nh->head, n); 330 up_write(&nh->rwsem); 331 return ret; 332 } 333 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 334 335 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, 336 unsigned long val_up, unsigned long val_down, void *v) 337 { 338 int ret = NOTIFY_DONE; 339 340 /* 341 * We check the head outside the lock, but if this access is 342 * racy then it does not matter what the result of the test 343 * is, we re-check the list after having taken the lock anyway: 344 */ 345 if (rcu_access_pointer(nh->head)) { 346 down_read(&nh->rwsem); 347 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 348 up_read(&nh->rwsem); 349 } 350 return ret; 351 } 352 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); 353 354 /** 355 * blocking_notifier_call_chain - Call functions in a blocking notifier chain 356 * @nh: Pointer to head of the blocking notifier chain 357 * @val: Value passed unmodified to notifier function 358 * @v: Pointer passed unmodified to notifier function 359 * 360 * Calls each function in a notifier chain in turn. The functions 361 * run in a process context, so they are allowed to block. 362 * 363 * If the return value of the notifier can be and'ed 364 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 365 * will return immediately, with the return value of 366 * the notifier function which halted execution. 367 * Otherwise the return value is the return value 368 * of the last notifier function called. 369 */ 370 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 371 unsigned long val, void *v) 372 { 373 int ret = NOTIFY_DONE; 374 375 /* 376 * We check the head outside the lock, but if this access is 377 * racy then it does not matter what the result of the test 378 * is, we re-check the list after having taken the lock anyway: 379 */ 380 if (rcu_access_pointer(nh->head)) { 381 down_read(&nh->rwsem); 382 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 383 up_read(&nh->rwsem); 384 } 385 return ret; 386 } 387 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 388 389 /* 390 * Raw notifier chain routines. There is no protection; 391 * the caller must provide it. Use at your own risk! 392 */ 393 394 /** 395 * raw_notifier_chain_register - Add notifier to a raw notifier chain 396 * @nh: Pointer to head of the raw notifier chain 397 * @n: New entry in notifier chain 398 * 399 * Adds a notifier to a raw notifier chain. 400 * All locking must be provided by the caller. 401 * 402 * Returns 0 on success, %-EEXIST on error. 403 */ 404 int raw_notifier_chain_register(struct raw_notifier_head *nh, 405 struct notifier_block *n) 406 { 407 return notifier_chain_register(&nh->head, n, false); 408 } 409 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 410 411 /** 412 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 413 * @nh: Pointer to head of the raw notifier chain 414 * @n: Entry to remove from notifier chain 415 * 416 * Removes a notifier from a raw notifier chain. 417 * All locking must be provided by the caller. 418 * 419 * Returns zero on success or %-ENOENT on failure. 420 */ 421 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 422 struct notifier_block *n) 423 { 424 return notifier_chain_unregister(&nh->head, n); 425 } 426 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 427 428 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, 429 unsigned long val_up, unsigned long val_down, void *v) 430 { 431 return notifier_call_chain_robust(&nh->head, val_up, val_down, v); 432 } 433 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); 434 435 /** 436 * raw_notifier_call_chain - Call functions in a raw notifier chain 437 * @nh: Pointer to head of the raw notifier chain 438 * @val: Value passed unmodified to notifier function 439 * @v: Pointer passed unmodified to notifier function 440 * 441 * Calls each function in a notifier chain in turn. The functions 442 * run in an undefined context. 443 * All locking must be provided by the caller. 444 * 445 * If the return value of the notifier can be and'ed 446 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 447 * will return immediately, with the return value of 448 * the notifier function which halted execution. 449 * Otherwise the return value is the return value 450 * of the last notifier function called. 451 */ 452 int raw_notifier_call_chain(struct raw_notifier_head *nh, 453 unsigned long val, void *v) 454 { 455 return notifier_call_chain(&nh->head, val, v, -1, NULL); 456 } 457 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 458 459 #ifdef CONFIG_SRCU 460 /* 461 * SRCU notifier chain routines. Registration and unregistration 462 * use a mutex, and call_chain is synchronized by SRCU (no locks). 463 */ 464 465 /** 466 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 467 * @nh: Pointer to head of the SRCU notifier chain 468 * @n: New entry in notifier chain 469 * 470 * Adds a notifier to an SRCU notifier chain. 471 * Must be called in process context. 472 * 473 * Returns 0 on success, %-EEXIST on error. 474 */ 475 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 476 struct notifier_block *n) 477 { 478 int ret; 479 480 /* 481 * This code gets used during boot-up, when task switching is 482 * not yet working and interrupts must remain disabled. At 483 * such times we must not call mutex_lock(). 484 */ 485 if (unlikely(system_state == SYSTEM_BOOTING)) 486 return notifier_chain_register(&nh->head, n, false); 487 488 mutex_lock(&nh->mutex); 489 ret = notifier_chain_register(&nh->head, n, false); 490 mutex_unlock(&nh->mutex); 491 return ret; 492 } 493 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 494 495 /** 496 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 497 * @nh: Pointer to head of the SRCU notifier chain 498 * @n: Entry to remove from notifier chain 499 * 500 * Removes a notifier from an SRCU notifier chain. 501 * Must be called from process context. 502 * 503 * Returns zero on success or %-ENOENT on failure. 504 */ 505 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 506 struct notifier_block *n) 507 { 508 int ret; 509 510 /* 511 * This code gets used during boot-up, when task switching is 512 * not yet working and interrupts must remain disabled. At 513 * such times we must not call mutex_lock(). 514 */ 515 if (unlikely(system_state == SYSTEM_BOOTING)) 516 return notifier_chain_unregister(&nh->head, n); 517 518 mutex_lock(&nh->mutex); 519 ret = notifier_chain_unregister(&nh->head, n); 520 mutex_unlock(&nh->mutex); 521 synchronize_srcu(&nh->srcu); 522 return ret; 523 } 524 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 525 526 /** 527 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain 528 * @nh: Pointer to head of the SRCU notifier chain 529 * @val: Value passed unmodified to notifier function 530 * @v: Pointer passed unmodified to notifier function 531 * 532 * Calls each function in a notifier chain in turn. The functions 533 * run in a process context, so they are allowed to block. 534 * 535 * If the return value of the notifier can be and'ed 536 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 537 * will return immediately, with the return value of 538 * the notifier function which halted execution. 539 * Otherwise the return value is the return value 540 * of the last notifier function called. 541 */ 542 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 543 unsigned long val, void *v) 544 { 545 int ret; 546 int idx; 547 548 idx = srcu_read_lock(&nh->srcu); 549 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 550 srcu_read_unlock(&nh->srcu, idx); 551 return ret; 552 } 553 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 554 555 /** 556 * srcu_init_notifier_head - Initialize an SRCU notifier head 557 * @nh: Pointer to head of the srcu notifier chain 558 * 559 * Unlike other sorts of notifier heads, SRCU notifier heads require 560 * dynamic initialization. Be sure to call this routine before 561 * calling any of the other SRCU notifier routines for this head. 562 * 563 * If an SRCU notifier head is deallocated, it must first be cleaned 564 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 565 * per-cpu data (used by the SRCU mechanism) will leak. 566 */ 567 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 568 { 569 mutex_init(&nh->mutex); 570 if (init_srcu_struct(&nh->srcu) < 0) 571 BUG(); 572 nh->head = NULL; 573 } 574 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 575 576 #endif /* CONFIG_SRCU */ 577 578 static ATOMIC_NOTIFIER_HEAD(die_chain); 579 580 int notrace notify_die(enum die_val val, const char *str, 581 struct pt_regs *regs, long err, int trap, int sig) 582 { 583 struct die_args args = { 584 .regs = regs, 585 .str = str, 586 .err = err, 587 .trapnr = trap, 588 .signr = sig, 589 590 }; 591 RCU_LOCKDEP_WARN(!rcu_is_watching(), 592 "notify_die called but RCU thinks we're quiescent"); 593 return atomic_notifier_call_chain(&die_chain, val, &args); 594 } 595 NOKPROBE_SYMBOL(notify_die); 596 597 int register_die_notifier(struct notifier_block *nb) 598 { 599 return atomic_notifier_chain_register(&die_chain, nb); 600 } 601 EXPORT_SYMBOL_GPL(register_die_notifier); 602 603 int unregister_die_notifier(struct notifier_block *nb) 604 { 605 return atomic_notifier_chain_unregister(&die_chain, nb); 606 } 607 EXPORT_SYMBOL_GPL(unregister_die_notifier); 608