1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 #include <linux/reboot.h> 9 10 /* 11 * Notifier list for kernel code which wants to be called 12 * at shutdown. This is used to stop any idling DMA operations 13 * and the like. 14 */ 15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 16 17 /* 18 * Notifier chain core routines. The exported routines below 19 * are layered on top of these, with appropriate locking added. 20 */ 21 22 static int notifier_chain_register(struct notifier_block **nl, 23 struct notifier_block *n, 24 bool unique_priority) 25 { 26 while ((*nl) != NULL) { 27 if (unlikely((*nl) == n)) { 28 WARN(1, "notifier callback %ps already registered", 29 n->notifier_call); 30 return -EEXIST; 31 } 32 if (n->priority > (*nl)->priority) 33 break; 34 if (n->priority == (*nl)->priority && unique_priority) 35 return -EBUSY; 36 nl = &((*nl)->next); 37 } 38 n->next = *nl; 39 rcu_assign_pointer(*nl, n); 40 return 0; 41 } 42 43 static int notifier_chain_unregister(struct notifier_block **nl, 44 struct notifier_block *n) 45 { 46 while ((*nl) != NULL) { 47 if ((*nl) == n) { 48 rcu_assign_pointer(*nl, n->next); 49 return 0; 50 } 51 nl = &((*nl)->next); 52 } 53 return -ENOENT; 54 } 55 56 /** 57 * notifier_call_chain - Informs the registered notifiers about an event. 58 * @nl: Pointer to head of the blocking notifier chain 59 * @val: Value passed unmodified to notifier function 60 * @v: Pointer passed unmodified to notifier function 61 * @nr_to_call: Number of notifier functions to be called. Don't care 62 * value of this parameter is -1. 63 * @nr_calls: Records the number of notifications sent. Don't care 64 * value of this field is NULL. 65 * Return: notifier_call_chain returns the value returned by the 66 * last notifier function called. 67 */ 68 static int notifier_call_chain(struct notifier_block **nl, 69 unsigned long val, void *v, 70 int nr_to_call, int *nr_calls) 71 { 72 int ret = NOTIFY_DONE; 73 struct notifier_block *nb, *next_nb; 74 75 nb = rcu_dereference_raw(*nl); 76 77 while (nb && nr_to_call) { 78 next_nb = rcu_dereference_raw(nb->next); 79 80 #ifdef CONFIG_DEBUG_NOTIFIERS 81 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 82 WARN(1, "Invalid notifier called!"); 83 nb = next_nb; 84 continue; 85 } 86 #endif 87 ret = nb->notifier_call(nb, val, v); 88 89 if (nr_calls) 90 (*nr_calls)++; 91 92 if (ret & NOTIFY_STOP_MASK) 93 break; 94 nb = next_nb; 95 nr_to_call--; 96 } 97 return ret; 98 } 99 NOKPROBE_SYMBOL(notifier_call_chain); 100 101 /** 102 * notifier_call_chain_robust - Inform the registered notifiers about an event 103 * and rollback on error. 104 * @nl: Pointer to head of the blocking notifier chain 105 * @val_up: Value passed unmodified to the notifier function 106 * @val_down: Value passed unmodified to the notifier function when recovering 107 * from an error on @val_up 108 * @v: Pointer passed unmodified to the notifier function 109 * 110 * NOTE: It is important the @nl chain doesn't change between the two 111 * invocations of notifier_call_chain() such that we visit the 112 * exact same notifier callbacks; this rules out any RCU usage. 113 * 114 * Return: the return value of the @val_up call. 115 */ 116 static int notifier_call_chain_robust(struct notifier_block **nl, 117 unsigned long val_up, unsigned long val_down, 118 void *v) 119 { 120 int ret, nr = 0; 121 122 ret = notifier_call_chain(nl, val_up, v, -1, &nr); 123 if (ret & NOTIFY_STOP_MASK) 124 notifier_call_chain(nl, val_down, v, nr-1, NULL); 125 126 return ret; 127 } 128 129 /* 130 * Atomic notifier chain routines. Registration and unregistration 131 * use a spinlock, and call_chain is synchronized by RCU (no locks). 132 */ 133 134 /** 135 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 136 * @nh: Pointer to head of the atomic notifier chain 137 * @n: New entry in notifier chain 138 * 139 * Adds a notifier to an atomic notifier chain. 140 * 141 * Returns 0 on success, %-EEXIST on error. 142 */ 143 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 144 struct notifier_block *n) 145 { 146 unsigned long flags; 147 int ret; 148 149 spin_lock_irqsave(&nh->lock, flags); 150 ret = notifier_chain_register(&nh->head, n, false); 151 spin_unlock_irqrestore(&nh->lock, flags); 152 return ret; 153 } 154 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 155 156 /** 157 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain 158 * @nh: Pointer to head of the atomic notifier chain 159 * @n: New entry in notifier chain 160 * 161 * Adds a notifier to an atomic notifier chain if there is no other 162 * notifier registered using the same priority. 163 * 164 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 165 */ 166 int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh, 167 struct notifier_block *n) 168 { 169 unsigned long flags; 170 int ret; 171 172 spin_lock_irqsave(&nh->lock, flags); 173 ret = notifier_chain_register(&nh->head, n, true); 174 spin_unlock_irqrestore(&nh->lock, flags); 175 return ret; 176 } 177 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio); 178 179 /** 180 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 181 * @nh: Pointer to head of the atomic notifier chain 182 * @n: Entry to remove from notifier chain 183 * 184 * Removes a notifier from an atomic notifier chain. 185 * 186 * Returns zero on success or %-ENOENT on failure. 187 */ 188 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 189 struct notifier_block *n) 190 { 191 unsigned long flags; 192 int ret; 193 194 spin_lock_irqsave(&nh->lock, flags); 195 ret = notifier_chain_unregister(&nh->head, n); 196 spin_unlock_irqrestore(&nh->lock, flags); 197 synchronize_rcu(); 198 return ret; 199 } 200 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 201 202 /** 203 * atomic_notifier_call_chain - Call functions in an atomic notifier chain 204 * @nh: Pointer to head of the atomic notifier chain 205 * @val: Value passed unmodified to notifier function 206 * @v: Pointer passed unmodified to notifier function 207 * 208 * Calls each function in a notifier chain in turn. The functions 209 * run in an atomic context, so they must not block. 210 * This routine uses RCU to synchronize with changes to the chain. 211 * 212 * If the return value of the notifier can be and'ed 213 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 214 * will return immediately, with the return value of 215 * the notifier function which halted execution. 216 * Otherwise the return value is the return value 217 * of the last notifier function called. 218 */ 219 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 220 unsigned long val, void *v) 221 { 222 int ret; 223 224 rcu_read_lock(); 225 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 226 rcu_read_unlock(); 227 228 return ret; 229 } 230 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 231 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 232 233 /** 234 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty 235 * @nh: Pointer to head of the atomic notifier chain 236 * 237 * Checks whether notifier chain is empty. 238 * 239 * Returns true is notifier chain is empty, false otherwise. 240 */ 241 bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh) 242 { 243 return !rcu_access_pointer(nh->head); 244 } 245 246 /* 247 * Blocking notifier chain routines. All access to the chain is 248 * synchronized by an rwsem. 249 */ 250 251 static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh, 252 struct notifier_block *n, 253 bool unique_priority) 254 { 255 int ret; 256 257 /* 258 * This code gets used during boot-up, when task switching is 259 * not yet working and interrupts must remain disabled. At 260 * such times we must not call down_write(). 261 */ 262 if (unlikely(system_state == SYSTEM_BOOTING)) 263 return notifier_chain_register(&nh->head, n, unique_priority); 264 265 down_write(&nh->rwsem); 266 ret = notifier_chain_register(&nh->head, n, unique_priority); 267 up_write(&nh->rwsem); 268 return ret; 269 } 270 271 /** 272 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 273 * @nh: Pointer to head of the blocking notifier chain 274 * @n: New entry in notifier chain 275 * 276 * Adds a notifier to a blocking notifier chain. 277 * Must be called in process context. 278 * 279 * Returns 0 on success, %-EEXIST on error. 280 */ 281 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 282 struct notifier_block *n) 283 { 284 return __blocking_notifier_chain_register(nh, n, false); 285 } 286 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 287 288 /** 289 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain 290 * @nh: Pointer to head of the blocking notifier chain 291 * @n: New entry in notifier chain 292 * 293 * Adds a notifier to an blocking notifier chain if there is no other 294 * notifier registered using the same priority. 295 * 296 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 297 */ 298 int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh, 299 struct notifier_block *n) 300 { 301 return __blocking_notifier_chain_register(nh, n, true); 302 } 303 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio); 304 305 /** 306 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 307 * @nh: Pointer to head of the blocking notifier chain 308 * @n: Entry to remove from notifier chain 309 * 310 * Removes a notifier from a blocking notifier chain. 311 * Must be called from process context. 312 * 313 * Returns zero on success or %-ENOENT on failure. 314 */ 315 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 316 struct notifier_block *n) 317 { 318 int ret; 319 320 /* 321 * This code gets used during boot-up, when task switching is 322 * not yet working and interrupts must remain disabled. At 323 * such times we must not call down_write(). 324 */ 325 if (unlikely(system_state == SYSTEM_BOOTING)) 326 return notifier_chain_unregister(&nh->head, n); 327 328 down_write(&nh->rwsem); 329 ret = notifier_chain_unregister(&nh->head, n); 330 up_write(&nh->rwsem); 331 return ret; 332 } 333 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 334 335 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, 336 unsigned long val_up, unsigned long val_down, void *v) 337 { 338 int ret = NOTIFY_DONE; 339 340 /* 341 * We check the head outside the lock, but if this access is 342 * racy then it does not matter what the result of the test 343 * is, we re-check the list after having taken the lock anyway: 344 */ 345 if (rcu_access_pointer(nh->head)) { 346 down_read(&nh->rwsem); 347 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 348 up_read(&nh->rwsem); 349 } 350 return ret; 351 } 352 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); 353 354 /** 355 * blocking_notifier_call_chain - Call functions in a blocking notifier chain 356 * @nh: Pointer to head of the blocking notifier chain 357 * @val: Value passed unmodified to notifier function 358 * @v: Pointer passed unmodified to notifier function 359 * 360 * Calls each function in a notifier chain in turn. The functions 361 * run in a process context, so they are allowed to block. 362 * 363 * If the return value of the notifier can be and'ed 364 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 365 * will return immediately, with the return value of 366 * the notifier function which halted execution. 367 * Otherwise the return value is the return value 368 * of the last notifier function called. 369 */ 370 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 371 unsigned long val, void *v) 372 { 373 int ret = NOTIFY_DONE; 374 375 /* 376 * We check the head outside the lock, but if this access is 377 * racy then it does not matter what the result of the test 378 * is, we re-check the list after having taken the lock anyway: 379 */ 380 if (rcu_access_pointer(nh->head)) { 381 down_read(&nh->rwsem); 382 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 383 up_read(&nh->rwsem); 384 } 385 return ret; 386 } 387 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 388 389 /* 390 * Raw notifier chain routines. There is no protection; 391 * the caller must provide it. Use at your own risk! 392 */ 393 394 /** 395 * raw_notifier_chain_register - Add notifier to a raw notifier chain 396 * @nh: Pointer to head of the raw notifier chain 397 * @n: New entry in notifier chain 398 * 399 * Adds a notifier to a raw notifier chain. 400 * All locking must be provided by the caller. 401 * 402 * Returns 0 on success, %-EEXIST on error. 403 */ 404 int raw_notifier_chain_register(struct raw_notifier_head *nh, 405 struct notifier_block *n) 406 { 407 return notifier_chain_register(&nh->head, n, false); 408 } 409 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 410 411 /** 412 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 413 * @nh: Pointer to head of the raw notifier chain 414 * @n: Entry to remove from notifier chain 415 * 416 * Removes a notifier from a raw notifier chain. 417 * All locking must be provided by the caller. 418 * 419 * Returns zero on success or %-ENOENT on failure. 420 */ 421 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 422 struct notifier_block *n) 423 { 424 return notifier_chain_unregister(&nh->head, n); 425 } 426 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 427 428 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, 429 unsigned long val_up, unsigned long val_down, void *v) 430 { 431 return notifier_call_chain_robust(&nh->head, val_up, val_down, v); 432 } 433 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); 434 435 /** 436 * raw_notifier_call_chain - Call functions in a raw notifier chain 437 * @nh: Pointer to head of the raw notifier chain 438 * @val: Value passed unmodified to notifier function 439 * @v: Pointer passed unmodified to notifier function 440 * 441 * Calls each function in a notifier chain in turn. The functions 442 * run in an undefined context. 443 * All locking must be provided by the caller. 444 * 445 * If the return value of the notifier can be and'ed 446 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 447 * will return immediately, with the return value of 448 * the notifier function which halted execution. 449 * Otherwise the return value is the return value 450 * of the last notifier function called. 451 */ 452 int raw_notifier_call_chain(struct raw_notifier_head *nh, 453 unsigned long val, void *v) 454 { 455 return notifier_call_chain(&nh->head, val, v, -1, NULL); 456 } 457 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 458 459 /* 460 * SRCU notifier chain routines. Registration and unregistration 461 * use a mutex, and call_chain is synchronized by SRCU (no locks). 462 */ 463 464 /** 465 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 466 * @nh: Pointer to head of the SRCU notifier chain 467 * @n: New entry in notifier chain 468 * 469 * Adds a notifier to an SRCU notifier chain. 470 * Must be called in process context. 471 * 472 * Returns 0 on success, %-EEXIST on error. 473 */ 474 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 475 struct notifier_block *n) 476 { 477 int ret; 478 479 /* 480 * This code gets used during boot-up, when task switching is 481 * not yet working and interrupts must remain disabled. At 482 * such times we must not call mutex_lock(). 483 */ 484 if (unlikely(system_state == SYSTEM_BOOTING)) 485 return notifier_chain_register(&nh->head, n, false); 486 487 mutex_lock(&nh->mutex); 488 ret = notifier_chain_register(&nh->head, n, false); 489 mutex_unlock(&nh->mutex); 490 return ret; 491 } 492 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 493 494 /** 495 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 496 * @nh: Pointer to head of the SRCU notifier chain 497 * @n: Entry to remove from notifier chain 498 * 499 * Removes a notifier from an SRCU notifier chain. 500 * Must be called from process context. 501 * 502 * Returns zero on success or %-ENOENT on failure. 503 */ 504 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 505 struct notifier_block *n) 506 { 507 int ret; 508 509 /* 510 * This code gets used during boot-up, when task switching is 511 * not yet working and interrupts must remain disabled. At 512 * such times we must not call mutex_lock(). 513 */ 514 if (unlikely(system_state == SYSTEM_BOOTING)) 515 return notifier_chain_unregister(&nh->head, n); 516 517 mutex_lock(&nh->mutex); 518 ret = notifier_chain_unregister(&nh->head, n); 519 mutex_unlock(&nh->mutex); 520 synchronize_srcu(&nh->srcu); 521 return ret; 522 } 523 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 524 525 /** 526 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain 527 * @nh: Pointer to head of the SRCU notifier chain 528 * @val: Value passed unmodified to notifier function 529 * @v: Pointer passed unmodified to notifier function 530 * 531 * Calls each function in a notifier chain in turn. The functions 532 * run in a process context, so they are allowed to block. 533 * 534 * If the return value of the notifier can be and'ed 535 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 536 * will return immediately, with the return value of 537 * the notifier function which halted execution. 538 * Otherwise the return value is the return value 539 * of the last notifier function called. 540 */ 541 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 542 unsigned long val, void *v) 543 { 544 int ret; 545 int idx; 546 547 idx = srcu_read_lock(&nh->srcu); 548 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 549 srcu_read_unlock(&nh->srcu, idx); 550 return ret; 551 } 552 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 553 554 /** 555 * srcu_init_notifier_head - Initialize an SRCU notifier head 556 * @nh: Pointer to head of the srcu notifier chain 557 * 558 * Unlike other sorts of notifier heads, SRCU notifier heads require 559 * dynamic initialization. Be sure to call this routine before 560 * calling any of the other SRCU notifier routines for this head. 561 * 562 * If an SRCU notifier head is deallocated, it must first be cleaned 563 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 564 * per-cpu data (used by the SRCU mechanism) will leak. 565 */ 566 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 567 { 568 mutex_init(&nh->mutex); 569 if (init_srcu_struct(&nh->srcu) < 0) 570 BUG(); 571 nh->head = NULL; 572 } 573 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 574 575 static ATOMIC_NOTIFIER_HEAD(die_chain); 576 577 int notrace notify_die(enum die_val val, const char *str, 578 struct pt_regs *regs, long err, int trap, int sig) 579 { 580 struct die_args args = { 581 .regs = regs, 582 .str = str, 583 .err = err, 584 .trapnr = trap, 585 .signr = sig, 586 587 }; 588 RCU_LOCKDEP_WARN(!rcu_is_watching(), 589 "notify_die called but RCU thinks we're quiescent"); 590 return atomic_notifier_call_chain(&die_chain, val, &args); 591 } 592 NOKPROBE_SYMBOL(notify_die); 593 594 int register_die_notifier(struct notifier_block *nb) 595 { 596 return atomic_notifier_chain_register(&die_chain, nb); 597 } 598 EXPORT_SYMBOL_GPL(register_die_notifier); 599 600 int unregister_die_notifier(struct notifier_block *nb) 601 { 602 return atomic_notifier_chain_unregister(&die_chain, nb); 603 } 604 EXPORT_SYMBOL_GPL(unregister_die_notifier); 605