1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 9 #define CREATE_TRACE_POINTS 10 #include <trace/events/notifier.h> 11 12 /* 13 * Notifier chain core routines. The exported routines below 14 * are layered on top of these, with appropriate locking added. 15 */ 16 17 static int notifier_chain_register(struct notifier_block **nl, 18 struct notifier_block *n, 19 bool unique_priority) 20 { 21 while ((*nl) != NULL) { 22 if (unlikely((*nl) == n)) { 23 WARN(1, "notifier callback %ps already registered", 24 n->notifier_call); 25 return -EEXIST; 26 } 27 if (n->priority > (*nl)->priority) 28 break; 29 if (n->priority == (*nl)->priority && unique_priority) 30 return -EBUSY; 31 nl = &((*nl)->next); 32 } 33 n->next = *nl; 34 rcu_assign_pointer(*nl, n); 35 trace_notifier_register((void *)n->notifier_call); 36 return 0; 37 } 38 39 static int notifier_chain_unregister(struct notifier_block **nl, 40 struct notifier_block *n) 41 { 42 while ((*nl) != NULL) { 43 if ((*nl) == n) { 44 rcu_assign_pointer(*nl, n->next); 45 trace_notifier_unregister((void *)n->notifier_call); 46 return 0; 47 } 48 nl = &((*nl)->next); 49 } 50 return -ENOENT; 51 } 52 53 /** 54 * notifier_call_chain - Informs the registered notifiers about an event. 55 * @nl: Pointer to head of the blocking notifier chain 56 * @val: Value passed unmodified to notifier function 57 * @v: Pointer passed unmodified to notifier function 58 * @nr_to_call: Number of notifier functions to be called. Don't care 59 * value of this parameter is -1. 60 * @nr_calls: Records the number of notifications sent. Don't care 61 * value of this field is NULL. 62 * Return: notifier_call_chain returns the value returned by the 63 * last notifier function called. 64 */ 65 static int notifier_call_chain(struct notifier_block **nl, 66 unsigned long val, void *v, 67 int nr_to_call, int *nr_calls) 68 { 69 int ret = NOTIFY_DONE; 70 struct notifier_block *nb, *next_nb; 71 72 nb = rcu_dereference_raw(*nl); 73 74 while (nb && nr_to_call) { 75 next_nb = rcu_dereference_raw(nb->next); 76 77 #ifdef CONFIG_DEBUG_NOTIFIERS 78 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 79 WARN(1, "Invalid notifier called!"); 80 nb = next_nb; 81 continue; 82 } 83 #endif 84 trace_notifier_run((void *)nb->notifier_call); 85 ret = nb->notifier_call(nb, val, v); 86 87 if (nr_calls) 88 (*nr_calls)++; 89 90 if (ret & NOTIFY_STOP_MASK) 91 break; 92 nb = next_nb; 93 nr_to_call--; 94 } 95 return ret; 96 } 97 NOKPROBE_SYMBOL(notifier_call_chain); 98 99 /** 100 * notifier_call_chain_robust - Inform the registered notifiers about an event 101 * and rollback on error. 102 * @nl: Pointer to head of the blocking notifier chain 103 * @val_up: Value passed unmodified to the notifier function 104 * @val_down: Value passed unmodified to the notifier function when recovering 105 * from an error on @val_up 106 * @v: Pointer passed unmodified to the notifier function 107 * 108 * NOTE: It is important the @nl chain doesn't change between the two 109 * invocations of notifier_call_chain() such that we visit the 110 * exact same notifier callbacks; this rules out any RCU usage. 111 * 112 * Return: the return value of the @val_up call. 113 */ 114 static int notifier_call_chain_robust(struct notifier_block **nl, 115 unsigned long val_up, unsigned long val_down, 116 void *v) 117 { 118 int ret, nr = 0; 119 120 ret = notifier_call_chain(nl, val_up, v, -1, &nr); 121 if (ret & NOTIFY_STOP_MASK) 122 notifier_call_chain(nl, val_down, v, nr-1, NULL); 123 124 return ret; 125 } 126 127 /* 128 * Atomic notifier chain routines. Registration and unregistration 129 * use a spinlock, and call_chain is synchronized by RCU (no locks). 130 */ 131 132 /** 133 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 134 * @nh: Pointer to head of the atomic notifier chain 135 * @n: New entry in notifier chain 136 * 137 * Adds a notifier to an atomic notifier chain. 138 * 139 * Returns 0 on success, %-EEXIST on error. 140 */ 141 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 142 struct notifier_block *n) 143 { 144 unsigned long flags; 145 int ret; 146 147 spin_lock_irqsave(&nh->lock, flags); 148 ret = notifier_chain_register(&nh->head, n, false); 149 spin_unlock_irqrestore(&nh->lock, flags); 150 return ret; 151 } 152 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 153 154 /** 155 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain 156 * @nh: Pointer to head of the atomic notifier chain 157 * @n: New entry in notifier chain 158 * 159 * Adds a notifier to an atomic notifier chain if there is no other 160 * notifier registered using the same priority. 161 * 162 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 163 */ 164 int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh, 165 struct notifier_block *n) 166 { 167 unsigned long flags; 168 int ret; 169 170 spin_lock_irqsave(&nh->lock, flags); 171 ret = notifier_chain_register(&nh->head, n, true); 172 spin_unlock_irqrestore(&nh->lock, flags); 173 return ret; 174 } 175 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio); 176 177 /** 178 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 179 * @nh: Pointer to head of the atomic notifier chain 180 * @n: Entry to remove from notifier chain 181 * 182 * Removes a notifier from an atomic notifier chain. 183 * 184 * Returns zero on success or %-ENOENT on failure. 185 */ 186 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 187 struct notifier_block *n) 188 { 189 unsigned long flags; 190 int ret; 191 192 spin_lock_irqsave(&nh->lock, flags); 193 ret = notifier_chain_unregister(&nh->head, n); 194 spin_unlock_irqrestore(&nh->lock, flags); 195 synchronize_rcu(); 196 return ret; 197 } 198 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 199 200 /** 201 * atomic_notifier_call_chain - Call functions in an atomic notifier chain 202 * @nh: Pointer to head of the atomic notifier chain 203 * @val: Value passed unmodified to notifier function 204 * @v: Pointer passed unmodified to notifier function 205 * 206 * Calls each function in a notifier chain in turn. The functions 207 * run in an atomic context, so they must not block. 208 * This routine uses RCU to synchronize with changes to the chain. 209 * 210 * If the return value of the notifier can be and'ed 211 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 212 * will return immediately, with the return value of 213 * the notifier function which halted execution. 214 * Otherwise the return value is the return value 215 * of the last notifier function called. 216 */ 217 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 218 unsigned long val, void *v) 219 { 220 int ret; 221 222 rcu_read_lock(); 223 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 224 rcu_read_unlock(); 225 226 return ret; 227 } 228 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 229 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 230 231 /** 232 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty 233 * @nh: Pointer to head of the atomic notifier chain 234 * 235 * Checks whether notifier chain is empty. 236 * 237 * Returns true is notifier chain is empty, false otherwise. 238 */ 239 bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh) 240 { 241 return !rcu_access_pointer(nh->head); 242 } 243 244 /* 245 * Blocking notifier chain routines. All access to the chain is 246 * synchronized by an rwsem. 247 */ 248 249 static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh, 250 struct notifier_block *n, 251 bool unique_priority) 252 { 253 int ret; 254 255 /* 256 * This code gets used during boot-up, when task switching is 257 * not yet working and interrupts must remain disabled. At 258 * such times we must not call down_write(). 259 */ 260 if (unlikely(system_state == SYSTEM_BOOTING)) 261 return notifier_chain_register(&nh->head, n, unique_priority); 262 263 down_write(&nh->rwsem); 264 ret = notifier_chain_register(&nh->head, n, unique_priority); 265 up_write(&nh->rwsem); 266 return ret; 267 } 268 269 /** 270 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 271 * @nh: Pointer to head of the blocking notifier chain 272 * @n: New entry in notifier chain 273 * 274 * Adds a notifier to a blocking notifier chain. 275 * Must be called in process context. 276 * 277 * Returns 0 on success, %-EEXIST on error. 278 */ 279 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 280 struct notifier_block *n) 281 { 282 return __blocking_notifier_chain_register(nh, n, false); 283 } 284 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 285 286 /** 287 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain 288 * @nh: Pointer to head of the blocking notifier chain 289 * @n: New entry in notifier chain 290 * 291 * Adds a notifier to an blocking notifier chain if there is no other 292 * notifier registered using the same priority. 293 * 294 * Returns 0 on success, %-EEXIST or %-EBUSY on error. 295 */ 296 int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh, 297 struct notifier_block *n) 298 { 299 return __blocking_notifier_chain_register(nh, n, true); 300 } 301 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio); 302 303 /** 304 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 305 * @nh: Pointer to head of the blocking notifier chain 306 * @n: Entry to remove from notifier chain 307 * 308 * Removes a notifier from a blocking notifier chain. 309 * Must be called from process context. 310 * 311 * Returns zero on success or %-ENOENT on failure. 312 */ 313 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 314 struct notifier_block *n) 315 { 316 int ret; 317 318 /* 319 * This code gets used during boot-up, when task switching is 320 * not yet working and interrupts must remain disabled. At 321 * such times we must not call down_write(). 322 */ 323 if (unlikely(system_state == SYSTEM_BOOTING)) 324 return notifier_chain_unregister(&nh->head, n); 325 326 down_write(&nh->rwsem); 327 ret = notifier_chain_unregister(&nh->head, n); 328 up_write(&nh->rwsem); 329 return ret; 330 } 331 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 332 333 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, 334 unsigned long val_up, unsigned long val_down, void *v) 335 { 336 int ret = NOTIFY_DONE; 337 338 /* 339 * We check the head outside the lock, but if this access is 340 * racy then it does not matter what the result of the test 341 * is, we re-check the list after having taken the lock anyway: 342 */ 343 if (rcu_access_pointer(nh->head)) { 344 down_read(&nh->rwsem); 345 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 346 up_read(&nh->rwsem); 347 } 348 return ret; 349 } 350 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); 351 352 /** 353 * blocking_notifier_call_chain - Call functions in a blocking notifier chain 354 * @nh: Pointer to head of the blocking notifier chain 355 * @val: Value passed unmodified to notifier function 356 * @v: Pointer passed unmodified to notifier function 357 * 358 * Calls each function in a notifier chain in turn. The functions 359 * run in a process context, so they are allowed to block. 360 * 361 * If the return value of the notifier can be and'ed 362 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 363 * will return immediately, with the return value of 364 * the notifier function which halted execution. 365 * Otherwise the return value is the return value 366 * of the last notifier function called. 367 */ 368 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 369 unsigned long val, void *v) 370 { 371 int ret = NOTIFY_DONE; 372 373 /* 374 * We check the head outside the lock, but if this access is 375 * racy then it does not matter what the result of the test 376 * is, we re-check the list after having taken the lock anyway: 377 */ 378 if (rcu_access_pointer(nh->head)) { 379 down_read(&nh->rwsem); 380 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 381 up_read(&nh->rwsem); 382 } 383 return ret; 384 } 385 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 386 387 /* 388 * Raw notifier chain routines. There is no protection; 389 * the caller must provide it. Use at your own risk! 390 */ 391 392 /** 393 * raw_notifier_chain_register - Add notifier to a raw notifier chain 394 * @nh: Pointer to head of the raw notifier chain 395 * @n: New entry in notifier chain 396 * 397 * Adds a notifier to a raw notifier chain. 398 * All locking must be provided by the caller. 399 * 400 * Returns 0 on success, %-EEXIST on error. 401 */ 402 int raw_notifier_chain_register(struct raw_notifier_head *nh, 403 struct notifier_block *n) 404 { 405 return notifier_chain_register(&nh->head, n, false); 406 } 407 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 408 409 /** 410 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 411 * @nh: Pointer to head of the raw notifier chain 412 * @n: Entry to remove from notifier chain 413 * 414 * Removes a notifier from a raw notifier chain. 415 * All locking must be provided by the caller. 416 * 417 * Returns zero on success or %-ENOENT on failure. 418 */ 419 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 420 struct notifier_block *n) 421 { 422 return notifier_chain_unregister(&nh->head, n); 423 } 424 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 425 426 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, 427 unsigned long val_up, unsigned long val_down, void *v) 428 { 429 return notifier_call_chain_robust(&nh->head, val_up, val_down, v); 430 } 431 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); 432 433 /** 434 * raw_notifier_call_chain - Call functions in a raw notifier chain 435 * @nh: Pointer to head of the raw notifier chain 436 * @val: Value passed unmodified to notifier function 437 * @v: Pointer passed unmodified to notifier function 438 * 439 * Calls each function in a notifier chain in turn. The functions 440 * run in an undefined context. 441 * All locking must be provided by the caller. 442 * 443 * If the return value of the notifier can be and'ed 444 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 445 * will return immediately, with the return value of 446 * the notifier function which halted execution. 447 * Otherwise the return value is the return value 448 * of the last notifier function called. 449 */ 450 int raw_notifier_call_chain(struct raw_notifier_head *nh, 451 unsigned long val, void *v) 452 { 453 return notifier_call_chain(&nh->head, val, v, -1, NULL); 454 } 455 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 456 457 /* 458 * SRCU notifier chain routines. Registration and unregistration 459 * use a mutex, and call_chain is synchronized by SRCU (no locks). 460 */ 461 462 /** 463 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 464 * @nh: Pointer to head of the SRCU notifier chain 465 * @n: New entry in notifier chain 466 * 467 * Adds a notifier to an SRCU notifier chain. 468 * Must be called in process context. 469 * 470 * Returns 0 on success, %-EEXIST on error. 471 */ 472 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 473 struct notifier_block *n) 474 { 475 int ret; 476 477 /* 478 * This code gets used during boot-up, when task switching is 479 * not yet working and interrupts must remain disabled. At 480 * such times we must not call mutex_lock(). 481 */ 482 if (unlikely(system_state == SYSTEM_BOOTING)) 483 return notifier_chain_register(&nh->head, n, false); 484 485 mutex_lock(&nh->mutex); 486 ret = notifier_chain_register(&nh->head, n, false); 487 mutex_unlock(&nh->mutex); 488 return ret; 489 } 490 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 491 492 /** 493 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 494 * @nh: Pointer to head of the SRCU notifier chain 495 * @n: Entry to remove from notifier chain 496 * 497 * Removes a notifier from an SRCU notifier chain. 498 * Must be called from process context. 499 * 500 * Returns zero on success or %-ENOENT on failure. 501 */ 502 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 503 struct notifier_block *n) 504 { 505 int ret; 506 507 /* 508 * This code gets used during boot-up, when task switching is 509 * not yet working and interrupts must remain disabled. At 510 * such times we must not call mutex_lock(). 511 */ 512 if (unlikely(system_state == SYSTEM_BOOTING)) 513 return notifier_chain_unregister(&nh->head, n); 514 515 mutex_lock(&nh->mutex); 516 ret = notifier_chain_unregister(&nh->head, n); 517 mutex_unlock(&nh->mutex); 518 synchronize_srcu(&nh->srcu); 519 return ret; 520 } 521 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 522 523 /** 524 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain 525 * @nh: Pointer to head of the SRCU notifier chain 526 * @val: Value passed unmodified to notifier function 527 * @v: Pointer passed unmodified to notifier function 528 * 529 * Calls each function in a notifier chain in turn. The functions 530 * run in a process context, so they are allowed to block. 531 * 532 * If the return value of the notifier can be and'ed 533 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 534 * will return immediately, with the return value of 535 * the notifier function which halted execution. 536 * Otherwise the return value is the return value 537 * of the last notifier function called. 538 */ 539 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 540 unsigned long val, void *v) 541 { 542 int ret; 543 int idx; 544 545 idx = srcu_read_lock(&nh->srcu); 546 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 547 srcu_read_unlock(&nh->srcu, idx); 548 return ret; 549 } 550 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 551 552 /** 553 * srcu_init_notifier_head - Initialize an SRCU notifier head 554 * @nh: Pointer to head of the srcu notifier chain 555 * 556 * Unlike other sorts of notifier heads, SRCU notifier heads require 557 * dynamic initialization. Be sure to call this routine before 558 * calling any of the other SRCU notifier routines for this head. 559 * 560 * If an SRCU notifier head is deallocated, it must first be cleaned 561 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 562 * per-cpu data (used by the SRCU mechanism) will leak. 563 */ 564 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 565 { 566 mutex_init(&nh->mutex); 567 if (init_srcu_struct(&nh->srcu) < 0) 568 BUG(); 569 nh->head = NULL; 570 } 571 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 572 573 static ATOMIC_NOTIFIER_HEAD(die_chain); 574 575 int notrace notify_die(enum die_val val, const char *str, 576 struct pt_regs *regs, long err, int trap, int sig) 577 { 578 struct die_args args = { 579 .regs = regs, 580 .str = str, 581 .err = err, 582 .trapnr = trap, 583 .signr = sig, 584 585 }; 586 RCU_LOCKDEP_WARN(!rcu_is_watching(), 587 "notify_die called but RCU thinks we're quiescent"); 588 return atomic_notifier_call_chain(&die_chain, val, &args); 589 } 590 NOKPROBE_SYMBOL(notify_die); 591 592 int register_die_notifier(struct notifier_block *nb) 593 { 594 return atomic_notifier_chain_register(&die_chain, nb); 595 } 596 EXPORT_SYMBOL_GPL(register_die_notifier); 597 598 int unregister_die_notifier(struct notifier_block *nb) 599 { 600 return atomic_notifier_chain_unregister(&die_chain, nb); 601 } 602 EXPORT_SYMBOL_GPL(unregister_die_notifier); 603