1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 #include <linux/reboot.h> 9 10 /* 11 * Notifier list for kernel code which wants to be called 12 * at shutdown. This is used to stop any idling DMA operations 13 * and the like. 14 */ 15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 16 17 /* 18 * Notifier chain core routines. The exported routines below 19 * are layered on top of these, with appropriate locking added. 20 */ 21 22 static int notifier_chain_register(struct notifier_block **nl, 23 struct notifier_block *n) 24 { 25 while ((*nl) != NULL) { 26 if (unlikely((*nl) == n)) { 27 WARN(1, "double register detected"); 28 return 0; 29 } 30 if (n->priority > (*nl)->priority) 31 break; 32 nl = &((*nl)->next); 33 } 34 n->next = *nl; 35 rcu_assign_pointer(*nl, n); 36 return 0; 37 } 38 39 static int notifier_chain_unregister(struct notifier_block **nl, 40 struct notifier_block *n) 41 { 42 while ((*nl) != NULL) { 43 if ((*nl) == n) { 44 rcu_assign_pointer(*nl, n->next); 45 return 0; 46 } 47 nl = &((*nl)->next); 48 } 49 return -ENOENT; 50 } 51 52 /** 53 * notifier_call_chain - Informs the registered notifiers about an event. 54 * @nl: Pointer to head of the blocking notifier chain 55 * @val: Value passed unmodified to notifier function 56 * @v: Pointer passed unmodified to notifier function 57 * @nr_to_call: Number of notifier functions to be called. Don't care 58 * value of this parameter is -1. 59 * @nr_calls: Records the number of notifications sent. Don't care 60 * value of this field is NULL. 61 * @returns: notifier_call_chain returns the value returned by the 62 * last notifier function called. 63 */ 64 static int notifier_call_chain(struct notifier_block **nl, 65 unsigned long val, void *v, 66 int nr_to_call, int *nr_calls) 67 { 68 int ret = NOTIFY_DONE; 69 struct notifier_block *nb, *next_nb; 70 71 nb = rcu_dereference_raw(*nl); 72 73 while (nb && nr_to_call) { 74 next_nb = rcu_dereference_raw(nb->next); 75 76 #ifdef CONFIG_DEBUG_NOTIFIERS 77 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 78 WARN(1, "Invalid notifier called!"); 79 nb = next_nb; 80 continue; 81 } 82 #endif 83 ret = nb->notifier_call(nb, val, v); 84 85 if (nr_calls) 86 (*nr_calls)++; 87 88 if (ret & NOTIFY_STOP_MASK) 89 break; 90 nb = next_nb; 91 nr_to_call--; 92 } 93 return ret; 94 } 95 NOKPROBE_SYMBOL(notifier_call_chain); 96 97 /** 98 * notifier_call_chain_robust - Inform the registered notifiers about an event 99 * and rollback on error. 100 * @nl: Pointer to head of the blocking notifier chain 101 * @val_up: Value passed unmodified to the notifier function 102 * @val_down: Value passed unmodified to the notifier function when recovering 103 * from an error on @val_up 104 * @v Pointer passed unmodified to the notifier function 105 * 106 * NOTE: It is important the @nl chain doesn't change between the two 107 * invocations of notifier_call_chain() such that we visit the 108 * exact same notifier callbacks; this rules out any RCU usage. 109 * 110 * Returns: the return value of the @val_up call. 111 */ 112 static int notifier_call_chain_robust(struct notifier_block **nl, 113 unsigned long val_up, unsigned long val_down, 114 void *v) 115 { 116 int ret, nr = 0; 117 118 ret = notifier_call_chain(nl, val_up, v, -1, &nr); 119 if (ret & NOTIFY_STOP_MASK) 120 notifier_call_chain(nl, val_down, v, nr-1, NULL); 121 122 return ret; 123 } 124 125 /* 126 * Atomic notifier chain routines. Registration and unregistration 127 * use a spinlock, and call_chain is synchronized by RCU (no locks). 128 */ 129 130 /** 131 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 132 * @nh: Pointer to head of the atomic notifier chain 133 * @n: New entry in notifier chain 134 * 135 * Adds a notifier to an atomic notifier chain. 136 * 137 * Currently always returns zero. 138 */ 139 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 140 struct notifier_block *n) 141 { 142 unsigned long flags; 143 int ret; 144 145 spin_lock_irqsave(&nh->lock, flags); 146 ret = notifier_chain_register(&nh->head, n); 147 spin_unlock_irqrestore(&nh->lock, flags); 148 return ret; 149 } 150 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 151 152 /** 153 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 154 * @nh: Pointer to head of the atomic notifier chain 155 * @n: Entry to remove from notifier chain 156 * 157 * Removes a notifier from an atomic notifier chain. 158 * 159 * Returns zero on success or %-ENOENT on failure. 160 */ 161 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 162 struct notifier_block *n) 163 { 164 unsigned long flags; 165 int ret; 166 167 spin_lock_irqsave(&nh->lock, flags); 168 ret = notifier_chain_unregister(&nh->head, n); 169 spin_unlock_irqrestore(&nh->lock, flags); 170 synchronize_rcu(); 171 return ret; 172 } 173 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 174 175 int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh, 176 unsigned long val_up, unsigned long val_down, void *v) 177 { 178 unsigned long flags; 179 int ret; 180 181 /* 182 * Musn't use RCU; because then the notifier list can 183 * change between the up and down traversal. 184 */ 185 spin_lock_irqsave(&nh->lock, flags); 186 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 187 spin_unlock_irqrestore(&nh->lock, flags); 188 189 return ret; 190 } 191 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust); 192 NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust); 193 194 /** 195 * atomic_notifier_call_chain - Call functions in an atomic notifier chain 196 * @nh: Pointer to head of the atomic notifier chain 197 * @val: Value passed unmodified to notifier function 198 * @v: Pointer passed unmodified to notifier function 199 * 200 * Calls each function in a notifier chain in turn. The functions 201 * run in an atomic context, so they must not block. 202 * This routine uses RCU to synchronize with changes to the chain. 203 * 204 * If the return value of the notifier can be and'ed 205 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 206 * will return immediately, with the return value of 207 * the notifier function which halted execution. 208 * Otherwise the return value is the return value 209 * of the last notifier function called. 210 */ 211 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 212 unsigned long val, void *v) 213 { 214 int ret; 215 216 rcu_read_lock(); 217 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 218 rcu_read_unlock(); 219 220 return ret; 221 } 222 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 223 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 224 225 /* 226 * Blocking notifier chain routines. All access to the chain is 227 * synchronized by an rwsem. 228 */ 229 230 /** 231 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 232 * @nh: Pointer to head of the blocking notifier chain 233 * @n: New entry in notifier chain 234 * 235 * Adds a notifier to a blocking notifier chain. 236 * Must be called in process context. 237 * 238 * Currently always returns zero. 239 */ 240 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 241 struct notifier_block *n) 242 { 243 int ret; 244 245 /* 246 * This code gets used during boot-up, when task switching is 247 * not yet working and interrupts must remain disabled. At 248 * such times we must not call down_write(). 249 */ 250 if (unlikely(system_state == SYSTEM_BOOTING)) 251 return notifier_chain_register(&nh->head, n); 252 253 down_write(&nh->rwsem); 254 ret = notifier_chain_register(&nh->head, n); 255 up_write(&nh->rwsem); 256 return ret; 257 } 258 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 259 260 /** 261 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 262 * @nh: Pointer to head of the blocking notifier chain 263 * @n: Entry to remove from notifier chain 264 * 265 * Removes a notifier from a blocking notifier chain. 266 * Must be called from process context. 267 * 268 * Returns zero on success or %-ENOENT on failure. 269 */ 270 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 271 struct notifier_block *n) 272 { 273 int ret; 274 275 /* 276 * This code gets used during boot-up, when task switching is 277 * not yet working and interrupts must remain disabled. At 278 * such times we must not call down_write(). 279 */ 280 if (unlikely(system_state == SYSTEM_BOOTING)) 281 return notifier_chain_unregister(&nh->head, n); 282 283 down_write(&nh->rwsem); 284 ret = notifier_chain_unregister(&nh->head, n); 285 up_write(&nh->rwsem); 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 289 290 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, 291 unsigned long val_up, unsigned long val_down, void *v) 292 { 293 int ret = NOTIFY_DONE; 294 295 /* 296 * We check the head outside the lock, but if this access is 297 * racy then it does not matter what the result of the test 298 * is, we re-check the list after having taken the lock anyway: 299 */ 300 if (rcu_access_pointer(nh->head)) { 301 down_read(&nh->rwsem); 302 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); 303 up_read(&nh->rwsem); 304 } 305 return ret; 306 } 307 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); 308 309 /** 310 * blocking_notifier_call_chain - Call functions in a blocking notifier chain 311 * @nh: Pointer to head of the blocking notifier chain 312 * @val: Value passed unmodified to notifier function 313 * @v: Pointer passed unmodified to notifier function 314 * 315 * Calls each function in a notifier chain in turn. The functions 316 * run in a process context, so they are allowed to block. 317 * 318 * If the return value of the notifier can be and'ed 319 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 320 * will return immediately, with the return value of 321 * the notifier function which halted execution. 322 * Otherwise the return value is the return value 323 * of the last notifier function called. 324 */ 325 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 326 unsigned long val, void *v) 327 { 328 int ret = NOTIFY_DONE; 329 330 /* 331 * We check the head outside the lock, but if this access is 332 * racy then it does not matter what the result of the test 333 * is, we re-check the list after having taken the lock anyway: 334 */ 335 if (rcu_access_pointer(nh->head)) { 336 down_read(&nh->rwsem); 337 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 338 up_read(&nh->rwsem); 339 } 340 return ret; 341 } 342 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 343 344 /* 345 * Raw notifier chain routines. There is no protection; 346 * the caller must provide it. Use at your own risk! 347 */ 348 349 /** 350 * raw_notifier_chain_register - Add notifier to a raw notifier chain 351 * @nh: Pointer to head of the raw notifier chain 352 * @n: New entry in notifier chain 353 * 354 * Adds a notifier to a raw notifier chain. 355 * All locking must be provided by the caller. 356 * 357 * Currently always returns zero. 358 */ 359 int raw_notifier_chain_register(struct raw_notifier_head *nh, 360 struct notifier_block *n) 361 { 362 return notifier_chain_register(&nh->head, n); 363 } 364 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 365 366 /** 367 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 368 * @nh: Pointer to head of the raw notifier chain 369 * @n: Entry to remove from notifier chain 370 * 371 * Removes a notifier from a raw notifier chain. 372 * All locking must be provided by the caller. 373 * 374 * Returns zero on success or %-ENOENT on failure. 375 */ 376 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 377 struct notifier_block *n) 378 { 379 return notifier_chain_unregister(&nh->head, n); 380 } 381 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 382 383 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, 384 unsigned long val_up, unsigned long val_down, void *v) 385 { 386 return notifier_call_chain_robust(&nh->head, val_up, val_down, v); 387 } 388 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); 389 390 /** 391 * raw_notifier_call_chain - Call functions in a raw notifier chain 392 * @nh: Pointer to head of the raw notifier chain 393 * @val: Value passed unmodified to notifier function 394 * @v: Pointer passed unmodified to notifier function 395 * 396 * Calls each function in a notifier chain in turn. The functions 397 * run in an undefined context. 398 * All locking must be provided by the caller. 399 * 400 * If the return value of the notifier can be and'ed 401 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 402 * will return immediately, with the return value of 403 * the notifier function which halted execution. 404 * Otherwise the return value is the return value 405 * of the last notifier function called. 406 */ 407 int raw_notifier_call_chain(struct raw_notifier_head *nh, 408 unsigned long val, void *v) 409 { 410 return notifier_call_chain(&nh->head, val, v, -1, NULL); 411 } 412 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 413 414 #ifdef CONFIG_SRCU 415 /* 416 * SRCU notifier chain routines. Registration and unregistration 417 * use a mutex, and call_chain is synchronized by SRCU (no locks). 418 */ 419 420 /** 421 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 422 * @nh: Pointer to head of the SRCU notifier chain 423 * @n: New entry in notifier chain 424 * 425 * Adds a notifier to an SRCU notifier chain. 426 * Must be called in process context. 427 * 428 * Currently always returns zero. 429 */ 430 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 431 struct notifier_block *n) 432 { 433 int ret; 434 435 /* 436 * This code gets used during boot-up, when task switching is 437 * not yet working and interrupts must remain disabled. At 438 * such times we must not call mutex_lock(). 439 */ 440 if (unlikely(system_state == SYSTEM_BOOTING)) 441 return notifier_chain_register(&nh->head, n); 442 443 mutex_lock(&nh->mutex); 444 ret = notifier_chain_register(&nh->head, n); 445 mutex_unlock(&nh->mutex); 446 return ret; 447 } 448 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 449 450 /** 451 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 452 * @nh: Pointer to head of the SRCU notifier chain 453 * @n: Entry to remove from notifier chain 454 * 455 * Removes a notifier from an SRCU notifier chain. 456 * Must be called from process context. 457 * 458 * Returns zero on success or %-ENOENT on failure. 459 */ 460 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 461 struct notifier_block *n) 462 { 463 int ret; 464 465 /* 466 * This code gets used during boot-up, when task switching is 467 * not yet working and interrupts must remain disabled. At 468 * such times we must not call mutex_lock(). 469 */ 470 if (unlikely(system_state == SYSTEM_BOOTING)) 471 return notifier_chain_unregister(&nh->head, n); 472 473 mutex_lock(&nh->mutex); 474 ret = notifier_chain_unregister(&nh->head, n); 475 mutex_unlock(&nh->mutex); 476 synchronize_srcu(&nh->srcu); 477 return ret; 478 } 479 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 480 481 /** 482 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain 483 * @nh: Pointer to head of the SRCU notifier chain 484 * @val: Value passed unmodified to notifier function 485 * @v: Pointer passed unmodified to notifier function 486 * 487 * Calls each function in a notifier chain in turn. The functions 488 * run in a process context, so they are allowed to block. 489 * 490 * If the return value of the notifier can be and'ed 491 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 492 * will return immediately, with the return value of 493 * the notifier function which halted execution. 494 * Otherwise the return value is the return value 495 * of the last notifier function called. 496 */ 497 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 498 unsigned long val, void *v) 499 { 500 int ret; 501 int idx; 502 503 idx = srcu_read_lock(&nh->srcu); 504 ret = notifier_call_chain(&nh->head, val, v, -1, NULL); 505 srcu_read_unlock(&nh->srcu, idx); 506 return ret; 507 } 508 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 509 510 /** 511 * srcu_init_notifier_head - Initialize an SRCU notifier head 512 * @nh: Pointer to head of the srcu notifier chain 513 * 514 * Unlike other sorts of notifier heads, SRCU notifier heads require 515 * dynamic initialization. Be sure to call this routine before 516 * calling any of the other SRCU notifier routines for this head. 517 * 518 * If an SRCU notifier head is deallocated, it must first be cleaned 519 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 520 * per-cpu data (used by the SRCU mechanism) will leak. 521 */ 522 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 523 { 524 mutex_init(&nh->mutex); 525 if (init_srcu_struct(&nh->srcu) < 0) 526 BUG(); 527 nh->head = NULL; 528 } 529 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 530 531 #endif /* CONFIG_SRCU */ 532 533 static ATOMIC_NOTIFIER_HEAD(die_chain); 534 535 int notrace notify_die(enum die_val val, const char *str, 536 struct pt_regs *regs, long err, int trap, int sig) 537 { 538 struct die_args args = { 539 .regs = regs, 540 .str = str, 541 .err = err, 542 .trapnr = trap, 543 .signr = sig, 544 545 }; 546 RCU_LOCKDEP_WARN(!rcu_is_watching(), 547 "notify_die called but RCU thinks we're quiescent"); 548 return atomic_notifier_call_chain(&die_chain, val, &args); 549 } 550 NOKPROBE_SYMBOL(notify_die); 551 552 int register_die_notifier(struct notifier_block *nb) 553 { 554 return atomic_notifier_chain_register(&die_chain, nb); 555 } 556 EXPORT_SYMBOL_GPL(register_die_notifier); 557 558 int unregister_die_notifier(struct notifier_block *nb) 559 { 560 return atomic_notifier_chain_unregister(&die_chain, nb); 561 } 562 EXPORT_SYMBOL_GPL(unregister_die_notifier); 563