1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/export.h> 5 #include <linux/notifier.h> 6 #include <linux/rcupdate.h> 7 #include <linux/vmalloc.h> 8 #include <linux/reboot.h> 9 10 /* 11 * Notifier list for kernel code which wants to be called 12 * at shutdown. This is used to stop any idling DMA operations 13 * and the like. 14 */ 15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 16 17 /* 18 * Notifier chain core routines. The exported routines below 19 * are layered on top of these, with appropriate locking added. 20 */ 21 22 static int notifier_chain_register(struct notifier_block **nl, 23 struct notifier_block *n) 24 { 25 while ((*nl) != NULL) { 26 if (unlikely((*nl) == n)) { 27 WARN(1, "double register detected"); 28 return 0; 29 } 30 if (n->priority > (*nl)->priority) 31 break; 32 nl = &((*nl)->next); 33 } 34 n->next = *nl; 35 rcu_assign_pointer(*nl, n); 36 return 0; 37 } 38 39 static int notifier_chain_unregister(struct notifier_block **nl, 40 struct notifier_block *n) 41 { 42 while ((*nl) != NULL) { 43 if ((*nl) == n) { 44 rcu_assign_pointer(*nl, n->next); 45 return 0; 46 } 47 nl = &((*nl)->next); 48 } 49 return -ENOENT; 50 } 51 52 /** 53 * notifier_call_chain - Informs the registered notifiers about an event. 54 * @nl: Pointer to head of the blocking notifier chain 55 * @val: Value passed unmodified to notifier function 56 * @v: Pointer passed unmodified to notifier function 57 * @nr_to_call: Number of notifier functions to be called. Don't care 58 * value of this parameter is -1. 59 * @nr_calls: Records the number of notifications sent. Don't care 60 * value of this field is NULL. 61 * @returns: notifier_call_chain returns the value returned by the 62 * last notifier function called. 63 */ 64 static int notifier_call_chain(struct notifier_block **nl, 65 unsigned long val, void *v, 66 int nr_to_call, int *nr_calls) 67 { 68 int ret = NOTIFY_DONE; 69 struct notifier_block *nb, *next_nb; 70 71 nb = rcu_dereference_raw(*nl); 72 73 while (nb && nr_to_call) { 74 next_nb = rcu_dereference_raw(nb->next); 75 76 #ifdef CONFIG_DEBUG_NOTIFIERS 77 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 78 WARN(1, "Invalid notifier called!"); 79 nb = next_nb; 80 continue; 81 } 82 #endif 83 ret = nb->notifier_call(nb, val, v); 84 85 if (nr_calls) 86 (*nr_calls)++; 87 88 if (ret & NOTIFY_STOP_MASK) 89 break; 90 nb = next_nb; 91 nr_to_call--; 92 } 93 return ret; 94 } 95 NOKPROBE_SYMBOL(notifier_call_chain); 96 97 /* 98 * Atomic notifier chain routines. Registration and unregistration 99 * use a spinlock, and call_chain is synchronized by RCU (no locks). 100 */ 101 102 /** 103 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 104 * @nh: Pointer to head of the atomic notifier chain 105 * @n: New entry in notifier chain 106 * 107 * Adds a notifier to an atomic notifier chain. 108 * 109 * Currently always returns zero. 110 */ 111 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 112 struct notifier_block *n) 113 { 114 unsigned long flags; 115 int ret; 116 117 spin_lock_irqsave(&nh->lock, flags); 118 ret = notifier_chain_register(&nh->head, n); 119 spin_unlock_irqrestore(&nh->lock, flags); 120 return ret; 121 } 122 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 123 124 /** 125 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 126 * @nh: Pointer to head of the atomic notifier chain 127 * @n: Entry to remove from notifier chain 128 * 129 * Removes a notifier from an atomic notifier chain. 130 * 131 * Returns zero on success or %-ENOENT on failure. 132 */ 133 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 134 struct notifier_block *n) 135 { 136 unsigned long flags; 137 int ret; 138 139 spin_lock_irqsave(&nh->lock, flags); 140 ret = notifier_chain_unregister(&nh->head, n); 141 spin_unlock_irqrestore(&nh->lock, flags); 142 synchronize_rcu(); 143 return ret; 144 } 145 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 146 147 /** 148 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain 149 * @nh: Pointer to head of the atomic notifier chain 150 * @val: Value passed unmodified to notifier function 151 * @v: Pointer passed unmodified to notifier function 152 * @nr_to_call: See the comment for notifier_call_chain. 153 * @nr_calls: See the comment for notifier_call_chain. 154 * 155 * Calls each function in a notifier chain in turn. The functions 156 * run in an atomic context, so they must not block. 157 * This routine uses RCU to synchronize with changes to the chain. 158 * 159 * If the return value of the notifier can be and'ed 160 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 161 * will return immediately, with the return value of 162 * the notifier function which halted execution. 163 * Otherwise the return value is the return value 164 * of the last notifier function called. 165 */ 166 int __atomic_notifier_call_chain(struct atomic_notifier_head *nh, 167 unsigned long val, void *v, 168 int nr_to_call, int *nr_calls) 169 { 170 int ret; 171 172 rcu_read_lock(); 173 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 174 rcu_read_unlock(); 175 return ret; 176 } 177 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain); 178 NOKPROBE_SYMBOL(__atomic_notifier_call_chain); 179 180 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 181 unsigned long val, void *v) 182 { 183 return __atomic_notifier_call_chain(nh, val, v, -1, NULL); 184 } 185 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 186 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 187 188 /* 189 * Blocking notifier chain routines. All access to the chain is 190 * synchronized by an rwsem. 191 */ 192 193 /** 194 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 195 * @nh: Pointer to head of the blocking notifier chain 196 * @n: New entry in notifier chain 197 * 198 * Adds a notifier to a blocking notifier chain. 199 * Must be called in process context. 200 * 201 * Currently always returns zero. 202 */ 203 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 204 struct notifier_block *n) 205 { 206 int ret; 207 208 /* 209 * This code gets used during boot-up, when task switching is 210 * not yet working and interrupts must remain disabled. At 211 * such times we must not call down_write(). 212 */ 213 if (unlikely(system_state == SYSTEM_BOOTING)) 214 return notifier_chain_register(&nh->head, n); 215 216 down_write(&nh->rwsem); 217 ret = notifier_chain_register(&nh->head, n); 218 up_write(&nh->rwsem); 219 return ret; 220 } 221 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 222 223 /** 224 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 225 * @nh: Pointer to head of the blocking notifier chain 226 * @n: Entry to remove from notifier chain 227 * 228 * Removes a notifier from a blocking notifier chain. 229 * Must be called from process context. 230 * 231 * Returns zero on success or %-ENOENT on failure. 232 */ 233 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 234 struct notifier_block *n) 235 { 236 int ret; 237 238 /* 239 * This code gets used during boot-up, when task switching is 240 * not yet working and interrupts must remain disabled. At 241 * such times we must not call down_write(). 242 */ 243 if (unlikely(system_state == SYSTEM_BOOTING)) 244 return notifier_chain_unregister(&nh->head, n); 245 246 down_write(&nh->rwsem); 247 ret = notifier_chain_unregister(&nh->head, n); 248 up_write(&nh->rwsem); 249 return ret; 250 } 251 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 252 253 /** 254 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain 255 * @nh: Pointer to head of the blocking notifier chain 256 * @val: Value passed unmodified to notifier function 257 * @v: Pointer passed unmodified to notifier function 258 * @nr_to_call: See comment for notifier_call_chain. 259 * @nr_calls: See comment for notifier_call_chain. 260 * 261 * Calls each function in a notifier chain in turn. The functions 262 * run in a process context, so they are allowed to block. 263 * 264 * If the return value of the notifier can be and'ed 265 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 266 * will return immediately, with the return value of 267 * the notifier function which halted execution. 268 * Otherwise the return value is the return value 269 * of the last notifier function called. 270 */ 271 int __blocking_notifier_call_chain(struct blocking_notifier_head *nh, 272 unsigned long val, void *v, 273 int nr_to_call, int *nr_calls) 274 { 275 int ret = NOTIFY_DONE; 276 277 /* 278 * We check the head outside the lock, but if this access is 279 * racy then it does not matter what the result of the test 280 * is, we re-check the list after having taken the lock anyway: 281 */ 282 if (rcu_access_pointer(nh->head)) { 283 down_read(&nh->rwsem); 284 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, 285 nr_calls); 286 up_read(&nh->rwsem); 287 } 288 return ret; 289 } 290 EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain); 291 292 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 293 unsigned long val, void *v) 294 { 295 return __blocking_notifier_call_chain(nh, val, v, -1, NULL); 296 } 297 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 298 299 /* 300 * Raw notifier chain routines. There is no protection; 301 * the caller must provide it. Use at your own risk! 302 */ 303 304 /** 305 * raw_notifier_chain_register - Add notifier to a raw notifier chain 306 * @nh: Pointer to head of the raw notifier chain 307 * @n: New entry in notifier chain 308 * 309 * Adds a notifier to a raw notifier chain. 310 * All locking must be provided by the caller. 311 * 312 * Currently always returns zero. 313 */ 314 int raw_notifier_chain_register(struct raw_notifier_head *nh, 315 struct notifier_block *n) 316 { 317 return notifier_chain_register(&nh->head, n); 318 } 319 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 320 321 /** 322 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 323 * @nh: Pointer to head of the raw notifier chain 324 * @n: Entry to remove from notifier chain 325 * 326 * Removes a notifier from a raw notifier chain. 327 * All locking must be provided by the caller. 328 * 329 * Returns zero on success or %-ENOENT on failure. 330 */ 331 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 332 struct notifier_block *n) 333 { 334 return notifier_chain_unregister(&nh->head, n); 335 } 336 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 337 338 /** 339 * __raw_notifier_call_chain - Call functions in a raw notifier chain 340 * @nh: Pointer to head of the raw notifier chain 341 * @val: Value passed unmodified to notifier function 342 * @v: Pointer passed unmodified to notifier function 343 * @nr_to_call: See comment for notifier_call_chain. 344 * @nr_calls: See comment for notifier_call_chain 345 * 346 * Calls each function in a notifier chain in turn. The functions 347 * run in an undefined context. 348 * All locking must be provided by the caller. 349 * 350 * If the return value of the notifier can be and'ed 351 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 352 * will return immediately, with the return value of 353 * the notifier function which halted execution. 354 * Otherwise the return value is the return value 355 * of the last notifier function called. 356 */ 357 int __raw_notifier_call_chain(struct raw_notifier_head *nh, 358 unsigned long val, void *v, 359 int nr_to_call, int *nr_calls) 360 { 361 return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 362 } 363 EXPORT_SYMBOL_GPL(__raw_notifier_call_chain); 364 365 int raw_notifier_call_chain(struct raw_notifier_head *nh, 366 unsigned long val, void *v) 367 { 368 return __raw_notifier_call_chain(nh, val, v, -1, NULL); 369 } 370 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 371 372 #ifdef CONFIG_SRCU 373 /* 374 * SRCU notifier chain routines. Registration and unregistration 375 * use a mutex, and call_chain is synchronized by SRCU (no locks). 376 */ 377 378 /** 379 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 380 * @nh: Pointer to head of the SRCU notifier chain 381 * @n: New entry in notifier chain 382 * 383 * Adds a notifier to an SRCU notifier chain. 384 * Must be called in process context. 385 * 386 * Currently always returns zero. 387 */ 388 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 389 struct notifier_block *n) 390 { 391 int ret; 392 393 /* 394 * This code gets used during boot-up, when task switching is 395 * not yet working and interrupts must remain disabled. At 396 * such times we must not call mutex_lock(). 397 */ 398 if (unlikely(system_state == SYSTEM_BOOTING)) 399 return notifier_chain_register(&nh->head, n); 400 401 mutex_lock(&nh->mutex); 402 ret = notifier_chain_register(&nh->head, n); 403 mutex_unlock(&nh->mutex); 404 return ret; 405 } 406 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 407 408 /** 409 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 410 * @nh: Pointer to head of the SRCU notifier chain 411 * @n: Entry to remove from notifier chain 412 * 413 * Removes a notifier from an SRCU notifier chain. 414 * Must be called from process context. 415 * 416 * Returns zero on success or %-ENOENT on failure. 417 */ 418 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 419 struct notifier_block *n) 420 { 421 int ret; 422 423 /* 424 * This code gets used during boot-up, when task switching is 425 * not yet working and interrupts must remain disabled. At 426 * such times we must not call mutex_lock(). 427 */ 428 if (unlikely(system_state == SYSTEM_BOOTING)) 429 return notifier_chain_unregister(&nh->head, n); 430 431 mutex_lock(&nh->mutex); 432 ret = notifier_chain_unregister(&nh->head, n); 433 mutex_unlock(&nh->mutex); 434 synchronize_srcu(&nh->srcu); 435 return ret; 436 } 437 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 438 439 /** 440 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain 441 * @nh: Pointer to head of the SRCU notifier chain 442 * @val: Value passed unmodified to notifier function 443 * @v: Pointer passed unmodified to notifier function 444 * @nr_to_call: See comment for notifier_call_chain. 445 * @nr_calls: See comment for notifier_call_chain 446 * 447 * Calls each function in a notifier chain in turn. The functions 448 * run in a process context, so they are allowed to block. 449 * 450 * If the return value of the notifier can be and'ed 451 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 452 * will return immediately, with the return value of 453 * the notifier function which halted execution. 454 * Otherwise the return value is the return value 455 * of the last notifier function called. 456 */ 457 int __srcu_notifier_call_chain(struct srcu_notifier_head *nh, 458 unsigned long val, void *v, 459 int nr_to_call, int *nr_calls) 460 { 461 int ret; 462 int idx; 463 464 idx = srcu_read_lock(&nh->srcu); 465 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 466 srcu_read_unlock(&nh->srcu, idx); 467 return ret; 468 } 469 EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain); 470 471 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 472 unsigned long val, void *v) 473 { 474 return __srcu_notifier_call_chain(nh, val, v, -1, NULL); 475 } 476 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 477 478 /** 479 * srcu_init_notifier_head - Initialize an SRCU notifier head 480 * @nh: Pointer to head of the srcu notifier chain 481 * 482 * Unlike other sorts of notifier heads, SRCU notifier heads require 483 * dynamic initialization. Be sure to call this routine before 484 * calling any of the other SRCU notifier routines for this head. 485 * 486 * If an SRCU notifier head is deallocated, it must first be cleaned 487 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 488 * per-cpu data (used by the SRCU mechanism) will leak. 489 */ 490 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 491 { 492 mutex_init(&nh->mutex); 493 if (init_srcu_struct(&nh->srcu) < 0) 494 BUG(); 495 nh->head = NULL; 496 } 497 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 498 499 #endif /* CONFIG_SRCU */ 500 501 static ATOMIC_NOTIFIER_HEAD(die_chain); 502 503 int notrace notify_die(enum die_val val, const char *str, 504 struct pt_regs *regs, long err, int trap, int sig) 505 { 506 struct die_args args = { 507 .regs = regs, 508 .str = str, 509 .err = err, 510 .trapnr = trap, 511 .signr = sig, 512 513 }; 514 RCU_LOCKDEP_WARN(!rcu_is_watching(), 515 "notify_die called but RCU thinks we're quiescent"); 516 return atomic_notifier_call_chain(&die_chain, val, &args); 517 } 518 NOKPROBE_SYMBOL(notify_die); 519 520 int register_die_notifier(struct notifier_block *nb) 521 { 522 return atomic_notifier_chain_register(&die_chain, nb); 523 } 524 EXPORT_SYMBOL_GPL(register_die_notifier); 525 526 int unregister_die_notifier(struct notifier_block *nb) 527 { 528 return atomic_notifier_chain_unregister(&die_chain, nb); 529 } 530 EXPORT_SYMBOL_GPL(unregister_die_notifier); 531