tree_plugin.h (385b599e8c04fa843c4d7f785478827cc512d720) tree_plugin.h (0864f057b050bc6dd68106b3185e02db5140012d)
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
5 * or preemptible semantics.
6 *
7 * Copyright Red Hat, 2009
8 * Copyright IBM Corporation, 2009

--- 574 unchanged lines hidden (view full) ---

583 t->rcu_read_lock_nesting -= RCU_NEST_BIAS;
584 local_irq_save(flags);
585 rcu_preempt_deferred_qs_irqrestore(t, flags);
586 if (couldrecurse)
587 t->rcu_read_lock_nesting += RCU_NEST_BIAS;
588}
589
590/*
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
5 * or preemptible semantics.
6 *
7 * Copyright Red Hat, 2009
8 * Copyright IBM Corporation, 2009

--- 574 unchanged lines hidden (view full) ---

583 t->rcu_read_lock_nesting -= RCU_NEST_BIAS;
584 local_irq_save(flags);
585 rcu_preempt_deferred_qs_irqrestore(t, flags);
586 if (couldrecurse)
587 t->rcu_read_lock_nesting += RCU_NEST_BIAS;
588}
589
590/*
591 * Minimal handler to give the scheduler a chance to re-evaluate.
592 */
593static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
594{
595 struct rcu_data *rdp;
596
597 rdp = container_of(iwp, struct rcu_data, defer_qs_iw);
598 rdp->defer_qs_iw_pending = false;
599}
600
601/*
591 * Handle special cases during rcu_read_unlock(), such as needing to
592 * notify RCU core processing or task having blocked during the RCU
593 * read-side critical section.
594 */
595static void rcu_read_unlock_special(struct task_struct *t)
596{
597 unsigned long flags;
598 bool preempt_bh_were_disabled =

--- 26 unchanged lines hidden (view full) ---

625 // Safe to awaken and we get no help from enabling
626 // irqs, unlike bh/preempt.
627 invoke_rcu_core();
628 } else {
629 // Enabling BH or preempt does reschedule, so...
630 // Also if no expediting or NO_HZ_FULL, slow is OK.
631 set_tsk_need_resched(current);
632 set_preempt_need_resched();
602 * Handle special cases during rcu_read_unlock(), such as needing to
603 * notify RCU core processing or task having blocked during the RCU
604 * read-side critical section.
605 */
606static void rcu_read_unlock_special(struct task_struct *t)
607{
608 unsigned long flags;
609 bool preempt_bh_were_disabled =

--- 26 unchanged lines hidden (view full) ---

636 // Safe to awaken and we get no help from enabling
637 // irqs, unlike bh/preempt.
638 invoke_rcu_core();
639 } else {
640 // Enabling BH or preempt does reschedule, so...
641 // Also if no expediting or NO_HZ_FULL, slow is OK.
642 set_tsk_need_resched(current);
643 set_preempt_need_resched();
644 if (IS_ENABLED(CONFIG_IRQ_WORK) &&
645 !rdp->defer_qs_iw_pending && exp) {
646 // Get scheduler to re-evaluate and call hooks.
647 // If !IRQ_WORK, FQS scan will eventually IPI.
648 init_irq_work(&rdp->defer_qs_iw,
649 rcu_preempt_deferred_qs_handler);
650 rdp->defer_qs_iw_pending = true;
651 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
652 }
633 }
634 t->rcu_read_unlock_special.b.deferred_qs = true;
635 local_irq_restore(flags);
636 return;
637 }
638 WRITE_ONCE(t->rcu_read_unlock_special.b.exp_hint, false);
639 rcu_preempt_deferred_qs_irqrestore(t, flags);
640}

--- 1685 unchanged lines hidden --- 		653 }
654 t->rcu_read_unlock_special.b.deferred_qs = true;
655 local_irq_restore(flags);
656 return;
657 }
658 WRITE_ONCE(t->rcu_read_unlock_special.b.exp_hint, false);
659 rcu_preempt_deferred_qs_irqrestore(t, flags);
660}

--- 1685 unchanged lines hidden ---