xref: /linux/drivers/android/binder/process.rs (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 // Copyright (C) 2025 Google LLC.
4 
5 //! This module defines the `Process` type, which represents a process using a particular binder
6 //! context.
7 //!
8 //! The `Process` object keeps track of all of the resources that this process owns in the binder
9 //! context.
10 //!
11 //! There is one `Process` object for each binder fd that a process has opened, so processes using
12 //! several binder contexts have several `Process` objects. This ensures that the contexts are
13 //! fully separated.
14 
15 use core::mem::take;
16 
17 use kernel::{
18     bindings,
19     bits::bit_u8,
20     cred::Credential,
21     error::Error,
22     fs::file::{self, File},
23     id_pool::IdPool,
24     list::{List, ListArc, ListArcField, ListLinks},
25     mm,
26     prelude::*,
27     rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation},
28     seq_file::SeqFile,
29     seq_print,
30     sync::poll::PollTable,
31     sync::{
32         aref::ARef,
33         lock::{spinlock::SpinLockBackend, Guard},
34         poll::PollCondVarBox,
35         Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc,
36     },
37     task::{Pid, Task},
38     uaccess::{UserSlice, UserSliceReader},
39     uapi,
40     workqueue::{self, Work},
41 };
42 
43 use crate::{
44     allocation::{Allocation, AllocationInfo, NewAllocation},
45     context::Context,
46     defs::*,
47     error::{BinderError, BinderResult},
48     node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef},
49     page_range::ShrinkablePageRange,
50     range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs},
51     stats::BinderStats,
52     thread::{PushWorkRes, Thread},
53     transaction::TransactionInfo,
54     BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead,
55 };
56 
57 #[path = "freeze.rs"]
58 mod freeze;
59 use self::freeze::{FreezeCookie, FreezeListener};
60 
61 struct Mapping {
62     address: usize,
63     alloc: RangeAllocator<AllocationInfo>,
64 }
65 
66 impl Mapping {
67     fn new(address: usize, size: usize) -> Self {
68         Self {
69             address,
70             alloc: RangeAllocator::new(size),
71         }
72     }
73 }
74 
75 kernel::impl_flags!(
76     /// Represents multiple deferred work flags.
77     #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)]
78     pub struct DeferWorks(u8);
79 
80     /// Represents a single deferred work category.
81     #[derive(Debug, Clone, Copy, PartialEq, Eq)]
82     pub enum DeferWork {
83         Flush = bit_u8(0),
84         Release = bit_u8(1),
85     }
86 );
87 
88 #[derive(Copy, Clone)]
89 pub(crate) enum IsFrozen {
90     Yes,
91     No,
92     InProgress,
93 }
94 
95 impl IsFrozen {
96     /// Whether incoming transactions should be rejected due to freeze.
97     pub(crate) fn is_frozen(self) -> bool {
98         match self {
99             IsFrozen::Yes => true,
100             IsFrozen::No => false,
101             IsFrozen::InProgress => true,
102         }
103     }
104 
105     /// Whether freeze notifications consider this process frozen.
106     pub(crate) fn is_fully_frozen(self) -> bool {
107         match self {
108             IsFrozen::Yes => true,
109             IsFrozen::No => false,
110             IsFrozen::InProgress => false,
111         }
112     }
113 }
114 
115 /// The fields of `Process` protected by the spinlock.
116 pub(crate) struct ProcessInner {
117     is_manager: bool,
118     pub(crate) is_dead: bool,
119     threads: RBTree<i32, Arc<Thread>>,
120     /// INVARIANT: Threads pushed to this list must be owned by this process.
121     ready_threads: List<Thread>,
122     nodes: RBTree<u64, DArc<Node>>,
123     mapping: Option<Mapping>,
124     work: List<DTRWrap<dyn DeliverToRead>>,
125     delivered_deaths: List<DTRWrap<NodeDeath>, 2>,
126 
127     /// The number of requested threads that haven't registered yet.
128     requested_thread_count: u32,
129     /// The maximum number of threads used by the process thread pool.
130     max_threads: u32,
131     /// The number of threads the started and registered with the thread pool.
132     started_thread_count: u32,
133 
134     /// Bitmap of deferred work to do.
135     defer_work: DeferWorks,
136 
137     /// Number of transactions to be transmitted before processes in freeze_wait
138     /// are woken up.
139     outstanding_txns: u32,
140     /// Process is frozen and unable to service binder transactions.
141     pub(crate) is_frozen: IsFrozen,
142     /// Process received sync transactions since last frozen.
143     pub(crate) sync_recv: bool,
144     /// Process received async transactions since last frozen.
145     pub(crate) async_recv: bool,
146     pub(crate) binderfs_file: Option<BinderfsProcFile>,
147     /// Check for oneway spam
148     oneway_spam_detection_enabled: bool,
149 }
150 
151 impl ProcessInner {
152     fn new() -> Self {
153         Self {
154             is_manager: false,
155             is_dead: false,
156             threads: RBTree::new(),
157             ready_threads: List::new(),
158             mapping: None,
159             nodes: RBTree::new(),
160             work: List::new(),
161             delivered_deaths: List::new(),
162             requested_thread_count: 0,
163             max_threads: 0,
164             started_thread_count: 0,
165             defer_work: DeferWorks::default(),
166             outstanding_txns: 0,
167             is_frozen: IsFrozen::No,
168             sync_recv: false,
169             async_recv: false,
170             binderfs_file: None,
171             oneway_spam_detection_enabled: false,
172         }
173     }
174 
175     /// Schedule the work item for execution on this process.
176     ///
177     /// If any threads are ready for work, then the work item is given directly to that thread and
178     /// it is woken up. Otherwise, it is pushed to the process work list.
179     ///
180     /// This call can fail only if the process is dead. In this case, the work item is returned to
181     /// the caller so that the caller can drop it after releasing the inner process lock. This is
182     /// necessary since the destructor of `Transaction` will take locks that can't necessarily be
183     /// taken while holding the inner process lock.
184     pub(crate) fn push_work(
185         &mut self,
186         proc: &Process,
187         work: DLArc<dyn DeliverToRead>,
188     ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> {
189         let sync = work.should_sync_wakeup();
190 
191         // Try to find a ready thread to which to push the work.
192         if let Some(thread) = self.ready_threads.pop_front() {
193             // Push to thread while holding state lock. This prevents the thread from giving up
194             // (for example, because of a signal) when we're about to deliver work.
195             match thread.push_work_inner(work, sync) {
196                 PushWorkRes::Ok => Ok(()),
197                 PushWorkRes::OkNotifyPoll => {
198                     proc.notify_poll(sync);
199                     Ok(())
200                 }
201                 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)),
202             }
203         } else if self.is_dead {
204             Err((BinderError::new_dead(), work))
205         } else {
206             // Didn't find a thread waiting for proc work; this can happen
207             // in two scenarios:
208             // 1. All threads are busy handling transactions
209             //    In that case, one of those threads should call back into
210             //    the kernel driver soon and pick up this work.
211             // 2. Threads are using the (e)poll interface, in which case
212             //    they may be blocked on the waitqueue without having been
213             //    added to waiting_threads. For this case, we wake it up
214             //    directly.
215             self.work.push_back(work);
216 
217             // Wake up polling threads, if any.
218             proc.notify_poll(sync);
219 
220             Ok(())
221         }
222     }
223 
224     pub(crate) fn remove_node(&mut self, ptr: u64) {
225         self.nodes.remove(&ptr);
226     }
227 
228     /// Updates the reference count on the given node.
229     pub(crate) fn update_node_refcount(
230         &mut self,
231         node: &DArc<Node>,
232         inc: bool,
233         strong: bool,
234         count: usize,
235         othread: Option<&Thread>,
236     ) {
237         let push = node.update_refcount_locked(inc, strong, count, self);
238 
239         // If we decided that we need to push work, push either to the process or to a thread if
240         // one is specified.
241         if let Some(pnode) = push {
242             if let Some(thread) = othread {
243                 thread.push_work_deferred(pnode);
244             } else {
245                 let _ = self.push_work(&node.owner, pnode);
246                 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in
247                 // that case, it doesn't care about the notification.
248             }
249         }
250     }
251 
252     pub(crate) fn new_node_ref(
253         &mut self,
254         node: DArc<Node>,
255         strong: bool,
256         thread: Option<&Thread>,
257     ) -> NodeRef {
258         self.update_node_refcount(&node, true, strong, 1, thread);
259         let strong_count = if strong { 1 } else { 0 };
260         NodeRef::new(node, strong_count, 1 - strong_count)
261     }
262 
263     pub(crate) fn new_node_ref_with_thread(
264         &mut self,
265         node: DArc<Node>,
266         strong: bool,
267         thread: &Thread,
268         wrapper: Option<CritIncrWrapper>,
269     ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> {
270         let push = match wrapper {
271             None => node
272                 .incr_refcount_allow_zero2one(strong, self)?
273                 .map(|node| node as DLArc<dyn DeliverToRead>),
274             Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self),
275         };
276         if let Some(node) = push {
277             thread.push_work_deferred(node);
278         }
279         let strong_count = if strong { 1 } else { 0 };
280         Ok(NodeRef::new(node, strong_count, 1 - strong_count))
281     }
282 
283     /// Returns an existing node with the given pointer and cookie, if one exists.
284     ///
285     /// Returns an error if a node with the given pointer but a different cookie exists.
286     fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> {
287         match self.nodes.get(&ptr) {
288             None => Ok(None),
289             Some(node) => {
290                 let (_, node_cookie) = node.get_id();
291                 if node_cookie == cookie {
292                     Ok(Some(node.clone()))
293                 } else {
294                     Err(EINVAL)
295                 }
296             }
297         }
298     }
299 
300     fn register_thread(&mut self) -> bool {
301         if self.requested_thread_count == 0 {
302             return false;
303         }
304 
305         self.requested_thread_count -= 1;
306         self.started_thread_count += 1;
307         true
308     }
309 
310     /// Finds a delivered death notification with the given cookie, removes it from the thread's
311     /// delivered list, and returns it.
312     fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> {
313         let mut cursor = self.delivered_deaths.cursor_front();
314         while let Some(next) = cursor.peek_next() {
315             if next.cookie == cookie {
316                 return Some(next.remove().into_arc());
317             }
318             cursor.move_next();
319         }
320         None
321     }
322 
323     pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) {
324         if let Some(death) = ListArc::try_from_arc_or_drop(death) {
325             self.delivered_deaths.push_back(death);
326         } else {
327             pr_warn_ratelimited!("Notification added to `delivered_deaths` twice.\n");
328         }
329     }
330 
331     pub(crate) fn add_outstanding_txn(&mut self) {
332         self.outstanding_txns += 1;
333     }
334 
335     fn txns_pending_locked(&self) -> bool {
336         if self.outstanding_txns > 0 {
337             return true;
338         }
339         for thread in self.threads.values() {
340             if thread.has_current_transaction() {
341                 return true;
342             }
343         }
344         false
345     }
346 }
347 
348 /// Used to keep track of a node that this process has a handle to.
349 #[pin_data]
350 pub(crate) struct NodeRefInfo {
351     debug_id: usize,
352     /// The refcount that this process owns to the node.
353     node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>,
354     death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>,
355     /// Cookie of the active freeze listener for this node.
356     freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>,
357     /// Used to store this `NodeRefInfo` in the node's `refs` list.
358     #[pin]
359     links: ListLinks<{ Self::LIST_NODE }>,
360     /// The handle for this `NodeRefInfo`.
361     handle: u32,
362     /// The process that has a handle to the node.
363     pub(crate) process: Arc<Process>,
364 }
365 
366 impl NodeRefInfo {
367     /// The id used for the `Node::refs` list.
368     pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10;
369     /// The id used for the `ListArc` in `ProcessNodeRefs`.
370     const LIST_PROC: u64 = 0xd703a5263dcc8650;
371 
372     fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> {
373         pin_init!(Self {
374             debug_id: super::next_debug_id(),
375             node_ref: ListArcField::new(node_ref),
376             death: ListArcField::new(None),
377             freeze: ListArcField::new(None),
378             links <- ListLinks::new(),
379             handle,
380             process,
381         })
382     }
383 
384     kernel::list::define_list_arc_field_getter! {
385         pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death }
386         pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze }
387         pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref }
388         pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref }
389     }
390 }
391 
392 kernel::list::impl_list_arc_safe! {
393     impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; }
394     impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; }
395 }
396 kernel::list::impl_list_item! {
397     impl ListItem<{Self::LIST_NODE}> for NodeRefInfo {
398         using ListLinks { self.links };
399     }
400 }
401 
402 /// Keeps track of references this process has to nodes owned by other processes.
403 ///
404 /// TODO: Currently, the rbtree requires two allocations per node reference, and two tree
405 /// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these
406 /// extra costs should be eliminated.
407 struct ProcessNodeRefs {
408     /// Used to look up nodes using the 32-bit id that this process knows it by.
409     by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>,
410     /// Used to quickly find unused ids in `by_handle`.
411     handle_is_present: IdPool,
412     /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of
413     /// the underlying `Node` struct as returned by `Node::global_id`.
414     by_node: RBTree<usize, u32>,
415     /// Used to look up a `FreezeListener` by cookie.
416     ///
417     /// There might be multiple freeze listeners for the same node, but at most one of them is
418     /// active.
419     freeze_listeners: RBTree<FreezeCookie, FreezeListener>,
420 }
421 
422 impl ProcessNodeRefs {
423     fn new() -> Self {
424         Self {
425             by_handle: RBTree::new(),
426             handle_is_present: IdPool::new(),
427             by_node: RBTree::new(),
428             freeze_listeners: RBTree::new(),
429         }
430     }
431 }
432 
433 use core::mem::offset_of;
434 use kernel::bindings::rb_process_layout;
435 pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout {
436     arc_offset: Arc::<Process>::DATA_OFFSET,
437     task: offset_of!(Process, task),
438 };
439 
440 /// A process using binder.
441 ///
442 /// Strictly speaking, there can be multiple of these per process. There is one for each binder fd
443 /// that a process has opened, so processes using several binder contexts have several `Process`
444 /// objects. This ensures that the contexts are fully separated.
445 #[pin_data]
446 pub(crate) struct Process {
447     pub(crate) ctx: Arc<Context>,
448 
449     // The task leader (process).
450     pub(crate) task: ARef<Task>,
451 
452     // Credential associated with file when `Process` is created.
453     pub(crate) cred: ARef<Credential>,
454 
455     #[pin]
456     pub(crate) inner: SpinLock<ProcessInner>,
457 
458     #[pin]
459     pub(crate) pages: ShrinkablePageRange,
460 
461     // Waitqueue of processes waiting for all outstanding transactions to be
462     // processed.
463     #[pin]
464     freeze_wait: CondVar,
465 
466     // Node references are in a different lock to avoid recursive acquisition when
467     // incrementing/decrementing a node in another process.
468     #[pin]
469     node_refs: SpinLock<ProcessNodeRefs>,
470 
471     // Synchronizes `register_wait` calls to the `PollCondVarBox`.
472     //
473     // The `PollCondVarBox` is not stored here because synchronization is
474     // done for `register_wait` only. Wakeups do not take this lock.
475     poll: SetOnce<PollCondVarBox>,
476 
477     // Work node for deferred work item.
478     #[pin]
479     defer_work: Work<Process>,
480 
481     // Links for process list in Context.
482     #[pin]
483     links: ListLinks,
484 
485     pub(crate) stats: BinderStats,
486 }
487 
488 kernel::impl_has_work! {
489     impl HasWork<Process> for Process { self.defer_work }
490 }
491 
492 kernel::list::impl_list_arc_safe! {
493     impl ListArcSafe<0> for Process { untracked; }
494 }
495 kernel::list::impl_list_item! {
496     impl ListItem<0> for Process {
497         using ListLinks { self.links };
498     }
499 }
500 
501 impl workqueue::WorkItem for Process {
502     type Pointer = Arc<Process>;
503 
504     fn run(me: Arc<Self>) {
505         let defer;
506         {
507             let mut inner = me.inner.lock();
508             defer = inner.defer_work;
509             inner.defer_work = DeferWorks::default();
510         }
511 
512         if defer.contains(DeferWork::Flush) {
513             me.deferred_flush();
514         }
515         if defer.contains(DeferWork::Release) {
516             me.deferred_release();
517         }
518     }
519 }
520 
521 impl Process {
522     fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> {
523         let current = kernel::current!();
524         let process = Arc::pin_init::<Error>(
525             try_pin_init!(Process {
526                 ctx,
527                 cred,
528                 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"),
529                 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER),
530                 node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"),
531                 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"),
532                 task: current.group_leader().into(),
533                 defer_work <- kernel::new_work!("Process::defer_work"),
534                 links <- ListLinks::new(),
535                 stats: BinderStats::new(),
536                 poll: SetOnce::new(),
537             }),
538             GFP_KERNEL,
539         )?;
540 
541         process.ctx.register_process(process.clone())?;
542 
543         Ok(process)
544     }
545 
546     pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid {
547         self.task.tgid_nr_ns(None)
548     }
549 
550     #[inline(never)]
551     pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result {
552         seq_print!(m, "proc {}\n", self.pid_in_current_ns());
553         seq_print!(m, "context {}\n", &*ctx.name);
554 
555         let inner = self.inner.lock();
556         seq_print!(m, "  threads: {}\n", inner.threads.iter().count());
557         seq_print!(
558             m,
559             "  requested threads: {}+{}/{}\n",
560             inner.requested_thread_count,
561             inner.started_thread_count,
562             inner.max_threads,
563         );
564         if let Some(mapping) = &inner.mapping {
565             seq_print!(
566                 m,
567                 "  free oneway space: {}\n",
568                 mapping.alloc.free_oneway_space()
569             );
570             seq_print!(m, "  buffers: {}\n", mapping.alloc.count_buffers());
571         }
572         seq_print!(
573             m,
574             "  outstanding transactions: {}\n",
575             inner.outstanding_txns
576         );
577         seq_print!(m, "  nodes: {}\n", inner.nodes.iter().count());
578         drop(inner);
579 
580         {
581             let mut refs = self.node_refs.lock();
582             let (mut count, mut weak, mut strong) = (0, 0, 0);
583             for r in refs.by_handle.values_mut() {
584                 let node_ref = r.node_ref();
585                 let (nstrong, nweak) = node_ref.get_count();
586                 count += 1;
587                 weak += nweak;
588                 strong += nstrong;
589             }
590             seq_print!(m, "  refs: {count} s {strong} w {weak}\n");
591         }
592 
593         self.stats.debug_print("  ", m);
594 
595         Ok(())
596     }
597 
598     #[inline(never)]
599     pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result {
600         seq_print!(m, "proc {}\n", self.pid_in_current_ns());
601         seq_print!(m, "context {}\n", &*ctx.name);
602 
603         let mut all_threads = KVec::new();
604         let mut all_nodes = KVec::new();
605         loop {
606             let inner = self.inner.lock();
607             let num_threads = inner.threads.iter().count();
608             let num_nodes = inner.nodes.iter().count();
609 
610             if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes {
611                 drop(inner);
612                 all_threads.reserve(num_threads, GFP_KERNEL)?;
613                 all_nodes.reserve(num_nodes, GFP_KERNEL)?;
614                 continue;
615             }
616 
617             for thread in inner.threads.values() {
618                 assert!(all_threads.len() < all_threads.capacity());
619                 let _ = all_threads.push(thread.clone(), GFP_ATOMIC);
620             }
621 
622             for node in inner.nodes.values() {
623                 assert!(all_nodes.len() < all_nodes.capacity());
624                 let _ = all_nodes.push(node.clone(), GFP_ATOMIC);
625             }
626 
627             break;
628         }
629 
630         for thread in all_threads {
631             thread.debug_print(m, print_all)?;
632         }
633 
634         let mut inner = self.inner.lock();
635         for node in all_nodes {
636             if print_all || node.has_oneway_transaction(&mut inner) {
637                 node.full_debug_print(m, &mut inner)?;
638             }
639         }
640         drop(inner);
641 
642         if print_all {
643             let mut refs = self.node_refs.lock();
644             for r in refs.by_handle.values_mut() {
645                 let node_ref = r.node_ref();
646                 let dead = node_ref.node.owner.inner.lock().is_dead;
647                 let (strong, weak) = node_ref.get_count();
648                 let debug_id = node_ref.node.debug_id;
649 
650                 seq_print!(
651                     m,
652                     "  ref {}: desc {} {}node {debug_id} s {strong} w {weak}",
653                     r.debug_id,
654                     r.handle,
655                     if dead { "dead " } else { "" }
656                 );
657             }
658         }
659 
660         let inner = self.inner.lock();
661         for work in &inner.work {
662             work.debug_print(m, "  ", "  pending transaction ")?;
663         }
664         for _death in &inner.delivered_deaths {
665             seq_print!(m, "  has delivered dead binder\n");
666         }
667         if let Some(mapping) = &inner.mapping {
668             mapping.alloc.debug_print(m)?;
669         }
670         drop(inner);
671 
672         Ok(())
673     }
674 
675     /// Attempts to fetch a work item from the process queue.
676     pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> {
677         self.inner.lock().work.pop_front()
678     }
679 
680     /// Attempts to fetch a work item from the process queue. If none is available, it registers the
681     /// given thread as ready to receive work directly.
682     ///
683     /// This must only be called when the thread is not participating in a transaction chain; when
684     /// it is, work will always be delivered directly to the thread (and not through the process
685     /// queue).
686     pub(crate) fn get_work_or_register<'a>(
687         &'a self,
688         thread: &'a Arc<Thread>,
689         thread_has_deferred_work: bool,
690     ) -> GetWorkOrRegister<'a> {
691         let mut inner = self.inner.lock();
692 
693         if thread_has_deferred_work && !inner.work.is_empty() {
694             if let Some(work) = thread.pop_work_even_if_deferred() {
695                 return GetWorkOrRegister::Work(work);
696             }
697         }
698 
699         // Try to get work from the process queue.
700         if let Some(work) = inner.work.pop_front() {
701             return GetWorkOrRegister::Work(work);
702         }
703 
704         // Register the thread as ready.
705         GetWorkOrRegister::Register(Registration::new(thread, &mut inner))
706     }
707 
708     fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> {
709         let id = {
710             let current = kernel::current!();
711             if self.task != current.group_leader() {
712                 pr_err_ratelimited!("get_current_thread was called from the wrong process.\n");
713                 return Err(EINVAL);
714             }
715             current.pid()
716         };
717 
718         {
719             let inner = self.inner.lock();
720             if let Some(thread) = inner.threads.get(&id) {
721                 return Ok(thread.clone());
722             }
723         }
724 
725         // Allocate a new `Thread` without holding any locks.
726         let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?;
727         let ta: Arc<Thread> = Thread::new(id, self.into())?;
728 
729         let mut inner = self.inner.lock();
730         match inner.threads.entry(id) {
731             rbtree::Entry::Vacant(entry) => {
732                 entry.insert(ta.clone(), reservation);
733                 Ok(ta)
734             }
735             rbtree::Entry::Occupied(_entry) => {
736                 pr_err_ratelimited!("Cannot create two threads with the same id.\n");
737                 Err(EINVAL)
738             }
739         }
740     }
741 
742     pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult {
743         // If push_work fails, drop the work item outside the lock.
744         let res = self.inner.lock().push_work(self, work);
745         match res {
746             Ok(()) => Ok(()),
747             Err((err, work)) => {
748                 drop(work);
749                 Err(err)
750             }
751         }
752     }
753 
754     fn set_as_manager(
755         self: ArcBorrow<'_, Self>,
756         info: Option<FlatBinderObject>,
757         thread: &Thread,
758     ) -> Result {
759         let (ptr, cookie, flags) = if let Some(obj) = info {
760             (
761                 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it
762                 // is safe to access the `binder` field.
763                 unsafe { obj.__bindgen_anon_1.binder },
764                 obj.cookie,
765                 obj.flags,
766             )
767         } else {
768             (0, 0, 0)
769         };
770         let node_ref = self.get_node(ptr, cookie, flags, true, thread)?;
771         let node = node_ref.node.clone();
772         self.ctx.set_manager_node(node_ref)?;
773         self.inner.lock().is_manager = true;
774 
775         // Force the state of the node to prevent the delivery of acquire/increfs.
776         let mut owner_inner = node.owner.inner.lock();
777         node.force_has_count(&mut owner_inner);
778         Ok(())
779     }
780 
781     fn get_node_inner(
782         self: ArcBorrow<'_, Self>,
783         ptr: u64,
784         cookie: u64,
785         flags: u32,
786         strong: bool,
787         thread: &Thread,
788         wrapper: Option<CritIncrWrapper>,
789     ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> {
790         // Try to find an existing node.
791         {
792             let mut inner = self.inner.lock();
793             if let Some(node) = inner.get_existing_node(ptr, cookie)? {
794                 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
795             }
796         }
797 
798         // Allocate the node before reacquiring the lock.
799         let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc();
800         let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?;
801         let mut inner = self.inner.lock();
802         if let Some(node) = inner.get_existing_node(ptr, cookie)? {
803             return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
804         }
805 
806         inner.nodes.insert(rbnode);
807         // This can only fail if someone has already pushed the node to a list, but we just created
808         // it and still hold the lock, so it can't fail right now.
809         let node_ref = inner
810             .new_node_ref_with_thread(node, strong, thread, wrapper)
811             .unwrap();
812 
813         Ok(Ok(node_ref))
814     }
815 
816     pub(crate) fn get_node(
817         self: ArcBorrow<'_, Self>,
818         ptr: u64,
819         cookie: u64,
820         flags: u32,
821         strong: bool,
822         thread: &Thread,
823     ) -> Result<NodeRef> {
824         let mut wrapper = None;
825         for _ in 0..2 {
826             match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) {
827                 Err(err) => return Err(err),
828                 Ok(Ok(node_ref)) => return Ok(node_ref),
829                 Ok(Err(CouldNotDeliverCriticalIncrement)) => {
830                     wrapper = Some(CritIncrWrapper::new()?);
831                 }
832             }
833         }
834         // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the
835         // loop should run at most twice.
836         unreachable!()
837     }
838 
839     pub(crate) fn insert_or_update_handle(
840         self: ArcBorrow<'_, Process>,
841         node_ref: NodeRef,
842         is_manager: bool,
843     ) -> Result<u32> {
844         {
845             let mut refs = self.node_refs.lock();
846 
847             // Do a lookup before inserting.
848             if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) {
849                 let handle = *handle_ref;
850                 let info = refs.by_handle.get_mut(&handle).unwrap();
851                 info.node_ref().absorb(node_ref);
852                 return Ok(handle);
853             }
854         }
855 
856         // Reserve memory for tree nodes.
857         let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?;
858         let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?;
859         let info = UniqueArc::new_uninit(GFP_KERNEL)?;
860 
861         let mut refs_lock = self.node_refs.lock();
862         let mut refs = &mut *refs_lock;
863 
864         let (unused_id, by_handle_slot) = loop {
865             // ID 0 may only be used by the manager.
866             let start = if is_manager { 0 } else { 1 };
867 
868             if let Some(res) = refs.handle_is_present.find_unused_id(start) {
869                 match refs.by_handle.entry(res.as_u32()) {
870                     rbtree::Entry::Vacant(entry) => break (res, entry),
871                     rbtree::Entry::Occupied(_) => {
872                         pr_err_ratelimited!(
873                             "Detected mismatch between handle_is_present and by_handle\n"
874                         );
875                         res.acquire();
876                         kernel::warn_on!(true);
877                         return Err(EINVAL);
878                     }
879                 }
880             }
881 
882             let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?;
883             drop(refs_lock);
884             let resizer = grow_request.realloc(GFP_KERNEL)?;
885             refs_lock = self.node_refs.lock();
886             refs = &mut *refs_lock;
887             refs.handle_is_present.grow(resizer);
888         };
889         let handle = unused_id.as_u32();
890 
891         // Do a lookup again as node may have been inserted before the lock was reacquired.
892         let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) {
893             rbtree::Entry::Vacant(by_node_slot) => by_node_slot,
894             rbtree::Entry::Occupied(handle_ref) => {
895                 // The node was inserted by another thread while we didn't hold the lock.
896                 let handle = handle_ref.get();
897                 let info = refs.by_handle.get_mut(handle).unwrap();
898                 info.node_ref().absorb(node_ref);
899                 return Ok(*handle);
900             }
901         };
902 
903         let (info_proc, info_node) = {
904             let info_init = NodeRefInfo::new(node_ref, handle, self.into());
905             let Ok(info) = info.pin_init_with(info_init);
906             ListArc::pair_from_pin_unique(info)
907         };
908 
909         // Ensure the process is still alive while we insert a new reference.
910         //
911         // This releases the lock before inserting the nodes, but since `is_dead` is set as the
912         // first thing in `deferred_release`, process cleanup will not miss the items inserted into
913         // `refs` below.
914         if self.inner.lock().is_dead {
915             // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of
916             // the lock.
917             drop(refs_lock);
918             return Err(ESRCH);
919         }
920 
921         // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting
922         // `info_node` into the right node's `refs` list.
923         unsafe { info_proc.node_ref2().node.insert_node_info(info_node) };
924 
925         by_node_slot.insert(handle, reserve1);
926         by_handle_slot.insert(info_proc, reserve2);
927         unused_id.acquire();
928         Ok(handle)
929     }
930 
931     pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> {
932         // When handle is zero, try to get the context manager.
933         if handle == 0 {
934             let node_ref = self.ctx.get_manager_node(true)?;
935             if core::ptr::eq(self, &*node_ref.node.owner) {
936                 return Err(EINVAL.into());
937             }
938             Ok(node_ref)
939         } else {
940             match self.get_node_from_handle(handle, true) {
941                 Ok(node_ref) => Ok(node_ref),
942                 Err(err) => {
943                     binder_debug!(UserError, "got transaction to invalid handle {handle}");
944                     Err(err.into())
945                 }
946             }
947         }
948     }
949 
950     pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> {
951         self.node_refs
952             .lock()
953             .by_handle
954             .get_mut(&handle)
955             .ok_or(ENOENT)?
956             .node_ref()
957             .clone(strong)
958     }
959 
960     pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) {
961         let mut inner = self.inner.lock();
962         // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list.
963         let removed = unsafe { inner.delivered_deaths.remove(death) };
964         drop(inner);
965         drop(removed);
966     }
967 
968     pub(crate) fn update_ref(
969         self: ArcBorrow<'_, Process>,
970         handle: u32,
971         inc: bool,
972         strong: bool,
973     ) -> Result {
974         if inc && handle == 0 {
975             if let Ok(node_ref) = self.ctx.get_manager_node(strong) {
976                 if core::ptr::eq(&*self, &*node_ref.node.owner) {
977                     return Err(EINVAL);
978                 }
979                 let _ = self.insert_or_update_handle(node_ref, true);
980                 return Ok(());
981             }
982         }
983 
984         // To preserve original binder behaviour, we only fail requests where the manager tries to
985         // increment references on itself.
986         let _to_free_by_handle;
987         let _to_free_by_node;
988         let _to_free_freeze_listener;
989         let _to_free_freeze_listener_cleanup;
990         let mut refs = self.node_refs.lock();
991         if let Some(info) = refs.by_handle.get_mut(&handle) {
992             if info.node_ref().update(inc, strong) {
993                 // Clean up death if there is one attached to this node reference.
994                 //
995                 // We remove the entire `info` below, so no need to remove `death` from `info`.
996                 if let Some(death) = info.death().as_ref() {
997                     death.set_cleared(true);
998                     self.remove_from_delivered_deaths(death);
999                 }
1000 
1001                 // Remove reference from process tables, and from the node's `refs` list.
1002 
1003                 // SAFETY: We are removing the `NodeRefInfo` from the right node.
1004                 unsafe { info.node_ref2().node.remove_node_info(info) };
1005 
1006                 let id = info.node_ref().node.global_id();
1007 
1008                 if let Some(freeze) = *info.freeze() {
1009                     if let Some(fl) = refs.freeze_listeners.remove(&freeze) {
1010                         _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self);
1011                         _to_free_freeze_listener = fl;
1012                     }
1013                 }
1014 
1015                 _to_free_by_handle = refs.by_handle.remove_node(&handle);
1016                 _to_free_by_node = refs.by_node.remove_node(&id);
1017                 refs.handle_is_present.release_id(handle as usize);
1018 
1019                 if let Some(shrink) = refs.handle_is_present.shrink_request() {
1020                     drop(refs);
1021                     // This intentionally ignores allocation failures.
1022                     if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) {
1023                         refs = self.node_refs.lock();
1024                         refs.handle_is_present.shrink(new_bitmap);
1025                     }
1026                 }
1027             }
1028         } else {
1029             // All refs are cleared in process exit, so this warning is expected in that case.
1030             if !self.inner.lock().is_dead {
1031                 binder_debug!(UserError, "no such ref {handle}");
1032             }
1033         }
1034         Ok(())
1035     }
1036 
1037     /// Decrements the refcount of the given node, if one exists.
1038     pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) {
1039         let mut inner = self.inner.lock();
1040         if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) {
1041             inner.update_node_refcount(&node, false, strong, 1, None);
1042         }
1043     }
1044 
1045     pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result {
1046         let ptr = reader.read::<u64>()?;
1047         let cookie = reader.read::<u64>()?;
1048         let mut inner = self.inner.lock();
1049         if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) {
1050             if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) {
1051                 // This only fails if the process is dead.
1052                 let _ = inner.push_work(self, node);
1053             }
1054         }
1055         Ok(())
1056     }
1057 
1058     pub(crate) fn buffer_alloc(
1059         self: &Arc<Self>,
1060         debug_id: usize,
1061         size: usize,
1062         info: &mut TransactionInfo,
1063     ) -> BinderResult<NewAllocation> {
1064         use kernel::page::PAGE_SIZE;
1065 
1066         let mut reserve_new_args = ReserveNewArgs {
1067             debug_id,
1068             size,
1069             is_oneway: info.is_oneway(),
1070             pid: info.from_pid,
1071             ..ReserveNewArgs::default()
1072         };
1073 
1074         let (new_alloc, addr) = loop {
1075             let mut inner = self.inner.lock();
1076             let mapping = inner
1077                 .mapping
1078                 .as_mut()
1079                 .ok_or_else(|| BinderError::new_dead())?;
1080             let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? {
1081                 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address),
1082                 ReserveNew::NeedAlloc(request) => request,
1083             };
1084             drop(inner);
1085             // We need to allocate memory and then call `reserve_new` again.
1086             reserve_new_args = alloc_request.make_alloc()?;
1087         };
1088 
1089         info.oneway_spam_suspect = new_alloc.oneway_spam_detected;
1090         let res = Allocation::new(
1091             self.clone(),
1092             debug_id,
1093             new_alloc.offset,
1094             size,
1095             addr + new_alloc.offset,
1096         );
1097 
1098         // This allocation will be marked as in use until the `Allocation` is used to free it.
1099         //
1100         // This method can't be called while holding a lock, so we release the lock first. It's
1101         // okay for several threads to use the method on the same index at the same time. In that
1102         // case, one of the calls will allocate the given page (if missing), and the other call
1103         // will wait for the other call to finish allocating the page.
1104         //
1105         // We will not call `stop_using_range` in parallel with this on the same page, because the
1106         // allocation can only be removed via the destructor of the `Allocation` object that we
1107         // currently own.
1108         match self.pages.use_range(
1109             new_alloc.offset / PAGE_SIZE,
1110             (new_alloc.offset + size).div_ceil(PAGE_SIZE),
1111         ) {
1112             Ok(()) => {}
1113             Err(err) => {
1114                 pr_warn_ratelimited!("use_range failure {:?}\n", err);
1115                 return Err(err.into());
1116             }
1117         }
1118 
1119         Ok(NewAllocation(res))
1120     }
1121 
1122     pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> {
1123         let mut inner = self.inner.lock();
1124         let mapping = inner.mapping.as_mut()?;
1125         let offset = ptr.checked_sub(mapping.address)?;
1126         let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?;
1127         let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr);
1128         if let Some(data) = odata {
1129             alloc.set_info(data);
1130         }
1131         Some(alloc)
1132     }
1133 
1134     pub(crate) fn buffer_raw_free(&self, ptr: usize) {
1135         let mut inner = self.inner.lock();
1136         if let Some(ref mut mapping) = &mut inner.mapping {
1137             let offset = match ptr.checked_sub(mapping.address) {
1138                 Some(offset) => offset,
1139                 None => return,
1140             };
1141 
1142             let freed_range = match mapping.alloc.reservation_abort(offset) {
1143                 Ok(freed_range) => freed_range,
1144                 Err(_) => {
1145                     pr_warn_ratelimited!(
1146                         "Pointer {:x} failed to free, base = {:x}\n",
1147                         ptr,
1148                         mapping.address
1149                     );
1150                     return;
1151                 }
1152             };
1153 
1154             // No more allocations in this range. Mark them as not in use.
1155             //
1156             // Must be done before we release the lock so that `use_range` is not used on these
1157             // indices until `stop_using_range` returns.
1158             self.pages
1159                 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx);
1160         }
1161     }
1162 
1163     pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) {
1164         let mut inner = self.inner.lock();
1165         if let Some(ref mut mapping) = &mut inner.mapping {
1166             if mapping.alloc.reservation_commit(offset, &mut data).is_err() {
1167                 pr_warn_ratelimited!("Offset {} failed to be marked freeable\n", offset);
1168             }
1169         }
1170     }
1171 
1172     fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result {
1173         use kernel::page::PAGE_SIZE;
1174         let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize);
1175         let mapping = Mapping::new(vma.start(), size);
1176         let page_count = self.pages.register_with_vma(vma)?;
1177         if page_count * PAGE_SIZE != size {
1178             return Err(EINVAL);
1179         }
1180 
1181         // Save range allocator for later.
1182         self.inner.lock().mapping = Some(mapping);
1183 
1184         Ok(())
1185     }
1186 
1187     fn version(&self, data: UserSlice) -> Result {
1188         data.writer().write(&BinderVersion::current())
1189     }
1190 
1191     pub(crate) fn register_thread(&self) -> bool {
1192         self.inner.lock().register_thread()
1193     }
1194 
1195     fn remove_thread(&self, thread: Arc<Thread>) {
1196         self.inner.lock().threads.remove(&thread.id);
1197         thread.release();
1198     }
1199 
1200     fn set_max_threads(&self, max: u32) {
1201         self.inner.lock().max_threads = max;
1202     }
1203 
1204     fn set_oneway_spam_detection_enabled(&self, enabled: u32) {
1205         self.inner.lock().oneway_spam_detection_enabled = enabled != 0;
1206     }
1207 
1208     pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool {
1209         self.inner.lock().oneway_spam_detection_enabled
1210     }
1211 
1212     fn get_node_debug_info(&self, data: UserSlice) -> Result {
1213         let (mut reader, mut writer) = data.reader_writer();
1214 
1215         // Read the starting point.
1216         let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr;
1217         let mut out = BinderNodeDebugInfo::default();
1218 
1219         if let Some(start_ptr) = ptr.checked_add(1) {
1220             let inner = self.inner.lock();
1221             // cursor_lower_bound retrieves the "key" passed or the next existing larger key
1222             if let Some(cursor) = inner.nodes.cursor_lower_bound(&start_ptr) {
1223                 let (_, node) = cursor.current();
1224                 node.populate_debug_info(&mut out, &inner);
1225             }
1226         }
1227 
1228         writer.write(&out)
1229     }
1230 
1231     fn get_node_info_from_ref(&self, data: UserSlice) -> Result {
1232         let (mut reader, mut writer) = data.reader_writer();
1233         let mut out = reader.read::<BinderNodeInfoForRef>()?;
1234 
1235         if out.strong_count != 0
1236             || out.weak_count != 0
1237             || out.reserved1 != 0
1238             || out.reserved2 != 0
1239             || out.reserved3 != 0
1240         {
1241             return Err(EINVAL);
1242         }
1243 
1244         // Only the context manager is allowed to use this ioctl.
1245         if !self.inner.lock().is_manager {
1246             return Err(EPERM);
1247         }
1248 
1249         {
1250             let mut node_refs = self.node_refs.lock();
1251             let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?;
1252             let node_ref = node_info.node_ref();
1253             let owner_inner = node_ref.node.owner.inner.lock();
1254             node_ref.node.populate_counts(&mut out, &owner_inner);
1255         }
1256 
1257         // Write the result back.
1258         writer.write(&out)
1259     }
1260 
1261     pub(crate) fn needs_thread(&self) -> bool {
1262         let mut inner = self.inner.lock();
1263         let ret = inner.requested_thread_count == 0
1264             && inner.ready_threads.is_empty()
1265             && inner.started_thread_count < inner.max_threads;
1266         if ret {
1267             inner.requested_thread_count += 1
1268         }
1269         ret
1270     }
1271 
1272     pub(crate) fn request_death(
1273         self: &Arc<Self>,
1274         reader: &mut UserSliceReader,
1275         thread: &Thread,
1276     ) -> Result {
1277         let handle: u32 = reader.read()?;
1278         let cookie: u64 = reader.read()?;
1279 
1280         // Queue BR_ERROR if we can't allocate memory for the death notification.
1281         let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| {
1282             thread.push_return_work(BR_ERROR);
1283             binder_debug!(
1284                 DeathNotification,
1285                 "BC_REQUEST_DEATH_NOTIFICATION failed due to memory allocation failure"
1286             );
1287         })?;
1288         let mut refs = self.node_refs.lock();
1289         let Some(info) = refs.by_handle.get_mut(&handle) else {
1290             binder_debug!(
1291                 UserError,
1292                 "BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}"
1293             );
1294             return Ok(());
1295         };
1296 
1297         // Nothing to do if there is already a death notification request for this handle.
1298         if info.death().is_some() {
1299             binder_debug!(
1300                 UserError,
1301                 "BC_REQUEST_DEATH_NOTIFICATION death notification already set"
1302             );
1303             return Ok(());
1304         }
1305 
1306         let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie);
1307         let Ok(death) = death.pin_init_with(death_init);
1308 
1309         // Register the death notification.
1310         {
1311             let owner = info.node_ref2().node.owner.clone();
1312             let mut owner_inner = owner.inner.lock();
1313             if owner_inner.is_dead {
1314                 let death = Arc::from(death);
1315                 *info.death() = Some(death.clone());
1316                 drop(owner_inner);
1317                 death.set_dead();
1318             } else {
1319                 let death = ListArc::from(death);
1320                 *info.death() = Some(death.clone_arc());
1321                 info.node_ref().node.add_death(death, &mut owner_inner);
1322             }
1323         }
1324         binder_debug!(
1325             DeathNotification,
1326             "BC_REQUEST_DEATH_NOTIFICATION handle {handle} cookie {:016x}",
1327             cookie
1328         );
1329         Ok(())
1330     }
1331 
1332     pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result {
1333         let handle: u32 = reader.read()?;
1334         let cookie: u64 = reader.read()?;
1335 
1336         let mut refs = self.node_refs.lock();
1337         let Some(info) = refs.by_handle.get_mut(&handle) else {
1338             binder_debug!(
1339                 UserError,
1340                 "BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}"
1341             );
1342             return Ok(());
1343         };
1344 
1345         let Some(death) = info.death().take() else {
1346             binder_debug!(
1347                 UserError,
1348                 "BC_CLEAR_DEATH_NOTIFICATION death notification not active"
1349             );
1350             return Ok(());
1351         };
1352         if death.cookie != cookie {
1353             *info.death() = Some(death);
1354             binder_debug!(
1355                 UserError,
1356                 "BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch"
1357             );
1358             return Ok(());
1359         }
1360 
1361         // Update state and determine if we need to queue a work item. We only need to do it when
1362         // the node is not dead or if the user already completed the death notification.
1363         let should_schedule = death.set_cleared(false);
1364         drop(refs);
1365 
1366         if should_schedule {
1367             if let Some(death) = ListArc::try_from_arc_or_drop(death) {
1368                 let _ = thread.push_work_if_looper(death);
1369             }
1370         }
1371 
1372         binder_debug!(
1373             DeathNotification,
1374             "BC_CLEAR_DEATH_NOTIFICATION handle {handle} cookie {:016x}",
1375             cookie
1376         );
1377         Ok(())
1378     }
1379 
1380     pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) {
1381         let death = self.inner.lock().pull_delivered_death(cookie);
1382         if let Some(death) = death {
1383             death.set_notification_done(thread);
1384         }
1385     }
1386 
1387     /// Locks the spinlock and move the `nodes` rbtree out.
1388     ///
1389     /// This allows you to iterate through `nodes` while also allowing you to give other parts of
1390     /// the codebase exclusive access to `ProcessInner`.
1391     pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> {
1392         let mut inner = self.inner.lock();
1393         WithNodes {
1394             nodes: take(&mut inner.nodes),
1395             inner,
1396         }
1397     }
1398 
1399     fn deferred_flush(&self) {
1400         binder_debug!(pid = self.task.pid(), OpenClose, "flushing process");
1401         let inner = self.inner.lock();
1402         for thread in inner.threads.values() {
1403             thread.exit_looper();
1404         }
1405     }
1406 
1407     fn deferred_release(self: Arc<Self>) {
1408         binder_debug!(pid = self.task.pid(), OpenClose, "releasing process");
1409 
1410         let is_manager = {
1411             let mut inner = self.inner.lock();
1412             inner.is_dead = true;
1413             inner.is_frozen = IsFrozen::No;
1414             inner.sync_recv = false;
1415             inner.async_recv = false;
1416             inner.is_manager
1417         };
1418 
1419         if is_manager {
1420             self.ctx.unset_manager_node();
1421         }
1422 
1423         self.ctx.deregister_process(&self);
1424 
1425         let binderfs_file = self.inner.lock().binderfs_file.take();
1426         drop(binderfs_file);
1427 
1428         // Release threads.
1429         let threads = {
1430             let mut inner = self.inner.lock();
1431             let threads = take(&mut inner.threads);
1432             let ready = take(&mut inner.ready_threads);
1433             drop(inner);
1434             drop(ready);
1435 
1436             for thread in threads.values() {
1437                 thread.release();
1438             }
1439             threads
1440         };
1441 
1442         // Release nodes.
1443         {
1444             while let Some(node) = {
1445                 let mut lock = self.inner.lock();
1446                 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1)
1447             } {
1448                 node.to_key_value().1.release();
1449             }
1450         }
1451 
1452         // Clean up death listeners and remove nodes from external node info lists.
1453         for info in self.node_refs.lock().by_handle.values_mut() {
1454             // SAFETY: We are removing the `NodeRefInfo` from the right node.
1455             unsafe { info.node_ref2().node.remove_node_info(info) };
1456 
1457             // Clear death notifications from the nodes (that belong to a different process).
1458             // No need to remove them from `info` as we clear info below.
1459             if let Some(death) = info.death().as_ref() {
1460                 death.set_cleared(false);
1461             }
1462         }
1463 
1464         // Clean up freeze listeners.
1465         let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners);
1466         for listener in freeze_listeners.values() {
1467             listener.on_process_cleanup(&self);
1468         }
1469         drop(freeze_listeners);
1470 
1471         // Release refs on foreign nodes.
1472         {
1473             let mut refs = self.node_refs.lock();
1474             let by_handle = take(&mut refs.by_handle);
1475             let by_node = take(&mut refs.by_node);
1476             drop(refs);
1477             drop(by_node);
1478             drop(by_handle);
1479         }
1480 
1481         // Cancel all pending work items.
1482         while let Some(work) = self.get_work() {
1483             work.into_arc().cancel();
1484         }
1485 
1486         // Clear delivered_deaths list.
1487         //
1488         // Scope ensures that MutexGuard is dropped while executing the body.
1489         while let Some(delivered_death) = {
1490             // Explicitly bind to avoid tail expression lifetime extension of the lockguard
1491             // Can be removed when the kernel moves to edition 2024
1492             let maybe_death = self.inner.lock().delivered_deaths.pop_front();
1493             maybe_death
1494         } {
1495             drop(delivered_death);
1496         }
1497 
1498         // Free any resources kept alive by allocated buffers.
1499         let omapping = self.inner.lock().mapping.take();
1500         if let Some(mut mapping) = omapping {
1501             let address = mapping.address;
1502             mapping
1503                 .alloc
1504                 .take_for_each(|offset, size, debug_id, odata| {
1505                     let ptr = offset + address;
1506                     let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr);
1507                     if let Some(data) = odata {
1508                         alloc.set_info(data);
1509                     }
1510                     drop(alloc)
1511                 });
1512         }
1513 
1514         // calls to synchronize_rcu() in thread drop will happen here
1515         drop(threads);
1516     }
1517 
1518     pub(crate) fn drop_outstanding_txn(&self) {
1519         let wake = {
1520             let mut inner = self.inner.lock();
1521             if inner.outstanding_txns == 0 {
1522                 pr_err_ratelimited!("outstanding_txns underflow\n");
1523                 return;
1524             }
1525             inner.outstanding_txns -= 1;
1526             inner.is_frozen.is_frozen() && inner.outstanding_txns == 0
1527         };
1528 
1529         if wake {
1530             self.freeze_wait.notify_all();
1531         }
1532     }
1533 
1534     // #[export_name] is a temporary workaround so that ps output does not become unreadable from
1535     // mangled symbol names.
1536     #[export_name = "rust_binder_freeze"]
1537     pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result {
1538         if info.enable == 0 {
1539             let msgs = self.prepare_freeze_messages()?;
1540             let mut inner = self.inner.lock();
1541             inner.sync_recv = false;
1542             inner.async_recv = false;
1543             inner.is_frozen = IsFrozen::No;
1544             drop(inner);
1545             msgs.send_messages();
1546             return Ok(());
1547         }
1548 
1549         let mut inner = self.inner.lock();
1550         inner.sync_recv = false;
1551         inner.async_recv = false;
1552         inner.is_frozen = IsFrozen::InProgress;
1553 
1554         if info.timeout_ms > 0 {
1555             let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms);
1556             while jiffies > 0 {
1557                 if inner.outstanding_txns == 0 {
1558                     break;
1559                 }
1560 
1561                 match self
1562                     .freeze_wait
1563                     .wait_interruptible_timeout(&mut inner, jiffies)
1564                 {
1565                     CondVarTimeoutResult::Signal { .. } => {
1566                         inner.is_frozen = IsFrozen::No;
1567                         return Err(ERESTARTSYS);
1568                     }
1569                     CondVarTimeoutResult::Woken { jiffies: remaining } => {
1570                         jiffies = remaining;
1571                     }
1572                     CondVarTimeoutResult::Timeout => {
1573                         jiffies = 0;
1574                     }
1575                 }
1576             }
1577         }
1578 
1579         if inner.txns_pending_locked() {
1580             inner.is_frozen = IsFrozen::No;
1581             Err(EAGAIN)
1582         } else {
1583             drop(inner);
1584             match self.prepare_freeze_messages() {
1585                 Ok(batch) => {
1586                     self.inner.lock().is_frozen = IsFrozen::Yes;
1587                     batch.send_messages();
1588                     Ok(())
1589                 }
1590                 Err(kernel::alloc::AllocError) => {
1591                     self.inner.lock().is_frozen = IsFrozen::No;
1592                     Err(ENOMEM)
1593                 }
1594             }
1595         }
1596     }
1597 
1598     pub(crate) fn notify_poll(&self, sync: bool) {
1599         if let Some(poll) = self.poll.as_ref() {
1600             if sync {
1601                 poll.notify_sync();
1602             }
1603             poll.notify_all();
1604         }
1605     }
1606 }
1607 
1608 fn get_frozen_status(data: UserSlice) -> Result {
1609     let (mut reader, mut writer) = data.reader_writer();
1610 
1611     let mut info = reader.read::<BinderFrozenStatusInfo>()?;
1612     info.sync_recv = 0;
1613     info.async_recv = 0;
1614     let mut found = false;
1615 
1616     for ctx in crate::context::get_all_contexts()? {
1617         ctx.for_each_proc(|proc| {
1618             if proc.task.pid() == info.pid as Pid {
1619                 found = true;
1620                 let inner = proc.inner.lock();
1621                 let txns_pending = inner.txns_pending_locked();
1622                 info.async_recv |= u32::from(inner.async_recv);
1623                 info.sync_recv |= u32::from(inner.sync_recv);
1624                 info.sync_recv |= u32::from(txns_pending) << 1;
1625             }
1626         });
1627     }
1628 
1629     if found {
1630         writer.write(&info)?;
1631         Ok(())
1632     } else {
1633         Err(EINVAL)
1634     }
1635 }
1636 
1637 fn ioctl_freeze(reader: &mut UserSliceReader) -> Result {
1638     let info = reader.read::<BinderFreezeInfo>()?;
1639 
1640     // Very unlikely for there to be more than 3, since a process normally uses at most binder and
1641     // hwbinder.
1642     let mut procs = KVec::with_capacity(3, GFP_KERNEL)?;
1643 
1644     let ctxs = crate::context::get_all_contexts()?;
1645     for ctx in ctxs {
1646         for proc in ctx.get_procs_with_pid(info.pid as i32)? {
1647             procs.push(proc, GFP_KERNEL)?;
1648         }
1649     }
1650 
1651     for proc in procs {
1652         proc.ioctl_freeze(&info)?;
1653     }
1654     Ok(())
1655 }
1656 
1657 /// The ioctl handler.
1658 impl Process {
1659     /// Ioctls that are write-only from the perspective of userspace.
1660     ///
1661     /// The kernel will only read from the pointer that userspace provided to us.
1662     fn ioctl_write_only(
1663         this: ArcBorrow<'_, Process>,
1664         _file: &File,
1665         cmd: u32,
1666         reader: &mut UserSliceReader,
1667     ) -> Result {
1668         if cmd == uapi::BINDER_FREEZE {
1669             return ioctl_freeze(reader);
1670         }
1671 
1672         let thread = this.get_current_thread()?;
1673         match cmd {
1674             uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?),
1675             uapi::BINDER_THREAD_EXIT => this.remove_thread(thread),
1676             uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?,
1677             uapi::BINDER_SET_CONTEXT_MGR_EXT => {
1678                 this.set_as_manager(Some(reader.read()?), &thread)?
1679             }
1680             uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => {
1681                 this.set_oneway_spam_detection_enabled(reader.read()?)
1682             }
1683             _ => return Err(EINVAL),
1684         }
1685         Ok(())
1686     }
1687 
1688     /// Ioctls that are read/write from the perspective of userspace.
1689     ///
1690     /// The kernel will both read from and write to the pointer that userspace provided to us.
1691     fn ioctl_write_read(
1692         this: ArcBorrow<'_, Process>,
1693         file: &File,
1694         cmd: u32,
1695         data: UserSlice,
1696     ) -> Result {
1697         let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0;
1698         match cmd {
1699             uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?,
1700             uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?,
1701             uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?,
1702             uapi::BINDER_VERSION => this.version(data)?,
1703             uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?,
1704             uapi::BINDER_GET_EXTENDED_ERROR => {
1705                 this.get_current_thread()?.get_extended_error(data)?
1706             }
1707             _ => return Err(EINVAL),
1708         }
1709         Ok(())
1710     }
1711 }
1712 
1713 /// The file operations supported by `Process`.
1714 impl Process {
1715     pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> {
1716         let proc = Self::new(ctx.into(), ARef::from(file.cred()))?;
1717         binder_debug!(OpenClose, "opened process");
1718         Ok(proc)
1719     }
1720 
1721     pub(crate) fn release(this: Arc<Process>, _file: &File) {
1722         let binderfs_file;
1723         let should_schedule;
1724         {
1725             let mut inner = this.inner.lock();
1726             should_schedule = inner.defer_work == DeferWorks::empty();
1727             inner.defer_work |= DeferWork::Release;
1728             binderfs_file = inner.binderfs_file.take();
1729         }
1730 
1731         if should_schedule {
1732             // Ignore failures to schedule to the workqueue. Those just mean that we're already
1733             // scheduled for execution.
1734             let _ = workqueue::system().enqueue(this);
1735         }
1736 
1737         drop(binderfs_file);
1738     }
1739 
1740     pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result {
1741         let should_schedule;
1742         {
1743             let mut inner = this.inner.lock();
1744             should_schedule = inner.defer_work == DeferWorks::empty();
1745             inner.defer_work |= DeferWork::Flush;
1746         }
1747 
1748         if should_schedule {
1749             // Ignore failures to schedule to the workqueue. Those just mean that we're already
1750             // scheduled for execution.
1751             let _ = workqueue::system().enqueue(Arc::from(this));
1752         }
1753         Ok(())
1754     }
1755 
1756     pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result {
1757         use kernel::ioctl::{_IOC_DIR, _IOC_SIZE};
1758         use kernel::uapi::{_IOC_READ, _IOC_WRITE};
1759 
1760         crate::trace::trace_ioctl(cmd, arg);
1761 
1762         let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd));
1763 
1764         const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE;
1765 
1766         let res = match _IOC_DIR(cmd) {
1767             _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()),
1768             _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice),
1769             _ => Err(EINVAL),
1770         };
1771 
1772         crate::trace::trace_ioctl_done(res);
1773         res
1774     }
1775 
1776     pub(crate) fn mmap(
1777         this: ArcBorrow<'_, Process>,
1778         _file: &File,
1779         vma: &mm::virt::VmaNew,
1780     ) -> Result {
1781         // We don't allow mmap to be used in a different process.
1782         if this.task != kernel::current!().group_leader() {
1783             return Err(EINVAL);
1784         }
1785         if vma.start() == 0 {
1786             return Err(EINVAL);
1787         }
1788 
1789         vma.try_clear_maywrite().map_err(|_| EPERM)?;
1790         vma.set_dontcopy();
1791         vma.set_mixedmap();
1792 
1793         // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it.
1794         this.create_mapping(vma)
1795     }
1796 
1797     pub(crate) fn poll(
1798         this: ArcBorrow<'_, Process>,
1799         file: &File,
1800         table: PollTable<'_>,
1801     ) -> Result<u32> {
1802         let thread = this.get_current_thread()?;
1803         {
1804             let poll = loop {
1805                 if let Some(poll) = this.poll.as_ref() {
1806                     break poll;
1807                 }
1808 
1809                 let poll = PollCondVarBox::new(c"Process::poll", kernel::static_lock_class!())?;
1810                 // Reuse our existing lock to synchronize callers initializing.
1811                 let _guard = this.node_refs.lock();
1812                 this.poll.populate(poll);
1813             };
1814 
1815             table.register_wait(file, poll);
1816         }
1817         let (from_proc, mut mask) = thread.poll()?;
1818         if mask == 0 && from_proc && !this.inner.lock().work.is_empty() {
1819             mask |= bindings::POLLIN;
1820         }
1821         Ok(mask)
1822     }
1823 }
1824 
1825 /// Represents that a thread has registered with the `ready_threads` list of its process.
1826 ///
1827 /// The destructor of this type will unregister the thread from the list of ready threads.
1828 pub(crate) struct Registration<'a> {
1829     thread: &'a Arc<Thread>,
1830 }
1831 
1832 impl<'a> Registration<'a> {
1833     fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self {
1834         assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref()));
1835         // INVARIANT: We are pushing this thread to the right `ready_threads` list.
1836         if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) {
1837             guard.ready_threads.push_front(list_arc);
1838         } else {
1839             // It is an error to hit this branch, and it should not be reachable. We try to do
1840             // something reasonable when the failure path happens. Most likely, the thread in
1841             // question will sleep forever.
1842             pr_err_ratelimited!("Same thread registered with `ready_threads` twice.\n");
1843         }
1844         Self { thread }
1845     }
1846 }
1847 
1848 impl Drop for Registration<'_> {
1849     fn drop(&mut self) {
1850         let mut inner = self.thread.process.inner.lock();
1851         // SAFETY: The thread has the invariant that we never push it to any other linked list than
1852         // the `ready_threads` list of its parent process. Therefore, the thread is either in that
1853         // list, or in no list.
1854         unsafe { inner.ready_threads.remove(self.thread) };
1855     }
1856 }
1857 
1858 pub(crate) struct WithNodes<'a> {
1859     pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>,
1860     pub(crate) nodes: RBTree<u64, DArc<Node>>,
1861 }
1862 
1863 impl Drop for WithNodes<'_> {
1864     fn drop(&mut self) {
1865         core::mem::swap(&mut self.nodes, &mut self.inner.nodes);
1866         if self.nodes.iter().next().is_some() {
1867             pr_err_ratelimited!("nodes array was modified while using lock_with_nodes\n");
1868         }
1869     }
1870 }
1871 
1872 pub(crate) enum GetWorkOrRegister<'a> {
1873     Work(DLArc<dyn DeliverToRead>),
1874     Register(Registration<'a>),
1875 }
1876