1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 //! This module defines the `Process` type, which represents a process using a particular binder 6 //! context. 7 //! 8 //! The `Process` object keeps track of all of the resources that this process owns in the binder 9 //! context. 10 //! 11 //! There is one `Process` object for each binder fd that a process has opened, so processes using 12 //! several binder contexts have several `Process` objects. This ensures that the contexts are 13 //! fully separated. 14 15 use core::mem::take; 16 17 use kernel::{ 18 bindings, 19 bits::bit_u8, 20 cred::Credential, 21 error::Error, 22 fs::file::{self, File}, 23 id_pool::IdPool, 24 list::{List, ListArc, ListArcField, ListLinks}, 25 mm, 26 prelude::*, 27 rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation}, 28 seq_file::SeqFile, 29 seq_print, 30 sync::poll::PollTable, 31 sync::{ 32 aref::ARef, 33 lock::{spinlock::SpinLockBackend, Guard}, 34 poll::PollCondVarBox, 35 Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc, 36 }, 37 task::{Pid, Task}, 38 uaccess::{UserSlice, UserSliceReader}, 39 uapi, 40 workqueue::{self, Work}, 41 }; 42 43 use crate::{ 44 allocation::{Allocation, AllocationInfo, NewAllocation}, 45 context::Context, 46 defs::*, 47 error::{BinderError, BinderResult}, 48 node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef}, 49 page_range::ShrinkablePageRange, 50 range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs}, 51 stats::BinderStats, 52 thread::{PushWorkRes, Thread}, 53 transaction::TransactionInfo, 54 BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead, 55 }; 56 57 #[path = "freeze.rs"] 58 mod freeze; 59 use self::freeze::{FreezeCookie, FreezeListener}; 60 61 struct Mapping { 62 address: usize, 63 alloc: RangeAllocator<AllocationInfo>, 64 } 65 66 impl Mapping { 67 fn new(address: usize, size: usize) -> Self { 68 Self { 69 address, 70 alloc: RangeAllocator::new(size), 71 } 72 } 73 } 74 75 kernel::impl_flags!( 76 /// Represents multiple deferred work flags. 77 #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] 78 pub struct DeferWorks(u8); 79 80 /// Represents a single deferred work category. 81 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 82 pub enum DeferWork { 83 Flush = bit_u8(0), 84 Release = bit_u8(1), 85 } 86 ); 87 88 #[derive(Copy, Clone)] 89 pub(crate) enum IsFrozen { 90 Yes, 91 No, 92 InProgress, 93 } 94 95 impl IsFrozen { 96 /// Whether incoming transactions should be rejected due to freeze. 97 pub(crate) fn is_frozen(self) -> bool { 98 match self { 99 IsFrozen::Yes => true, 100 IsFrozen::No => false, 101 IsFrozen::InProgress => true, 102 } 103 } 104 105 /// Whether freeze notifications consider this process frozen. 106 pub(crate) fn is_fully_frozen(self) -> bool { 107 match self { 108 IsFrozen::Yes => true, 109 IsFrozen::No => false, 110 IsFrozen::InProgress => false, 111 } 112 } 113 } 114 115 /// The fields of `Process` protected by the spinlock. 116 pub(crate) struct ProcessInner { 117 is_manager: bool, 118 pub(crate) is_dead: bool, 119 threads: RBTree<i32, Arc<Thread>>, 120 /// INVARIANT: Threads pushed to this list must be owned by this process. 121 ready_threads: List<Thread>, 122 nodes: RBTree<u64, DArc<Node>>, 123 mapping: Option<Mapping>, 124 work: List<DTRWrap<dyn DeliverToRead>>, 125 delivered_deaths: List<DTRWrap<NodeDeath>, 2>, 126 127 /// The number of requested threads that haven't registered yet. 128 requested_thread_count: u32, 129 /// The maximum number of threads used by the process thread pool. 130 max_threads: u32, 131 /// The number of threads the started and registered with the thread pool. 132 started_thread_count: u32, 133 134 /// Bitmap of deferred work to do. 135 defer_work: DeferWorks, 136 137 /// Number of transactions to be transmitted before processes in freeze_wait 138 /// are woken up. 139 outstanding_txns: u32, 140 /// Process is frozen and unable to service binder transactions. 141 pub(crate) is_frozen: IsFrozen, 142 /// Process received sync transactions since last frozen. 143 pub(crate) sync_recv: bool, 144 /// Process received async transactions since last frozen. 145 pub(crate) async_recv: bool, 146 pub(crate) binderfs_file: Option<BinderfsProcFile>, 147 /// Check for oneway spam 148 oneway_spam_detection_enabled: bool, 149 } 150 151 impl ProcessInner { 152 fn new() -> Self { 153 Self { 154 is_manager: false, 155 is_dead: false, 156 threads: RBTree::new(), 157 ready_threads: List::new(), 158 mapping: None, 159 nodes: RBTree::new(), 160 work: List::new(), 161 delivered_deaths: List::new(), 162 requested_thread_count: 0, 163 max_threads: 0, 164 started_thread_count: 0, 165 defer_work: DeferWorks::default(), 166 outstanding_txns: 0, 167 is_frozen: IsFrozen::No, 168 sync_recv: false, 169 async_recv: false, 170 binderfs_file: None, 171 oneway_spam_detection_enabled: false, 172 } 173 } 174 175 /// Schedule the work item for execution on this process. 176 /// 177 /// If any threads are ready for work, then the work item is given directly to that thread and 178 /// it is woken up. Otherwise, it is pushed to the process work list. 179 /// 180 /// This call can fail only if the process is dead. In this case, the work item is returned to 181 /// the caller so that the caller can drop it after releasing the inner process lock. This is 182 /// necessary since the destructor of `Transaction` will take locks that can't necessarily be 183 /// taken while holding the inner process lock. 184 pub(crate) fn push_work( 185 &mut self, 186 proc: &Process, 187 work: DLArc<dyn DeliverToRead>, 188 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> { 189 let sync = work.should_sync_wakeup(); 190 191 // Try to find a ready thread to which to push the work. 192 if let Some(thread) = self.ready_threads.pop_front() { 193 // Push to thread while holding state lock. This prevents the thread from giving up 194 // (for example, because of a signal) when we're about to deliver work. 195 match thread.push_work_inner(work, sync) { 196 PushWorkRes::Ok => Ok(()), 197 PushWorkRes::OkNotifyPoll => { 198 proc.notify_poll(sync); 199 Ok(()) 200 } 201 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)), 202 } 203 } else if self.is_dead { 204 Err((BinderError::new_dead(), work)) 205 } else { 206 // Didn't find a thread waiting for proc work; this can happen 207 // in two scenarios: 208 // 1. All threads are busy handling transactions 209 // In that case, one of those threads should call back into 210 // the kernel driver soon and pick up this work. 211 // 2. Threads are using the (e)poll interface, in which case 212 // they may be blocked on the waitqueue without having been 213 // added to waiting_threads. For this case, we wake it up 214 // directly. 215 self.work.push_back(work); 216 217 // Wake up polling threads, if any. 218 proc.notify_poll(sync); 219 220 Ok(()) 221 } 222 } 223 224 pub(crate) fn remove_node(&mut self, ptr: u64) { 225 self.nodes.remove(&ptr); 226 } 227 228 /// Updates the reference count on the given node. 229 pub(crate) fn update_node_refcount( 230 &mut self, 231 node: &DArc<Node>, 232 inc: bool, 233 strong: bool, 234 count: usize, 235 othread: Option<&Thread>, 236 ) { 237 let push = node.update_refcount_locked(inc, strong, count, self); 238 239 // If we decided that we need to push work, push either to the process or to a thread if 240 // one is specified. 241 if let Some(pnode) = push { 242 if let Some(thread) = othread { 243 thread.push_work_deferred(pnode); 244 } else { 245 let _ = self.push_work(&node.owner, pnode); 246 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in 247 // that case, it doesn't care about the notification. 248 } 249 } 250 } 251 252 pub(crate) fn new_node_ref( 253 &mut self, 254 node: DArc<Node>, 255 strong: bool, 256 thread: Option<&Thread>, 257 ) -> NodeRef { 258 self.update_node_refcount(&node, true, strong, 1, thread); 259 let strong_count = if strong { 1 } else { 0 }; 260 NodeRef::new(node, strong_count, 1 - strong_count) 261 } 262 263 pub(crate) fn new_node_ref_with_thread( 264 &mut self, 265 node: DArc<Node>, 266 strong: bool, 267 thread: &Thread, 268 wrapper: Option<CritIncrWrapper>, 269 ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> { 270 let push = match wrapper { 271 None => node 272 .incr_refcount_allow_zero2one(strong, self)? 273 .map(|node| node as DLArc<dyn DeliverToRead>), 274 Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self), 275 }; 276 if let Some(node) = push { 277 thread.push_work_deferred(node); 278 } 279 let strong_count = if strong { 1 } else { 0 }; 280 Ok(NodeRef::new(node, strong_count, 1 - strong_count)) 281 } 282 283 /// Returns an existing node with the given pointer and cookie, if one exists. 284 /// 285 /// Returns an error if a node with the given pointer but a different cookie exists. 286 fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> { 287 match self.nodes.get(&ptr) { 288 None => Ok(None), 289 Some(node) => { 290 let (_, node_cookie) = node.get_id(); 291 if node_cookie == cookie { 292 Ok(Some(node.clone())) 293 } else { 294 Err(EINVAL) 295 } 296 } 297 } 298 } 299 300 fn register_thread(&mut self) -> bool { 301 if self.requested_thread_count == 0 { 302 return false; 303 } 304 305 self.requested_thread_count -= 1; 306 self.started_thread_count += 1; 307 true 308 } 309 310 /// Finds a delivered death notification with the given cookie, removes it from the thread's 311 /// delivered list, and returns it. 312 fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> { 313 let mut cursor = self.delivered_deaths.cursor_front(); 314 while let Some(next) = cursor.peek_next() { 315 if next.cookie == cookie { 316 return Some(next.remove().into_arc()); 317 } 318 cursor.move_next(); 319 } 320 None 321 } 322 323 pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) { 324 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 325 self.delivered_deaths.push_back(death); 326 } else { 327 pr_warn_ratelimited!("Notification added to `delivered_deaths` twice.\n"); 328 } 329 } 330 331 pub(crate) fn add_outstanding_txn(&mut self) { 332 self.outstanding_txns += 1; 333 } 334 335 fn txns_pending_locked(&self) -> bool { 336 if self.outstanding_txns > 0 { 337 return true; 338 } 339 for thread in self.threads.values() { 340 if thread.has_current_transaction() { 341 return true; 342 } 343 } 344 false 345 } 346 } 347 348 /// Used to keep track of a node that this process has a handle to. 349 #[pin_data] 350 pub(crate) struct NodeRefInfo { 351 debug_id: usize, 352 /// The refcount that this process owns to the node. 353 node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>, 354 death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>, 355 /// Cookie of the active freeze listener for this node. 356 freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>, 357 /// Used to store this `NodeRefInfo` in the node's `refs` list. 358 #[pin] 359 links: ListLinks<{ Self::LIST_NODE }>, 360 /// The handle for this `NodeRefInfo`. 361 handle: u32, 362 /// The process that has a handle to the node. 363 pub(crate) process: Arc<Process>, 364 } 365 366 impl NodeRefInfo { 367 /// The id used for the `Node::refs` list. 368 pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10; 369 /// The id used for the `ListArc` in `ProcessNodeRefs`. 370 const LIST_PROC: u64 = 0xd703a5263dcc8650; 371 372 fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> { 373 pin_init!(Self { 374 debug_id: super::next_debug_id(), 375 node_ref: ListArcField::new(node_ref), 376 death: ListArcField::new(None), 377 freeze: ListArcField::new(None), 378 links <- ListLinks::new(), 379 handle, 380 process, 381 }) 382 } 383 384 kernel::list::define_list_arc_field_getter! { 385 pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death } 386 pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze } 387 pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref } 388 pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref } 389 } 390 } 391 392 kernel::list::impl_list_arc_safe! { 393 impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; } 394 impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; } 395 } 396 kernel::list::impl_list_item! { 397 impl ListItem<{Self::LIST_NODE}> for NodeRefInfo { 398 using ListLinks { self.links }; 399 } 400 } 401 402 /// Keeps track of references this process has to nodes owned by other processes. 403 /// 404 /// TODO: Currently, the rbtree requires two allocations per node reference, and two tree 405 /// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these 406 /// extra costs should be eliminated. 407 struct ProcessNodeRefs { 408 /// Used to look up nodes using the 32-bit id that this process knows it by. 409 by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>, 410 /// Used to quickly find unused ids in `by_handle`. 411 handle_is_present: IdPool, 412 /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of 413 /// the underlying `Node` struct as returned by `Node::global_id`. 414 by_node: RBTree<usize, u32>, 415 /// Used to look up a `FreezeListener` by cookie. 416 /// 417 /// There might be multiple freeze listeners for the same node, but at most one of them is 418 /// active. 419 freeze_listeners: RBTree<FreezeCookie, FreezeListener>, 420 } 421 422 impl ProcessNodeRefs { 423 fn new() -> Self { 424 Self { 425 by_handle: RBTree::new(), 426 handle_is_present: IdPool::new(), 427 by_node: RBTree::new(), 428 freeze_listeners: RBTree::new(), 429 } 430 } 431 } 432 433 use core::mem::offset_of; 434 use kernel::bindings::rb_process_layout; 435 pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout { 436 arc_offset: Arc::<Process>::DATA_OFFSET, 437 task: offset_of!(Process, task), 438 }; 439 440 /// A process using binder. 441 /// 442 /// Strictly speaking, there can be multiple of these per process. There is one for each binder fd 443 /// that a process has opened, so processes using several binder contexts have several `Process` 444 /// objects. This ensures that the contexts are fully separated. 445 #[pin_data] 446 pub(crate) struct Process { 447 pub(crate) ctx: Arc<Context>, 448 449 // The task leader (process). 450 pub(crate) task: ARef<Task>, 451 452 // Credential associated with file when `Process` is created. 453 pub(crate) cred: ARef<Credential>, 454 455 #[pin] 456 pub(crate) inner: SpinLock<ProcessInner>, 457 458 #[pin] 459 pub(crate) pages: ShrinkablePageRange, 460 461 // Waitqueue of processes waiting for all outstanding transactions to be 462 // processed. 463 #[pin] 464 freeze_wait: CondVar, 465 466 // Node references are in a different lock to avoid recursive acquisition when 467 // incrementing/decrementing a node in another process. 468 #[pin] 469 node_refs: SpinLock<ProcessNodeRefs>, 470 471 // Synchronizes `register_wait` calls to the `PollCondVarBox`. 472 // 473 // The `PollCondVarBox` is not stored here because synchronization is 474 // done for `register_wait` only. Wakeups do not take this lock. 475 poll: SetOnce<PollCondVarBox>, 476 477 // Work node for deferred work item. 478 #[pin] 479 defer_work: Work<Process>, 480 481 // Links for process list in Context. 482 #[pin] 483 links: ListLinks, 484 485 pub(crate) stats: BinderStats, 486 } 487 488 kernel::impl_has_work! { 489 impl HasWork<Process> for Process { self.defer_work } 490 } 491 492 kernel::list::impl_list_arc_safe! { 493 impl ListArcSafe<0> for Process { untracked; } 494 } 495 kernel::list::impl_list_item! { 496 impl ListItem<0> for Process { 497 using ListLinks { self.links }; 498 } 499 } 500 501 impl workqueue::WorkItem for Process { 502 type Pointer = Arc<Process>; 503 504 fn run(me: Arc<Self>) { 505 let defer; 506 { 507 let mut inner = me.inner.lock(); 508 defer = inner.defer_work; 509 inner.defer_work = DeferWorks::default(); 510 } 511 512 if defer.contains(DeferWork::Flush) { 513 me.deferred_flush(); 514 } 515 if defer.contains(DeferWork::Release) { 516 me.deferred_release(); 517 } 518 } 519 } 520 521 impl Process { 522 fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> { 523 let current = kernel::current!(); 524 let process = Arc::pin_init::<Error>( 525 try_pin_init!(Process { 526 ctx, 527 cred, 528 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"), 529 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER), 530 node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"), 531 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"), 532 task: current.group_leader().into(), 533 defer_work <- kernel::new_work!("Process::defer_work"), 534 links <- ListLinks::new(), 535 stats: BinderStats::new(), 536 poll: SetOnce::new(), 537 }), 538 GFP_KERNEL, 539 )?; 540 541 process.ctx.register_process(process.clone())?; 542 543 Ok(process) 544 } 545 546 pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid { 547 self.task.tgid_nr_ns(None) 548 } 549 550 #[inline(never)] 551 pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result { 552 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 553 seq_print!(m, "context {}\n", &*ctx.name); 554 555 let inner = self.inner.lock(); 556 seq_print!(m, " threads: {}\n", inner.threads.iter().count()); 557 seq_print!( 558 m, 559 " requested threads: {}+{}/{}\n", 560 inner.requested_thread_count, 561 inner.started_thread_count, 562 inner.max_threads, 563 ); 564 if let Some(mapping) = &inner.mapping { 565 seq_print!( 566 m, 567 " free oneway space: {}\n", 568 mapping.alloc.free_oneway_space() 569 ); 570 seq_print!(m, " buffers: {}\n", mapping.alloc.count_buffers()); 571 } 572 seq_print!( 573 m, 574 " outstanding transactions: {}\n", 575 inner.outstanding_txns 576 ); 577 seq_print!(m, " nodes: {}\n", inner.nodes.iter().count()); 578 drop(inner); 579 580 { 581 let mut refs = self.node_refs.lock(); 582 let (mut count, mut weak, mut strong) = (0, 0, 0); 583 for r in refs.by_handle.values_mut() { 584 let node_ref = r.node_ref(); 585 let (nstrong, nweak) = node_ref.get_count(); 586 count += 1; 587 weak += nweak; 588 strong += nstrong; 589 } 590 seq_print!(m, " refs: {count} s {strong} w {weak}\n"); 591 } 592 593 self.stats.debug_print(" ", m); 594 595 Ok(()) 596 } 597 598 #[inline(never)] 599 pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result { 600 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 601 seq_print!(m, "context {}\n", &*ctx.name); 602 603 let mut all_threads = KVec::new(); 604 let mut all_nodes = KVec::new(); 605 loop { 606 let inner = self.inner.lock(); 607 let num_threads = inner.threads.iter().count(); 608 let num_nodes = inner.nodes.iter().count(); 609 610 if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes { 611 drop(inner); 612 all_threads.reserve(num_threads, GFP_KERNEL)?; 613 all_nodes.reserve(num_nodes, GFP_KERNEL)?; 614 continue; 615 } 616 617 for thread in inner.threads.values() { 618 assert!(all_threads.len() < all_threads.capacity()); 619 let _ = all_threads.push(thread.clone(), GFP_ATOMIC); 620 } 621 622 for node in inner.nodes.values() { 623 assert!(all_nodes.len() < all_nodes.capacity()); 624 let _ = all_nodes.push(node.clone(), GFP_ATOMIC); 625 } 626 627 break; 628 } 629 630 for thread in all_threads { 631 thread.debug_print(m, print_all)?; 632 } 633 634 let mut inner = self.inner.lock(); 635 for node in all_nodes { 636 if print_all || node.has_oneway_transaction(&mut inner) { 637 node.full_debug_print(m, &mut inner)?; 638 } 639 } 640 drop(inner); 641 642 if print_all { 643 let mut refs = self.node_refs.lock(); 644 for r in refs.by_handle.values_mut() { 645 let node_ref = r.node_ref(); 646 let dead = node_ref.node.owner.inner.lock().is_dead; 647 let (strong, weak) = node_ref.get_count(); 648 let debug_id = node_ref.node.debug_id; 649 650 seq_print!( 651 m, 652 " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", 653 r.debug_id, 654 r.handle, 655 if dead { "dead " } else { "" } 656 ); 657 } 658 } 659 660 let inner = self.inner.lock(); 661 for work in &inner.work { 662 work.debug_print(m, " ", " pending transaction ")?; 663 } 664 for _death in &inner.delivered_deaths { 665 seq_print!(m, " has delivered dead binder\n"); 666 } 667 if let Some(mapping) = &inner.mapping { 668 mapping.alloc.debug_print(m)?; 669 } 670 drop(inner); 671 672 Ok(()) 673 } 674 675 /// Attempts to fetch a work item from the process queue. 676 pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> { 677 self.inner.lock().work.pop_front() 678 } 679 680 /// Attempts to fetch a work item from the process queue. If none is available, it registers the 681 /// given thread as ready to receive work directly. 682 /// 683 /// This must only be called when the thread is not participating in a transaction chain; when 684 /// it is, work will always be delivered directly to the thread (and not through the process 685 /// queue). 686 pub(crate) fn get_work_or_register<'a>( 687 &'a self, 688 thread: &'a Arc<Thread>, 689 thread_has_deferred_work: bool, 690 ) -> GetWorkOrRegister<'a> { 691 let mut inner = self.inner.lock(); 692 693 if thread_has_deferred_work && !inner.work.is_empty() { 694 if let Some(work) = thread.pop_work_even_if_deferred() { 695 return GetWorkOrRegister::Work(work); 696 } 697 } 698 699 // Try to get work from the process queue. 700 if let Some(work) = inner.work.pop_front() { 701 return GetWorkOrRegister::Work(work); 702 } 703 704 // Register the thread as ready. 705 GetWorkOrRegister::Register(Registration::new(thread, &mut inner)) 706 } 707 708 fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> { 709 let id = { 710 let current = kernel::current!(); 711 if self.task != current.group_leader() { 712 pr_err_ratelimited!("get_current_thread was called from the wrong process.\n"); 713 return Err(EINVAL); 714 } 715 current.pid() 716 }; 717 718 { 719 let inner = self.inner.lock(); 720 if let Some(thread) = inner.threads.get(&id) { 721 return Ok(thread.clone()); 722 } 723 } 724 725 // Allocate a new `Thread` without holding any locks. 726 let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?; 727 let ta: Arc<Thread> = Thread::new(id, self.into())?; 728 729 let mut inner = self.inner.lock(); 730 match inner.threads.entry(id) { 731 rbtree::Entry::Vacant(entry) => { 732 entry.insert(ta.clone(), reservation); 733 Ok(ta) 734 } 735 rbtree::Entry::Occupied(_entry) => { 736 pr_err_ratelimited!("Cannot create two threads with the same id.\n"); 737 Err(EINVAL) 738 } 739 } 740 } 741 742 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { 743 // If push_work fails, drop the work item outside the lock. 744 let res = self.inner.lock().push_work(self, work); 745 match res { 746 Ok(()) => Ok(()), 747 Err((err, work)) => { 748 drop(work); 749 Err(err) 750 } 751 } 752 } 753 754 fn set_as_manager( 755 self: ArcBorrow<'_, Self>, 756 info: Option<FlatBinderObject>, 757 thread: &Thread, 758 ) -> Result { 759 let (ptr, cookie, flags) = if let Some(obj) = info { 760 ( 761 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it 762 // is safe to access the `binder` field. 763 unsafe { obj.__bindgen_anon_1.binder }, 764 obj.cookie, 765 obj.flags, 766 ) 767 } else { 768 (0, 0, 0) 769 }; 770 let node_ref = self.get_node(ptr, cookie, flags, true, thread)?; 771 let node = node_ref.node.clone(); 772 self.ctx.set_manager_node(node_ref)?; 773 self.inner.lock().is_manager = true; 774 775 // Force the state of the node to prevent the delivery of acquire/increfs. 776 let mut owner_inner = node.owner.inner.lock(); 777 node.force_has_count(&mut owner_inner); 778 Ok(()) 779 } 780 781 fn get_node_inner( 782 self: ArcBorrow<'_, Self>, 783 ptr: u64, 784 cookie: u64, 785 flags: u32, 786 strong: bool, 787 thread: &Thread, 788 wrapper: Option<CritIncrWrapper>, 789 ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> { 790 // Try to find an existing node. 791 { 792 let mut inner = self.inner.lock(); 793 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 794 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 795 } 796 } 797 798 // Allocate the node before reacquiring the lock. 799 let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc(); 800 let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?; 801 let mut inner = self.inner.lock(); 802 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 803 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 804 } 805 806 inner.nodes.insert(rbnode); 807 // This can only fail if someone has already pushed the node to a list, but we just created 808 // it and still hold the lock, so it can't fail right now. 809 let node_ref = inner 810 .new_node_ref_with_thread(node, strong, thread, wrapper) 811 .unwrap(); 812 813 Ok(Ok(node_ref)) 814 } 815 816 pub(crate) fn get_node( 817 self: ArcBorrow<'_, Self>, 818 ptr: u64, 819 cookie: u64, 820 flags: u32, 821 strong: bool, 822 thread: &Thread, 823 ) -> Result<NodeRef> { 824 let mut wrapper = None; 825 for _ in 0..2 { 826 match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) { 827 Err(err) => return Err(err), 828 Ok(Ok(node_ref)) => return Ok(node_ref), 829 Ok(Err(CouldNotDeliverCriticalIncrement)) => { 830 wrapper = Some(CritIncrWrapper::new()?); 831 } 832 } 833 } 834 // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the 835 // loop should run at most twice. 836 unreachable!() 837 } 838 839 pub(crate) fn insert_or_update_handle( 840 self: ArcBorrow<'_, Process>, 841 node_ref: NodeRef, 842 is_manager: bool, 843 ) -> Result<u32> { 844 { 845 let mut refs = self.node_refs.lock(); 846 847 // Do a lookup before inserting. 848 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { 849 let handle = *handle_ref; 850 let info = refs.by_handle.get_mut(&handle).unwrap(); 851 info.node_ref().absorb(node_ref); 852 return Ok(handle); 853 } 854 } 855 856 // Reserve memory for tree nodes. 857 let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?; 858 let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?; 859 let info = UniqueArc::new_uninit(GFP_KERNEL)?; 860 861 let mut refs_lock = self.node_refs.lock(); 862 let mut refs = &mut *refs_lock; 863 864 let (unused_id, by_handle_slot) = loop { 865 // ID 0 may only be used by the manager. 866 let start = if is_manager { 0 } else { 1 }; 867 868 if let Some(res) = refs.handle_is_present.find_unused_id(start) { 869 match refs.by_handle.entry(res.as_u32()) { 870 rbtree::Entry::Vacant(entry) => break (res, entry), 871 rbtree::Entry::Occupied(_) => { 872 pr_err_ratelimited!( 873 "Detected mismatch between handle_is_present and by_handle\n" 874 ); 875 res.acquire(); 876 kernel::warn_on!(true); 877 return Err(EINVAL); 878 } 879 } 880 } 881 882 let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?; 883 drop(refs_lock); 884 let resizer = grow_request.realloc(GFP_KERNEL)?; 885 refs_lock = self.node_refs.lock(); 886 refs = &mut *refs_lock; 887 refs.handle_is_present.grow(resizer); 888 }; 889 let handle = unused_id.as_u32(); 890 891 // Do a lookup again as node may have been inserted before the lock was reacquired. 892 let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) { 893 rbtree::Entry::Vacant(by_node_slot) => by_node_slot, 894 rbtree::Entry::Occupied(handle_ref) => { 895 // The node was inserted by another thread while we didn't hold the lock. 896 let handle = handle_ref.get(); 897 let info = refs.by_handle.get_mut(handle).unwrap(); 898 info.node_ref().absorb(node_ref); 899 return Ok(*handle); 900 } 901 }; 902 903 let (info_proc, info_node) = { 904 let info_init = NodeRefInfo::new(node_ref, handle, self.into()); 905 let Ok(info) = info.pin_init_with(info_init); 906 ListArc::pair_from_pin_unique(info) 907 }; 908 909 // Ensure the process is still alive while we insert a new reference. 910 // 911 // This releases the lock before inserting the nodes, but since `is_dead` is set as the 912 // first thing in `deferred_release`, process cleanup will not miss the items inserted into 913 // `refs` below. 914 if self.inner.lock().is_dead { 915 // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of 916 // the lock. 917 drop(refs_lock); 918 return Err(ESRCH); 919 } 920 921 // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting 922 // `info_node` into the right node's `refs` list. 923 unsafe { info_proc.node_ref2().node.insert_node_info(info_node) }; 924 925 by_node_slot.insert(handle, reserve1); 926 by_handle_slot.insert(info_proc, reserve2); 927 unused_id.acquire(); 928 Ok(handle) 929 } 930 931 pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { 932 // When handle is zero, try to get the context manager. 933 if handle == 0 { 934 let node_ref = self.ctx.get_manager_node(true)?; 935 if core::ptr::eq(self, &*node_ref.node.owner) { 936 return Err(EINVAL.into()); 937 } 938 Ok(node_ref) 939 } else { 940 match self.get_node_from_handle(handle, true) { 941 Ok(node_ref) => Ok(node_ref), 942 Err(err) => { 943 binder_debug!(UserError, "got transaction to invalid handle {handle}"); 944 Err(err.into()) 945 } 946 } 947 } 948 } 949 950 pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> { 951 self.node_refs 952 .lock() 953 .by_handle 954 .get_mut(&handle) 955 .ok_or(ENOENT)? 956 .node_ref() 957 .clone(strong) 958 } 959 960 pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) { 961 let mut inner = self.inner.lock(); 962 // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list. 963 let removed = unsafe { inner.delivered_deaths.remove(death) }; 964 drop(inner); 965 drop(removed); 966 } 967 968 pub(crate) fn update_ref( 969 self: ArcBorrow<'_, Process>, 970 handle: u32, 971 inc: bool, 972 strong: bool, 973 ) -> Result { 974 if inc && handle == 0 { 975 if let Ok(node_ref) = self.ctx.get_manager_node(strong) { 976 if core::ptr::eq(&*self, &*node_ref.node.owner) { 977 return Err(EINVAL); 978 } 979 let _ = self.insert_or_update_handle(node_ref, true); 980 return Ok(()); 981 } 982 } 983 984 // To preserve original binder behaviour, we only fail requests where the manager tries to 985 // increment references on itself. 986 let _to_free_by_handle; 987 let _to_free_by_node; 988 let _to_free_freeze_listener; 989 let _to_free_freeze_listener_cleanup; 990 let mut refs = self.node_refs.lock(); 991 if let Some(info) = refs.by_handle.get_mut(&handle) { 992 if info.node_ref().update(inc, strong) { 993 // Clean up death if there is one attached to this node reference. 994 // 995 // We remove the entire `info` below, so no need to remove `death` from `info`. 996 if let Some(death) = info.death().as_ref() { 997 death.set_cleared(true); 998 self.remove_from_delivered_deaths(death); 999 } 1000 1001 // Remove reference from process tables, and from the node's `refs` list. 1002 1003 // SAFETY: We are removing the `NodeRefInfo` from the right node. 1004 unsafe { info.node_ref2().node.remove_node_info(info) }; 1005 1006 let id = info.node_ref().node.global_id(); 1007 1008 if let Some(freeze) = *info.freeze() { 1009 if let Some(fl) = refs.freeze_listeners.remove(&freeze) { 1010 _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self); 1011 _to_free_freeze_listener = fl; 1012 } 1013 } 1014 1015 _to_free_by_handle = refs.by_handle.remove_node(&handle); 1016 _to_free_by_node = refs.by_node.remove_node(&id); 1017 refs.handle_is_present.release_id(handle as usize); 1018 1019 if let Some(shrink) = refs.handle_is_present.shrink_request() { 1020 drop(refs); 1021 // This intentionally ignores allocation failures. 1022 if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) { 1023 refs = self.node_refs.lock(); 1024 refs.handle_is_present.shrink(new_bitmap); 1025 } 1026 } 1027 } 1028 } else { 1029 // All refs are cleared in process exit, so this warning is expected in that case. 1030 if !self.inner.lock().is_dead { 1031 binder_debug!(UserError, "no such ref {handle}"); 1032 } 1033 } 1034 Ok(()) 1035 } 1036 1037 /// Decrements the refcount of the given node, if one exists. 1038 pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) { 1039 let mut inner = self.inner.lock(); 1040 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1041 inner.update_node_refcount(&node, false, strong, 1, None); 1042 } 1043 } 1044 1045 pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result { 1046 let ptr = reader.read::<u64>()?; 1047 let cookie = reader.read::<u64>()?; 1048 let mut inner = self.inner.lock(); 1049 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1050 if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) { 1051 // This only fails if the process is dead. 1052 let _ = inner.push_work(self, node); 1053 } 1054 } 1055 Ok(()) 1056 } 1057 1058 pub(crate) fn buffer_alloc( 1059 self: &Arc<Self>, 1060 debug_id: usize, 1061 size: usize, 1062 info: &mut TransactionInfo, 1063 ) -> BinderResult<NewAllocation> { 1064 use kernel::page::PAGE_SIZE; 1065 1066 let mut reserve_new_args = ReserveNewArgs { 1067 debug_id, 1068 size, 1069 is_oneway: info.is_oneway(), 1070 pid: info.from_pid, 1071 ..ReserveNewArgs::default() 1072 }; 1073 1074 let (new_alloc, addr) = loop { 1075 let mut inner = self.inner.lock(); 1076 let mapping = inner 1077 .mapping 1078 .as_mut() 1079 .ok_or_else(|| BinderError::new_dead())?; 1080 let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? { 1081 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address), 1082 ReserveNew::NeedAlloc(request) => request, 1083 }; 1084 drop(inner); 1085 // We need to allocate memory and then call `reserve_new` again. 1086 reserve_new_args = alloc_request.make_alloc()?; 1087 }; 1088 1089 info.oneway_spam_suspect = new_alloc.oneway_spam_detected; 1090 let res = Allocation::new( 1091 self.clone(), 1092 debug_id, 1093 new_alloc.offset, 1094 size, 1095 addr + new_alloc.offset, 1096 ); 1097 1098 // This allocation will be marked as in use until the `Allocation` is used to free it. 1099 // 1100 // This method can't be called while holding a lock, so we release the lock first. It's 1101 // okay for several threads to use the method on the same index at the same time. In that 1102 // case, one of the calls will allocate the given page (if missing), and the other call 1103 // will wait for the other call to finish allocating the page. 1104 // 1105 // We will not call `stop_using_range` in parallel with this on the same page, because the 1106 // allocation can only be removed via the destructor of the `Allocation` object that we 1107 // currently own. 1108 match self.pages.use_range( 1109 new_alloc.offset / PAGE_SIZE, 1110 (new_alloc.offset + size).div_ceil(PAGE_SIZE), 1111 ) { 1112 Ok(()) => {} 1113 Err(err) => { 1114 pr_warn_ratelimited!("use_range failure {:?}\n", err); 1115 return Err(err.into()); 1116 } 1117 } 1118 1119 Ok(NewAllocation(res)) 1120 } 1121 1122 pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> { 1123 let mut inner = self.inner.lock(); 1124 let mapping = inner.mapping.as_mut()?; 1125 let offset = ptr.checked_sub(mapping.address)?; 1126 let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?; 1127 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1128 if let Some(data) = odata { 1129 alloc.set_info(data); 1130 } 1131 Some(alloc) 1132 } 1133 1134 pub(crate) fn buffer_raw_free(&self, ptr: usize) { 1135 let mut inner = self.inner.lock(); 1136 if let Some(ref mut mapping) = &mut inner.mapping { 1137 let offset = match ptr.checked_sub(mapping.address) { 1138 Some(offset) => offset, 1139 None => return, 1140 }; 1141 1142 let freed_range = match mapping.alloc.reservation_abort(offset) { 1143 Ok(freed_range) => freed_range, 1144 Err(_) => { 1145 pr_warn_ratelimited!( 1146 "Pointer {:x} failed to free, base = {:x}\n", 1147 ptr, 1148 mapping.address 1149 ); 1150 return; 1151 } 1152 }; 1153 1154 // No more allocations in this range. Mark them as not in use. 1155 // 1156 // Must be done before we release the lock so that `use_range` is not used on these 1157 // indices until `stop_using_range` returns. 1158 self.pages 1159 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx); 1160 } 1161 } 1162 1163 pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) { 1164 let mut inner = self.inner.lock(); 1165 if let Some(ref mut mapping) = &mut inner.mapping { 1166 if mapping.alloc.reservation_commit(offset, &mut data).is_err() { 1167 pr_warn_ratelimited!("Offset {} failed to be marked freeable\n", offset); 1168 } 1169 } 1170 } 1171 1172 fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result { 1173 use kernel::page::PAGE_SIZE; 1174 let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); 1175 let mapping = Mapping::new(vma.start(), size); 1176 let page_count = self.pages.register_with_vma(vma)?; 1177 if page_count * PAGE_SIZE != size { 1178 return Err(EINVAL); 1179 } 1180 1181 // Save range allocator for later. 1182 self.inner.lock().mapping = Some(mapping); 1183 1184 Ok(()) 1185 } 1186 1187 fn version(&self, data: UserSlice) -> Result { 1188 data.writer().write(&BinderVersion::current()) 1189 } 1190 1191 pub(crate) fn register_thread(&self) -> bool { 1192 self.inner.lock().register_thread() 1193 } 1194 1195 fn remove_thread(&self, thread: Arc<Thread>) { 1196 self.inner.lock().threads.remove(&thread.id); 1197 thread.release(); 1198 } 1199 1200 fn set_max_threads(&self, max: u32) { 1201 self.inner.lock().max_threads = max; 1202 } 1203 1204 fn set_oneway_spam_detection_enabled(&self, enabled: u32) { 1205 self.inner.lock().oneway_spam_detection_enabled = enabled != 0; 1206 } 1207 1208 pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool { 1209 self.inner.lock().oneway_spam_detection_enabled 1210 } 1211 1212 fn get_node_debug_info(&self, data: UserSlice) -> Result { 1213 let (mut reader, mut writer) = data.reader_writer(); 1214 1215 // Read the starting point. 1216 let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr; 1217 let mut out = BinderNodeDebugInfo::default(); 1218 1219 if let Some(start_ptr) = ptr.checked_add(1) { 1220 let inner = self.inner.lock(); 1221 // cursor_lower_bound retrieves the "key" passed or the next existing larger key 1222 if let Some(cursor) = inner.nodes.cursor_lower_bound(&start_ptr) { 1223 let (_, node) = cursor.current(); 1224 node.populate_debug_info(&mut out, &inner); 1225 } 1226 } 1227 1228 writer.write(&out) 1229 } 1230 1231 fn get_node_info_from_ref(&self, data: UserSlice) -> Result { 1232 let (mut reader, mut writer) = data.reader_writer(); 1233 let mut out = reader.read::<BinderNodeInfoForRef>()?; 1234 1235 if out.strong_count != 0 1236 || out.weak_count != 0 1237 || out.reserved1 != 0 1238 || out.reserved2 != 0 1239 || out.reserved3 != 0 1240 { 1241 return Err(EINVAL); 1242 } 1243 1244 // Only the context manager is allowed to use this ioctl. 1245 if !self.inner.lock().is_manager { 1246 return Err(EPERM); 1247 } 1248 1249 { 1250 let mut node_refs = self.node_refs.lock(); 1251 let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?; 1252 let node_ref = node_info.node_ref(); 1253 let owner_inner = node_ref.node.owner.inner.lock(); 1254 node_ref.node.populate_counts(&mut out, &owner_inner); 1255 } 1256 1257 // Write the result back. 1258 writer.write(&out) 1259 } 1260 1261 pub(crate) fn needs_thread(&self) -> bool { 1262 let mut inner = self.inner.lock(); 1263 let ret = inner.requested_thread_count == 0 1264 && inner.ready_threads.is_empty() 1265 && inner.started_thread_count < inner.max_threads; 1266 if ret { 1267 inner.requested_thread_count += 1 1268 } 1269 ret 1270 } 1271 1272 pub(crate) fn request_death( 1273 self: &Arc<Self>, 1274 reader: &mut UserSliceReader, 1275 thread: &Thread, 1276 ) -> Result { 1277 let handle: u32 = reader.read()?; 1278 let cookie: u64 = reader.read()?; 1279 1280 // Queue BR_ERROR if we can't allocate memory for the death notification. 1281 let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| { 1282 thread.push_return_work(BR_ERROR); 1283 binder_debug!( 1284 DeathNotification, 1285 "BC_REQUEST_DEATH_NOTIFICATION failed due to memory allocation failure" 1286 ); 1287 })?; 1288 let mut refs = self.node_refs.lock(); 1289 let Some(info) = refs.by_handle.get_mut(&handle) else { 1290 binder_debug!( 1291 UserError, 1292 "BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}" 1293 ); 1294 return Ok(()); 1295 }; 1296 1297 // Nothing to do if there is already a death notification request for this handle. 1298 if info.death().is_some() { 1299 binder_debug!( 1300 UserError, 1301 "BC_REQUEST_DEATH_NOTIFICATION death notification already set" 1302 ); 1303 return Ok(()); 1304 } 1305 1306 let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie); 1307 let Ok(death) = death.pin_init_with(death_init); 1308 1309 // Register the death notification. 1310 { 1311 let owner = info.node_ref2().node.owner.clone(); 1312 let mut owner_inner = owner.inner.lock(); 1313 if owner_inner.is_dead { 1314 let death = Arc::from(death); 1315 *info.death() = Some(death.clone()); 1316 drop(owner_inner); 1317 death.set_dead(); 1318 } else { 1319 let death = ListArc::from(death); 1320 *info.death() = Some(death.clone_arc()); 1321 info.node_ref().node.add_death(death, &mut owner_inner); 1322 } 1323 } 1324 binder_debug!( 1325 DeathNotification, 1326 "BC_REQUEST_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1327 cookie 1328 ); 1329 Ok(()) 1330 } 1331 1332 pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result { 1333 let handle: u32 = reader.read()?; 1334 let cookie: u64 = reader.read()?; 1335 1336 let mut refs = self.node_refs.lock(); 1337 let Some(info) = refs.by_handle.get_mut(&handle) else { 1338 binder_debug!( 1339 UserError, 1340 "BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}" 1341 ); 1342 return Ok(()); 1343 }; 1344 1345 let Some(death) = info.death().take() else { 1346 binder_debug!( 1347 UserError, 1348 "BC_CLEAR_DEATH_NOTIFICATION death notification not active" 1349 ); 1350 return Ok(()); 1351 }; 1352 if death.cookie != cookie { 1353 *info.death() = Some(death); 1354 binder_debug!( 1355 UserError, 1356 "BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch" 1357 ); 1358 return Ok(()); 1359 } 1360 1361 // Update state and determine if we need to queue a work item. We only need to do it when 1362 // the node is not dead or if the user already completed the death notification. 1363 let should_schedule = death.set_cleared(false); 1364 drop(refs); 1365 1366 if should_schedule { 1367 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 1368 let _ = thread.push_work_if_looper(death); 1369 } 1370 } 1371 1372 binder_debug!( 1373 DeathNotification, 1374 "BC_CLEAR_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1375 cookie 1376 ); 1377 Ok(()) 1378 } 1379 1380 pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { 1381 let death = self.inner.lock().pull_delivered_death(cookie); 1382 if let Some(death) = death { 1383 death.set_notification_done(thread); 1384 } 1385 } 1386 1387 /// Locks the spinlock and move the `nodes` rbtree out. 1388 /// 1389 /// This allows you to iterate through `nodes` while also allowing you to give other parts of 1390 /// the codebase exclusive access to `ProcessInner`. 1391 pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> { 1392 let mut inner = self.inner.lock(); 1393 WithNodes { 1394 nodes: take(&mut inner.nodes), 1395 inner, 1396 } 1397 } 1398 1399 fn deferred_flush(&self) { 1400 binder_debug!(pid = self.task.pid(), OpenClose, "flushing process"); 1401 let inner = self.inner.lock(); 1402 for thread in inner.threads.values() { 1403 thread.exit_looper(); 1404 } 1405 } 1406 1407 fn deferred_release(self: Arc<Self>) { 1408 binder_debug!(pid = self.task.pid(), OpenClose, "releasing process"); 1409 1410 let is_manager = { 1411 let mut inner = self.inner.lock(); 1412 inner.is_dead = true; 1413 inner.is_frozen = IsFrozen::No; 1414 inner.sync_recv = false; 1415 inner.async_recv = false; 1416 inner.is_manager 1417 }; 1418 1419 if is_manager { 1420 self.ctx.unset_manager_node(); 1421 } 1422 1423 self.ctx.deregister_process(&self); 1424 1425 let binderfs_file = self.inner.lock().binderfs_file.take(); 1426 drop(binderfs_file); 1427 1428 // Release threads. 1429 let threads = { 1430 let mut inner = self.inner.lock(); 1431 let threads = take(&mut inner.threads); 1432 let ready = take(&mut inner.ready_threads); 1433 drop(inner); 1434 drop(ready); 1435 1436 for thread in threads.values() { 1437 thread.release(); 1438 } 1439 threads 1440 }; 1441 1442 // Release nodes. 1443 { 1444 while let Some(node) = { 1445 let mut lock = self.inner.lock(); 1446 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) 1447 } { 1448 node.to_key_value().1.release(); 1449 } 1450 } 1451 1452 // Clean up death listeners and remove nodes from external node info lists. 1453 for info in self.node_refs.lock().by_handle.values_mut() { 1454 // SAFETY: We are removing the `NodeRefInfo` from the right node. 1455 unsafe { info.node_ref2().node.remove_node_info(info) }; 1456 1457 // Clear death notifications from the nodes (that belong to a different process). 1458 // No need to remove them from `info` as we clear info below. 1459 if let Some(death) = info.death().as_ref() { 1460 death.set_cleared(false); 1461 } 1462 } 1463 1464 // Clean up freeze listeners. 1465 let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners); 1466 for listener in freeze_listeners.values() { 1467 listener.on_process_cleanup(&self); 1468 } 1469 drop(freeze_listeners); 1470 1471 // Release refs on foreign nodes. 1472 { 1473 let mut refs = self.node_refs.lock(); 1474 let by_handle = take(&mut refs.by_handle); 1475 let by_node = take(&mut refs.by_node); 1476 drop(refs); 1477 drop(by_node); 1478 drop(by_handle); 1479 } 1480 1481 // Cancel all pending work items. 1482 while let Some(work) = self.get_work() { 1483 work.into_arc().cancel(); 1484 } 1485 1486 // Clear delivered_deaths list. 1487 // 1488 // Scope ensures that MutexGuard is dropped while executing the body. 1489 while let Some(delivered_death) = { 1490 // Explicitly bind to avoid tail expression lifetime extension of the lockguard 1491 // Can be removed when the kernel moves to edition 2024 1492 let maybe_death = self.inner.lock().delivered_deaths.pop_front(); 1493 maybe_death 1494 } { 1495 drop(delivered_death); 1496 } 1497 1498 // Free any resources kept alive by allocated buffers. 1499 let omapping = self.inner.lock().mapping.take(); 1500 if let Some(mut mapping) = omapping { 1501 let address = mapping.address; 1502 mapping 1503 .alloc 1504 .take_for_each(|offset, size, debug_id, odata| { 1505 let ptr = offset + address; 1506 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1507 if let Some(data) = odata { 1508 alloc.set_info(data); 1509 } 1510 drop(alloc) 1511 }); 1512 } 1513 1514 // calls to synchronize_rcu() in thread drop will happen here 1515 drop(threads); 1516 } 1517 1518 pub(crate) fn drop_outstanding_txn(&self) { 1519 let wake = { 1520 let mut inner = self.inner.lock(); 1521 if inner.outstanding_txns == 0 { 1522 pr_err_ratelimited!("outstanding_txns underflow\n"); 1523 return; 1524 } 1525 inner.outstanding_txns -= 1; 1526 inner.is_frozen.is_frozen() && inner.outstanding_txns == 0 1527 }; 1528 1529 if wake { 1530 self.freeze_wait.notify_all(); 1531 } 1532 } 1533 1534 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 1535 // mangled symbol names. 1536 #[export_name = "rust_binder_freeze"] 1537 pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result { 1538 if info.enable == 0 { 1539 let msgs = self.prepare_freeze_messages()?; 1540 let mut inner = self.inner.lock(); 1541 inner.sync_recv = false; 1542 inner.async_recv = false; 1543 inner.is_frozen = IsFrozen::No; 1544 drop(inner); 1545 msgs.send_messages(); 1546 return Ok(()); 1547 } 1548 1549 let mut inner = self.inner.lock(); 1550 inner.sync_recv = false; 1551 inner.async_recv = false; 1552 inner.is_frozen = IsFrozen::InProgress; 1553 1554 if info.timeout_ms > 0 { 1555 let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms); 1556 while jiffies > 0 { 1557 if inner.outstanding_txns == 0 { 1558 break; 1559 } 1560 1561 match self 1562 .freeze_wait 1563 .wait_interruptible_timeout(&mut inner, jiffies) 1564 { 1565 CondVarTimeoutResult::Signal { .. } => { 1566 inner.is_frozen = IsFrozen::No; 1567 return Err(ERESTARTSYS); 1568 } 1569 CondVarTimeoutResult::Woken { jiffies: remaining } => { 1570 jiffies = remaining; 1571 } 1572 CondVarTimeoutResult::Timeout => { 1573 jiffies = 0; 1574 } 1575 } 1576 } 1577 } 1578 1579 if inner.txns_pending_locked() { 1580 inner.is_frozen = IsFrozen::No; 1581 Err(EAGAIN) 1582 } else { 1583 drop(inner); 1584 match self.prepare_freeze_messages() { 1585 Ok(batch) => { 1586 self.inner.lock().is_frozen = IsFrozen::Yes; 1587 batch.send_messages(); 1588 Ok(()) 1589 } 1590 Err(kernel::alloc::AllocError) => { 1591 self.inner.lock().is_frozen = IsFrozen::No; 1592 Err(ENOMEM) 1593 } 1594 } 1595 } 1596 } 1597 1598 pub(crate) fn notify_poll(&self, sync: bool) { 1599 if let Some(poll) = self.poll.as_ref() { 1600 if sync { 1601 poll.notify_sync(); 1602 } 1603 poll.notify_all(); 1604 } 1605 } 1606 } 1607 1608 fn get_frozen_status(data: UserSlice) -> Result { 1609 let (mut reader, mut writer) = data.reader_writer(); 1610 1611 let mut info = reader.read::<BinderFrozenStatusInfo>()?; 1612 info.sync_recv = 0; 1613 info.async_recv = 0; 1614 let mut found = false; 1615 1616 for ctx in crate::context::get_all_contexts()? { 1617 ctx.for_each_proc(|proc| { 1618 if proc.task.pid() == info.pid as Pid { 1619 found = true; 1620 let inner = proc.inner.lock(); 1621 let txns_pending = inner.txns_pending_locked(); 1622 info.async_recv |= u32::from(inner.async_recv); 1623 info.sync_recv |= u32::from(inner.sync_recv); 1624 info.sync_recv |= u32::from(txns_pending) << 1; 1625 } 1626 }); 1627 } 1628 1629 if found { 1630 writer.write(&info)?; 1631 Ok(()) 1632 } else { 1633 Err(EINVAL) 1634 } 1635 } 1636 1637 fn ioctl_freeze(reader: &mut UserSliceReader) -> Result { 1638 let info = reader.read::<BinderFreezeInfo>()?; 1639 1640 // Very unlikely for there to be more than 3, since a process normally uses at most binder and 1641 // hwbinder. 1642 let mut procs = KVec::with_capacity(3, GFP_KERNEL)?; 1643 1644 let ctxs = crate::context::get_all_contexts()?; 1645 for ctx in ctxs { 1646 for proc in ctx.get_procs_with_pid(info.pid as i32)? { 1647 procs.push(proc, GFP_KERNEL)?; 1648 } 1649 } 1650 1651 for proc in procs { 1652 proc.ioctl_freeze(&info)?; 1653 } 1654 Ok(()) 1655 } 1656 1657 /// The ioctl handler. 1658 impl Process { 1659 /// Ioctls that are write-only from the perspective of userspace. 1660 /// 1661 /// The kernel will only read from the pointer that userspace provided to us. 1662 fn ioctl_write_only( 1663 this: ArcBorrow<'_, Process>, 1664 _file: &File, 1665 cmd: u32, 1666 reader: &mut UserSliceReader, 1667 ) -> Result { 1668 if cmd == uapi::BINDER_FREEZE { 1669 return ioctl_freeze(reader); 1670 } 1671 1672 let thread = this.get_current_thread()?; 1673 match cmd { 1674 uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), 1675 uapi::BINDER_THREAD_EXIT => this.remove_thread(thread), 1676 uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?, 1677 uapi::BINDER_SET_CONTEXT_MGR_EXT => { 1678 this.set_as_manager(Some(reader.read()?), &thread)? 1679 } 1680 uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => { 1681 this.set_oneway_spam_detection_enabled(reader.read()?) 1682 } 1683 _ => return Err(EINVAL), 1684 } 1685 Ok(()) 1686 } 1687 1688 /// Ioctls that are read/write from the perspective of userspace. 1689 /// 1690 /// The kernel will both read from and write to the pointer that userspace provided to us. 1691 fn ioctl_write_read( 1692 this: ArcBorrow<'_, Process>, 1693 file: &File, 1694 cmd: u32, 1695 data: UserSlice, 1696 ) -> Result { 1697 let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0; 1698 match cmd { 1699 uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?, 1700 uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?, 1701 uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?, 1702 uapi::BINDER_VERSION => this.version(data)?, 1703 uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?, 1704 uapi::BINDER_GET_EXTENDED_ERROR => { 1705 this.get_current_thread()?.get_extended_error(data)? 1706 } 1707 _ => return Err(EINVAL), 1708 } 1709 Ok(()) 1710 } 1711 } 1712 1713 /// The file operations supported by `Process`. 1714 impl Process { 1715 pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> { 1716 let proc = Self::new(ctx.into(), ARef::from(file.cred()))?; 1717 binder_debug!(OpenClose, "opened process"); 1718 Ok(proc) 1719 } 1720 1721 pub(crate) fn release(this: Arc<Process>, _file: &File) { 1722 let binderfs_file; 1723 let should_schedule; 1724 { 1725 let mut inner = this.inner.lock(); 1726 should_schedule = inner.defer_work == DeferWorks::empty(); 1727 inner.defer_work |= DeferWork::Release; 1728 binderfs_file = inner.binderfs_file.take(); 1729 } 1730 1731 if should_schedule { 1732 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1733 // scheduled for execution. 1734 let _ = workqueue::system().enqueue(this); 1735 } 1736 1737 drop(binderfs_file); 1738 } 1739 1740 pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result { 1741 let should_schedule; 1742 { 1743 let mut inner = this.inner.lock(); 1744 should_schedule = inner.defer_work == DeferWorks::empty(); 1745 inner.defer_work |= DeferWork::Flush; 1746 } 1747 1748 if should_schedule { 1749 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1750 // scheduled for execution. 1751 let _ = workqueue::system().enqueue(Arc::from(this)); 1752 } 1753 Ok(()) 1754 } 1755 1756 pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result { 1757 use kernel::ioctl::{_IOC_DIR, _IOC_SIZE}; 1758 use kernel::uapi::{_IOC_READ, _IOC_WRITE}; 1759 1760 crate::trace::trace_ioctl(cmd, arg); 1761 1762 let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd)); 1763 1764 const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE; 1765 1766 let res = match _IOC_DIR(cmd) { 1767 _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()), 1768 _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice), 1769 _ => Err(EINVAL), 1770 }; 1771 1772 crate::trace::trace_ioctl_done(res); 1773 res 1774 } 1775 1776 pub(crate) fn mmap( 1777 this: ArcBorrow<'_, Process>, 1778 _file: &File, 1779 vma: &mm::virt::VmaNew, 1780 ) -> Result { 1781 // We don't allow mmap to be used in a different process. 1782 if this.task != kernel::current!().group_leader() { 1783 return Err(EINVAL); 1784 } 1785 if vma.start() == 0 { 1786 return Err(EINVAL); 1787 } 1788 1789 vma.try_clear_maywrite().map_err(|_| EPERM)?; 1790 vma.set_dontcopy(); 1791 vma.set_mixedmap(); 1792 1793 // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it. 1794 this.create_mapping(vma) 1795 } 1796 1797 pub(crate) fn poll( 1798 this: ArcBorrow<'_, Process>, 1799 file: &File, 1800 table: PollTable<'_>, 1801 ) -> Result<u32> { 1802 let thread = this.get_current_thread()?; 1803 { 1804 let poll = loop { 1805 if let Some(poll) = this.poll.as_ref() { 1806 break poll; 1807 } 1808 1809 let poll = PollCondVarBox::new(c"Process::poll", kernel::static_lock_class!())?; 1810 // Reuse our existing lock to synchronize callers initializing. 1811 let _guard = this.node_refs.lock(); 1812 this.poll.populate(poll); 1813 }; 1814 1815 table.register_wait(file, poll); 1816 } 1817 let (from_proc, mut mask) = thread.poll()?; 1818 if mask == 0 && from_proc && !this.inner.lock().work.is_empty() { 1819 mask |= bindings::POLLIN; 1820 } 1821 Ok(mask) 1822 } 1823 } 1824 1825 /// Represents that a thread has registered with the `ready_threads` list of its process. 1826 /// 1827 /// The destructor of this type will unregister the thread from the list of ready threads. 1828 pub(crate) struct Registration<'a> { 1829 thread: &'a Arc<Thread>, 1830 } 1831 1832 impl<'a> Registration<'a> { 1833 fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self { 1834 assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref())); 1835 // INVARIANT: We are pushing this thread to the right `ready_threads` list. 1836 if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) { 1837 guard.ready_threads.push_front(list_arc); 1838 } else { 1839 // It is an error to hit this branch, and it should not be reachable. We try to do 1840 // something reasonable when the failure path happens. Most likely, the thread in 1841 // question will sleep forever. 1842 pr_err_ratelimited!("Same thread registered with `ready_threads` twice.\n"); 1843 } 1844 Self { thread } 1845 } 1846 } 1847 1848 impl Drop for Registration<'_> { 1849 fn drop(&mut self) { 1850 let mut inner = self.thread.process.inner.lock(); 1851 // SAFETY: The thread has the invariant that we never push it to any other linked list than 1852 // the `ready_threads` list of its parent process. Therefore, the thread is either in that 1853 // list, or in no list. 1854 unsafe { inner.ready_threads.remove(self.thread) }; 1855 } 1856 } 1857 1858 pub(crate) struct WithNodes<'a> { 1859 pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>, 1860 pub(crate) nodes: RBTree<u64, DArc<Node>>, 1861 } 1862 1863 impl Drop for WithNodes<'_> { 1864 fn drop(&mut self) { 1865 core::mem::swap(&mut self.nodes, &mut self.inner.nodes); 1866 if self.nodes.iter().next().is_some() { 1867 pr_err_ratelimited!("nodes array was modified while using lock_with_nodes\n"); 1868 } 1869 } 1870 } 1871 1872 pub(crate) enum GetWorkOrRegister<'a> { 1873 Work(DLArc<dyn DeliverToRead>), 1874 Register(Registration<'a>), 1875 } 1876