1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 //! This module defines the `Process` type, which represents a process using a particular binder 6 //! context. 7 //! 8 //! The `Process` object keeps track of all of the resources that this process owns in the binder 9 //! context. 10 //! 11 //! There is one `Process` object for each binder fd that a process has opened, so processes using 12 //! several binder contexts have several `Process` objects. This ensures that the contexts are 13 //! fully separated. 14 15 use core::mem::take; 16 17 use kernel::{ 18 bindings, 19 bits::bit_u8, 20 cred::Credential, 21 error::Error, 22 fs::file::{self, File}, 23 id_pool::IdPool, 24 list::{List, ListArc, ListArcField, ListLinks}, 25 mm, 26 prelude::*, 27 rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation}, 28 seq_file::SeqFile, 29 seq_print, 30 sync::poll::PollTable, 31 sync::{ 32 aref::ARef, 33 lock::{spinlock::SpinLockBackend, Guard}, 34 poll::PollCondVarBox, 35 Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc, 36 }, 37 task::{Pid, Task}, 38 uaccess::{UserSlice, UserSliceReader}, 39 uapi, 40 workqueue::{self, Work}, 41 }; 42 43 use crate::{ 44 allocation::{Allocation, AllocationInfo, NewAllocation}, 45 context::Context, 46 defs::*, 47 error::{BinderError, BinderResult}, 48 node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef}, 49 page_range::ShrinkablePageRange, 50 range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs}, 51 stats::BinderStats, 52 thread::{PushWorkRes, Thread}, 53 transaction::TransactionInfo, 54 BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead, 55 }; 56 57 #[path = "freeze.rs"] 58 mod freeze; 59 use self::freeze::{FreezeCookie, FreezeListener}; 60 61 struct Mapping { 62 address: usize, 63 alloc: RangeAllocator<AllocationInfo>, 64 } 65 66 impl Mapping { 67 fn new(address: usize, size: usize) -> Self { 68 Self { 69 address, 70 alloc: RangeAllocator::new(size), 71 } 72 } 73 } 74 75 kernel::impl_flags!( 76 /// Represents multiple deferred work flags. 77 #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] 78 pub struct DeferWorks(u8); 79 80 /// Represents a single deferred work category. 81 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 82 pub enum DeferWork { 83 Flush = bit_u8(0), 84 Release = bit_u8(1), 85 } 86 ); 87 88 #[derive(Copy, Clone)] 89 pub(crate) enum IsFrozen { 90 Yes, 91 No, 92 InProgress, 93 } 94 95 impl IsFrozen { 96 /// Whether incoming transactions should be rejected due to freeze. 97 pub(crate) fn is_frozen(self) -> bool { 98 match self { 99 IsFrozen::Yes => true, 100 IsFrozen::No => false, 101 IsFrozen::InProgress => true, 102 } 103 } 104 105 /// Whether freeze notifications consider this process frozen. 106 pub(crate) fn is_fully_frozen(self) -> bool { 107 match self { 108 IsFrozen::Yes => true, 109 IsFrozen::No => false, 110 IsFrozen::InProgress => false, 111 } 112 } 113 } 114 115 /// The fields of `Process` protected by the spinlock. 116 pub(crate) struct ProcessInner { 117 is_manager: bool, 118 pub(crate) is_dead: bool, 119 threads: RBTree<i32, Arc<Thread>>, 120 /// INVARIANT: Threads pushed to this list must be owned by this process. 121 ready_threads: List<Thread>, 122 nodes: RBTree<u64, DArc<Node>>, 123 mapping: Option<Mapping>, 124 work: List<DTRWrap<dyn DeliverToRead>>, 125 delivered_deaths: List<DTRWrap<NodeDeath>, 2>, 126 127 /// The number of requested threads that haven't registered yet. 128 requested_thread_count: u32, 129 /// The maximum number of threads used by the process thread pool. 130 max_threads: u32, 131 /// The number of threads the started and registered with the thread pool. 132 started_thread_count: u32, 133 134 /// Bitmap of deferred work to do. 135 defer_work: DeferWorks, 136 137 /// Number of transactions to be transmitted before processes in freeze_wait 138 /// are woken up. 139 outstanding_txns: u32, 140 /// Process is frozen and unable to service binder transactions. 141 pub(crate) is_frozen: IsFrozen, 142 /// Process received sync transactions since last frozen. 143 pub(crate) sync_recv: bool, 144 /// Process received async transactions since last frozen. 145 pub(crate) async_recv: bool, 146 pub(crate) binderfs_file: Option<BinderfsProcFile>, 147 /// Check for oneway spam 148 oneway_spam_detection_enabled: bool, 149 } 150 151 impl ProcessInner { 152 fn new() -> Self { 153 Self { 154 is_manager: false, 155 is_dead: false, 156 threads: RBTree::new(), 157 ready_threads: List::new(), 158 mapping: None, 159 nodes: RBTree::new(), 160 work: List::new(), 161 delivered_deaths: List::new(), 162 requested_thread_count: 0, 163 max_threads: 0, 164 started_thread_count: 0, 165 defer_work: DeferWorks::default(), 166 outstanding_txns: 0, 167 is_frozen: IsFrozen::No, 168 sync_recv: false, 169 async_recv: false, 170 binderfs_file: None, 171 oneway_spam_detection_enabled: false, 172 } 173 } 174 175 /// Schedule the work item for execution on this process. 176 /// 177 /// If any threads are ready for work, then the work item is given directly to that thread and 178 /// it is woken up. Otherwise, it is pushed to the process work list. 179 /// 180 /// This call can fail only if the process is dead. In this case, the work item is returned to 181 /// the caller so that the caller can drop it after releasing the inner process lock. This is 182 /// necessary since the destructor of `Transaction` will take locks that can't necessarily be 183 /// taken while holding the inner process lock. 184 pub(crate) fn push_work( 185 &mut self, 186 proc: &Process, 187 work: DLArc<dyn DeliverToRead>, 188 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> { 189 let sync = work.should_sync_wakeup(); 190 191 // Try to find a ready thread to which to push the work. 192 if let Some(thread) = self.ready_threads.pop_front() { 193 // Push to thread while holding state lock. This prevents the thread from giving up 194 // (for example, because of a signal) when we're about to deliver work. 195 match thread.push_work_inner(work, sync) { 196 PushWorkRes::Ok => Ok(()), 197 PushWorkRes::OkNotifyPoll => { 198 proc.notify_poll(sync); 199 Ok(()) 200 } 201 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)), 202 } 203 } else if self.is_dead { 204 Err((BinderError::new_dead(), work)) 205 } else { 206 // Didn't find a thread waiting for proc work; this can happen 207 // in two scenarios: 208 // 1. All threads are busy handling transactions 209 // In that case, one of those threads should call back into 210 // the kernel driver soon and pick up this work. 211 // 2. Threads are using the (e)poll interface, in which case 212 // they may be blocked on the waitqueue without having been 213 // added to waiting_threads. For this case, we wake it up 214 // directly. 215 self.work.push_back(work); 216 217 // Wake up polling threads, if any. 218 proc.notify_poll(sync); 219 220 Ok(()) 221 } 222 } 223 224 pub(crate) fn remove_node(&mut self, ptr: u64) { 225 self.nodes.remove(&ptr); 226 } 227 228 /// Updates the reference count on the given node. 229 pub(crate) fn update_node_refcount( 230 &mut self, 231 node: &DArc<Node>, 232 inc: bool, 233 strong: bool, 234 count: usize, 235 othread: Option<&Thread>, 236 ) { 237 let push = node.update_refcount_locked(inc, strong, count, self); 238 239 // If we decided that we need to push work, push either to the process or to a thread if 240 // one is specified. 241 if let Some(pnode) = push { 242 if let Some(thread) = othread { 243 thread.push_work_deferred(pnode); 244 } else { 245 let _ = self.push_work(&node.owner, pnode); 246 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in 247 // that case, it doesn't care about the notification. 248 } 249 } 250 } 251 252 pub(crate) fn new_node_ref( 253 &mut self, 254 node: DArc<Node>, 255 strong: bool, 256 thread: Option<&Thread>, 257 ) -> NodeRef { 258 self.update_node_refcount(&node, true, strong, 1, thread); 259 let strong_count = if strong { 1 } else { 0 }; 260 NodeRef::new(node, strong_count, 1 - strong_count) 261 } 262 263 pub(crate) fn new_node_ref_with_thread( 264 &mut self, 265 node: DArc<Node>, 266 strong: bool, 267 thread: &Thread, 268 wrapper: Option<CritIncrWrapper>, 269 ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> { 270 let push = match wrapper { 271 None => node 272 .incr_refcount_allow_zero2one(strong, self)? 273 .map(|node| node as DLArc<dyn DeliverToRead>), 274 Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self), 275 }; 276 if let Some(node) = push { 277 thread.push_work_deferred(node); 278 } 279 let strong_count = if strong { 1 } else { 0 }; 280 Ok(NodeRef::new(node, strong_count, 1 - strong_count)) 281 } 282 283 /// Returns an existing node with the given pointer and cookie, if one exists. 284 /// 285 /// Returns an error if a node with the given pointer but a different cookie exists. 286 fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> { 287 match self.nodes.get(&ptr) { 288 None => Ok(None), 289 Some(node) => { 290 let (_, node_cookie) = node.get_id(); 291 if node_cookie == cookie { 292 Ok(Some(node.clone())) 293 } else { 294 Err(EINVAL) 295 } 296 } 297 } 298 } 299 300 fn register_thread(&mut self) -> bool { 301 if self.requested_thread_count == 0 { 302 return false; 303 } 304 305 self.requested_thread_count -= 1; 306 self.started_thread_count += 1; 307 true 308 } 309 310 /// Finds a delivered death notification with the given cookie, removes it from the thread's 311 /// delivered list, and returns it. 312 fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> { 313 let mut cursor = self.delivered_deaths.cursor_front(); 314 while let Some(next) = cursor.peek_next() { 315 if next.cookie == cookie { 316 return Some(next.remove().into_arc()); 317 } 318 cursor.move_next(); 319 } 320 None 321 } 322 323 pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) { 324 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 325 self.delivered_deaths.push_back(death); 326 } else { 327 pr_warn!("Notification added to `delivered_deaths` twice."); 328 } 329 } 330 331 pub(crate) fn add_outstanding_txn(&mut self) { 332 self.outstanding_txns += 1; 333 } 334 335 fn txns_pending_locked(&self) -> bool { 336 if self.outstanding_txns > 0 { 337 return true; 338 } 339 for thread in self.threads.values() { 340 if thread.has_current_transaction() { 341 return true; 342 } 343 } 344 false 345 } 346 } 347 348 /// Used to keep track of a node that this process has a handle to. 349 #[pin_data] 350 pub(crate) struct NodeRefInfo { 351 debug_id: usize, 352 /// The refcount that this process owns to the node. 353 node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>, 354 death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>, 355 /// Cookie of the active freeze listener for this node. 356 freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>, 357 /// Used to store this `NodeRefInfo` in the node's `refs` list. 358 #[pin] 359 links: ListLinks<{ Self::LIST_NODE }>, 360 /// The handle for this `NodeRefInfo`. 361 handle: u32, 362 /// The process that has a handle to the node. 363 pub(crate) process: Arc<Process>, 364 } 365 366 impl NodeRefInfo { 367 /// The id used for the `Node::refs` list. 368 pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10; 369 /// The id used for the `ListArc` in `ProcessNodeRefs`. 370 const LIST_PROC: u64 = 0xd703a5263dcc8650; 371 372 fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> { 373 pin_init!(Self { 374 debug_id: super::next_debug_id(), 375 node_ref: ListArcField::new(node_ref), 376 death: ListArcField::new(None), 377 freeze: ListArcField::new(None), 378 links <- ListLinks::new(), 379 handle, 380 process, 381 }) 382 } 383 384 kernel::list::define_list_arc_field_getter! { 385 pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death } 386 pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze } 387 pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref } 388 pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref } 389 } 390 } 391 392 kernel::list::impl_list_arc_safe! { 393 impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; } 394 impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; } 395 } 396 kernel::list::impl_list_item! { 397 impl ListItem<{Self::LIST_NODE}> for NodeRefInfo { 398 using ListLinks { self.links }; 399 } 400 } 401 402 /// Keeps track of references this process has to nodes owned by other processes. 403 /// 404 /// TODO: Currently, the rbtree requires two allocations per node reference, and two tree 405 /// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these 406 /// extra costs should be eliminated. 407 struct ProcessNodeRefs { 408 /// Used to look up nodes using the 32-bit id that this process knows it by. 409 by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>, 410 /// Used to quickly find unused ids in `by_handle`. 411 handle_is_present: IdPool, 412 /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of 413 /// the underlying `Node` struct as returned by `Node::global_id`. 414 by_node: RBTree<usize, u32>, 415 /// Used to look up a `FreezeListener` by cookie. 416 /// 417 /// There might be multiple freeze listeners for the same node, but at most one of them is 418 /// active. 419 freeze_listeners: RBTree<FreezeCookie, FreezeListener>, 420 } 421 422 impl ProcessNodeRefs { 423 fn new() -> Self { 424 Self { 425 by_handle: RBTree::new(), 426 handle_is_present: IdPool::new(), 427 by_node: RBTree::new(), 428 freeze_listeners: RBTree::new(), 429 } 430 } 431 } 432 433 use core::mem::offset_of; 434 use kernel::bindings::rb_process_layout; 435 pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout { 436 arc_offset: Arc::<Process>::DATA_OFFSET, 437 task: offset_of!(Process, task), 438 }; 439 440 /// A process using binder. 441 /// 442 /// Strictly speaking, there can be multiple of these per process. There is one for each binder fd 443 /// that a process has opened, so processes using several binder contexts have several `Process` 444 /// objects. This ensures that the contexts are fully separated. 445 #[pin_data] 446 pub(crate) struct Process { 447 pub(crate) ctx: Arc<Context>, 448 449 // The task leader (process). 450 pub(crate) task: ARef<Task>, 451 452 // Credential associated with file when `Process` is created. 453 pub(crate) cred: ARef<Credential>, 454 455 #[pin] 456 pub(crate) inner: SpinLock<ProcessInner>, 457 458 #[pin] 459 pub(crate) pages: ShrinkablePageRange, 460 461 // Waitqueue of processes waiting for all outstanding transactions to be 462 // processed. 463 #[pin] 464 freeze_wait: CondVar, 465 466 // Node references are in a different lock to avoid recursive acquisition when 467 // incrementing/decrementing a node in another process. 468 #[pin] 469 node_refs: SpinLock<ProcessNodeRefs>, 470 471 // Synchronizes `register_wait` calls to the `PollCondVarBox`. 472 // 473 // The `PollCondVarBox` is not stored here because synchronization is 474 // done for `register_wait` only. Wakeups do not take this lock. 475 poll: SetOnce<PollCondVarBox>, 476 477 // Work node for deferred work item. 478 #[pin] 479 defer_work: Work<Process>, 480 481 // Links for process list in Context. 482 #[pin] 483 links: ListLinks, 484 485 pub(crate) stats: BinderStats, 486 } 487 488 kernel::impl_has_work! { 489 impl HasWork<Process> for Process { self.defer_work } 490 } 491 492 kernel::list::impl_list_arc_safe! { 493 impl ListArcSafe<0> for Process { untracked; } 494 } 495 kernel::list::impl_list_item! { 496 impl ListItem<0> for Process { 497 using ListLinks { self.links }; 498 } 499 } 500 501 impl workqueue::WorkItem for Process { 502 type Pointer = Arc<Process>; 503 504 fn run(me: Arc<Self>) { 505 let defer; 506 { 507 let mut inner = me.inner.lock(); 508 defer = inner.defer_work; 509 inner.defer_work = DeferWorks::default(); 510 } 511 512 if defer.contains(DeferWork::Flush) { 513 me.deferred_flush(); 514 } 515 if defer.contains(DeferWork::Release) { 516 me.deferred_release(); 517 } 518 } 519 } 520 521 impl Process { 522 fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> { 523 let current = kernel::current!(); 524 let process = Arc::pin_init::<Error>( 525 try_pin_init!(Process { 526 ctx, 527 cred, 528 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"), 529 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER), 530 node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"), 531 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"), 532 task: current.group_leader().into(), 533 defer_work <- kernel::new_work!("Process::defer_work"), 534 links <- ListLinks::new(), 535 stats: BinderStats::new(), 536 poll: SetOnce::new(), 537 }), 538 GFP_KERNEL, 539 )?; 540 541 process.ctx.register_process(process.clone())?; 542 543 Ok(process) 544 } 545 546 pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid { 547 self.task.tgid_nr_ns(None) 548 } 549 550 #[inline(never)] 551 pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> { 552 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 553 seq_print!(m, "context {}\n", &*ctx.name); 554 555 let inner = self.inner.lock(); 556 seq_print!(m, " threads: {}\n", inner.threads.iter().count()); 557 seq_print!( 558 m, 559 " requested threads: {}+{}/{}\n", 560 inner.requested_thread_count, 561 inner.started_thread_count, 562 inner.max_threads, 563 ); 564 if let Some(mapping) = &inner.mapping { 565 seq_print!( 566 m, 567 " free oneway space: {}\n", 568 mapping.alloc.free_oneway_space() 569 ); 570 seq_print!(m, " buffers: {}\n", mapping.alloc.count_buffers()); 571 } 572 seq_print!( 573 m, 574 " outstanding transactions: {}\n", 575 inner.outstanding_txns 576 ); 577 seq_print!(m, " nodes: {}\n", inner.nodes.iter().count()); 578 drop(inner); 579 580 { 581 let mut refs = self.node_refs.lock(); 582 let (mut count, mut weak, mut strong) = (0, 0, 0); 583 for r in refs.by_handle.values_mut() { 584 let node_ref = r.node_ref(); 585 let (nstrong, nweak) = node_ref.get_count(); 586 count += 1; 587 weak += nweak; 588 strong += nstrong; 589 } 590 seq_print!(m, " refs: {count} s {strong} w {weak}\n"); 591 } 592 593 self.stats.debug_print(" ", m); 594 595 Ok(()) 596 } 597 598 #[inline(never)] 599 pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> { 600 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 601 seq_print!(m, "context {}\n", &*ctx.name); 602 603 let mut all_threads = KVec::new(); 604 let mut all_nodes = KVec::new(); 605 loop { 606 let inner = self.inner.lock(); 607 let num_threads = inner.threads.iter().count(); 608 let num_nodes = inner.nodes.iter().count(); 609 610 if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes { 611 drop(inner); 612 all_threads.reserve(num_threads, GFP_KERNEL)?; 613 all_nodes.reserve(num_nodes, GFP_KERNEL)?; 614 continue; 615 } 616 617 for thread in inner.threads.values() { 618 assert!(all_threads.len() < all_threads.capacity()); 619 let _ = all_threads.push(thread.clone(), GFP_ATOMIC); 620 } 621 622 for node in inner.nodes.values() { 623 assert!(all_nodes.len() < all_nodes.capacity()); 624 let _ = all_nodes.push(node.clone(), GFP_ATOMIC); 625 } 626 627 break; 628 } 629 630 for thread in all_threads { 631 thread.debug_print(m, print_all)?; 632 } 633 634 let mut inner = self.inner.lock(); 635 for node in all_nodes { 636 if print_all || node.has_oneway_transaction(&mut inner) { 637 node.full_debug_print(m, &mut inner)?; 638 } 639 } 640 drop(inner); 641 642 if print_all { 643 let mut refs = self.node_refs.lock(); 644 for r in refs.by_handle.values_mut() { 645 let node_ref = r.node_ref(); 646 let dead = node_ref.node.owner.inner.lock().is_dead; 647 let (strong, weak) = node_ref.get_count(); 648 let debug_id = node_ref.node.debug_id; 649 650 seq_print!( 651 m, 652 " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", 653 r.debug_id, 654 r.handle, 655 if dead { "dead " } else { "" } 656 ); 657 } 658 } 659 660 let inner = self.inner.lock(); 661 for work in &inner.work { 662 work.debug_print(m, " ", " pending transaction ")?; 663 } 664 for _death in &inner.delivered_deaths { 665 seq_print!(m, " has delivered dead binder\n"); 666 } 667 if let Some(mapping) = &inner.mapping { 668 mapping.alloc.debug_print(m)?; 669 } 670 drop(inner); 671 672 Ok(()) 673 } 674 675 /// Attempts to fetch a work item from the process queue. 676 pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> { 677 self.inner.lock().work.pop_front() 678 } 679 680 /// Attempts to fetch a work item from the process queue. If none is available, it registers the 681 /// given thread as ready to receive work directly. 682 /// 683 /// This must only be called when the thread is not participating in a transaction chain; when 684 /// it is, work will always be delivered directly to the thread (and not through the process 685 /// queue). 686 pub(crate) fn get_work_or_register<'a>( 687 &'a self, 688 thread: &'a Arc<Thread>, 689 ) -> GetWorkOrRegister<'a> { 690 let mut inner = self.inner.lock(); 691 // Try to get work from the process queue. 692 if let Some(work) = inner.work.pop_front() { 693 return GetWorkOrRegister::Work(work); 694 } 695 696 // Register the thread as ready. 697 GetWorkOrRegister::Register(Registration::new(thread, &mut inner)) 698 } 699 700 fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> { 701 let id = { 702 let current = kernel::current!(); 703 if self.task != current.group_leader() { 704 pr_err!("get_current_thread was called from the wrong process."); 705 return Err(EINVAL); 706 } 707 current.pid() 708 }; 709 710 { 711 let inner = self.inner.lock(); 712 if let Some(thread) = inner.threads.get(&id) { 713 return Ok(thread.clone()); 714 } 715 } 716 717 // Allocate a new `Thread` without holding any locks. 718 let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?; 719 let ta: Arc<Thread> = Thread::new(id, self.into())?; 720 721 let mut inner = self.inner.lock(); 722 match inner.threads.entry(id) { 723 rbtree::Entry::Vacant(entry) => { 724 entry.insert(ta.clone(), reservation); 725 Ok(ta) 726 } 727 rbtree::Entry::Occupied(_entry) => { 728 pr_err!("Cannot create two threads with the same id."); 729 Err(EINVAL) 730 } 731 } 732 } 733 734 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { 735 // If push_work fails, drop the work item outside the lock. 736 let res = self.inner.lock().push_work(self, work); 737 match res { 738 Ok(()) => Ok(()), 739 Err((err, work)) => { 740 drop(work); 741 Err(err) 742 } 743 } 744 } 745 746 fn set_as_manager( 747 self: ArcBorrow<'_, Self>, 748 info: Option<FlatBinderObject>, 749 thread: &Thread, 750 ) -> Result { 751 let (ptr, cookie, flags) = if let Some(obj) = info { 752 ( 753 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it 754 // is safe to access the `binder` field. 755 unsafe { obj.__bindgen_anon_1.binder }, 756 obj.cookie, 757 obj.flags, 758 ) 759 } else { 760 (0, 0, 0) 761 }; 762 let node_ref = self.get_node(ptr, cookie, flags, true, thread)?; 763 let node = node_ref.node.clone(); 764 self.ctx.set_manager_node(node_ref)?; 765 self.inner.lock().is_manager = true; 766 767 // Force the state of the node to prevent the delivery of acquire/increfs. 768 let mut owner_inner = node.owner.inner.lock(); 769 node.force_has_count(&mut owner_inner); 770 Ok(()) 771 } 772 773 fn get_node_inner( 774 self: ArcBorrow<'_, Self>, 775 ptr: u64, 776 cookie: u64, 777 flags: u32, 778 strong: bool, 779 thread: &Thread, 780 wrapper: Option<CritIncrWrapper>, 781 ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> { 782 // Try to find an existing node. 783 { 784 let mut inner = self.inner.lock(); 785 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 786 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 787 } 788 } 789 790 // Allocate the node before reacquiring the lock. 791 let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc(); 792 let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?; 793 let mut inner = self.inner.lock(); 794 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 795 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 796 } 797 798 inner.nodes.insert(rbnode); 799 // This can only fail if someone has already pushed the node to a list, but we just created 800 // it and still hold the lock, so it can't fail right now. 801 let node_ref = inner 802 .new_node_ref_with_thread(node, strong, thread, wrapper) 803 .unwrap(); 804 805 Ok(Ok(node_ref)) 806 } 807 808 pub(crate) fn get_node( 809 self: ArcBorrow<'_, Self>, 810 ptr: u64, 811 cookie: u64, 812 flags: u32, 813 strong: bool, 814 thread: &Thread, 815 ) -> Result<NodeRef> { 816 let mut wrapper = None; 817 for _ in 0..2 { 818 match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) { 819 Err(err) => return Err(err), 820 Ok(Ok(node_ref)) => return Ok(node_ref), 821 Ok(Err(CouldNotDeliverCriticalIncrement)) => { 822 wrapper = Some(CritIncrWrapper::new()?); 823 } 824 } 825 } 826 // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the 827 // loop should run at most twice. 828 unreachable!() 829 } 830 831 pub(crate) fn insert_or_update_handle( 832 self: ArcBorrow<'_, Process>, 833 node_ref: NodeRef, 834 is_manager: bool, 835 ) -> Result<u32> { 836 { 837 let mut refs = self.node_refs.lock(); 838 839 // Do a lookup before inserting. 840 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { 841 let handle = *handle_ref; 842 let info = refs.by_handle.get_mut(&handle).unwrap(); 843 info.node_ref().absorb(node_ref); 844 return Ok(handle); 845 } 846 } 847 848 // Reserve memory for tree nodes. 849 let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?; 850 let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?; 851 let info = UniqueArc::new_uninit(GFP_KERNEL)?; 852 853 let mut refs_lock = self.node_refs.lock(); 854 let mut refs = &mut *refs_lock; 855 856 let (unused_id, by_handle_slot) = loop { 857 // ID 0 may only be used by the manager. 858 let start = if is_manager { 0 } else { 1 }; 859 860 if let Some(res) = refs.handle_is_present.find_unused_id(start) { 861 match refs.by_handle.entry(res.as_u32()) { 862 rbtree::Entry::Vacant(entry) => break (res, entry), 863 rbtree::Entry::Occupied(_) => { 864 pr_err!("Detected mismatch between handle_is_present and by_handle"); 865 res.acquire(); 866 kernel::warn_on!(true); 867 return Err(EINVAL); 868 } 869 } 870 } 871 872 let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?; 873 drop(refs_lock); 874 let resizer = grow_request.realloc(GFP_KERNEL)?; 875 refs_lock = self.node_refs.lock(); 876 refs = &mut *refs_lock; 877 refs.handle_is_present.grow(resizer); 878 }; 879 let handle = unused_id.as_u32(); 880 881 // Do a lookup again as node may have been inserted before the lock was reacquired. 882 let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) { 883 rbtree::Entry::Vacant(by_node_slot) => by_node_slot, 884 rbtree::Entry::Occupied(handle_ref) => { 885 // The node was inserted by another thread while we didn't hold the lock. 886 let handle = handle_ref.get(); 887 let info = refs.by_handle.get_mut(handle).unwrap(); 888 info.node_ref().absorb(node_ref); 889 return Ok(*handle); 890 } 891 }; 892 893 let (info_proc, info_node) = { 894 let info_init = NodeRefInfo::new(node_ref, handle, self.into()); 895 match info.pin_init_with(info_init) { 896 Ok(info) => ListArc::pair_from_pin_unique(info), 897 // error is infallible 898 Err(err) => match err {}, 899 } 900 }; 901 902 // Ensure the process is still alive while we insert a new reference. 903 // 904 // This releases the lock before inserting the nodes, but since `is_dead` is set as the 905 // first thing in `deferred_release`, process cleanup will not miss the items inserted into 906 // `refs` below. 907 if self.inner.lock().is_dead { 908 // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of 909 // the lock. 910 drop(refs_lock); 911 return Err(ESRCH); 912 } 913 914 // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting 915 // `info_node` into the right node's `refs` list. 916 unsafe { info_proc.node_ref2().node.insert_node_info(info_node) }; 917 918 by_node_slot.insert(handle, reserve1); 919 by_handle_slot.insert(info_proc, reserve2); 920 unused_id.acquire(); 921 Ok(handle) 922 } 923 924 pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { 925 // When handle is zero, try to get the context manager. 926 if handle == 0 { 927 let node_ref = self.ctx.get_manager_node(true)?; 928 if core::ptr::eq(self, &*node_ref.node.owner) { 929 return Err(EINVAL.into()); 930 } 931 Ok(node_ref) 932 } else { 933 match self.get_node_from_handle(handle, true) { 934 Ok(node_ref) => Ok(node_ref), 935 Err(err) => { 936 binder_debug!(UserError, "got transaction to invalid handle {handle}"); 937 Err(err.into()) 938 } 939 } 940 } 941 } 942 943 pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> { 944 self.node_refs 945 .lock() 946 .by_handle 947 .get_mut(&handle) 948 .ok_or(ENOENT)? 949 .node_ref() 950 .clone(strong) 951 } 952 953 pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) { 954 let mut inner = self.inner.lock(); 955 // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list. 956 let removed = unsafe { inner.delivered_deaths.remove(death) }; 957 drop(inner); 958 drop(removed); 959 } 960 961 pub(crate) fn update_ref( 962 self: ArcBorrow<'_, Process>, 963 handle: u32, 964 inc: bool, 965 strong: bool, 966 ) -> Result { 967 if inc && handle == 0 { 968 if let Ok(node_ref) = self.ctx.get_manager_node(strong) { 969 if core::ptr::eq(&*self, &*node_ref.node.owner) { 970 return Err(EINVAL); 971 } 972 let _ = self.insert_or_update_handle(node_ref, true); 973 return Ok(()); 974 } 975 } 976 977 // To preserve original binder behaviour, we only fail requests where the manager tries to 978 // increment references on itself. 979 let _to_free_by_handle; 980 let _to_free_by_node; 981 let _to_free_freeze_listener; 982 let _to_free_freeze_listener_cleanup; 983 let mut refs = self.node_refs.lock(); 984 if let Some(info) = refs.by_handle.get_mut(&handle) { 985 if info.node_ref().update(inc, strong) { 986 // Clean up death if there is one attached to this node reference. 987 // 988 // We remove the entire `info` below, so no need to remove `death` from `info`. 989 if let Some(death) = info.death().as_ref() { 990 death.set_cleared(true); 991 self.remove_from_delivered_deaths(death); 992 } 993 994 // Remove reference from process tables, and from the node's `refs` list. 995 996 // SAFETY: We are removing the `NodeRefInfo` from the right node. 997 unsafe { info.node_ref2().node.remove_node_info(info) }; 998 999 let id = info.node_ref().node.global_id(); 1000 1001 if let Some(freeze) = *info.freeze() { 1002 if let Some(fl) = refs.freeze_listeners.remove(&freeze) { 1003 _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self); 1004 _to_free_freeze_listener = fl; 1005 } 1006 } 1007 1008 _to_free_by_handle = refs.by_handle.remove_node(&handle); 1009 _to_free_by_node = refs.by_node.remove_node(&id); 1010 refs.handle_is_present.release_id(handle as usize); 1011 1012 if let Some(shrink) = refs.handle_is_present.shrink_request() { 1013 drop(refs); 1014 // This intentionally ignores allocation failures. 1015 if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) { 1016 refs = self.node_refs.lock(); 1017 refs.handle_is_present.shrink(new_bitmap); 1018 } 1019 } 1020 } 1021 } else { 1022 // All refs are cleared in process exit, so this warning is expected in that case. 1023 if !self.inner.lock().is_dead { 1024 binder_debug!(UserError, "no such ref {handle}"); 1025 } 1026 } 1027 Ok(()) 1028 } 1029 1030 /// Decrements the refcount of the given node, if one exists. 1031 pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) { 1032 let mut inner = self.inner.lock(); 1033 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1034 inner.update_node_refcount(&node, false, strong, 1, None); 1035 } 1036 } 1037 1038 pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result { 1039 let ptr = reader.read::<u64>()?; 1040 let cookie = reader.read::<u64>()?; 1041 let mut inner = self.inner.lock(); 1042 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1043 if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) { 1044 // This only fails if the process is dead. 1045 let _ = inner.push_work(self, node); 1046 } 1047 } 1048 Ok(()) 1049 } 1050 1051 pub(crate) fn buffer_alloc( 1052 self: &Arc<Self>, 1053 debug_id: usize, 1054 size: usize, 1055 info: &mut TransactionInfo, 1056 ) -> BinderResult<NewAllocation> { 1057 use kernel::page::PAGE_SIZE; 1058 1059 let mut reserve_new_args = ReserveNewArgs { 1060 debug_id, 1061 size, 1062 is_oneway: info.is_oneway(), 1063 pid: info.from_pid, 1064 ..ReserveNewArgs::default() 1065 }; 1066 1067 let (new_alloc, addr) = loop { 1068 let mut inner = self.inner.lock(); 1069 let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?; 1070 let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? { 1071 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address), 1072 ReserveNew::NeedAlloc(request) => request, 1073 }; 1074 drop(inner); 1075 // We need to allocate memory and then call `reserve_new` again. 1076 reserve_new_args = alloc_request.make_alloc()?; 1077 }; 1078 1079 info.oneway_spam_suspect = new_alloc.oneway_spam_detected; 1080 let res = Allocation::new( 1081 self.clone(), 1082 debug_id, 1083 new_alloc.offset, 1084 size, 1085 addr + new_alloc.offset, 1086 ); 1087 1088 // This allocation will be marked as in use until the `Allocation` is used to free it. 1089 // 1090 // This method can't be called while holding a lock, so we release the lock first. It's 1091 // okay for several threads to use the method on the same index at the same time. In that 1092 // case, one of the calls will allocate the given page (if missing), and the other call 1093 // will wait for the other call to finish allocating the page. 1094 // 1095 // We will not call `stop_using_range` in parallel with this on the same page, because the 1096 // allocation can only be removed via the destructor of the `Allocation` object that we 1097 // currently own. 1098 match self.pages.use_range( 1099 new_alloc.offset / PAGE_SIZE, 1100 (new_alloc.offset + size).div_ceil(PAGE_SIZE), 1101 ) { 1102 Ok(()) => {} 1103 Err(err) => { 1104 pr_warn!("use_range failure {:?}", err); 1105 return Err(err.into()); 1106 } 1107 } 1108 1109 Ok(NewAllocation(res)) 1110 } 1111 1112 pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> { 1113 let mut inner = self.inner.lock(); 1114 let mapping = inner.mapping.as_mut()?; 1115 let offset = ptr.checked_sub(mapping.address)?; 1116 let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?; 1117 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1118 if let Some(data) = odata { 1119 alloc.set_info(data); 1120 } 1121 Some(alloc) 1122 } 1123 1124 pub(crate) fn buffer_raw_free(&self, ptr: usize) { 1125 let mut inner = self.inner.lock(); 1126 if let Some(ref mut mapping) = &mut inner.mapping { 1127 let offset = match ptr.checked_sub(mapping.address) { 1128 Some(offset) => offset, 1129 None => return, 1130 }; 1131 1132 let freed_range = match mapping.alloc.reservation_abort(offset) { 1133 Ok(freed_range) => freed_range, 1134 Err(_) => { 1135 pr_warn!( 1136 "Pointer {:x} failed to free, base = {:x}\n", 1137 ptr, 1138 mapping.address 1139 ); 1140 return; 1141 } 1142 }; 1143 1144 // No more allocations in this range. Mark them as not in use. 1145 // 1146 // Must be done before we release the lock so that `use_range` is not used on these 1147 // indices until `stop_using_range` returns. 1148 self.pages 1149 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx); 1150 } 1151 } 1152 1153 pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) { 1154 let mut inner = self.inner.lock(); 1155 if let Some(ref mut mapping) = &mut inner.mapping { 1156 if mapping.alloc.reservation_commit(offset, &mut data).is_err() { 1157 pr_warn!("Offset {} failed to be marked freeable\n", offset); 1158 } 1159 } 1160 } 1161 1162 fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result { 1163 use kernel::page::PAGE_SIZE; 1164 let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); 1165 let mapping = Mapping::new(vma.start(), size); 1166 let page_count = self.pages.register_with_vma(vma)?; 1167 if page_count * PAGE_SIZE != size { 1168 return Err(EINVAL); 1169 } 1170 1171 // Save range allocator for later. 1172 self.inner.lock().mapping = Some(mapping); 1173 1174 Ok(()) 1175 } 1176 1177 fn version(&self, data: UserSlice) -> Result { 1178 data.writer().write(&BinderVersion::current()) 1179 } 1180 1181 pub(crate) fn register_thread(&self) -> bool { 1182 self.inner.lock().register_thread() 1183 } 1184 1185 fn remove_thread(&self, thread: Arc<Thread>) { 1186 self.inner.lock().threads.remove(&thread.id); 1187 thread.release(); 1188 } 1189 1190 fn set_max_threads(&self, max: u32) { 1191 self.inner.lock().max_threads = max; 1192 } 1193 1194 fn set_oneway_spam_detection_enabled(&self, enabled: u32) { 1195 self.inner.lock().oneway_spam_detection_enabled = enabled != 0; 1196 } 1197 1198 pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool { 1199 self.inner.lock().oneway_spam_detection_enabled 1200 } 1201 1202 fn get_node_debug_info(&self, data: UserSlice) -> Result { 1203 let (mut reader, mut writer) = data.reader_writer(); 1204 1205 // Read the starting point. 1206 let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr; 1207 let mut out = BinderNodeDebugInfo::default(); 1208 1209 { 1210 let inner = self.inner.lock(); 1211 for (node_ptr, node) in &inner.nodes { 1212 if *node_ptr > ptr { 1213 node.populate_debug_info(&mut out, &inner); 1214 break; 1215 } 1216 } 1217 } 1218 1219 writer.write(&out) 1220 } 1221 1222 fn get_node_info_from_ref(&self, data: UserSlice) -> Result { 1223 let (mut reader, mut writer) = data.reader_writer(); 1224 let mut out = reader.read::<BinderNodeInfoForRef>()?; 1225 1226 if out.strong_count != 0 1227 || out.weak_count != 0 1228 || out.reserved1 != 0 1229 || out.reserved2 != 0 1230 || out.reserved3 != 0 1231 { 1232 return Err(EINVAL); 1233 } 1234 1235 // Only the context manager is allowed to use this ioctl. 1236 if !self.inner.lock().is_manager { 1237 return Err(EPERM); 1238 } 1239 1240 { 1241 let mut node_refs = self.node_refs.lock(); 1242 let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?; 1243 let node_ref = node_info.node_ref(); 1244 let owner_inner = node_ref.node.owner.inner.lock(); 1245 node_ref.node.populate_counts(&mut out, &owner_inner); 1246 } 1247 1248 // Write the result back. 1249 writer.write(&out) 1250 } 1251 1252 pub(crate) fn needs_thread(&self) -> bool { 1253 let mut inner = self.inner.lock(); 1254 let ret = inner.requested_thread_count == 0 1255 && inner.ready_threads.is_empty() 1256 && inner.started_thread_count < inner.max_threads; 1257 if ret { 1258 inner.requested_thread_count += 1 1259 } 1260 ret 1261 } 1262 1263 pub(crate) fn request_death( 1264 self: &Arc<Self>, 1265 reader: &mut UserSliceReader, 1266 thread: &Thread, 1267 ) -> Result { 1268 let handle: u32 = reader.read()?; 1269 let cookie: u64 = reader.read()?; 1270 1271 // Queue BR_ERROR if we can't allocate memory for the death notification. 1272 let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| { 1273 thread.push_return_work(BR_ERROR); 1274 binder_debug!( 1275 DeathNotification, 1276 "BC_REQUEST_DEATH_NOTIFICATION failed due to memory allocation failure" 1277 ); 1278 })?; 1279 let mut refs = self.node_refs.lock(); 1280 let Some(info) = refs.by_handle.get_mut(&handle) else { 1281 binder_debug!( 1282 UserError, 1283 "BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}" 1284 ); 1285 return Ok(()); 1286 }; 1287 1288 // Nothing to do if there is already a death notification request for this handle. 1289 if info.death().is_some() { 1290 binder_debug!( 1291 UserError, 1292 "BC_REQUEST_DEATH_NOTIFICATION death notification already set" 1293 ); 1294 return Ok(()); 1295 } 1296 1297 let death = { 1298 let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie); 1299 match death.pin_init_with(death_init) { 1300 Ok(death) => death, 1301 // error is infallible 1302 Err(err) => match err {}, 1303 } 1304 }; 1305 1306 // Register the death notification. 1307 { 1308 let owner = info.node_ref2().node.owner.clone(); 1309 let mut owner_inner = owner.inner.lock(); 1310 if owner_inner.is_dead { 1311 let death = Arc::from(death); 1312 *info.death() = Some(death.clone()); 1313 drop(owner_inner); 1314 death.set_dead(); 1315 } else { 1316 let death = ListArc::from(death); 1317 *info.death() = Some(death.clone_arc()); 1318 info.node_ref().node.add_death(death, &mut owner_inner); 1319 } 1320 } 1321 binder_debug!( 1322 DeathNotification, 1323 "BC_REQUEST_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1324 cookie 1325 ); 1326 Ok(()) 1327 } 1328 1329 pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result { 1330 let handle: u32 = reader.read()?; 1331 let cookie: u64 = reader.read()?; 1332 1333 let mut refs = self.node_refs.lock(); 1334 let Some(info) = refs.by_handle.get_mut(&handle) else { 1335 binder_debug!( 1336 UserError, 1337 "BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}" 1338 ); 1339 return Ok(()); 1340 }; 1341 1342 let Some(death) = info.death().take() else { 1343 binder_debug!( 1344 UserError, 1345 "BC_CLEAR_DEATH_NOTIFICATION death notification not active" 1346 ); 1347 return Ok(()); 1348 }; 1349 if death.cookie != cookie { 1350 *info.death() = Some(death); 1351 binder_debug!( 1352 UserError, 1353 "BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch" 1354 ); 1355 return Ok(()); 1356 } 1357 1358 // Update state and determine if we need to queue a work item. We only need to do it when 1359 // the node is not dead or if the user already completed the death notification. 1360 let should_schedule = death.set_cleared(false); 1361 drop(refs); 1362 1363 if should_schedule { 1364 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 1365 let _ = thread.push_work_if_looper(death); 1366 } 1367 } 1368 1369 binder_debug!( 1370 DeathNotification, 1371 "BC_CLEAR_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1372 cookie 1373 ); 1374 Ok(()) 1375 } 1376 1377 pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { 1378 let death = self.inner.lock().pull_delivered_death(cookie); 1379 if let Some(death) = death { 1380 death.set_notification_done(thread); 1381 } 1382 } 1383 1384 /// Locks the spinlock and move the `nodes` rbtree out. 1385 /// 1386 /// This allows you to iterate through `nodes` while also allowing you to give other parts of 1387 /// the codebase exclusive access to `ProcessInner`. 1388 pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> { 1389 let mut inner = self.inner.lock(); 1390 WithNodes { 1391 nodes: take(&mut inner.nodes), 1392 inner, 1393 } 1394 } 1395 1396 fn deferred_flush(&self) { 1397 binder_debug!(pid = self.task.pid(), OpenClose, "flushing process"); 1398 let inner = self.inner.lock(); 1399 for thread in inner.threads.values() { 1400 thread.exit_looper(); 1401 } 1402 } 1403 1404 fn deferred_release(self: Arc<Self>) { 1405 binder_debug!(pid = self.task.pid(), OpenClose, "releasing process"); 1406 1407 let is_manager = { 1408 let mut inner = self.inner.lock(); 1409 inner.is_dead = true; 1410 inner.is_frozen = IsFrozen::No; 1411 inner.sync_recv = false; 1412 inner.async_recv = false; 1413 inner.is_manager 1414 }; 1415 1416 if is_manager { 1417 self.ctx.unset_manager_node(); 1418 } 1419 1420 self.ctx.deregister_process(&self); 1421 1422 let binderfs_file = self.inner.lock().binderfs_file.take(); 1423 drop(binderfs_file); 1424 1425 // Release threads. 1426 let threads = { 1427 let mut inner = self.inner.lock(); 1428 let threads = take(&mut inner.threads); 1429 let ready = take(&mut inner.ready_threads); 1430 drop(inner); 1431 drop(ready); 1432 1433 for thread in threads.values() { 1434 thread.release(); 1435 } 1436 threads 1437 }; 1438 1439 // Release nodes. 1440 { 1441 while let Some(node) = { 1442 let mut lock = self.inner.lock(); 1443 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) 1444 } { 1445 node.to_key_value().1.release(); 1446 } 1447 } 1448 1449 // Clean up death listeners and remove nodes from external node info lists. 1450 for info in self.node_refs.lock().by_handle.values_mut() { 1451 // SAFETY: We are removing the `NodeRefInfo` from the right node. 1452 unsafe { info.node_ref2().node.remove_node_info(info) }; 1453 1454 // Clear death notifications from the nodes (that belong to a different process). 1455 // No need to remove them from `info` as we clear info below. 1456 if let Some(death) = info.death().as_ref() { 1457 death.set_cleared(false); 1458 } 1459 } 1460 1461 // Clean up freeze listeners. 1462 let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners); 1463 for listener in freeze_listeners.values() { 1464 listener.on_process_cleanup(&self); 1465 } 1466 drop(freeze_listeners); 1467 1468 // Release refs on foreign nodes. 1469 { 1470 let mut refs = self.node_refs.lock(); 1471 let by_handle = take(&mut refs.by_handle); 1472 let by_node = take(&mut refs.by_node); 1473 drop(refs); 1474 drop(by_node); 1475 drop(by_handle); 1476 } 1477 1478 // Cancel all pending work items. 1479 while let Some(work) = self.get_work() { 1480 work.into_arc().cancel(); 1481 } 1482 1483 // Clear delivered_deaths list. 1484 // 1485 // Scope ensures that MutexGuard is dropped while executing the body. 1486 while let Some(delivered_death) = { 1487 // Explicitly bind to avoid tail expression lifetime extension of the lockguard 1488 // Can be removed when the kernel moves to edition 2024 1489 let maybe_death = self.inner.lock().delivered_deaths.pop_front(); 1490 maybe_death 1491 } { 1492 drop(delivered_death); 1493 } 1494 1495 // Free any resources kept alive by allocated buffers. 1496 let omapping = self.inner.lock().mapping.take(); 1497 if let Some(mut mapping) = omapping { 1498 let address = mapping.address; 1499 mapping 1500 .alloc 1501 .take_for_each(|offset, size, debug_id, odata| { 1502 let ptr = offset + address; 1503 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1504 if let Some(data) = odata { 1505 alloc.set_info(data); 1506 } 1507 drop(alloc) 1508 }); 1509 } 1510 1511 // calls to synchronize_rcu() in thread drop will happen here 1512 drop(threads); 1513 } 1514 1515 pub(crate) fn drop_outstanding_txn(&self) { 1516 let wake = { 1517 let mut inner = self.inner.lock(); 1518 if inner.outstanding_txns == 0 { 1519 pr_err!("outstanding_txns underflow"); 1520 return; 1521 } 1522 inner.outstanding_txns -= 1; 1523 inner.is_frozen.is_frozen() && inner.outstanding_txns == 0 1524 }; 1525 1526 if wake { 1527 self.freeze_wait.notify_all(); 1528 } 1529 } 1530 1531 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 1532 // mangled symbol names. 1533 #[export_name = "rust_binder_freeze"] 1534 pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result { 1535 if info.enable == 0 { 1536 let msgs = self.prepare_freeze_messages()?; 1537 let mut inner = self.inner.lock(); 1538 inner.sync_recv = false; 1539 inner.async_recv = false; 1540 inner.is_frozen = IsFrozen::No; 1541 drop(inner); 1542 msgs.send_messages(); 1543 return Ok(()); 1544 } 1545 1546 let mut inner = self.inner.lock(); 1547 inner.sync_recv = false; 1548 inner.async_recv = false; 1549 inner.is_frozen = IsFrozen::InProgress; 1550 1551 if info.timeout_ms > 0 { 1552 let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms); 1553 while jiffies > 0 { 1554 if inner.outstanding_txns == 0 { 1555 break; 1556 } 1557 1558 match self 1559 .freeze_wait 1560 .wait_interruptible_timeout(&mut inner, jiffies) 1561 { 1562 CondVarTimeoutResult::Signal { .. } => { 1563 inner.is_frozen = IsFrozen::No; 1564 return Err(ERESTARTSYS); 1565 } 1566 CondVarTimeoutResult::Woken { jiffies: remaining } => { 1567 jiffies = remaining; 1568 } 1569 CondVarTimeoutResult::Timeout => { 1570 jiffies = 0; 1571 } 1572 } 1573 } 1574 } 1575 1576 if inner.txns_pending_locked() { 1577 inner.is_frozen = IsFrozen::No; 1578 Err(EAGAIN) 1579 } else { 1580 drop(inner); 1581 match self.prepare_freeze_messages() { 1582 Ok(batch) => { 1583 self.inner.lock().is_frozen = IsFrozen::Yes; 1584 batch.send_messages(); 1585 Ok(()) 1586 } 1587 Err(kernel::alloc::AllocError) => { 1588 self.inner.lock().is_frozen = IsFrozen::No; 1589 Err(ENOMEM) 1590 } 1591 } 1592 } 1593 } 1594 1595 pub(crate) fn notify_poll(&self, sync: bool) { 1596 if let Some(poll) = self.poll.as_ref() { 1597 if sync { 1598 poll.notify_sync(); 1599 } 1600 poll.notify_all(); 1601 } 1602 } 1603 } 1604 1605 fn get_frozen_status(data: UserSlice) -> Result { 1606 let (mut reader, mut writer) = data.reader_writer(); 1607 1608 let mut info = reader.read::<BinderFrozenStatusInfo>()?; 1609 info.sync_recv = 0; 1610 info.async_recv = 0; 1611 let mut found = false; 1612 1613 for ctx in crate::context::get_all_contexts()? { 1614 ctx.for_each_proc(|proc| { 1615 if proc.task.pid() == info.pid as Pid { 1616 found = true; 1617 let inner = proc.inner.lock(); 1618 let txns_pending = inner.txns_pending_locked(); 1619 info.async_recv |= u32::from(inner.async_recv); 1620 info.sync_recv |= u32::from(inner.sync_recv); 1621 info.sync_recv |= u32::from(txns_pending) << 1; 1622 } 1623 }); 1624 } 1625 1626 if found { 1627 writer.write(&info)?; 1628 Ok(()) 1629 } else { 1630 Err(EINVAL) 1631 } 1632 } 1633 1634 fn ioctl_freeze(reader: &mut UserSliceReader) -> Result { 1635 let info = reader.read::<BinderFreezeInfo>()?; 1636 1637 // Very unlikely for there to be more than 3, since a process normally uses at most binder and 1638 // hwbinder. 1639 let mut procs = KVec::with_capacity(3, GFP_KERNEL)?; 1640 1641 let ctxs = crate::context::get_all_contexts()?; 1642 for ctx in ctxs { 1643 for proc in ctx.get_procs_with_pid(info.pid as i32)? { 1644 procs.push(proc, GFP_KERNEL)?; 1645 } 1646 } 1647 1648 for proc in procs { 1649 proc.ioctl_freeze(&info)?; 1650 } 1651 Ok(()) 1652 } 1653 1654 /// The ioctl handler. 1655 impl Process { 1656 /// Ioctls that are write-only from the perspective of userspace. 1657 /// 1658 /// The kernel will only read from the pointer that userspace provided to us. 1659 fn ioctl_write_only( 1660 this: ArcBorrow<'_, Process>, 1661 _file: &File, 1662 cmd: u32, 1663 reader: &mut UserSliceReader, 1664 ) -> Result { 1665 let thread = this.get_current_thread()?; 1666 match cmd { 1667 uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), 1668 uapi::BINDER_THREAD_EXIT => this.remove_thread(thread), 1669 uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?, 1670 uapi::BINDER_SET_CONTEXT_MGR_EXT => { 1671 this.set_as_manager(Some(reader.read()?), &thread)? 1672 } 1673 uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => { 1674 this.set_oneway_spam_detection_enabled(reader.read()?) 1675 } 1676 uapi::BINDER_FREEZE => ioctl_freeze(reader)?, 1677 _ => return Err(EINVAL), 1678 } 1679 Ok(()) 1680 } 1681 1682 /// Ioctls that are read/write from the perspective of userspace. 1683 /// 1684 /// The kernel will both read from and write to the pointer that userspace provided to us. 1685 fn ioctl_write_read( 1686 this: ArcBorrow<'_, Process>, 1687 file: &File, 1688 cmd: u32, 1689 data: UserSlice, 1690 ) -> Result { 1691 let thread = this.get_current_thread()?; 1692 let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0; 1693 match cmd { 1694 uapi::BINDER_WRITE_READ => thread.write_read(data, blocking)?, 1695 uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?, 1696 uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?, 1697 uapi::BINDER_VERSION => this.version(data)?, 1698 uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?, 1699 uapi::BINDER_GET_EXTENDED_ERROR => thread.get_extended_error(data)?, 1700 _ => return Err(EINVAL), 1701 } 1702 Ok(()) 1703 } 1704 } 1705 1706 /// The file operations supported by `Process`. 1707 impl Process { 1708 pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> { 1709 let proc = Self::new(ctx.into(), ARef::from(file.cred()))?; 1710 binder_debug!(OpenClose, "opened process"); 1711 Ok(proc) 1712 } 1713 1714 pub(crate) fn release(this: Arc<Process>, _file: &File) { 1715 let binderfs_file; 1716 let should_schedule; 1717 { 1718 let mut inner = this.inner.lock(); 1719 should_schedule = inner.defer_work == DeferWorks::empty(); 1720 inner.defer_work |= DeferWork::Release; 1721 binderfs_file = inner.binderfs_file.take(); 1722 } 1723 1724 if should_schedule { 1725 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1726 // scheduled for execution. 1727 let _ = workqueue::system().enqueue(this); 1728 } 1729 1730 drop(binderfs_file); 1731 } 1732 1733 pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result { 1734 let should_schedule; 1735 { 1736 let mut inner = this.inner.lock(); 1737 should_schedule = inner.defer_work == DeferWorks::empty(); 1738 inner.defer_work |= DeferWork::Flush; 1739 } 1740 1741 if should_schedule { 1742 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1743 // scheduled for execution. 1744 let _ = workqueue::system().enqueue(Arc::from(this)); 1745 } 1746 Ok(()) 1747 } 1748 1749 pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result { 1750 use kernel::ioctl::{_IOC_DIR, _IOC_SIZE}; 1751 use kernel::uapi::{_IOC_READ, _IOC_WRITE}; 1752 1753 crate::trace::trace_ioctl(cmd, arg); 1754 1755 let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd)); 1756 1757 const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE; 1758 1759 let res = match _IOC_DIR(cmd) { 1760 _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()), 1761 _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice), 1762 _ => Err(EINVAL), 1763 }; 1764 1765 crate::trace::trace_ioctl_done(res); 1766 res 1767 } 1768 1769 pub(crate) fn mmap( 1770 this: ArcBorrow<'_, Process>, 1771 _file: &File, 1772 vma: &mm::virt::VmaNew, 1773 ) -> Result { 1774 // We don't allow mmap to be used in a different process. 1775 if this.task != kernel::current!().group_leader() { 1776 return Err(EINVAL); 1777 } 1778 if vma.start() == 0 { 1779 return Err(EINVAL); 1780 } 1781 1782 vma.try_clear_maywrite().map_err(|_| EPERM)?; 1783 vma.set_dontcopy(); 1784 vma.set_mixedmap(); 1785 1786 // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it. 1787 this.create_mapping(vma) 1788 } 1789 1790 pub(crate) fn poll( 1791 this: ArcBorrow<'_, Process>, 1792 file: &File, 1793 table: PollTable<'_>, 1794 ) -> Result<u32> { 1795 let thread = this.get_current_thread()?; 1796 { 1797 let poll = loop { 1798 if let Some(poll) = this.poll.as_ref() { 1799 break poll; 1800 } 1801 1802 let poll = PollCondVarBox::new(c"Process::poll", kernel::static_lock_class!())?; 1803 // Reuse our existing lock to synchronize callers initializing. 1804 let _guard = this.node_refs.lock(); 1805 this.poll.populate(poll); 1806 }; 1807 1808 table.register_wait(file, poll); 1809 } 1810 let (from_proc, mut mask) = thread.poll()?; 1811 if mask == 0 && from_proc && !this.inner.lock().work.is_empty() { 1812 mask |= bindings::POLLIN; 1813 } 1814 Ok(mask) 1815 } 1816 } 1817 1818 /// Represents that a thread has registered with the `ready_threads` list of its process. 1819 /// 1820 /// The destructor of this type will unregister the thread from the list of ready threads. 1821 pub(crate) struct Registration<'a> { 1822 thread: &'a Arc<Thread>, 1823 } 1824 1825 impl<'a> Registration<'a> { 1826 fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self { 1827 assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref())); 1828 // INVARIANT: We are pushing this thread to the right `ready_threads` list. 1829 if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) { 1830 guard.ready_threads.push_front(list_arc); 1831 } else { 1832 // It is an error to hit this branch, and it should not be reachable. We try to do 1833 // something reasonable when the failure path happens. Most likely, the thread in 1834 // question will sleep forever. 1835 pr_err!("Same thread registered with `ready_threads` twice."); 1836 } 1837 Self { thread } 1838 } 1839 } 1840 1841 impl Drop for Registration<'_> { 1842 fn drop(&mut self) { 1843 let mut inner = self.thread.process.inner.lock(); 1844 // SAFETY: The thread has the invariant that we never push it to any other linked list than 1845 // the `ready_threads` list of its parent process. Therefore, the thread is either in that 1846 // list, or in no list. 1847 unsafe { inner.ready_threads.remove(self.thread) }; 1848 } 1849 } 1850 1851 pub(crate) struct WithNodes<'a> { 1852 pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>, 1853 pub(crate) nodes: RBTree<u64, DArc<Node>>, 1854 } 1855 1856 impl Drop for WithNodes<'_> { 1857 fn drop(&mut self) { 1858 core::mem::swap(&mut self.nodes, &mut self.inner.nodes); 1859 if self.nodes.iter().next().is_some() { 1860 pr_err!("nodes array was modified while using lock_with_nodes\n"); 1861 } 1862 } 1863 } 1864 1865 pub(crate) enum GetWorkOrRegister<'a> { 1866 Work(DLArc<dyn DeliverToRead>), 1867 Register(Registration<'a>), 1868 } 1869