1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 //! This module defines the `Process` type, which represents a process using a particular binder 6 //! context. 7 //! 8 //! The `Process` object keeps track of all of the resources that this process owns in the binder 9 //! context. 10 //! 11 //! There is one `Process` object for each binder fd that a process has opened, so processes using 12 //! several binder contexts have several `Process` objects. This ensures that the contexts are 13 //! fully separated. 14 15 use core::mem::take; 16 17 use kernel::{ 18 bindings, 19 bits::bit_u8, 20 cred::Credential, 21 error::Error, 22 fs::file::{self, File}, 23 id_pool::IdPool, 24 list::{List, ListArc, ListArcField, ListLinks}, 25 mm, 26 prelude::*, 27 rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation}, 28 seq_file::SeqFile, 29 seq_print, 30 sync::poll::PollTable, 31 sync::{ 32 aref::ARef, 33 lock::{spinlock::SpinLockBackend, Guard}, 34 poll::PollCondVarBox, 35 Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc, 36 }, 37 task::{Pid, Task}, 38 uaccess::{UserSlice, UserSliceReader}, 39 uapi, 40 workqueue::{self, Work}, 41 }; 42 43 use crate::{ 44 allocation::{Allocation, AllocationInfo, NewAllocation}, 45 context::Context, 46 defs::*, 47 error::{BinderError, BinderResult}, 48 node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef}, 49 page_range::ShrinkablePageRange, 50 range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs}, 51 stats::BinderStats, 52 thread::{PushWorkRes, Thread}, 53 transaction::TransactionInfo, 54 BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead, 55 }; 56 57 #[path = "freeze.rs"] 58 mod freeze; 59 use self::freeze::{FreezeCookie, FreezeListener}; 60 61 struct Mapping { 62 address: usize, 63 alloc: RangeAllocator<AllocationInfo>, 64 } 65 66 impl Mapping { 67 fn new(address: usize, size: usize) -> Self { 68 Self { 69 address, 70 alloc: RangeAllocator::new(size), 71 } 72 } 73 } 74 75 kernel::impl_flags!( 76 /// Represents multiple deferred work flags. 77 #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] 78 pub struct DeferWorks(u8); 79 80 /// Represents a single deferred work category. 81 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 82 pub enum DeferWork { 83 Flush = bit_u8(0), 84 Release = bit_u8(1), 85 } 86 ); 87 88 #[derive(Copy, Clone)] 89 pub(crate) enum IsFrozen { 90 Yes, 91 No, 92 InProgress, 93 } 94 95 impl IsFrozen { 96 /// Whether incoming transactions should be rejected due to freeze. 97 pub(crate) fn is_frozen(self) -> bool { 98 match self { 99 IsFrozen::Yes => true, 100 IsFrozen::No => false, 101 IsFrozen::InProgress => true, 102 } 103 } 104 105 /// Whether freeze notifications consider this process frozen. 106 pub(crate) fn is_fully_frozen(self) -> bool { 107 match self { 108 IsFrozen::Yes => true, 109 IsFrozen::No => false, 110 IsFrozen::InProgress => false, 111 } 112 } 113 } 114 115 /// The fields of `Process` protected by the spinlock. 116 pub(crate) struct ProcessInner { 117 is_manager: bool, 118 pub(crate) is_dead: bool, 119 threads: RBTree<i32, Arc<Thread>>, 120 /// INVARIANT: Threads pushed to this list must be owned by this process. 121 ready_threads: List<Thread>, 122 nodes: RBTree<u64, DArc<Node>>, 123 mapping: Option<Mapping>, 124 work: List<DTRWrap<dyn DeliverToRead>>, 125 delivered_deaths: List<DTRWrap<NodeDeath>, 2>, 126 127 /// The number of requested threads that haven't registered yet. 128 requested_thread_count: u32, 129 /// The maximum number of threads used by the process thread pool. 130 max_threads: u32, 131 /// The number of threads the started and registered with the thread pool. 132 started_thread_count: u32, 133 134 /// Bitmap of deferred work to do. 135 defer_work: DeferWorks, 136 137 /// Number of transactions to be transmitted before processes in freeze_wait 138 /// are woken up. 139 outstanding_txns: u32, 140 /// Process is frozen and unable to service binder transactions. 141 pub(crate) is_frozen: IsFrozen, 142 /// Process received sync transactions since last frozen. 143 pub(crate) sync_recv: bool, 144 /// Process received async transactions since last frozen. 145 pub(crate) async_recv: bool, 146 pub(crate) binderfs_file: Option<BinderfsProcFile>, 147 /// Check for oneway spam 148 oneway_spam_detection_enabled: bool, 149 } 150 151 impl ProcessInner { 152 fn new() -> Self { 153 Self { 154 is_manager: false, 155 is_dead: false, 156 threads: RBTree::new(), 157 ready_threads: List::new(), 158 mapping: None, 159 nodes: RBTree::new(), 160 work: List::new(), 161 delivered_deaths: List::new(), 162 requested_thread_count: 0, 163 max_threads: 0, 164 started_thread_count: 0, 165 defer_work: DeferWorks::default(), 166 outstanding_txns: 0, 167 is_frozen: IsFrozen::No, 168 sync_recv: false, 169 async_recv: false, 170 binderfs_file: None, 171 oneway_spam_detection_enabled: false, 172 } 173 } 174 175 /// Schedule the work item for execution on this process. 176 /// 177 /// If any threads are ready for work, then the work item is given directly to that thread and 178 /// it is woken up. Otherwise, it is pushed to the process work list. 179 /// 180 /// This call can fail only if the process is dead. In this case, the work item is returned to 181 /// the caller so that the caller can drop it after releasing the inner process lock. This is 182 /// necessary since the destructor of `Transaction` will take locks that can't necessarily be 183 /// taken while holding the inner process lock. 184 pub(crate) fn push_work( 185 &mut self, 186 proc: &Process, 187 work: DLArc<dyn DeliverToRead>, 188 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> { 189 let sync = work.should_sync_wakeup(); 190 191 // Try to find a ready thread to which to push the work. 192 if let Some(thread) = self.ready_threads.pop_front() { 193 // Push to thread while holding state lock. This prevents the thread from giving up 194 // (for example, because of a signal) when we're about to deliver work. 195 match thread.push_work_inner(work, sync) { 196 PushWorkRes::Ok => Ok(()), 197 PushWorkRes::OkNotifyPoll => { 198 proc.notify_poll(sync); 199 Ok(()) 200 } 201 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)), 202 } 203 } else if self.is_dead { 204 Err((BinderError::new_dead(), work)) 205 } else { 206 // Didn't find a thread waiting for proc work; this can happen 207 // in two scenarios: 208 // 1. All threads are busy handling transactions 209 // In that case, one of those threads should call back into 210 // the kernel driver soon and pick up this work. 211 // 2. Threads are using the (e)poll interface, in which case 212 // they may be blocked on the waitqueue without having been 213 // added to waiting_threads. For this case, we wake it up 214 // directly. 215 self.work.push_back(work); 216 217 // Wake up polling threads, if any. 218 proc.notify_poll(sync); 219 220 Ok(()) 221 } 222 } 223 224 pub(crate) fn remove_node(&mut self, ptr: u64) { 225 self.nodes.remove(&ptr); 226 } 227 228 /// Updates the reference count on the given node. 229 pub(crate) fn update_node_refcount( 230 &mut self, 231 node: &DArc<Node>, 232 inc: bool, 233 strong: bool, 234 count: usize, 235 othread: Option<&Thread>, 236 ) { 237 let push = node.update_refcount_locked(inc, strong, count, self); 238 239 // If we decided that we need to push work, push either to the process or to a thread if 240 // one is specified. 241 if let Some(pnode) = push { 242 if let Some(thread) = othread { 243 thread.push_work_deferred(pnode); 244 } else { 245 let _ = self.push_work(&node.owner, pnode); 246 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in 247 // that case, it doesn't care about the notification. 248 } 249 } 250 } 251 252 pub(crate) fn new_node_ref( 253 &mut self, 254 node: DArc<Node>, 255 strong: bool, 256 thread: Option<&Thread>, 257 ) -> NodeRef { 258 self.update_node_refcount(&node, true, strong, 1, thread); 259 let strong_count = if strong { 1 } else { 0 }; 260 NodeRef::new(node, strong_count, 1 - strong_count) 261 } 262 263 pub(crate) fn new_node_ref_with_thread( 264 &mut self, 265 node: DArc<Node>, 266 strong: bool, 267 thread: &Thread, 268 wrapper: Option<CritIncrWrapper>, 269 ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> { 270 let push = match wrapper { 271 None => node 272 .incr_refcount_allow_zero2one(strong, self)? 273 .map(|node| node as DLArc<dyn DeliverToRead>), 274 Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self), 275 }; 276 if let Some(node) = push { 277 thread.push_work_deferred(node); 278 } 279 let strong_count = if strong { 1 } else { 0 }; 280 Ok(NodeRef::new(node, strong_count, 1 - strong_count)) 281 } 282 283 /// Returns an existing node with the given pointer and cookie, if one exists. 284 /// 285 /// Returns an error if a node with the given pointer but a different cookie exists. 286 fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> { 287 match self.nodes.get(&ptr) { 288 None => Ok(None), 289 Some(node) => { 290 let (_, node_cookie) = node.get_id(); 291 if node_cookie == cookie { 292 Ok(Some(node.clone())) 293 } else { 294 Err(EINVAL) 295 } 296 } 297 } 298 } 299 300 fn register_thread(&mut self) -> bool { 301 if self.requested_thread_count == 0 { 302 return false; 303 } 304 305 self.requested_thread_count -= 1; 306 self.started_thread_count += 1; 307 true 308 } 309 310 /// Finds a delivered death notification with the given cookie, removes it from the thread's 311 /// delivered list, and returns it. 312 fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> { 313 let mut cursor = self.delivered_deaths.cursor_front(); 314 while let Some(next) = cursor.peek_next() { 315 if next.cookie == cookie { 316 return Some(next.remove().into_arc()); 317 } 318 cursor.move_next(); 319 } 320 None 321 } 322 323 pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) { 324 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 325 self.delivered_deaths.push_back(death); 326 } else { 327 pr_warn_ratelimited!("Notification added to `delivered_deaths` twice.\n"); 328 } 329 } 330 331 pub(crate) fn add_outstanding_txn(&mut self) { 332 self.outstanding_txns += 1; 333 } 334 335 fn txns_pending_locked(&self) -> bool { 336 if self.outstanding_txns > 0 { 337 return true; 338 } 339 for thread in self.threads.values() { 340 if thread.has_current_transaction() { 341 return true; 342 } 343 } 344 false 345 } 346 } 347 348 /// Used to keep track of a node that this process has a handle to. 349 #[pin_data] 350 pub(crate) struct NodeRefInfo { 351 debug_id: usize, 352 /// The refcount that this process owns to the node. 353 node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>, 354 death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>, 355 /// Cookie of the active freeze listener for this node. 356 freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>, 357 /// Used to store this `NodeRefInfo` in the node's `refs` list. 358 #[pin] 359 links: ListLinks<{ Self::LIST_NODE }>, 360 /// The handle for this `NodeRefInfo`. 361 handle: u32, 362 /// The process that has a handle to the node. 363 pub(crate) process: Arc<Process>, 364 } 365 366 impl NodeRefInfo { 367 /// The id used for the `Node::refs` list. 368 pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10; 369 /// The id used for the `ListArc` in `ProcessNodeRefs`. 370 const LIST_PROC: u64 = 0xd703a5263dcc8650; 371 372 fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> { 373 pin_init!(Self { 374 debug_id: super::next_debug_id(), 375 node_ref: ListArcField::new(node_ref), 376 death: ListArcField::new(None), 377 freeze: ListArcField::new(None), 378 links <- ListLinks::new(), 379 handle, 380 process, 381 }) 382 } 383 384 kernel::list::define_list_arc_field_getter! { 385 pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death } 386 pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze } 387 pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref } 388 pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref } 389 } 390 } 391 392 kernel::list::impl_list_arc_safe! { 393 impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; } 394 impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; } 395 } 396 kernel::list::impl_list_item! { 397 impl ListItem<{Self::LIST_NODE}> for NodeRefInfo { 398 using ListLinks { self.links }; 399 } 400 } 401 402 /// Keeps track of references this process has to nodes owned by other processes. 403 /// 404 /// TODO: Currently, the rbtree requires two allocations per node reference, and two tree 405 /// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these 406 /// extra costs should be eliminated. 407 struct ProcessNodeRefs { 408 /// Used to look up nodes using the 32-bit id that this process knows it by. 409 by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>, 410 /// Used to quickly find unused ids in `by_handle`. 411 handle_is_present: IdPool, 412 /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of 413 /// the underlying `Node` struct as returned by `Node::global_id`. 414 by_node: RBTree<usize, u32>, 415 /// Used to look up a `FreezeListener` by cookie. 416 /// 417 /// There might be multiple freeze listeners for the same node, but at most one of them is 418 /// active. 419 freeze_listeners: RBTree<FreezeCookie, FreezeListener>, 420 } 421 422 impl ProcessNodeRefs { 423 fn new() -> Self { 424 Self { 425 by_handle: RBTree::new(), 426 handle_is_present: IdPool::new(), 427 by_node: RBTree::new(), 428 freeze_listeners: RBTree::new(), 429 } 430 } 431 } 432 433 use core::mem::offset_of; 434 use kernel::bindings::rb_process_layout; 435 pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout { 436 arc_offset: Arc::<Process>::DATA_OFFSET, 437 task: offset_of!(Process, task), 438 }; 439 440 /// A process using binder. 441 /// 442 /// Strictly speaking, there can be multiple of these per process. There is one for each binder fd 443 /// that a process has opened, so processes using several binder contexts have several `Process` 444 /// objects. This ensures that the contexts are fully separated. 445 #[pin_data] 446 pub(crate) struct Process { 447 pub(crate) ctx: Arc<Context>, 448 449 // The task leader (process). 450 pub(crate) task: ARef<Task>, 451 452 // Credential associated with file when `Process` is created. 453 pub(crate) cred: ARef<Credential>, 454 455 #[pin] 456 pub(crate) inner: SpinLock<ProcessInner>, 457 458 #[pin] 459 pub(crate) pages: ShrinkablePageRange, 460 461 // Waitqueue of processes waiting for all outstanding transactions to be 462 // processed. 463 #[pin] 464 freeze_wait: CondVar, 465 466 // Node references are in a different lock to avoid recursive acquisition when 467 // incrementing/decrementing a node in another process. 468 #[pin] 469 node_refs: SpinLock<ProcessNodeRefs>, 470 471 // Synchronizes `register_wait` calls to the `PollCondVarBox`. 472 // 473 // The `PollCondVarBox` is not stored here because synchronization is 474 // done for `register_wait` only. Wakeups do not take this lock. 475 poll: SetOnce<PollCondVarBox>, 476 477 // Work node for deferred work item. 478 #[pin] 479 defer_work: Work<Process>, 480 481 // Links for process list in Context. 482 #[pin] 483 links: ListLinks, 484 485 pub(crate) stats: BinderStats, 486 } 487 488 kernel::impl_has_work! { 489 impl HasWork<Process> for Process { self.defer_work } 490 } 491 492 kernel::list::impl_list_arc_safe! { 493 impl ListArcSafe<0> for Process { untracked; } 494 } 495 kernel::list::impl_list_item! { 496 impl ListItem<0> for Process { 497 using ListLinks { self.links }; 498 } 499 } 500 501 impl workqueue::WorkItem for Process { 502 type Pointer = Arc<Process>; 503 504 fn run(me: Arc<Self>) { 505 let defer; 506 { 507 let mut inner = me.inner.lock(); 508 defer = inner.defer_work; 509 inner.defer_work = DeferWorks::default(); 510 } 511 512 if defer.contains(DeferWork::Flush) { 513 me.deferred_flush(); 514 } 515 if defer.contains(DeferWork::Release) { 516 me.deferred_release(); 517 } 518 } 519 } 520 521 impl Process { 522 fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> { 523 let current = kernel::current!(); 524 let process = Arc::pin_init::<Error>( 525 try_pin_init!(Process { 526 ctx, 527 cred, 528 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"), 529 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER), 530 node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"), 531 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"), 532 task: current.group_leader().into(), 533 defer_work <- kernel::new_work!("Process::defer_work"), 534 links <- ListLinks::new(), 535 stats: BinderStats::new(), 536 poll: SetOnce::new(), 537 }), 538 GFP_KERNEL, 539 )?; 540 541 process.ctx.register_process(process.clone())?; 542 543 Ok(process) 544 } 545 546 pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid { 547 self.task.tgid_nr_ns(None) 548 } 549 550 #[inline(never)] 551 pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> { 552 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 553 seq_print!(m, "context {}\n", &*ctx.name); 554 555 let inner = self.inner.lock(); 556 seq_print!(m, " threads: {}\n", inner.threads.iter().count()); 557 seq_print!( 558 m, 559 " requested threads: {}+{}/{}\n", 560 inner.requested_thread_count, 561 inner.started_thread_count, 562 inner.max_threads, 563 ); 564 if let Some(mapping) = &inner.mapping { 565 seq_print!( 566 m, 567 " free oneway space: {}\n", 568 mapping.alloc.free_oneway_space() 569 ); 570 seq_print!(m, " buffers: {}\n", mapping.alloc.count_buffers()); 571 } 572 seq_print!( 573 m, 574 " outstanding transactions: {}\n", 575 inner.outstanding_txns 576 ); 577 seq_print!(m, " nodes: {}\n", inner.nodes.iter().count()); 578 drop(inner); 579 580 { 581 let mut refs = self.node_refs.lock(); 582 let (mut count, mut weak, mut strong) = (0, 0, 0); 583 for r in refs.by_handle.values_mut() { 584 let node_ref = r.node_ref(); 585 let (nstrong, nweak) = node_ref.get_count(); 586 count += 1; 587 weak += nweak; 588 strong += nstrong; 589 } 590 seq_print!(m, " refs: {count} s {strong} w {weak}\n"); 591 } 592 593 self.stats.debug_print(" ", m); 594 595 Ok(()) 596 } 597 598 #[inline(never)] 599 pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> { 600 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 601 seq_print!(m, "context {}\n", &*ctx.name); 602 603 let mut all_threads = KVec::new(); 604 let mut all_nodes = KVec::new(); 605 loop { 606 let inner = self.inner.lock(); 607 let num_threads = inner.threads.iter().count(); 608 let num_nodes = inner.nodes.iter().count(); 609 610 if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes { 611 drop(inner); 612 all_threads.reserve(num_threads, GFP_KERNEL)?; 613 all_nodes.reserve(num_nodes, GFP_KERNEL)?; 614 continue; 615 } 616 617 for thread in inner.threads.values() { 618 assert!(all_threads.len() < all_threads.capacity()); 619 let _ = all_threads.push(thread.clone(), GFP_ATOMIC); 620 } 621 622 for node in inner.nodes.values() { 623 assert!(all_nodes.len() < all_nodes.capacity()); 624 let _ = all_nodes.push(node.clone(), GFP_ATOMIC); 625 } 626 627 break; 628 } 629 630 for thread in all_threads { 631 thread.debug_print(m, print_all)?; 632 } 633 634 let mut inner = self.inner.lock(); 635 for node in all_nodes { 636 if print_all || node.has_oneway_transaction(&mut inner) { 637 node.full_debug_print(m, &mut inner)?; 638 } 639 } 640 drop(inner); 641 642 if print_all { 643 let mut refs = self.node_refs.lock(); 644 for r in refs.by_handle.values_mut() { 645 let node_ref = r.node_ref(); 646 let dead = node_ref.node.owner.inner.lock().is_dead; 647 let (strong, weak) = node_ref.get_count(); 648 let debug_id = node_ref.node.debug_id; 649 650 seq_print!( 651 m, 652 " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", 653 r.debug_id, 654 r.handle, 655 if dead { "dead " } else { "" } 656 ); 657 } 658 } 659 660 let inner = self.inner.lock(); 661 for work in &inner.work { 662 work.debug_print(m, " ", " pending transaction ")?; 663 } 664 for _death in &inner.delivered_deaths { 665 seq_print!(m, " has delivered dead binder\n"); 666 } 667 if let Some(mapping) = &inner.mapping { 668 mapping.alloc.debug_print(m)?; 669 } 670 drop(inner); 671 672 Ok(()) 673 } 674 675 /// Attempts to fetch a work item from the process queue. 676 pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> { 677 self.inner.lock().work.pop_front() 678 } 679 680 /// Attempts to fetch a work item from the process queue. If none is available, it registers the 681 /// given thread as ready to receive work directly. 682 /// 683 /// This must only be called when the thread is not participating in a transaction chain; when 684 /// it is, work will always be delivered directly to the thread (and not through the process 685 /// queue). 686 pub(crate) fn get_work_or_register<'a>( 687 &'a self, 688 thread: &'a Arc<Thread>, 689 ) -> GetWorkOrRegister<'a> { 690 let mut inner = self.inner.lock(); 691 // Try to get work from the process queue. 692 if let Some(work) = inner.work.pop_front() { 693 return GetWorkOrRegister::Work(work); 694 } 695 696 // Register the thread as ready. 697 GetWorkOrRegister::Register(Registration::new(thread, &mut inner)) 698 } 699 700 fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> { 701 let id = { 702 let current = kernel::current!(); 703 if self.task != current.group_leader() { 704 pr_err_ratelimited!("get_current_thread was called from the wrong process.\n"); 705 return Err(EINVAL); 706 } 707 current.pid() 708 }; 709 710 { 711 let inner = self.inner.lock(); 712 if let Some(thread) = inner.threads.get(&id) { 713 return Ok(thread.clone()); 714 } 715 } 716 717 // Allocate a new `Thread` without holding any locks. 718 let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?; 719 let ta: Arc<Thread> = Thread::new(id, self.into())?; 720 721 let mut inner = self.inner.lock(); 722 match inner.threads.entry(id) { 723 rbtree::Entry::Vacant(entry) => { 724 entry.insert(ta.clone(), reservation); 725 Ok(ta) 726 } 727 rbtree::Entry::Occupied(_entry) => { 728 pr_err_ratelimited!("Cannot create two threads with the same id.\n"); 729 Err(EINVAL) 730 } 731 } 732 } 733 734 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { 735 // If push_work fails, drop the work item outside the lock. 736 let res = self.inner.lock().push_work(self, work); 737 match res { 738 Ok(()) => Ok(()), 739 Err((err, work)) => { 740 drop(work); 741 Err(err) 742 } 743 } 744 } 745 746 fn set_as_manager( 747 self: ArcBorrow<'_, Self>, 748 info: Option<FlatBinderObject>, 749 thread: &Thread, 750 ) -> Result { 751 let (ptr, cookie, flags) = if let Some(obj) = info { 752 ( 753 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it 754 // is safe to access the `binder` field. 755 unsafe { obj.__bindgen_anon_1.binder }, 756 obj.cookie, 757 obj.flags, 758 ) 759 } else { 760 (0, 0, 0) 761 }; 762 let node_ref = self.get_node(ptr, cookie, flags, true, thread)?; 763 let node = node_ref.node.clone(); 764 self.ctx.set_manager_node(node_ref)?; 765 self.inner.lock().is_manager = true; 766 767 // Force the state of the node to prevent the delivery of acquire/increfs. 768 let mut owner_inner = node.owner.inner.lock(); 769 node.force_has_count(&mut owner_inner); 770 Ok(()) 771 } 772 773 fn get_node_inner( 774 self: ArcBorrow<'_, Self>, 775 ptr: u64, 776 cookie: u64, 777 flags: u32, 778 strong: bool, 779 thread: &Thread, 780 wrapper: Option<CritIncrWrapper>, 781 ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> { 782 // Try to find an existing node. 783 { 784 let mut inner = self.inner.lock(); 785 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 786 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 787 } 788 } 789 790 // Allocate the node before reacquiring the lock. 791 let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc(); 792 let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?; 793 let mut inner = self.inner.lock(); 794 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 795 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 796 } 797 798 inner.nodes.insert(rbnode); 799 // This can only fail if someone has already pushed the node to a list, but we just created 800 // it and still hold the lock, so it can't fail right now. 801 let node_ref = inner 802 .new_node_ref_with_thread(node, strong, thread, wrapper) 803 .unwrap(); 804 805 Ok(Ok(node_ref)) 806 } 807 808 pub(crate) fn get_node( 809 self: ArcBorrow<'_, Self>, 810 ptr: u64, 811 cookie: u64, 812 flags: u32, 813 strong: bool, 814 thread: &Thread, 815 ) -> Result<NodeRef> { 816 let mut wrapper = None; 817 for _ in 0..2 { 818 match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) { 819 Err(err) => return Err(err), 820 Ok(Ok(node_ref)) => return Ok(node_ref), 821 Ok(Err(CouldNotDeliverCriticalIncrement)) => { 822 wrapper = Some(CritIncrWrapper::new()?); 823 } 824 } 825 } 826 // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the 827 // loop should run at most twice. 828 unreachable!() 829 } 830 831 pub(crate) fn insert_or_update_handle( 832 self: ArcBorrow<'_, Process>, 833 node_ref: NodeRef, 834 is_manager: bool, 835 ) -> Result<u32> { 836 { 837 let mut refs = self.node_refs.lock(); 838 839 // Do a lookup before inserting. 840 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { 841 let handle = *handle_ref; 842 let info = refs.by_handle.get_mut(&handle).unwrap(); 843 info.node_ref().absorb(node_ref); 844 return Ok(handle); 845 } 846 } 847 848 // Reserve memory for tree nodes. 849 let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?; 850 let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?; 851 let info = UniqueArc::new_uninit(GFP_KERNEL)?; 852 853 let mut refs_lock = self.node_refs.lock(); 854 let mut refs = &mut *refs_lock; 855 856 let (unused_id, by_handle_slot) = loop { 857 // ID 0 may only be used by the manager. 858 let start = if is_manager { 0 } else { 1 }; 859 860 if let Some(res) = refs.handle_is_present.find_unused_id(start) { 861 match refs.by_handle.entry(res.as_u32()) { 862 rbtree::Entry::Vacant(entry) => break (res, entry), 863 rbtree::Entry::Occupied(_) => { 864 pr_err_ratelimited!( 865 "Detected mismatch between handle_is_present and by_handle\n" 866 ); 867 res.acquire(); 868 kernel::warn_on!(true); 869 return Err(EINVAL); 870 } 871 } 872 } 873 874 let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?; 875 drop(refs_lock); 876 let resizer = grow_request.realloc(GFP_KERNEL)?; 877 refs_lock = self.node_refs.lock(); 878 refs = &mut *refs_lock; 879 refs.handle_is_present.grow(resizer); 880 }; 881 let handle = unused_id.as_u32(); 882 883 // Do a lookup again as node may have been inserted before the lock was reacquired. 884 let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) { 885 rbtree::Entry::Vacant(by_node_slot) => by_node_slot, 886 rbtree::Entry::Occupied(handle_ref) => { 887 // The node was inserted by another thread while we didn't hold the lock. 888 let handle = handle_ref.get(); 889 let info = refs.by_handle.get_mut(handle).unwrap(); 890 info.node_ref().absorb(node_ref); 891 return Ok(*handle); 892 } 893 }; 894 895 let (info_proc, info_node) = { 896 let info_init = NodeRefInfo::new(node_ref, handle, self.into()); 897 match info.pin_init_with(info_init) { 898 Ok(info) => ListArc::pair_from_pin_unique(info), 899 // error is infallible 900 Err(err) => match err {}, 901 } 902 }; 903 904 // Ensure the process is still alive while we insert a new reference. 905 // 906 // This releases the lock before inserting the nodes, but since `is_dead` is set as the 907 // first thing in `deferred_release`, process cleanup will not miss the items inserted into 908 // `refs` below. 909 if self.inner.lock().is_dead { 910 // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of 911 // the lock. 912 drop(refs_lock); 913 return Err(ESRCH); 914 } 915 916 // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting 917 // `info_node` into the right node's `refs` list. 918 unsafe { info_proc.node_ref2().node.insert_node_info(info_node) }; 919 920 by_node_slot.insert(handle, reserve1); 921 by_handle_slot.insert(info_proc, reserve2); 922 unused_id.acquire(); 923 Ok(handle) 924 } 925 926 pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { 927 // When handle is zero, try to get the context manager. 928 if handle == 0 { 929 let node_ref = self.ctx.get_manager_node(true)?; 930 if core::ptr::eq(self, &*node_ref.node.owner) { 931 return Err(EINVAL.into()); 932 } 933 Ok(node_ref) 934 } else { 935 match self.get_node_from_handle(handle, true) { 936 Ok(node_ref) => Ok(node_ref), 937 Err(err) => { 938 binder_debug!(UserError, "got transaction to invalid handle {handle}"); 939 Err(err.into()) 940 } 941 } 942 } 943 } 944 945 pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> { 946 self.node_refs 947 .lock() 948 .by_handle 949 .get_mut(&handle) 950 .ok_or(ENOENT)? 951 .node_ref() 952 .clone(strong) 953 } 954 955 pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) { 956 let mut inner = self.inner.lock(); 957 // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list. 958 let removed = unsafe { inner.delivered_deaths.remove(death) }; 959 drop(inner); 960 drop(removed); 961 } 962 963 pub(crate) fn update_ref( 964 self: ArcBorrow<'_, Process>, 965 handle: u32, 966 inc: bool, 967 strong: bool, 968 ) -> Result { 969 if inc && handle == 0 { 970 if let Ok(node_ref) = self.ctx.get_manager_node(strong) { 971 if core::ptr::eq(&*self, &*node_ref.node.owner) { 972 return Err(EINVAL); 973 } 974 let _ = self.insert_or_update_handle(node_ref, true); 975 return Ok(()); 976 } 977 } 978 979 // To preserve original binder behaviour, we only fail requests where the manager tries to 980 // increment references on itself. 981 let _to_free_by_handle; 982 let _to_free_by_node; 983 let _to_free_freeze_listener; 984 let _to_free_freeze_listener_cleanup; 985 let mut refs = self.node_refs.lock(); 986 if let Some(info) = refs.by_handle.get_mut(&handle) { 987 if info.node_ref().update(inc, strong) { 988 // Clean up death if there is one attached to this node reference. 989 // 990 // We remove the entire `info` below, so no need to remove `death` from `info`. 991 if let Some(death) = info.death().as_ref() { 992 death.set_cleared(true); 993 self.remove_from_delivered_deaths(death); 994 } 995 996 // Remove reference from process tables, and from the node's `refs` list. 997 998 // SAFETY: We are removing the `NodeRefInfo` from the right node. 999 unsafe { info.node_ref2().node.remove_node_info(info) }; 1000 1001 let id = info.node_ref().node.global_id(); 1002 1003 if let Some(freeze) = *info.freeze() { 1004 if let Some(fl) = refs.freeze_listeners.remove(&freeze) { 1005 _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self); 1006 _to_free_freeze_listener = fl; 1007 } 1008 } 1009 1010 _to_free_by_handle = refs.by_handle.remove_node(&handle); 1011 _to_free_by_node = refs.by_node.remove_node(&id); 1012 refs.handle_is_present.release_id(handle as usize); 1013 1014 if let Some(shrink) = refs.handle_is_present.shrink_request() { 1015 drop(refs); 1016 // This intentionally ignores allocation failures. 1017 if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) { 1018 refs = self.node_refs.lock(); 1019 refs.handle_is_present.shrink(new_bitmap); 1020 } 1021 } 1022 } 1023 } else { 1024 // All refs are cleared in process exit, so this warning is expected in that case. 1025 if !self.inner.lock().is_dead { 1026 binder_debug!(UserError, "no such ref {handle}"); 1027 } 1028 } 1029 Ok(()) 1030 } 1031 1032 /// Decrements the refcount of the given node, if one exists. 1033 pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) { 1034 let mut inner = self.inner.lock(); 1035 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1036 inner.update_node_refcount(&node, false, strong, 1, None); 1037 } 1038 } 1039 1040 pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result { 1041 let ptr = reader.read::<u64>()?; 1042 let cookie = reader.read::<u64>()?; 1043 let mut inner = self.inner.lock(); 1044 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1045 if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) { 1046 // This only fails if the process is dead. 1047 let _ = inner.push_work(self, node); 1048 } 1049 } 1050 Ok(()) 1051 } 1052 1053 pub(crate) fn buffer_alloc( 1054 self: &Arc<Self>, 1055 debug_id: usize, 1056 size: usize, 1057 info: &mut TransactionInfo, 1058 ) -> BinderResult<NewAllocation> { 1059 use kernel::page::PAGE_SIZE; 1060 1061 let mut reserve_new_args = ReserveNewArgs { 1062 debug_id, 1063 size, 1064 is_oneway: info.is_oneway(), 1065 pid: info.from_pid, 1066 ..ReserveNewArgs::default() 1067 }; 1068 1069 let (new_alloc, addr) = loop { 1070 let mut inner = self.inner.lock(); 1071 let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?; 1072 let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? { 1073 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address), 1074 ReserveNew::NeedAlloc(request) => request, 1075 }; 1076 drop(inner); 1077 // We need to allocate memory and then call `reserve_new` again. 1078 reserve_new_args = alloc_request.make_alloc()?; 1079 }; 1080 1081 info.oneway_spam_suspect = new_alloc.oneway_spam_detected; 1082 let res = Allocation::new( 1083 self.clone(), 1084 debug_id, 1085 new_alloc.offset, 1086 size, 1087 addr + new_alloc.offset, 1088 ); 1089 1090 // This allocation will be marked as in use until the `Allocation` is used to free it. 1091 // 1092 // This method can't be called while holding a lock, so we release the lock first. It's 1093 // okay for several threads to use the method on the same index at the same time. In that 1094 // case, one of the calls will allocate the given page (if missing), and the other call 1095 // will wait for the other call to finish allocating the page. 1096 // 1097 // We will not call `stop_using_range` in parallel with this on the same page, because the 1098 // allocation can only be removed via the destructor of the `Allocation` object that we 1099 // currently own. 1100 match self.pages.use_range( 1101 new_alloc.offset / PAGE_SIZE, 1102 (new_alloc.offset + size).div_ceil(PAGE_SIZE), 1103 ) { 1104 Ok(()) => {} 1105 Err(err) => { 1106 pr_warn_ratelimited!("use_range failure {:?}\n", err); 1107 return Err(err.into()); 1108 } 1109 } 1110 1111 Ok(NewAllocation(res)) 1112 } 1113 1114 pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> { 1115 let mut inner = self.inner.lock(); 1116 let mapping = inner.mapping.as_mut()?; 1117 let offset = ptr.checked_sub(mapping.address)?; 1118 let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?; 1119 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1120 if let Some(data) = odata { 1121 alloc.set_info(data); 1122 } 1123 Some(alloc) 1124 } 1125 1126 pub(crate) fn buffer_raw_free(&self, ptr: usize) { 1127 let mut inner = self.inner.lock(); 1128 if let Some(ref mut mapping) = &mut inner.mapping { 1129 let offset = match ptr.checked_sub(mapping.address) { 1130 Some(offset) => offset, 1131 None => return, 1132 }; 1133 1134 let freed_range = match mapping.alloc.reservation_abort(offset) { 1135 Ok(freed_range) => freed_range, 1136 Err(_) => { 1137 pr_warn_ratelimited!( 1138 "Pointer {:x} failed to free, base = {:x}\n", 1139 ptr, 1140 mapping.address 1141 ); 1142 return; 1143 } 1144 }; 1145 1146 // No more allocations in this range. Mark them as not in use. 1147 // 1148 // Must be done before we release the lock so that `use_range` is not used on these 1149 // indices until `stop_using_range` returns. 1150 self.pages 1151 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx); 1152 } 1153 } 1154 1155 pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) { 1156 let mut inner = self.inner.lock(); 1157 if let Some(ref mut mapping) = &mut inner.mapping { 1158 if mapping.alloc.reservation_commit(offset, &mut data).is_err() { 1159 pr_warn_ratelimited!("Offset {} failed to be marked freeable\n", offset); 1160 } 1161 } 1162 } 1163 1164 fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result { 1165 use kernel::page::PAGE_SIZE; 1166 let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); 1167 let mapping = Mapping::new(vma.start(), size); 1168 let page_count = self.pages.register_with_vma(vma)?; 1169 if page_count * PAGE_SIZE != size { 1170 return Err(EINVAL); 1171 } 1172 1173 // Save range allocator for later. 1174 self.inner.lock().mapping = Some(mapping); 1175 1176 Ok(()) 1177 } 1178 1179 fn version(&self, data: UserSlice) -> Result { 1180 data.writer().write(&BinderVersion::current()) 1181 } 1182 1183 pub(crate) fn register_thread(&self) -> bool { 1184 self.inner.lock().register_thread() 1185 } 1186 1187 fn remove_thread(&self, thread: Arc<Thread>) { 1188 self.inner.lock().threads.remove(&thread.id); 1189 thread.release(); 1190 } 1191 1192 fn set_max_threads(&self, max: u32) { 1193 self.inner.lock().max_threads = max; 1194 } 1195 1196 fn set_oneway_spam_detection_enabled(&self, enabled: u32) { 1197 self.inner.lock().oneway_spam_detection_enabled = enabled != 0; 1198 } 1199 1200 pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool { 1201 self.inner.lock().oneway_spam_detection_enabled 1202 } 1203 1204 fn get_node_debug_info(&self, data: UserSlice) -> Result { 1205 let (mut reader, mut writer) = data.reader_writer(); 1206 1207 // Read the starting point. 1208 let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr; 1209 let mut out = BinderNodeDebugInfo::default(); 1210 1211 { 1212 let inner = self.inner.lock(); 1213 for (node_ptr, node) in &inner.nodes { 1214 if *node_ptr > ptr { 1215 node.populate_debug_info(&mut out, &inner); 1216 break; 1217 } 1218 } 1219 } 1220 1221 writer.write(&out) 1222 } 1223 1224 fn get_node_info_from_ref(&self, data: UserSlice) -> Result { 1225 let (mut reader, mut writer) = data.reader_writer(); 1226 let mut out = reader.read::<BinderNodeInfoForRef>()?; 1227 1228 if out.strong_count != 0 1229 || out.weak_count != 0 1230 || out.reserved1 != 0 1231 || out.reserved2 != 0 1232 || out.reserved3 != 0 1233 { 1234 return Err(EINVAL); 1235 } 1236 1237 // Only the context manager is allowed to use this ioctl. 1238 if !self.inner.lock().is_manager { 1239 return Err(EPERM); 1240 } 1241 1242 { 1243 let mut node_refs = self.node_refs.lock(); 1244 let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?; 1245 let node_ref = node_info.node_ref(); 1246 let owner_inner = node_ref.node.owner.inner.lock(); 1247 node_ref.node.populate_counts(&mut out, &owner_inner); 1248 } 1249 1250 // Write the result back. 1251 writer.write(&out) 1252 } 1253 1254 pub(crate) fn needs_thread(&self) -> bool { 1255 let mut inner = self.inner.lock(); 1256 let ret = inner.requested_thread_count == 0 1257 && inner.ready_threads.is_empty() 1258 && inner.started_thread_count < inner.max_threads; 1259 if ret { 1260 inner.requested_thread_count += 1 1261 } 1262 ret 1263 } 1264 1265 pub(crate) fn request_death( 1266 self: &Arc<Self>, 1267 reader: &mut UserSliceReader, 1268 thread: &Thread, 1269 ) -> Result { 1270 let handle: u32 = reader.read()?; 1271 let cookie: u64 = reader.read()?; 1272 1273 // Queue BR_ERROR if we can't allocate memory for the death notification. 1274 let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| { 1275 thread.push_return_work(BR_ERROR); 1276 binder_debug!( 1277 DeathNotification, 1278 "BC_REQUEST_DEATH_NOTIFICATION failed due to memory allocation failure" 1279 ); 1280 })?; 1281 let mut refs = self.node_refs.lock(); 1282 let Some(info) = refs.by_handle.get_mut(&handle) else { 1283 binder_debug!( 1284 UserError, 1285 "BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}" 1286 ); 1287 return Ok(()); 1288 }; 1289 1290 // Nothing to do if there is already a death notification request for this handle. 1291 if info.death().is_some() { 1292 binder_debug!( 1293 UserError, 1294 "BC_REQUEST_DEATH_NOTIFICATION death notification already set" 1295 ); 1296 return Ok(()); 1297 } 1298 1299 let death = { 1300 let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie); 1301 match death.pin_init_with(death_init) { 1302 Ok(death) => death, 1303 // error is infallible 1304 Err(err) => match err {}, 1305 } 1306 }; 1307 1308 // Register the death notification. 1309 { 1310 let owner = info.node_ref2().node.owner.clone(); 1311 let mut owner_inner = owner.inner.lock(); 1312 if owner_inner.is_dead { 1313 let death = Arc::from(death); 1314 *info.death() = Some(death.clone()); 1315 drop(owner_inner); 1316 death.set_dead(); 1317 } else { 1318 let death = ListArc::from(death); 1319 *info.death() = Some(death.clone_arc()); 1320 info.node_ref().node.add_death(death, &mut owner_inner); 1321 } 1322 } 1323 binder_debug!( 1324 DeathNotification, 1325 "BC_REQUEST_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1326 cookie 1327 ); 1328 Ok(()) 1329 } 1330 1331 pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result { 1332 let handle: u32 = reader.read()?; 1333 let cookie: u64 = reader.read()?; 1334 1335 let mut refs = self.node_refs.lock(); 1336 let Some(info) = refs.by_handle.get_mut(&handle) else { 1337 binder_debug!( 1338 UserError, 1339 "BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}" 1340 ); 1341 return Ok(()); 1342 }; 1343 1344 let Some(death) = info.death().take() else { 1345 binder_debug!( 1346 UserError, 1347 "BC_CLEAR_DEATH_NOTIFICATION death notification not active" 1348 ); 1349 return Ok(()); 1350 }; 1351 if death.cookie != cookie { 1352 *info.death() = Some(death); 1353 binder_debug!( 1354 UserError, 1355 "BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch" 1356 ); 1357 return Ok(()); 1358 } 1359 1360 // Update state and determine if we need to queue a work item. We only need to do it when 1361 // the node is not dead or if the user already completed the death notification. 1362 let should_schedule = death.set_cleared(false); 1363 drop(refs); 1364 1365 if should_schedule { 1366 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 1367 let _ = thread.push_work_if_looper(death); 1368 } 1369 } 1370 1371 binder_debug!( 1372 DeathNotification, 1373 "BC_CLEAR_DEATH_NOTIFICATION handle {handle} cookie {:016x}", 1374 cookie 1375 ); 1376 Ok(()) 1377 } 1378 1379 pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { 1380 let death = self.inner.lock().pull_delivered_death(cookie); 1381 if let Some(death) = death { 1382 death.set_notification_done(thread); 1383 } 1384 } 1385 1386 /// Locks the spinlock and move the `nodes` rbtree out. 1387 /// 1388 /// This allows you to iterate through `nodes` while also allowing you to give other parts of 1389 /// the codebase exclusive access to `ProcessInner`. 1390 pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> { 1391 let mut inner = self.inner.lock(); 1392 WithNodes { 1393 nodes: take(&mut inner.nodes), 1394 inner, 1395 } 1396 } 1397 1398 fn deferred_flush(&self) { 1399 binder_debug!(pid = self.task.pid(), OpenClose, "flushing process"); 1400 let inner = self.inner.lock(); 1401 for thread in inner.threads.values() { 1402 thread.exit_looper(); 1403 } 1404 } 1405 1406 fn deferred_release(self: Arc<Self>) { 1407 binder_debug!(pid = self.task.pid(), OpenClose, "releasing process"); 1408 1409 let is_manager = { 1410 let mut inner = self.inner.lock(); 1411 inner.is_dead = true; 1412 inner.is_frozen = IsFrozen::No; 1413 inner.sync_recv = false; 1414 inner.async_recv = false; 1415 inner.is_manager 1416 }; 1417 1418 if is_manager { 1419 self.ctx.unset_manager_node(); 1420 } 1421 1422 self.ctx.deregister_process(&self); 1423 1424 let binderfs_file = self.inner.lock().binderfs_file.take(); 1425 drop(binderfs_file); 1426 1427 // Release threads. 1428 let threads = { 1429 let mut inner = self.inner.lock(); 1430 let threads = take(&mut inner.threads); 1431 let ready = take(&mut inner.ready_threads); 1432 drop(inner); 1433 drop(ready); 1434 1435 for thread in threads.values() { 1436 thread.release(); 1437 } 1438 threads 1439 }; 1440 1441 // Release nodes. 1442 { 1443 while let Some(node) = { 1444 let mut lock = self.inner.lock(); 1445 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) 1446 } { 1447 node.to_key_value().1.release(); 1448 } 1449 } 1450 1451 // Clean up death listeners and remove nodes from external node info lists. 1452 for info in self.node_refs.lock().by_handle.values_mut() { 1453 // SAFETY: We are removing the `NodeRefInfo` from the right node. 1454 unsafe { info.node_ref2().node.remove_node_info(info) }; 1455 1456 // Clear death notifications from the nodes (that belong to a different process). 1457 // No need to remove them from `info` as we clear info below. 1458 if let Some(death) = info.death().as_ref() { 1459 death.set_cleared(false); 1460 } 1461 } 1462 1463 // Clean up freeze listeners. 1464 let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners); 1465 for listener in freeze_listeners.values() { 1466 listener.on_process_cleanup(&self); 1467 } 1468 drop(freeze_listeners); 1469 1470 // Release refs on foreign nodes. 1471 { 1472 let mut refs = self.node_refs.lock(); 1473 let by_handle = take(&mut refs.by_handle); 1474 let by_node = take(&mut refs.by_node); 1475 drop(refs); 1476 drop(by_node); 1477 drop(by_handle); 1478 } 1479 1480 // Cancel all pending work items. 1481 while let Some(work) = self.get_work() { 1482 work.into_arc().cancel(); 1483 } 1484 1485 // Clear delivered_deaths list. 1486 // 1487 // Scope ensures that MutexGuard is dropped while executing the body. 1488 while let Some(delivered_death) = { 1489 // Explicitly bind to avoid tail expression lifetime extension of the lockguard 1490 // Can be removed when the kernel moves to edition 2024 1491 let maybe_death = self.inner.lock().delivered_deaths.pop_front(); 1492 maybe_death 1493 } { 1494 drop(delivered_death); 1495 } 1496 1497 // Free any resources kept alive by allocated buffers. 1498 let omapping = self.inner.lock().mapping.take(); 1499 if let Some(mut mapping) = omapping { 1500 let address = mapping.address; 1501 mapping 1502 .alloc 1503 .take_for_each(|offset, size, debug_id, odata| { 1504 let ptr = offset + address; 1505 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1506 if let Some(data) = odata { 1507 alloc.set_info(data); 1508 } 1509 drop(alloc) 1510 }); 1511 } 1512 1513 // calls to synchronize_rcu() in thread drop will happen here 1514 drop(threads); 1515 } 1516 1517 pub(crate) fn drop_outstanding_txn(&self) { 1518 let wake = { 1519 let mut inner = self.inner.lock(); 1520 if inner.outstanding_txns == 0 { 1521 pr_err_ratelimited!("outstanding_txns underflow\n"); 1522 return; 1523 } 1524 inner.outstanding_txns -= 1; 1525 inner.is_frozen.is_frozen() && inner.outstanding_txns == 0 1526 }; 1527 1528 if wake { 1529 self.freeze_wait.notify_all(); 1530 } 1531 } 1532 1533 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 1534 // mangled symbol names. 1535 #[export_name = "rust_binder_freeze"] 1536 pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result { 1537 if info.enable == 0 { 1538 let msgs = self.prepare_freeze_messages()?; 1539 let mut inner = self.inner.lock(); 1540 inner.sync_recv = false; 1541 inner.async_recv = false; 1542 inner.is_frozen = IsFrozen::No; 1543 drop(inner); 1544 msgs.send_messages(); 1545 return Ok(()); 1546 } 1547 1548 let mut inner = self.inner.lock(); 1549 inner.sync_recv = false; 1550 inner.async_recv = false; 1551 inner.is_frozen = IsFrozen::InProgress; 1552 1553 if info.timeout_ms > 0 { 1554 let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms); 1555 while jiffies > 0 { 1556 if inner.outstanding_txns == 0 { 1557 break; 1558 } 1559 1560 match self 1561 .freeze_wait 1562 .wait_interruptible_timeout(&mut inner, jiffies) 1563 { 1564 CondVarTimeoutResult::Signal { .. } => { 1565 inner.is_frozen = IsFrozen::No; 1566 return Err(ERESTARTSYS); 1567 } 1568 CondVarTimeoutResult::Woken { jiffies: remaining } => { 1569 jiffies = remaining; 1570 } 1571 CondVarTimeoutResult::Timeout => { 1572 jiffies = 0; 1573 } 1574 } 1575 } 1576 } 1577 1578 if inner.txns_pending_locked() { 1579 inner.is_frozen = IsFrozen::No; 1580 Err(EAGAIN) 1581 } else { 1582 drop(inner); 1583 match self.prepare_freeze_messages() { 1584 Ok(batch) => { 1585 self.inner.lock().is_frozen = IsFrozen::Yes; 1586 batch.send_messages(); 1587 Ok(()) 1588 } 1589 Err(kernel::alloc::AllocError) => { 1590 self.inner.lock().is_frozen = IsFrozen::No; 1591 Err(ENOMEM) 1592 } 1593 } 1594 } 1595 } 1596 1597 pub(crate) fn notify_poll(&self, sync: bool) { 1598 if let Some(poll) = self.poll.as_ref() { 1599 if sync { 1600 poll.notify_sync(); 1601 } 1602 poll.notify_all(); 1603 } 1604 } 1605 } 1606 1607 fn get_frozen_status(data: UserSlice) -> Result { 1608 let (mut reader, mut writer) = data.reader_writer(); 1609 1610 let mut info = reader.read::<BinderFrozenStatusInfo>()?; 1611 info.sync_recv = 0; 1612 info.async_recv = 0; 1613 let mut found = false; 1614 1615 for ctx in crate::context::get_all_contexts()? { 1616 ctx.for_each_proc(|proc| { 1617 if proc.task.pid() == info.pid as Pid { 1618 found = true; 1619 let inner = proc.inner.lock(); 1620 let txns_pending = inner.txns_pending_locked(); 1621 info.async_recv |= u32::from(inner.async_recv); 1622 info.sync_recv |= u32::from(inner.sync_recv); 1623 info.sync_recv |= u32::from(txns_pending) << 1; 1624 } 1625 }); 1626 } 1627 1628 if found { 1629 writer.write(&info)?; 1630 Ok(()) 1631 } else { 1632 Err(EINVAL) 1633 } 1634 } 1635 1636 fn ioctl_freeze(reader: &mut UserSliceReader) -> Result { 1637 let info = reader.read::<BinderFreezeInfo>()?; 1638 1639 // Very unlikely for there to be more than 3, since a process normally uses at most binder and 1640 // hwbinder. 1641 let mut procs = KVec::with_capacity(3, GFP_KERNEL)?; 1642 1643 let ctxs = crate::context::get_all_contexts()?; 1644 for ctx in ctxs { 1645 for proc in ctx.get_procs_with_pid(info.pid as i32)? { 1646 procs.push(proc, GFP_KERNEL)?; 1647 } 1648 } 1649 1650 for proc in procs { 1651 proc.ioctl_freeze(&info)?; 1652 } 1653 Ok(()) 1654 } 1655 1656 /// The ioctl handler. 1657 impl Process { 1658 /// Ioctls that are write-only from the perspective of userspace. 1659 /// 1660 /// The kernel will only read from the pointer that userspace provided to us. 1661 fn ioctl_write_only( 1662 this: ArcBorrow<'_, Process>, 1663 _file: &File, 1664 cmd: u32, 1665 reader: &mut UserSliceReader, 1666 ) -> Result { 1667 if cmd == uapi::BINDER_FREEZE { 1668 return ioctl_freeze(reader); 1669 } 1670 1671 let thread = this.get_current_thread()?; 1672 match cmd { 1673 uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), 1674 uapi::BINDER_THREAD_EXIT => this.remove_thread(thread), 1675 uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?, 1676 uapi::BINDER_SET_CONTEXT_MGR_EXT => { 1677 this.set_as_manager(Some(reader.read()?), &thread)? 1678 } 1679 uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => { 1680 this.set_oneway_spam_detection_enabled(reader.read()?) 1681 } 1682 _ => return Err(EINVAL), 1683 } 1684 Ok(()) 1685 } 1686 1687 /// Ioctls that are read/write from the perspective of userspace. 1688 /// 1689 /// The kernel will both read from and write to the pointer that userspace provided to us. 1690 fn ioctl_write_read( 1691 this: ArcBorrow<'_, Process>, 1692 file: &File, 1693 cmd: u32, 1694 data: UserSlice, 1695 ) -> Result { 1696 let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0; 1697 match cmd { 1698 uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?, 1699 uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?, 1700 uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?, 1701 uapi::BINDER_VERSION => this.version(data)?, 1702 uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?, 1703 uapi::BINDER_GET_EXTENDED_ERROR => { 1704 this.get_current_thread()?.get_extended_error(data)? 1705 } 1706 _ => return Err(EINVAL), 1707 } 1708 Ok(()) 1709 } 1710 } 1711 1712 /// The file operations supported by `Process`. 1713 impl Process { 1714 pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> { 1715 let proc = Self::new(ctx.into(), ARef::from(file.cred()))?; 1716 binder_debug!(OpenClose, "opened process"); 1717 Ok(proc) 1718 } 1719 1720 pub(crate) fn release(this: Arc<Process>, _file: &File) { 1721 let binderfs_file; 1722 let should_schedule; 1723 { 1724 let mut inner = this.inner.lock(); 1725 should_schedule = inner.defer_work == DeferWorks::empty(); 1726 inner.defer_work |= DeferWork::Release; 1727 binderfs_file = inner.binderfs_file.take(); 1728 } 1729 1730 if should_schedule { 1731 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1732 // scheduled for execution. 1733 let _ = workqueue::system().enqueue(this); 1734 } 1735 1736 drop(binderfs_file); 1737 } 1738 1739 pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result { 1740 let should_schedule; 1741 { 1742 let mut inner = this.inner.lock(); 1743 should_schedule = inner.defer_work == DeferWorks::empty(); 1744 inner.defer_work |= DeferWork::Flush; 1745 } 1746 1747 if should_schedule { 1748 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1749 // scheduled for execution. 1750 let _ = workqueue::system().enqueue(Arc::from(this)); 1751 } 1752 Ok(()) 1753 } 1754 1755 pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result { 1756 use kernel::ioctl::{_IOC_DIR, _IOC_SIZE}; 1757 use kernel::uapi::{_IOC_READ, _IOC_WRITE}; 1758 1759 crate::trace::trace_ioctl(cmd, arg); 1760 1761 let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd)); 1762 1763 const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE; 1764 1765 let res = match _IOC_DIR(cmd) { 1766 _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()), 1767 _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice), 1768 _ => Err(EINVAL), 1769 }; 1770 1771 crate::trace::trace_ioctl_done(res); 1772 res 1773 } 1774 1775 pub(crate) fn mmap( 1776 this: ArcBorrow<'_, Process>, 1777 _file: &File, 1778 vma: &mm::virt::VmaNew, 1779 ) -> Result { 1780 // We don't allow mmap to be used in a different process. 1781 if this.task != kernel::current!().group_leader() { 1782 return Err(EINVAL); 1783 } 1784 if vma.start() == 0 { 1785 return Err(EINVAL); 1786 } 1787 1788 vma.try_clear_maywrite().map_err(|_| EPERM)?; 1789 vma.set_dontcopy(); 1790 vma.set_mixedmap(); 1791 1792 // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it. 1793 this.create_mapping(vma) 1794 } 1795 1796 pub(crate) fn poll( 1797 this: ArcBorrow<'_, Process>, 1798 file: &File, 1799 table: PollTable<'_>, 1800 ) -> Result<u32> { 1801 let thread = this.get_current_thread()?; 1802 { 1803 let poll = loop { 1804 if let Some(poll) = this.poll.as_ref() { 1805 break poll; 1806 } 1807 1808 let poll = PollCondVarBox::new(c"Process::poll", kernel::static_lock_class!())?; 1809 // Reuse our existing lock to synchronize callers initializing. 1810 let _guard = this.node_refs.lock(); 1811 this.poll.populate(poll); 1812 }; 1813 1814 table.register_wait(file, poll); 1815 } 1816 let (from_proc, mut mask) = thread.poll()?; 1817 if mask == 0 && from_proc && !this.inner.lock().work.is_empty() { 1818 mask |= bindings::POLLIN; 1819 } 1820 Ok(mask) 1821 } 1822 } 1823 1824 /// Represents that a thread has registered with the `ready_threads` list of its process. 1825 /// 1826 /// The destructor of this type will unregister the thread from the list of ready threads. 1827 pub(crate) struct Registration<'a> { 1828 thread: &'a Arc<Thread>, 1829 } 1830 1831 impl<'a> Registration<'a> { 1832 fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self { 1833 assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref())); 1834 // INVARIANT: We are pushing this thread to the right `ready_threads` list. 1835 if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) { 1836 guard.ready_threads.push_front(list_arc); 1837 } else { 1838 // It is an error to hit this branch, and it should not be reachable. We try to do 1839 // something reasonable when the failure path happens. Most likely, the thread in 1840 // question will sleep forever. 1841 pr_err_ratelimited!("Same thread registered with `ready_threads` twice.\n"); 1842 } 1843 Self { thread } 1844 } 1845 } 1846 1847 impl Drop for Registration<'_> { 1848 fn drop(&mut self) { 1849 let mut inner = self.thread.process.inner.lock(); 1850 // SAFETY: The thread has the invariant that we never push it to any other linked list than 1851 // the `ready_threads` list of its parent process. Therefore, the thread is either in that 1852 // list, or in no list. 1853 unsafe { inner.ready_threads.remove(self.thread) }; 1854 } 1855 } 1856 1857 pub(crate) struct WithNodes<'a> { 1858 pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>, 1859 pub(crate) nodes: RBTree<u64, DArc<Node>>, 1860 } 1861 1862 impl Drop for WithNodes<'_> { 1863 fn drop(&mut self) { 1864 core::mem::swap(&mut self.nodes, &mut self.inner.nodes); 1865 if self.nodes.iter().next().is_some() { 1866 pr_err_ratelimited!("nodes array was modified while using lock_with_nodes\n"); 1867 } 1868 } 1869 } 1870 1871 pub(crate) enum GetWorkOrRegister<'a> { 1872 Work(DLArc<dyn DeliverToRead>), 1873 Register(Registration<'a>), 1874 } 1875