1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 //! This module defines the `Process` type, which represents a process using a particular binder 6 //! context. 7 //! 8 //! The `Process` object keeps track of all of the resources that this process owns in the binder 9 //! context. 10 //! 11 //! There is one `Process` object for each binder fd that a process has opened, so processes using 12 //! several binder contexts have several `Process` objects. This ensures that the contexts are 13 //! fully separated. 14 15 use core::mem::take; 16 17 use kernel::{ 18 bindings, 19 cred::Credential, 20 error::Error, 21 fs::file::{self, File}, 22 id_pool::IdPool, 23 list::{List, ListArc, ListArcField, ListLinks}, 24 mm, 25 prelude::*, 26 rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation}, 27 seq_file::SeqFile, 28 seq_print, 29 sync::poll::PollTable, 30 sync::{ 31 aref::ARef, 32 lock::{spinlock::SpinLockBackend, Guard}, 33 Arc, ArcBorrow, CondVar, CondVarTimeoutResult, Mutex, SpinLock, UniqueArc, 34 }, 35 task::Task, 36 uaccess::{UserSlice, UserSliceReader}, 37 uapi, 38 workqueue::{self, Work}, 39 }; 40 41 use crate::{ 42 allocation::{Allocation, AllocationInfo, NewAllocation}, 43 context::Context, 44 defs::*, 45 error::{BinderError, BinderResult}, 46 node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef}, 47 page_range::ShrinkablePageRange, 48 range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs}, 49 stats::BinderStats, 50 thread::{PushWorkRes, Thread}, 51 transaction::TransactionInfo, 52 BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead, 53 }; 54 55 #[path = "freeze.rs"] 56 mod freeze; 57 use self::freeze::{FreezeCookie, FreezeListener}; 58 59 struct Mapping { 60 address: usize, 61 alloc: RangeAllocator<AllocationInfo>, 62 } 63 64 impl Mapping { 65 fn new(address: usize, size: usize) -> Self { 66 Self { 67 address, 68 alloc: RangeAllocator::new(size), 69 } 70 } 71 } 72 73 // bitflags for defer_work. 74 const PROC_DEFER_FLUSH: u8 = 1; 75 const PROC_DEFER_RELEASE: u8 = 2; 76 77 #[derive(Copy, Clone)] 78 pub(crate) enum IsFrozen { 79 Yes, 80 No, 81 InProgress, 82 } 83 84 impl IsFrozen { 85 /// Whether incoming transactions should be rejected due to freeze. 86 pub(crate) fn is_frozen(self) -> bool { 87 match self { 88 IsFrozen::Yes => true, 89 IsFrozen::No => false, 90 IsFrozen::InProgress => true, 91 } 92 } 93 94 /// Whether freeze notifications consider this process frozen. 95 pub(crate) fn is_fully_frozen(self) -> bool { 96 match self { 97 IsFrozen::Yes => true, 98 IsFrozen::No => false, 99 IsFrozen::InProgress => false, 100 } 101 } 102 } 103 104 /// The fields of `Process` protected by the spinlock. 105 pub(crate) struct ProcessInner { 106 is_manager: bool, 107 pub(crate) is_dead: bool, 108 threads: RBTree<i32, Arc<Thread>>, 109 /// INVARIANT: Threads pushed to this list must be owned by this process. 110 ready_threads: List<Thread>, 111 nodes: RBTree<u64, DArc<Node>>, 112 mapping: Option<Mapping>, 113 work: List<DTRWrap<dyn DeliverToRead>>, 114 delivered_deaths: List<DTRWrap<NodeDeath>, 2>, 115 116 /// The number of requested threads that haven't registered yet. 117 requested_thread_count: u32, 118 /// The maximum number of threads used by the process thread pool. 119 max_threads: u32, 120 /// The number of threads the started and registered with the thread pool. 121 started_thread_count: u32, 122 123 /// Bitmap of deferred work to do. 124 defer_work: u8, 125 126 /// Number of transactions to be transmitted before processes in freeze_wait 127 /// are woken up. 128 outstanding_txns: u32, 129 /// Process is frozen and unable to service binder transactions. 130 pub(crate) is_frozen: IsFrozen, 131 /// Process received sync transactions since last frozen. 132 pub(crate) sync_recv: bool, 133 /// Process received async transactions since last frozen. 134 pub(crate) async_recv: bool, 135 pub(crate) binderfs_file: Option<BinderfsProcFile>, 136 /// Check for oneway spam 137 oneway_spam_detection_enabled: bool, 138 } 139 140 impl ProcessInner { 141 fn new() -> Self { 142 Self { 143 is_manager: false, 144 is_dead: false, 145 threads: RBTree::new(), 146 ready_threads: List::new(), 147 mapping: None, 148 nodes: RBTree::new(), 149 work: List::new(), 150 delivered_deaths: List::new(), 151 requested_thread_count: 0, 152 max_threads: 0, 153 started_thread_count: 0, 154 defer_work: 0, 155 outstanding_txns: 0, 156 is_frozen: IsFrozen::No, 157 sync_recv: false, 158 async_recv: false, 159 binderfs_file: None, 160 oneway_spam_detection_enabled: false, 161 } 162 } 163 164 /// Schedule the work item for execution on this process. 165 /// 166 /// If any threads are ready for work, then the work item is given directly to that thread and 167 /// it is woken up. Otherwise, it is pushed to the process work list. 168 /// 169 /// This call can fail only if the process is dead. In this case, the work item is returned to 170 /// the caller so that the caller can drop it after releasing the inner process lock. This is 171 /// necessary since the destructor of `Transaction` will take locks that can't necessarily be 172 /// taken while holding the inner process lock. 173 pub(crate) fn push_work( 174 &mut self, 175 work: DLArc<dyn DeliverToRead>, 176 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> { 177 // Try to find a ready thread to which to push the work. 178 if let Some(thread) = self.ready_threads.pop_front() { 179 // Push to thread while holding state lock. This prevents the thread from giving up 180 // (for example, because of a signal) when we're about to deliver work. 181 match thread.push_work(work) { 182 PushWorkRes::Ok => Ok(()), 183 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)), 184 } 185 } else if self.is_dead { 186 Err((BinderError::new_dead(), work)) 187 } else { 188 let sync = work.should_sync_wakeup(); 189 190 // Didn't find a thread waiting for proc work; this can happen 191 // in two scenarios: 192 // 1. All threads are busy handling transactions 193 // In that case, one of those threads should call back into 194 // the kernel driver soon and pick up this work. 195 // 2. Threads are using the (e)poll interface, in which case 196 // they may be blocked on the waitqueue without having been 197 // added to waiting_threads. For this case, we just iterate 198 // over all threads not handling transaction work, and 199 // wake them all up. We wake all because we don't know whether 200 // a thread that called into (e)poll is handling non-binder 201 // work currently. 202 self.work.push_back(work); 203 204 // Wake up polling threads, if any. 205 for thread in self.threads.values() { 206 thread.notify_if_poll_ready(sync); 207 } 208 209 Ok(()) 210 } 211 } 212 213 pub(crate) fn remove_node(&mut self, ptr: u64) { 214 self.nodes.remove(&ptr); 215 } 216 217 /// Updates the reference count on the given node. 218 pub(crate) fn update_node_refcount( 219 &mut self, 220 node: &DArc<Node>, 221 inc: bool, 222 strong: bool, 223 count: usize, 224 othread: Option<&Thread>, 225 ) { 226 let push = node.update_refcount_locked(inc, strong, count, self); 227 228 // If we decided that we need to push work, push either to the process or to a thread if 229 // one is specified. 230 if let Some(node) = push { 231 if let Some(thread) = othread { 232 thread.push_work_deferred(node); 233 } else { 234 let _ = self.push_work(node); 235 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in 236 // that case, it doesn't care about the notification. 237 } 238 } 239 } 240 241 pub(crate) fn new_node_ref( 242 &mut self, 243 node: DArc<Node>, 244 strong: bool, 245 thread: Option<&Thread>, 246 ) -> NodeRef { 247 self.update_node_refcount(&node, true, strong, 1, thread); 248 let strong_count = if strong { 1 } else { 0 }; 249 NodeRef::new(node, strong_count, 1 - strong_count) 250 } 251 252 pub(crate) fn new_node_ref_with_thread( 253 &mut self, 254 node: DArc<Node>, 255 strong: bool, 256 thread: &Thread, 257 wrapper: Option<CritIncrWrapper>, 258 ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> { 259 let push = match wrapper { 260 None => node 261 .incr_refcount_allow_zero2one(strong, self)? 262 .map(|node| node as _), 263 Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self), 264 }; 265 if let Some(node) = push { 266 thread.push_work_deferred(node); 267 } 268 let strong_count = if strong { 1 } else { 0 }; 269 Ok(NodeRef::new(node, strong_count, 1 - strong_count)) 270 } 271 272 /// Returns an existing node with the given pointer and cookie, if one exists. 273 /// 274 /// Returns an error if a node with the given pointer but a different cookie exists. 275 fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> { 276 match self.nodes.get(&ptr) { 277 None => Ok(None), 278 Some(node) => { 279 let (_, node_cookie) = node.get_id(); 280 if node_cookie == cookie { 281 Ok(Some(node.clone())) 282 } else { 283 Err(EINVAL) 284 } 285 } 286 } 287 } 288 289 fn register_thread(&mut self) -> bool { 290 if self.requested_thread_count == 0 { 291 return false; 292 } 293 294 self.requested_thread_count -= 1; 295 self.started_thread_count += 1; 296 true 297 } 298 299 /// Finds a delivered death notification with the given cookie, removes it from the thread's 300 /// delivered list, and returns it. 301 fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> { 302 let mut cursor = self.delivered_deaths.cursor_front(); 303 while let Some(next) = cursor.peek_next() { 304 if next.cookie == cookie { 305 return Some(next.remove().into_arc()); 306 } 307 cursor.move_next(); 308 } 309 None 310 } 311 312 pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) { 313 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 314 self.delivered_deaths.push_back(death); 315 } else { 316 pr_warn!("Notification added to `delivered_deaths` twice."); 317 } 318 } 319 320 pub(crate) fn add_outstanding_txn(&mut self) { 321 self.outstanding_txns += 1; 322 } 323 324 fn txns_pending_locked(&self) -> bool { 325 if self.outstanding_txns > 0 { 326 return true; 327 } 328 for thread in self.threads.values() { 329 if thread.has_current_transaction() { 330 return true; 331 } 332 } 333 false 334 } 335 } 336 337 /// Used to keep track of a node that this process has a handle to. 338 #[pin_data] 339 pub(crate) struct NodeRefInfo { 340 debug_id: usize, 341 /// The refcount that this process owns to the node. 342 node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>, 343 death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>, 344 /// Cookie of the active freeze listener for this node. 345 freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>, 346 /// Used to store this `NodeRefInfo` in the node's `refs` list. 347 #[pin] 348 links: ListLinks<{ Self::LIST_NODE }>, 349 /// The handle for this `NodeRefInfo`. 350 handle: u32, 351 /// The process that has a handle to the node. 352 pub(crate) process: Arc<Process>, 353 } 354 355 impl NodeRefInfo { 356 /// The id used for the `Node::refs` list. 357 pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10; 358 /// The id used for the `ListArc` in `ProcessNodeRefs`. 359 const LIST_PROC: u64 = 0xd703a5263dcc8650; 360 361 fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> { 362 pin_init!(Self { 363 debug_id: super::next_debug_id(), 364 node_ref: ListArcField::new(node_ref), 365 death: ListArcField::new(None), 366 freeze: ListArcField::new(None), 367 links <- ListLinks::new(), 368 handle, 369 process, 370 }) 371 } 372 373 kernel::list::define_list_arc_field_getter! { 374 pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death } 375 pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze } 376 pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref } 377 pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref } 378 } 379 } 380 381 kernel::list::impl_list_arc_safe! { 382 impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; } 383 impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; } 384 } 385 kernel::list::impl_list_item! { 386 impl ListItem<{Self::LIST_NODE}> for NodeRefInfo { 387 using ListLinks { self.links }; 388 } 389 } 390 391 /// Keeps track of references this process has to nodes owned by other processes. 392 /// 393 /// TODO: Currently, the rbtree requires two allocations per node reference, and two tree 394 /// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these 395 /// extra costs should be eliminated. 396 struct ProcessNodeRefs { 397 /// Used to look up nodes using the 32-bit id that this process knows it by. 398 by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>, 399 /// Used to quickly find unused ids in `by_handle`. 400 handle_is_present: IdPool, 401 /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of 402 /// the underlying `Node` struct as returned by `Node::global_id`. 403 by_node: RBTree<usize, u32>, 404 /// Used to look up a `FreezeListener` by cookie. 405 /// 406 /// There might be multiple freeze listeners for the same node, but at most one of them is 407 /// active. 408 freeze_listeners: RBTree<FreezeCookie, FreezeListener>, 409 } 410 411 impl ProcessNodeRefs { 412 fn new() -> Self { 413 Self { 414 by_handle: RBTree::new(), 415 handle_is_present: IdPool::new(), 416 by_node: RBTree::new(), 417 freeze_listeners: RBTree::new(), 418 } 419 } 420 } 421 422 use core::mem::offset_of; 423 use kernel::bindings::rb_process_layout; 424 pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout { 425 arc_offset: Arc::<Process>::DATA_OFFSET, 426 task: offset_of!(Process, task), 427 }; 428 429 /// A process using binder. 430 /// 431 /// Strictly speaking, there can be multiple of these per process. There is one for each binder fd 432 /// that a process has opened, so processes using several binder contexts have several `Process` 433 /// objects. This ensures that the contexts are fully separated. 434 #[pin_data] 435 pub(crate) struct Process { 436 pub(crate) ctx: Arc<Context>, 437 438 // The task leader (process). 439 pub(crate) task: ARef<Task>, 440 441 // Credential associated with file when `Process` is created. 442 pub(crate) cred: ARef<Credential>, 443 444 #[pin] 445 pub(crate) inner: SpinLock<ProcessInner>, 446 447 #[pin] 448 pub(crate) pages: ShrinkablePageRange, 449 450 // Waitqueue of processes waiting for all outstanding transactions to be 451 // processed. 452 #[pin] 453 freeze_wait: CondVar, 454 455 // Node references are in a different lock to avoid recursive acquisition when 456 // incrementing/decrementing a node in another process. 457 #[pin] 458 node_refs: Mutex<ProcessNodeRefs>, 459 460 // Work node for deferred work item. 461 #[pin] 462 defer_work: Work<Process>, 463 464 // Links for process list in Context. 465 #[pin] 466 links: ListLinks, 467 468 pub(crate) stats: BinderStats, 469 } 470 471 kernel::impl_has_work! { 472 impl HasWork<Process> for Process { self.defer_work } 473 } 474 475 kernel::list::impl_list_arc_safe! { 476 impl ListArcSafe<0> for Process { untracked; } 477 } 478 kernel::list::impl_list_item! { 479 impl ListItem<0> for Process { 480 using ListLinks { self.links }; 481 } 482 } 483 484 impl workqueue::WorkItem for Process { 485 type Pointer = Arc<Process>; 486 487 fn run(me: Arc<Self>) { 488 let defer; 489 { 490 let mut inner = me.inner.lock(); 491 defer = inner.defer_work; 492 inner.defer_work = 0; 493 } 494 495 if defer & PROC_DEFER_FLUSH != 0 { 496 me.deferred_flush(); 497 } 498 if defer & PROC_DEFER_RELEASE != 0 { 499 me.deferred_release(); 500 } 501 } 502 } 503 504 impl Process { 505 fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> { 506 let current = kernel::current!(); 507 let process = Arc::pin_init::<Error>( 508 try_pin_init!(Process { 509 ctx, 510 cred, 511 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"), 512 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER), 513 node_refs <- kernel::new_mutex!(ProcessNodeRefs::new(), "Process::node_refs"), 514 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"), 515 task: current.group_leader().into(), 516 defer_work <- kernel::new_work!("Process::defer_work"), 517 links <- ListLinks::new(), 518 stats: BinderStats::new(), 519 }), 520 GFP_KERNEL, 521 )?; 522 523 process.ctx.register_process(process.clone())?; 524 525 Ok(process) 526 } 527 528 pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid { 529 self.task.tgid_nr_ns(None) 530 } 531 532 #[inline(never)] 533 pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> { 534 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 535 seq_print!(m, "context {}\n", &*ctx.name); 536 537 let inner = self.inner.lock(); 538 seq_print!(m, " threads: {}\n", inner.threads.iter().count()); 539 seq_print!( 540 m, 541 " requested threads: {}+{}/{}\n", 542 inner.requested_thread_count, 543 inner.started_thread_count, 544 inner.max_threads, 545 ); 546 if let Some(mapping) = &inner.mapping { 547 seq_print!( 548 m, 549 " free oneway space: {}\n", 550 mapping.alloc.free_oneway_space() 551 ); 552 seq_print!(m, " buffers: {}\n", mapping.alloc.count_buffers()); 553 } 554 seq_print!( 555 m, 556 " outstanding transactions: {}\n", 557 inner.outstanding_txns 558 ); 559 seq_print!(m, " nodes: {}\n", inner.nodes.iter().count()); 560 drop(inner); 561 562 { 563 let mut refs = self.node_refs.lock(); 564 let (mut count, mut weak, mut strong) = (0, 0, 0); 565 for r in refs.by_handle.values_mut() { 566 let node_ref = r.node_ref(); 567 let (nstrong, nweak) = node_ref.get_count(); 568 count += 1; 569 weak += nweak; 570 strong += nstrong; 571 } 572 seq_print!(m, " refs: {count} s {strong} w {weak}\n"); 573 } 574 575 self.stats.debug_print(" ", m); 576 577 Ok(()) 578 } 579 580 #[inline(never)] 581 pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> { 582 seq_print!(m, "proc {}\n", self.pid_in_current_ns()); 583 seq_print!(m, "context {}\n", &*ctx.name); 584 585 let mut all_threads = KVec::new(); 586 let mut all_nodes = KVec::new(); 587 loop { 588 let inner = self.inner.lock(); 589 let num_threads = inner.threads.iter().count(); 590 let num_nodes = inner.nodes.iter().count(); 591 592 if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes { 593 drop(inner); 594 all_threads.reserve(num_threads, GFP_KERNEL)?; 595 all_nodes.reserve(num_nodes, GFP_KERNEL)?; 596 continue; 597 } 598 599 for thread in inner.threads.values() { 600 assert!(all_threads.len() < all_threads.capacity()); 601 let _ = all_threads.push(thread.clone(), GFP_ATOMIC); 602 } 603 604 for node in inner.nodes.values() { 605 assert!(all_nodes.len() < all_nodes.capacity()); 606 let _ = all_nodes.push(node.clone(), GFP_ATOMIC); 607 } 608 609 break; 610 } 611 612 for thread in all_threads { 613 thread.debug_print(m, print_all)?; 614 } 615 616 let mut inner = self.inner.lock(); 617 for node in all_nodes { 618 if print_all || node.has_oneway_transaction(&mut inner) { 619 node.full_debug_print(m, &mut inner)?; 620 } 621 } 622 drop(inner); 623 624 if print_all { 625 let mut refs = self.node_refs.lock(); 626 for r in refs.by_handle.values_mut() { 627 let node_ref = r.node_ref(); 628 let dead = node_ref.node.owner.inner.lock().is_dead; 629 let (strong, weak) = node_ref.get_count(); 630 let debug_id = node_ref.node.debug_id; 631 632 seq_print!( 633 m, 634 " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", 635 r.debug_id, 636 r.handle, 637 if dead { "dead " } else { "" } 638 ); 639 } 640 } 641 642 let inner = self.inner.lock(); 643 for work in &inner.work { 644 work.debug_print(m, " ", " pending transaction ")?; 645 } 646 for _death in &inner.delivered_deaths { 647 seq_print!(m, " has delivered dead binder\n"); 648 } 649 if let Some(mapping) = &inner.mapping { 650 mapping.alloc.debug_print(m)?; 651 } 652 drop(inner); 653 654 Ok(()) 655 } 656 657 /// Attempts to fetch a work item from the process queue. 658 pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> { 659 self.inner.lock().work.pop_front() 660 } 661 662 /// Attempts to fetch a work item from the process queue. If none is available, it registers the 663 /// given thread as ready to receive work directly. 664 /// 665 /// This must only be called when the thread is not participating in a transaction chain; when 666 /// it is, work will always be delivered directly to the thread (and not through the process 667 /// queue). 668 pub(crate) fn get_work_or_register<'a>( 669 &'a self, 670 thread: &'a Arc<Thread>, 671 ) -> GetWorkOrRegister<'a> { 672 let mut inner = self.inner.lock(); 673 // Try to get work from the process queue. 674 if let Some(work) = inner.work.pop_front() { 675 return GetWorkOrRegister::Work(work); 676 } 677 678 // Register the thread as ready. 679 GetWorkOrRegister::Register(Registration::new(thread, &mut inner)) 680 } 681 682 fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> { 683 let id = { 684 let current = kernel::current!(); 685 if self.task != current.group_leader() { 686 pr_err!("get_current_thread was called from the wrong process."); 687 return Err(EINVAL); 688 } 689 current.pid() 690 }; 691 692 { 693 let inner = self.inner.lock(); 694 if let Some(thread) = inner.threads.get(&id) { 695 return Ok(thread.clone()); 696 } 697 } 698 699 // Allocate a new `Thread` without holding any locks. 700 let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?; 701 let ta: Arc<Thread> = Thread::new(id, self.into())?; 702 703 let mut inner = self.inner.lock(); 704 match inner.threads.entry(id) { 705 rbtree::Entry::Vacant(entry) => { 706 entry.insert(ta.clone(), reservation); 707 Ok(ta) 708 } 709 rbtree::Entry::Occupied(_entry) => { 710 pr_err!("Cannot create two threads with the same id."); 711 Err(EINVAL) 712 } 713 } 714 } 715 716 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { 717 // If push_work fails, drop the work item outside the lock. 718 let res = self.inner.lock().push_work(work); 719 match res { 720 Ok(()) => Ok(()), 721 Err((err, work)) => { 722 drop(work); 723 Err(err) 724 } 725 } 726 } 727 728 fn set_as_manager( 729 self: ArcBorrow<'_, Self>, 730 info: Option<FlatBinderObject>, 731 thread: &Thread, 732 ) -> Result { 733 let (ptr, cookie, flags) = if let Some(obj) = info { 734 ( 735 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it 736 // is safe to access the `binder` field. 737 unsafe { obj.__bindgen_anon_1.binder }, 738 obj.cookie, 739 obj.flags, 740 ) 741 } else { 742 (0, 0, 0) 743 }; 744 let node_ref = self.get_node(ptr, cookie, flags as _, true, thread)?; 745 let node = node_ref.node.clone(); 746 self.ctx.set_manager_node(node_ref)?; 747 self.inner.lock().is_manager = true; 748 749 // Force the state of the node to prevent the delivery of acquire/increfs. 750 let mut owner_inner = node.owner.inner.lock(); 751 node.force_has_count(&mut owner_inner); 752 Ok(()) 753 } 754 755 fn get_node_inner( 756 self: ArcBorrow<'_, Self>, 757 ptr: u64, 758 cookie: u64, 759 flags: u32, 760 strong: bool, 761 thread: &Thread, 762 wrapper: Option<CritIncrWrapper>, 763 ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> { 764 // Try to find an existing node. 765 { 766 let mut inner = self.inner.lock(); 767 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 768 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 769 } 770 } 771 772 // Allocate the node before reacquiring the lock. 773 let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc(); 774 let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?; 775 let mut inner = self.inner.lock(); 776 if let Some(node) = inner.get_existing_node(ptr, cookie)? { 777 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper)); 778 } 779 780 inner.nodes.insert(rbnode); 781 // This can only fail if someone has already pushed the node to a list, but we just created 782 // it and still hold the lock, so it can't fail right now. 783 let node_ref = inner 784 .new_node_ref_with_thread(node, strong, thread, wrapper) 785 .unwrap(); 786 787 Ok(Ok(node_ref)) 788 } 789 790 pub(crate) fn get_node( 791 self: ArcBorrow<'_, Self>, 792 ptr: u64, 793 cookie: u64, 794 flags: u32, 795 strong: bool, 796 thread: &Thread, 797 ) -> Result<NodeRef> { 798 let mut wrapper = None; 799 for _ in 0..2 { 800 match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) { 801 Err(err) => return Err(err), 802 Ok(Ok(node_ref)) => return Ok(node_ref), 803 Ok(Err(CouldNotDeliverCriticalIncrement)) => { 804 wrapper = Some(CritIncrWrapper::new()?); 805 } 806 } 807 } 808 // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the 809 // loop should run at most twice. 810 unreachable!() 811 } 812 813 pub(crate) fn insert_or_update_handle( 814 self: ArcBorrow<'_, Process>, 815 node_ref: NodeRef, 816 is_manager: bool, 817 ) -> Result<u32> { 818 { 819 let mut refs = self.node_refs.lock(); 820 821 // Do a lookup before inserting. 822 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { 823 let handle = *handle_ref; 824 let info = refs.by_handle.get_mut(&handle).unwrap(); 825 info.node_ref().absorb(node_ref); 826 return Ok(handle); 827 } 828 } 829 830 // Reserve memory for tree nodes. 831 let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?; 832 let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?; 833 let info = UniqueArc::new_uninit(GFP_KERNEL)?; 834 835 let mut refs_lock = self.node_refs.lock(); 836 let mut refs = &mut *refs_lock; 837 838 let (unused_id, by_handle_slot) = loop { 839 // ID 0 may only be used by the manager. 840 let start = if is_manager { 0 } else { 1 }; 841 842 if let Some(res) = refs.handle_is_present.find_unused_id(start) { 843 match refs.by_handle.entry(res.as_u32()) { 844 rbtree::Entry::Vacant(entry) => break (res, entry), 845 rbtree::Entry::Occupied(_) => { 846 pr_err!("Detected mismatch between handle_is_present and by_handle"); 847 res.acquire(); 848 kernel::warn_on!(true); 849 return Err(EINVAL); 850 } 851 } 852 } 853 854 let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?; 855 drop(refs_lock); 856 let resizer = grow_request.realloc(GFP_KERNEL)?; 857 refs_lock = self.node_refs.lock(); 858 refs = &mut *refs_lock; 859 refs.handle_is_present.grow(resizer); 860 }; 861 let handle = unused_id.as_u32(); 862 863 // Do a lookup again as node may have been inserted before the lock was reacquired. 864 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { 865 let handle = *handle_ref; 866 let info = refs.by_handle.get_mut(&handle).unwrap(); 867 info.node_ref().absorb(node_ref); 868 return Ok(handle); 869 } 870 871 let gid = node_ref.node.global_id(); 872 let (info_proc, info_node) = { 873 let info_init = NodeRefInfo::new(node_ref, handle, self.into()); 874 match info.pin_init_with(info_init) { 875 Ok(info) => ListArc::pair_from_pin_unique(info), 876 // error is infallible 877 Err(err) => match err {}, 878 } 879 }; 880 881 // Ensure the process is still alive while we insert a new reference. 882 // 883 // This releases the lock before inserting the nodes, but since `is_dead` is set as the 884 // first thing in `deferred_release`, process cleanup will not miss the items inserted into 885 // `refs` below. 886 if self.inner.lock().is_dead { 887 return Err(ESRCH); 888 } 889 890 // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting 891 // `info_node` into the right node's `refs` list. 892 unsafe { info_proc.node_ref2().node.insert_node_info(info_node) }; 893 894 refs.by_node.insert(reserve1.into_node(gid, handle)); 895 by_handle_slot.insert(info_proc, reserve2); 896 unused_id.acquire(); 897 Ok(handle) 898 } 899 900 pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { 901 // When handle is zero, try to get the context manager. 902 if handle == 0 { 903 let node_ref = self.ctx.get_manager_node(true)?; 904 if core::ptr::eq(self, &*node_ref.node.owner) { 905 return Err(EINVAL.into()); 906 } 907 Ok(node_ref) 908 } else { 909 Ok(self.get_node_from_handle(handle, true)?) 910 } 911 } 912 913 pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> { 914 self.node_refs 915 .lock() 916 .by_handle 917 .get_mut(&handle) 918 .ok_or(ENOENT)? 919 .node_ref() 920 .clone(strong) 921 } 922 923 pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) { 924 let mut inner = self.inner.lock(); 925 // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list. 926 let removed = unsafe { inner.delivered_deaths.remove(death) }; 927 drop(inner); 928 drop(removed); 929 } 930 931 pub(crate) fn update_ref( 932 self: ArcBorrow<'_, Process>, 933 handle: u32, 934 inc: bool, 935 strong: bool, 936 ) -> Result { 937 if inc && handle == 0 { 938 if let Ok(node_ref) = self.ctx.get_manager_node(strong) { 939 if core::ptr::eq(&*self, &*node_ref.node.owner) { 940 return Err(EINVAL); 941 } 942 let _ = self.insert_or_update_handle(node_ref, true); 943 return Ok(()); 944 } 945 } 946 947 // To preserve original binder behaviour, we only fail requests where the manager tries to 948 // increment references on itself. 949 let _to_free_freeze_listener; 950 let _to_free_freeze_listener_cleanup; 951 let mut refs = self.node_refs.lock(); 952 if let Some(info) = refs.by_handle.get_mut(&handle) { 953 if info.node_ref().update(inc, strong) { 954 // Clean up death if there is one attached to this node reference. 955 if let Some(death) = info.death().take() { 956 death.set_cleared(true); 957 self.remove_from_delivered_deaths(&death); 958 } 959 960 // Remove reference from process tables, and from the node's `refs` list. 961 962 // SAFETY: We are removing the `NodeRefInfo` from the right node. 963 unsafe { info.node_ref2().node.remove_node_info(info) }; 964 965 let id = info.node_ref().node.global_id(); 966 967 if let Some(freeze) = *info.freeze() { 968 if let Some(fl) = refs.freeze_listeners.remove(&freeze) { 969 _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self); 970 _to_free_freeze_listener = fl; 971 } 972 } 973 974 refs.by_handle.remove(&handle); 975 refs.by_node.remove(&id); 976 refs.handle_is_present.release_id(handle as usize); 977 978 if let Some(shrink) = refs.handle_is_present.shrink_request() { 979 drop(refs); 980 // This intentionally ignores allocation failures. 981 if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) { 982 refs = self.node_refs.lock(); 983 refs.handle_is_present.shrink(new_bitmap); 984 } 985 } 986 } 987 } else { 988 // All refs are cleared in process exit, so this warning is expected in that case. 989 if !self.inner.lock().is_dead { 990 pr_warn!("{}: no such ref {handle}\n", self.pid_in_current_ns()); 991 } 992 } 993 Ok(()) 994 } 995 996 /// Decrements the refcount of the given node, if one exists. 997 pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) { 998 let mut inner = self.inner.lock(); 999 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1000 inner.update_node_refcount(&node, false, strong, 1, None); 1001 } 1002 } 1003 1004 pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result { 1005 let ptr = reader.read::<u64>()?; 1006 let cookie = reader.read::<u64>()?; 1007 let mut inner = self.inner.lock(); 1008 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { 1009 if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) { 1010 // This only fails if the process is dead. 1011 let _ = inner.push_work(node); 1012 } 1013 } 1014 Ok(()) 1015 } 1016 1017 pub(crate) fn buffer_alloc( 1018 self: &Arc<Self>, 1019 debug_id: usize, 1020 size: usize, 1021 info: &mut TransactionInfo, 1022 ) -> BinderResult<NewAllocation> { 1023 use kernel::page::PAGE_SIZE; 1024 1025 let mut reserve_new_args = ReserveNewArgs { 1026 debug_id, 1027 size, 1028 is_oneway: info.is_oneway(), 1029 pid: info.from_pid, 1030 ..ReserveNewArgs::default() 1031 }; 1032 1033 let (new_alloc, addr) = loop { 1034 let mut inner = self.inner.lock(); 1035 let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?; 1036 let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? { 1037 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address), 1038 ReserveNew::NeedAlloc(request) => request, 1039 }; 1040 drop(inner); 1041 // We need to allocate memory and then call `reserve_new` again. 1042 reserve_new_args = alloc_request.make_alloc()?; 1043 }; 1044 1045 info.oneway_spam_suspect = new_alloc.oneway_spam_detected; 1046 let res = Allocation::new( 1047 self.clone(), 1048 debug_id, 1049 new_alloc.offset, 1050 size, 1051 addr + new_alloc.offset, 1052 ); 1053 1054 // This allocation will be marked as in use until the `Allocation` is used to free it. 1055 // 1056 // This method can't be called while holding a lock, so we release the lock first. It's 1057 // okay for several threads to use the method on the same index at the same time. In that 1058 // case, one of the calls will allocate the given page (if missing), and the other call 1059 // will wait for the other call to finish allocating the page. 1060 // 1061 // We will not call `stop_using_range` in parallel with this on the same page, because the 1062 // allocation can only be removed via the destructor of the `Allocation` object that we 1063 // currently own. 1064 match self.pages.use_range( 1065 new_alloc.offset / PAGE_SIZE, 1066 (new_alloc.offset + size).div_ceil(PAGE_SIZE), 1067 ) { 1068 Ok(()) => {} 1069 Err(err) => { 1070 pr_warn!("use_range failure {:?}", err); 1071 return Err(err.into()); 1072 } 1073 } 1074 1075 Ok(NewAllocation(res)) 1076 } 1077 1078 pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> { 1079 let mut inner = self.inner.lock(); 1080 let mapping = inner.mapping.as_mut()?; 1081 let offset = ptr.checked_sub(mapping.address)?; 1082 let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?; 1083 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1084 if let Some(data) = odata { 1085 alloc.set_info(data); 1086 } 1087 Some(alloc) 1088 } 1089 1090 pub(crate) fn buffer_raw_free(&self, ptr: usize) { 1091 let mut inner = self.inner.lock(); 1092 if let Some(ref mut mapping) = &mut inner.mapping { 1093 let offset = match ptr.checked_sub(mapping.address) { 1094 Some(offset) => offset, 1095 None => return, 1096 }; 1097 1098 let freed_range = match mapping.alloc.reservation_abort(offset) { 1099 Ok(freed_range) => freed_range, 1100 Err(_) => { 1101 pr_warn!( 1102 "Pointer {:x} failed to free, base = {:x}\n", 1103 ptr, 1104 mapping.address 1105 ); 1106 return; 1107 } 1108 }; 1109 1110 // No more allocations in this range. Mark them as not in use. 1111 // 1112 // Must be done before we release the lock so that `use_range` is not used on these 1113 // indices until `stop_using_range` returns. 1114 self.pages 1115 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx); 1116 } 1117 } 1118 1119 pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) { 1120 let mut inner = self.inner.lock(); 1121 if let Some(ref mut mapping) = &mut inner.mapping { 1122 if mapping.alloc.reservation_commit(offset, &mut data).is_err() { 1123 pr_warn!("Offset {} failed to be marked freeable\n", offset); 1124 } 1125 } 1126 } 1127 1128 fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result { 1129 use kernel::page::PAGE_SIZE; 1130 let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); 1131 let mapping = Mapping::new(vma.start(), size); 1132 let page_count = self.pages.register_with_vma(vma)?; 1133 if page_count * PAGE_SIZE != size { 1134 return Err(EINVAL); 1135 } 1136 1137 // Save range allocator for later. 1138 self.inner.lock().mapping = Some(mapping); 1139 1140 Ok(()) 1141 } 1142 1143 fn version(&self, data: UserSlice) -> Result { 1144 data.writer().write(&BinderVersion::current()) 1145 } 1146 1147 pub(crate) fn register_thread(&self) -> bool { 1148 self.inner.lock().register_thread() 1149 } 1150 1151 fn remove_thread(&self, thread: Arc<Thread>) { 1152 self.inner.lock().threads.remove(&thread.id); 1153 thread.release(); 1154 } 1155 1156 fn set_max_threads(&self, max: u32) { 1157 self.inner.lock().max_threads = max; 1158 } 1159 1160 fn set_oneway_spam_detection_enabled(&self, enabled: u32) { 1161 self.inner.lock().oneway_spam_detection_enabled = enabled != 0; 1162 } 1163 1164 pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool { 1165 self.inner.lock().oneway_spam_detection_enabled 1166 } 1167 1168 fn get_node_debug_info(&self, data: UserSlice) -> Result { 1169 let (mut reader, mut writer) = data.reader_writer(); 1170 1171 // Read the starting point. 1172 let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr; 1173 let mut out = BinderNodeDebugInfo::default(); 1174 1175 { 1176 let inner = self.inner.lock(); 1177 for (node_ptr, node) in &inner.nodes { 1178 if *node_ptr > ptr { 1179 node.populate_debug_info(&mut out, &inner); 1180 break; 1181 } 1182 } 1183 } 1184 1185 writer.write(&out) 1186 } 1187 1188 fn get_node_info_from_ref(&self, data: UserSlice) -> Result { 1189 let (mut reader, mut writer) = data.reader_writer(); 1190 let mut out = reader.read::<BinderNodeInfoForRef>()?; 1191 1192 if out.strong_count != 0 1193 || out.weak_count != 0 1194 || out.reserved1 != 0 1195 || out.reserved2 != 0 1196 || out.reserved3 != 0 1197 { 1198 return Err(EINVAL); 1199 } 1200 1201 // Only the context manager is allowed to use this ioctl. 1202 if !self.inner.lock().is_manager { 1203 return Err(EPERM); 1204 } 1205 1206 { 1207 let mut node_refs = self.node_refs.lock(); 1208 let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?; 1209 let node_ref = node_info.node_ref(); 1210 let owner_inner = node_ref.node.owner.inner.lock(); 1211 node_ref.node.populate_counts(&mut out, &owner_inner); 1212 } 1213 1214 // Write the result back. 1215 writer.write(&out) 1216 } 1217 1218 pub(crate) fn needs_thread(&self) -> bool { 1219 let mut inner = self.inner.lock(); 1220 let ret = inner.requested_thread_count == 0 1221 && inner.ready_threads.is_empty() 1222 && inner.started_thread_count < inner.max_threads; 1223 if ret { 1224 inner.requested_thread_count += 1 1225 } 1226 ret 1227 } 1228 1229 pub(crate) fn request_death( 1230 self: &Arc<Self>, 1231 reader: &mut UserSliceReader, 1232 thread: &Thread, 1233 ) -> Result { 1234 let handle: u32 = reader.read()?; 1235 let cookie: u64 = reader.read()?; 1236 1237 // Queue BR_ERROR if we can't allocate memory for the death notification. 1238 let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| { 1239 thread.push_return_work(BR_ERROR); 1240 })?; 1241 let mut refs = self.node_refs.lock(); 1242 let Some(info) = refs.by_handle.get_mut(&handle) else { 1243 pr_warn!("BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}\n"); 1244 return Ok(()); 1245 }; 1246 1247 // Nothing to do if there is already a death notification request for this handle. 1248 if info.death().is_some() { 1249 pr_warn!("BC_REQUEST_DEATH_NOTIFICATION death notification already set\n"); 1250 return Ok(()); 1251 } 1252 1253 let death = { 1254 let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie); 1255 match death.pin_init_with(death_init) { 1256 Ok(death) => death, 1257 // error is infallible 1258 Err(err) => match err {}, 1259 } 1260 }; 1261 1262 // Register the death notification. 1263 { 1264 let owner = info.node_ref2().node.owner.clone(); 1265 let mut owner_inner = owner.inner.lock(); 1266 if owner_inner.is_dead { 1267 let death = Arc::from(death); 1268 *info.death() = Some(death.clone()); 1269 drop(owner_inner); 1270 death.set_dead(); 1271 } else { 1272 let death = ListArc::from(death); 1273 *info.death() = Some(death.clone_arc()); 1274 info.node_ref().node.add_death(death, &mut owner_inner); 1275 } 1276 } 1277 Ok(()) 1278 } 1279 1280 pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result { 1281 let handle: u32 = reader.read()?; 1282 let cookie: u64 = reader.read()?; 1283 1284 let mut refs = self.node_refs.lock(); 1285 let Some(info) = refs.by_handle.get_mut(&handle) else { 1286 pr_warn!("BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}\n"); 1287 return Ok(()); 1288 }; 1289 1290 let Some(death) = info.death().take() else { 1291 pr_warn!("BC_CLEAR_DEATH_NOTIFICATION death notification not active\n"); 1292 return Ok(()); 1293 }; 1294 if death.cookie != cookie { 1295 *info.death() = Some(death); 1296 pr_warn!("BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch\n"); 1297 return Ok(()); 1298 } 1299 1300 // Update state and determine if we need to queue a work item. We only need to do it when 1301 // the node is not dead or if the user already completed the death notification. 1302 if death.set_cleared(false) { 1303 if let Some(death) = ListArc::try_from_arc_or_drop(death) { 1304 let _ = thread.push_work_if_looper(death); 1305 } 1306 } 1307 1308 Ok(()) 1309 } 1310 1311 pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { 1312 let death = self.inner.lock().pull_delivered_death(cookie); 1313 if let Some(death) = death { 1314 death.set_notification_done(thread); 1315 } 1316 } 1317 1318 /// Locks the spinlock and move the `nodes` rbtree out. 1319 /// 1320 /// This allows you to iterate through `nodes` while also allowing you to give other parts of 1321 /// the codebase exclusive access to `ProcessInner`. 1322 pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> { 1323 let mut inner = self.inner.lock(); 1324 WithNodes { 1325 nodes: take(&mut inner.nodes), 1326 inner, 1327 } 1328 } 1329 1330 fn deferred_flush(&self) { 1331 let inner = self.inner.lock(); 1332 for thread in inner.threads.values() { 1333 thread.exit_looper(); 1334 } 1335 } 1336 1337 fn deferred_release(self: Arc<Self>) { 1338 let is_manager = { 1339 let mut inner = self.inner.lock(); 1340 inner.is_dead = true; 1341 inner.is_frozen = IsFrozen::No; 1342 inner.sync_recv = false; 1343 inner.async_recv = false; 1344 inner.is_manager 1345 }; 1346 1347 if is_manager { 1348 self.ctx.unset_manager_node(); 1349 } 1350 1351 self.ctx.deregister_process(&self); 1352 1353 let binderfs_file = self.inner.lock().binderfs_file.take(); 1354 drop(binderfs_file); 1355 1356 // Release threads. 1357 let threads = { 1358 let mut inner = self.inner.lock(); 1359 let threads = take(&mut inner.threads); 1360 let ready = take(&mut inner.ready_threads); 1361 drop(inner); 1362 drop(ready); 1363 1364 for thread in threads.values() { 1365 thread.release(); 1366 } 1367 threads 1368 }; 1369 1370 // Release nodes. 1371 { 1372 while let Some(node) = { 1373 let mut lock = self.inner.lock(); 1374 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) 1375 } { 1376 node.to_key_value().1.release(); 1377 } 1378 } 1379 1380 // Clean up death listeners and remove nodes from external node info lists. 1381 for info in self.node_refs.lock().by_handle.values_mut() { 1382 // SAFETY: We are removing the `NodeRefInfo` from the right node. 1383 unsafe { info.node_ref2().node.remove_node_info(info) }; 1384 1385 // Remove all death notifications from the nodes (that belong to a different process). 1386 let death = if let Some(existing) = info.death().take() { 1387 existing 1388 } else { 1389 continue; 1390 }; 1391 death.set_cleared(false); 1392 } 1393 1394 // Clean up freeze listeners. 1395 let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners); 1396 for listener in freeze_listeners.values() { 1397 listener.on_process_cleanup(&self); 1398 } 1399 drop(freeze_listeners); 1400 1401 // Release refs on foreign nodes. 1402 { 1403 let mut refs = self.node_refs.lock(); 1404 let by_handle = take(&mut refs.by_handle); 1405 let by_node = take(&mut refs.by_node); 1406 drop(refs); 1407 drop(by_node); 1408 drop(by_handle); 1409 } 1410 1411 // Cancel all pending work items. 1412 while let Some(work) = self.get_work() { 1413 work.into_arc().cancel(); 1414 } 1415 1416 // Clear delivered_deaths list. 1417 // 1418 // Scope ensures that MutexGuard is dropped while executing the body. 1419 while let Some(delivered_death) = { 1420 // Explicitly bind to avoid tail expression lifetime extension of the lockguard 1421 // Can be removed when the kernel moves to edition 2024 1422 let maybe_death = self.inner.lock().delivered_deaths.pop_front(); 1423 maybe_death 1424 } { 1425 drop(delivered_death); 1426 } 1427 1428 // Free any resources kept alive by allocated buffers. 1429 let omapping = self.inner.lock().mapping.take(); 1430 if let Some(mut mapping) = omapping { 1431 let address = mapping.address; 1432 mapping 1433 .alloc 1434 .take_for_each(|offset, size, debug_id, odata| { 1435 let ptr = offset + address; 1436 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr); 1437 if let Some(data) = odata { 1438 alloc.set_info(data); 1439 } 1440 drop(alloc) 1441 }); 1442 } 1443 1444 // calls to synchronize_rcu() in thread drop will happen here 1445 drop(threads); 1446 } 1447 1448 pub(crate) fn drop_outstanding_txn(&self) { 1449 let wake = { 1450 let mut inner = self.inner.lock(); 1451 if inner.outstanding_txns == 0 { 1452 pr_err!("outstanding_txns underflow"); 1453 return; 1454 } 1455 inner.outstanding_txns -= 1; 1456 inner.is_frozen.is_frozen() && inner.outstanding_txns == 0 1457 }; 1458 1459 if wake { 1460 self.freeze_wait.notify_all(); 1461 } 1462 } 1463 1464 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 1465 // mangled symbol names. 1466 #[export_name = "rust_binder_freeze"] 1467 pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result { 1468 if info.enable == 0 { 1469 let msgs = self.prepare_freeze_messages()?; 1470 let mut inner = self.inner.lock(); 1471 inner.sync_recv = false; 1472 inner.async_recv = false; 1473 inner.is_frozen = IsFrozen::No; 1474 drop(inner); 1475 msgs.send_messages(); 1476 return Ok(()); 1477 } 1478 1479 let mut inner = self.inner.lock(); 1480 inner.sync_recv = false; 1481 inner.async_recv = false; 1482 inner.is_frozen = IsFrozen::InProgress; 1483 1484 if info.timeout_ms > 0 { 1485 let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms); 1486 while jiffies > 0 { 1487 if inner.outstanding_txns == 0 { 1488 break; 1489 } 1490 1491 match self 1492 .freeze_wait 1493 .wait_interruptible_timeout(&mut inner, jiffies) 1494 { 1495 CondVarTimeoutResult::Signal { .. } => { 1496 inner.is_frozen = IsFrozen::No; 1497 return Err(ERESTARTSYS); 1498 } 1499 CondVarTimeoutResult::Woken { jiffies: remaining } => { 1500 jiffies = remaining; 1501 } 1502 CondVarTimeoutResult::Timeout => { 1503 jiffies = 0; 1504 } 1505 } 1506 } 1507 } 1508 1509 if inner.txns_pending_locked() { 1510 inner.is_frozen = IsFrozen::No; 1511 Err(EAGAIN) 1512 } else { 1513 drop(inner); 1514 match self.prepare_freeze_messages() { 1515 Ok(batch) => { 1516 self.inner.lock().is_frozen = IsFrozen::Yes; 1517 batch.send_messages(); 1518 Ok(()) 1519 } 1520 Err(kernel::alloc::AllocError) => { 1521 self.inner.lock().is_frozen = IsFrozen::No; 1522 Err(ENOMEM) 1523 } 1524 } 1525 } 1526 } 1527 } 1528 1529 fn get_frozen_status(data: UserSlice) -> Result { 1530 let (mut reader, mut writer) = data.reader_writer(); 1531 1532 let mut info = reader.read::<BinderFrozenStatusInfo>()?; 1533 info.sync_recv = 0; 1534 info.async_recv = 0; 1535 let mut found = false; 1536 1537 for ctx in crate::context::get_all_contexts()? { 1538 ctx.for_each_proc(|proc| { 1539 if proc.task.pid() == info.pid as _ { 1540 found = true; 1541 let inner = proc.inner.lock(); 1542 let txns_pending = inner.txns_pending_locked(); 1543 info.async_recv |= inner.async_recv as u32; 1544 info.sync_recv |= inner.sync_recv as u32; 1545 info.sync_recv |= (txns_pending as u32) << 1; 1546 } 1547 }); 1548 } 1549 1550 if found { 1551 writer.write(&info)?; 1552 Ok(()) 1553 } else { 1554 Err(EINVAL) 1555 } 1556 } 1557 1558 fn ioctl_freeze(reader: &mut UserSliceReader) -> Result { 1559 let info = reader.read::<BinderFreezeInfo>()?; 1560 1561 // Very unlikely for there to be more than 3, since a process normally uses at most binder and 1562 // hwbinder. 1563 let mut procs = KVec::with_capacity(3, GFP_KERNEL)?; 1564 1565 let ctxs = crate::context::get_all_contexts()?; 1566 for ctx in ctxs { 1567 for proc in ctx.get_procs_with_pid(info.pid as i32)? { 1568 procs.push(proc, GFP_KERNEL)?; 1569 } 1570 } 1571 1572 for proc in procs { 1573 proc.ioctl_freeze(&info)?; 1574 } 1575 Ok(()) 1576 } 1577 1578 /// The ioctl handler. 1579 impl Process { 1580 /// Ioctls that are write-only from the perspective of userspace. 1581 /// 1582 /// The kernel will only read from the pointer that userspace provided to us. 1583 fn ioctl_write_only( 1584 this: ArcBorrow<'_, Process>, 1585 _file: &File, 1586 cmd: u32, 1587 reader: &mut UserSliceReader, 1588 ) -> Result { 1589 if cmd == uapi::BINDER_FREEZE { 1590 return ioctl_freeze(reader); 1591 } 1592 1593 let thread = this.get_current_thread()?; 1594 match cmd { 1595 uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), 1596 uapi::BINDER_THREAD_EXIT => this.remove_thread(thread), 1597 uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?, 1598 uapi::BINDER_SET_CONTEXT_MGR_EXT => { 1599 this.set_as_manager(Some(reader.read()?), &thread)? 1600 } 1601 uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => { 1602 this.set_oneway_spam_detection_enabled(reader.read()?) 1603 } 1604 _ => return Err(EINVAL), 1605 } 1606 Ok(()) 1607 } 1608 1609 /// Ioctls that are read/write from the perspective of userspace. 1610 /// 1611 /// The kernel will both read from and write to the pointer that userspace provided to us. 1612 fn ioctl_write_read( 1613 this: ArcBorrow<'_, Process>, 1614 file: &File, 1615 cmd: u32, 1616 data: UserSlice, 1617 ) -> Result { 1618 let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0; 1619 match cmd { 1620 uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?, 1621 uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?, 1622 uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?, 1623 uapi::BINDER_VERSION => this.version(data)?, 1624 uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?, 1625 uapi::BINDER_GET_EXTENDED_ERROR => { 1626 this.get_current_thread()?.get_extended_error(data)? 1627 } 1628 _ => return Err(EINVAL), 1629 } 1630 Ok(()) 1631 } 1632 } 1633 1634 /// The file operations supported by `Process`. 1635 impl Process { 1636 pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> { 1637 Self::new(ctx.into(), ARef::from(file.cred())) 1638 } 1639 1640 pub(crate) fn release(this: Arc<Process>, _file: &File) { 1641 let binderfs_file; 1642 let should_schedule; 1643 { 1644 let mut inner = this.inner.lock(); 1645 should_schedule = inner.defer_work == 0; 1646 inner.defer_work |= PROC_DEFER_RELEASE; 1647 binderfs_file = inner.binderfs_file.take(); 1648 } 1649 1650 if should_schedule { 1651 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1652 // scheduled for execution. 1653 let _ = workqueue::system().enqueue(this); 1654 } 1655 1656 drop(binderfs_file); 1657 } 1658 1659 pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result { 1660 let should_schedule; 1661 { 1662 let mut inner = this.inner.lock(); 1663 should_schedule = inner.defer_work == 0; 1664 inner.defer_work |= PROC_DEFER_FLUSH; 1665 } 1666 1667 if should_schedule { 1668 // Ignore failures to schedule to the workqueue. Those just mean that we're already 1669 // scheduled for execution. 1670 let _ = workqueue::system().enqueue(Arc::from(this)); 1671 } 1672 Ok(()) 1673 } 1674 1675 pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result { 1676 use kernel::ioctl::{_IOC_DIR, _IOC_SIZE}; 1677 use kernel::uapi::{_IOC_READ, _IOC_WRITE}; 1678 1679 crate::trace::trace_ioctl(cmd, arg); 1680 1681 let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd)); 1682 1683 const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE; 1684 1685 let res = match _IOC_DIR(cmd) { 1686 _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()), 1687 _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice), 1688 _ => Err(EINVAL), 1689 }; 1690 1691 crate::trace::trace_ioctl_done(res); 1692 res 1693 } 1694 1695 pub(crate) fn mmap( 1696 this: ArcBorrow<'_, Process>, 1697 _file: &File, 1698 vma: &mm::virt::VmaNew, 1699 ) -> Result { 1700 // We don't allow mmap to be used in a different process. 1701 if this.task != kernel::current!().group_leader() { 1702 return Err(EINVAL); 1703 } 1704 if vma.start() == 0 { 1705 return Err(EINVAL); 1706 } 1707 1708 vma.try_clear_maywrite().map_err(|_| EPERM)?; 1709 vma.set_dontcopy(); 1710 vma.set_mixedmap(); 1711 1712 // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it. 1713 this.create_mapping(vma) 1714 } 1715 1716 pub(crate) fn poll( 1717 this: ArcBorrow<'_, Process>, 1718 file: &File, 1719 table: PollTable<'_>, 1720 ) -> Result<u32> { 1721 let thread = this.get_current_thread()?; 1722 let (from_proc, mut mask) = thread.poll(file, table); 1723 if mask == 0 && from_proc && !this.inner.lock().work.is_empty() { 1724 mask |= bindings::POLLIN; 1725 } 1726 Ok(mask) 1727 } 1728 } 1729 1730 /// Represents that a thread has registered with the `ready_threads` list of its process. 1731 /// 1732 /// The destructor of this type will unregister the thread from the list of ready threads. 1733 pub(crate) struct Registration<'a> { 1734 thread: &'a Arc<Thread>, 1735 } 1736 1737 impl<'a> Registration<'a> { 1738 fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self { 1739 assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref())); 1740 // INVARIANT: We are pushing this thread to the right `ready_threads` list. 1741 if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) { 1742 guard.ready_threads.push_front(list_arc); 1743 } else { 1744 // It is an error to hit this branch, and it should not be reachable. We try to do 1745 // something reasonable when the failure path happens. Most likely, the thread in 1746 // question will sleep forever. 1747 pr_err!("Same thread registered with `ready_threads` twice."); 1748 } 1749 Self { thread } 1750 } 1751 } 1752 1753 impl Drop for Registration<'_> { 1754 fn drop(&mut self) { 1755 let mut inner = self.thread.process.inner.lock(); 1756 // SAFETY: The thread has the invariant that we never push it to any other linked list than 1757 // the `ready_threads` list of its parent process. Therefore, the thread is either in that 1758 // list, or in no list. 1759 unsafe { inner.ready_threads.remove(self.thread) }; 1760 } 1761 } 1762 1763 pub(crate) struct WithNodes<'a> { 1764 pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>, 1765 pub(crate) nodes: RBTree<u64, DArc<Node>>, 1766 } 1767 1768 impl Drop for WithNodes<'_> { 1769 fn drop(&mut self) { 1770 core::mem::swap(&mut self.nodes, &mut self.inner.nodes); 1771 if self.nodes.iter().next().is_some() { 1772 pr_err!("nodes array was modified while using lock_with_nodes\n"); 1773 } 1774 } 1775 } 1776 1777 pub(crate) enum GetWorkOrRegister<'a> { 1778 Work(DLArc<dyn DeliverToRead>), 1779 Register(Registration<'a>), 1780 } 1781