1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 use kernel::{ 6 list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, 7 prelude::*, 8 seq_file::SeqFile, 9 seq_print, 10 sync::lock::{spinlock::SpinLockBackend, Guard}, 11 sync::{Arc, LockedBy, SpinLock}, 12 uapi, 13 }; 14 15 use crate::{ 16 defs::*, 17 error::BinderError, 18 process::{NodeRefInfo, Process, ProcessInner}, 19 thread::Thread, 20 transaction::Transaction, 21 BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead, 22 }; 23 24 use core::mem; 25 use core::ptr; 26 27 mod wrapper; 28 pub(crate) use self::wrapper::CritIncrWrapper; 29 30 #[derive(Debug)] 31 pub(crate) struct CouldNotDeliverCriticalIncrement; 32 33 /// Keeps track of how this node is scheduled. 34 /// 35 /// There are two ways to schedule a node to a work list. Just schedule the node itself, or 36 /// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to 37 /// make it possible to "move" a node from one list to another - when `do_work` is called directly 38 /// on the `Node`, then it's a no-op if there's also a pending wrapper. 39 /// 40 /// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases 41 /// of this: weak increments and strong increments. We call such increments "critical" because it 42 /// is critical that they are delivered to the thread doing the increment. Some examples: 43 /// 44 /// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak 45 /// increment. Delivering the node to the thread doing the weak increment is wrong, since the 46 /// thread doing the strong increment may have ended a long time ago when the command is actually 47 /// processed by userspace. 48 /// 49 /// * We have a weak reference and are about to drop it on one thread. But then another thread does 50 /// a zero-to-one strong increment. If the strong increment gets sent to the thread that was 51 /// about to drop the weak reference, then the strong increment could be processed after the 52 /// other thread has already exited, which would be too late. 53 /// 54 /// Note that trying to create a `ListArc` to the node can succeed even if `has_pushed_node` is 55 /// set. This is because another thread might just have popped the node from a todo list, but not 56 /// yet called `do_work`. However, if `has_pushed_node` is false, then creating a `ListArc` should 57 /// always succeed. 58 /// 59 /// Like the other fields in `NodeInner`, the delivery state is protected by the process lock. 60 struct DeliveryState { 61 /// Is the `Node` currently scheduled? 62 has_pushed_node: bool, 63 64 /// Is a wrapper currently scheduled? 65 /// 66 /// The wrapper is used only for strong zero2one increments. 67 has_pushed_wrapper: bool, 68 69 /// Is the currently scheduled `Node` scheduled due to a weak zero2one increment? 70 /// 71 /// Weak zero2one operations are always scheduled using the `Node`. 72 has_weak_zero2one: bool, 73 74 /// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment? 75 /// 76 /// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the 77 /// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`. 78 has_strong_zero2one: bool, 79 } 80 81 impl DeliveryState { 82 fn should_normal_push(&self) -> bool { 83 !self.has_pushed_node && !self.has_pushed_wrapper 84 } 85 86 fn did_normal_push(&mut self) { 87 assert!(self.should_normal_push()); 88 self.has_pushed_node = true; 89 } 90 91 fn should_push_weak_zero2one(&self) -> bool { 92 !self.has_weak_zero2one && !self.has_strong_zero2one 93 } 94 95 fn can_push_weak_zero2one_normally(&self) -> bool { 96 !self.has_pushed_node 97 } 98 99 fn did_push_weak_zero2one(&mut self) { 100 assert!(self.should_push_weak_zero2one()); 101 assert!(self.can_push_weak_zero2one_normally()); 102 self.has_pushed_node = true; 103 self.has_weak_zero2one = true; 104 } 105 106 fn should_push_strong_zero2one(&self) -> bool { 107 !self.has_strong_zero2one 108 } 109 110 fn can_push_strong_zero2one_normally(&self) -> bool { 111 !self.has_pushed_node 112 } 113 114 fn did_push_strong_zero2one(&mut self) { 115 assert!(self.should_push_strong_zero2one()); 116 assert!(self.can_push_strong_zero2one_normally()); 117 self.has_pushed_node = true; 118 self.has_strong_zero2one = true; 119 } 120 121 fn did_push_strong_zero2one_wrapper(&mut self) { 122 assert!(self.should_push_strong_zero2one()); 123 assert!(!self.can_push_strong_zero2one_normally()); 124 self.has_pushed_wrapper = true; 125 self.has_strong_zero2one = true; 126 } 127 } 128 129 struct CountState { 130 /// The reference count. 131 count: usize, 132 /// Whether the process that owns this node thinks that we hold a refcount on it. (Note that 133 /// even if count is greater than one, we only increment it once in the owning process.) 134 has_count: bool, 135 } 136 137 impl CountState { 138 fn new() -> Self { 139 Self { 140 count: 0, 141 has_count: false, 142 } 143 } 144 } 145 146 struct NodeInner { 147 /// Strong refcounts held on this node by `NodeRef` objects. 148 strong: CountState, 149 /// Weak refcounts held on this node by `NodeRef` objects. 150 weak: CountState, 151 delivery_state: DeliveryState, 152 /// The binder driver guarantees that oneway transactions sent to the same node are serialized, 153 /// that is, userspace will not be given the next one until it has finished processing the 154 /// previous oneway transaction. This is done to avoid the case where two oneway transactions 155 /// arrive in opposite order from the order in which they were sent. (E.g., they could be 156 /// delivered to two different threads, which could appear as-if they were sent in opposite 157 /// order.) 158 /// 159 /// To fix that, we store pending oneway transactions in a separate list in the node, and don't 160 /// deliver the next oneway transaction until userspace signals that it has finished processing 161 /// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl. 162 oneway_todo: List<DTRWrap<Transaction>>, 163 /// Keeps track of whether this node has a pending oneway transaction. 164 /// 165 /// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of 166 /// being delivered directly to the process. 167 has_oneway_transaction: bool, 168 /// List of processes to deliver a notification to when this node is destroyed (usually due to 169 /// the process dying). 170 death_list: List<DTRWrap<NodeDeath>, 1>, 171 /// List of processes to deliver freeze notifications to. 172 freeze_list: KVVec<Arc<Process>>, 173 /// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two) 174 /// 175 /// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the 176 /// active operations have ended. This avoids the situation an increment and decrement get 177 /// reordered from userspace's perspective. 178 active_inc_refs: u8, 179 /// List of `NodeRefInfo` objects that reference this node. 180 refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>, 181 } 182 183 use kernel::bindings::rb_node_layout; 184 use mem::offset_of; 185 pub(crate) const NODE_LAYOUT: rb_node_layout = rb_node_layout { 186 arc_offset: Arc::<Node>::DATA_OFFSET + offset_of!(DTRWrap<Node>, wrapped), 187 debug_id: offset_of!(Node, debug_id), 188 ptr: offset_of!(Node, ptr), 189 }; 190 191 #[pin_data] 192 pub(crate) struct Node { 193 pub(crate) debug_id: usize, 194 ptr: u64, 195 pub(crate) cookie: u64, 196 pub(crate) flags: u32, 197 pub(crate) owner: Arc<Process>, 198 inner: LockedBy<NodeInner, ProcessInner>, 199 #[pin] 200 links_track: AtomicTracker, 201 } 202 203 kernel::list::impl_list_arc_safe! { 204 impl ListArcSafe<0> for Node { 205 tracked_by links_track: AtomicTracker; 206 } 207 } 208 209 // Make `oneway_todo` work. 210 kernel::list::impl_list_item! { 211 impl ListItem<0> for DTRWrap<Transaction> { 212 using ListLinks { self.links.inner }; 213 } 214 } 215 216 impl Node { 217 pub(crate) fn new( 218 ptr: u64, 219 cookie: u64, 220 flags: u32, 221 owner: Arc<Process>, 222 ) -> impl PinInit<Self> { 223 pin_init!(Self { 224 inner: LockedBy::new( 225 &owner.inner, 226 NodeInner { 227 strong: CountState::new(), 228 weak: CountState::new(), 229 delivery_state: DeliveryState { 230 has_pushed_node: false, 231 has_pushed_wrapper: false, 232 has_weak_zero2one: false, 233 has_strong_zero2one: false, 234 }, 235 death_list: List::new(), 236 oneway_todo: List::new(), 237 freeze_list: KVVec::new(), 238 has_oneway_transaction: false, 239 active_inc_refs: 0, 240 refs: List::new(), 241 }, 242 ), 243 debug_id: super::next_debug_id(), 244 ptr, 245 cookie, 246 flags, 247 owner, 248 links_track <- AtomicTracker::new(), 249 }) 250 } 251 252 pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool { 253 let inner = self.inner.access_mut(owner_inner); 254 inner.has_oneway_transaction 255 } 256 257 #[inline(never)] 258 pub(crate) fn full_debug_print(&self, m: &SeqFile, owner_inner: &mut ProcessInner) -> Result { 259 let inner = self.inner.access_mut(owner_inner); 260 seq_print!( 261 m, 262 " node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}", 263 self.debug_id, 264 self.ptr, 265 self.cookie, 266 inner.strong.has_count, 267 inner.weak.has_count, 268 inner.strong.count, 269 inner.weak.count, 270 ); 271 if !inner.refs.is_empty() { 272 seq_print!(m, " proc"); 273 for node_ref in &inner.refs { 274 seq_print!(m, " {}", node_ref.process.task.pid()); 275 } 276 } 277 seq_print!(m, "\n"); 278 for t in &inner.oneway_todo { 279 t.debug_print_inner(m, " pending async transaction "); 280 } 281 Ok(()) 282 } 283 284 /// Insert the `NodeRef` into this `refs` list. 285 /// 286 /// # Safety 287 /// 288 /// It must be the case that `info.node_ref.node` is this node. 289 pub(crate) unsafe fn insert_node_info( 290 &self, 291 info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>, 292 ) { 293 self.inner 294 .access_mut(&mut self.owner.inner.lock()) 295 .refs 296 .push_front(info); 297 } 298 299 /// Insert the `NodeRef` into this `refs` list. 300 /// 301 /// # Safety 302 /// 303 /// It must be the case that `info.node_ref.node` is this node. 304 pub(crate) unsafe fn remove_node_info( 305 &self, 306 info: &NodeRefInfo, 307 ) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> { 308 // SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it 309 // references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in 310 // the `refs` list of another node. 311 unsafe { 312 self.inner 313 .access_mut(&mut self.owner.inner.lock()) 314 .refs 315 .remove(info) 316 } 317 } 318 319 /// An id that is unique across all binder nodes on the system. Used as the key in the 320 /// `by_node` map. 321 pub(crate) fn global_id(&self) -> usize { 322 ptr::from_ref(self).addr() 323 } 324 325 pub(crate) fn get_id(&self) -> (u64, u64) { 326 (self.ptr, self.cookie) 327 } 328 329 pub(crate) fn add_death( 330 &self, 331 death: ListArc<DTRWrap<NodeDeath>, 1>, 332 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>, 333 ) { 334 assert!( 335 core::ptr::eq(self, &**death.node), 336 "attempt to add NodeDeath to the wrong death list" 337 ); 338 self.inner.access_mut(guard).death_list.push_back(death); 339 } 340 341 pub(crate) fn inc_ref_done_locked( 342 self: &DArc<Node>, 343 _strong: bool, 344 owner_inner: &mut ProcessInner, 345 ) -> Option<DLArc<Node>> { 346 let inner = self.inner.access_mut(owner_inner); 347 if inner.active_inc_refs == 0 { 348 binder_debug!(UserError, "inc_ref_done called when no active inc_refs"); 349 return None; 350 } 351 352 inner.active_inc_refs -= 1; 353 if inner.active_inc_refs == 0 { 354 // Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we 355 // would send a refcount decrement, and if so, tell the caller to schedule us. 356 let strong = inner.strong.count > 0; 357 let has_strong = inner.strong.has_count; 358 let weak = strong || inner.weak.count > 0; 359 let has_weak = inner.weak.has_count; 360 361 let should_drop_weak = !weak && has_weak; 362 let should_drop_strong = !strong && has_strong; 363 364 // If we want to drop the ref-count again, tell the caller to schedule a work node for 365 // that. 366 let need_push = should_drop_weak || should_drop_strong; 367 368 if need_push && inner.delivery_state.should_normal_push() { 369 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); 370 inner.delivery_state.did_normal_push(); 371 Some(list_arc) 372 } else { 373 None 374 } 375 } else { 376 None 377 } 378 } 379 380 pub(crate) fn update_refcount_locked( 381 self: &DArc<Node>, 382 inc: bool, 383 strong: bool, 384 count: usize, 385 owner_inner: &mut ProcessInner, 386 ) -> Option<DLArc<Node>> { 387 let is_dead = owner_inner.is_dead; 388 let inner = self.inner.access_mut(owner_inner); 389 390 // Get a reference to the state we'll update. 391 let state = if strong { 392 &mut inner.strong 393 } else { 394 &mut inner.weak 395 }; 396 397 // Update the count and determine whether we need to push work. 398 let need_push = if inc { 399 state.count += count; 400 // TODO: This method shouldn't be used for zero-to-one increments. 401 !is_dead && !state.has_count 402 } else { 403 if state.count < count { 404 pr_err_ratelimited!("Failure: refcount underflow!\n"); 405 return None; 406 } 407 state.count -= count; 408 !is_dead && state.count == 0 && state.has_count 409 }; 410 411 if need_push && inner.delivery_state.should_normal_push() { 412 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); 413 inner.delivery_state.did_normal_push(); 414 Some(list_arc) 415 } else { 416 None 417 } 418 } 419 420 pub(crate) fn incr_refcount_allow_zero2one( 421 self: &DArc<Self>, 422 strong: bool, 423 owner_inner: &mut ProcessInner, 424 ) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> { 425 let is_dead = owner_inner.is_dead; 426 let inner = self.inner.access_mut(owner_inner); 427 428 // Get a reference to the state we'll update. 429 let state = if strong { 430 &mut inner.strong 431 } else { 432 &mut inner.weak 433 }; 434 435 // Update the count and determine whether we need to push work. 436 state.count += 1; 437 if is_dead || state.has_count { 438 return Ok(None); 439 } 440 441 // Userspace needs to be notified of this. 442 if !strong && inner.delivery_state.should_push_weak_zero2one() { 443 assert!(inner.delivery_state.can_push_weak_zero2one_normally()); 444 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); 445 inner.delivery_state.did_push_weak_zero2one(); 446 Ok(Some(list_arc)) 447 } else if strong && inner.delivery_state.should_push_strong_zero2one() { 448 if inner.delivery_state.can_push_strong_zero2one_normally() { 449 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); 450 inner.delivery_state.did_push_strong_zero2one(); 451 Ok(Some(list_arc)) 452 } else { 453 state.count -= 1; 454 Err(CouldNotDeliverCriticalIncrement) 455 } 456 } else { 457 // Work is already pushed, and we don't need to push again. 458 Ok(None) 459 } 460 } 461 462 pub(crate) fn incr_refcount_allow_zero2one_with_wrapper( 463 self: &DArc<Self>, 464 strong: bool, 465 wrapper: CritIncrWrapper, 466 owner_inner: &mut ProcessInner, 467 ) -> Option<DLArc<dyn DeliverToRead>> { 468 match self.incr_refcount_allow_zero2one(strong, owner_inner) { 469 Ok(Some(node)) => Some(node as DLArc<dyn DeliverToRead>), 470 Ok(None) => None, 471 Err(CouldNotDeliverCriticalIncrement) => { 472 assert!(strong); 473 let inner = self.inner.access_mut(owner_inner); 474 inner.strong.count += 1; 475 inner.delivery_state.did_push_strong_zero2one_wrapper(); 476 Some(wrapper.init(self.clone())) 477 } 478 } 479 } 480 481 pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) { 482 self.owner 483 .inner 484 .lock() 485 .update_node_refcount(self, inc, strong, count, None); 486 } 487 488 pub(crate) fn populate_counts( 489 &self, 490 out: &mut BinderNodeInfoForRef, 491 guard: &Guard<'_, ProcessInner, SpinLockBackend>, 492 ) { 493 let inner = self.inner.access(guard); 494 out.strong_count = inner.strong.count as u32; 495 out.weak_count = inner.weak.count as u32; 496 } 497 498 pub(crate) fn populate_debug_info( 499 &self, 500 out: &mut BinderNodeDebugInfo, 501 guard: &Guard<'_, ProcessInner, SpinLockBackend>, 502 ) { 503 out.ptr = self.ptr as uapi::binder_uintptr_t; 504 out.cookie = self.cookie as uapi::binder_uintptr_t; 505 let inner = self.inner.access(guard); 506 if inner.strong.has_count { 507 out.has_strong_ref = 1; 508 } 509 if inner.weak.has_count { 510 out.has_weak_ref = 1; 511 } 512 } 513 514 pub(crate) fn force_has_count(&self, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) { 515 let inner = self.inner.access_mut(guard); 516 inner.strong.has_count = true; 517 inner.weak.has_count = true; 518 } 519 520 fn write(&self, writer: &mut BinderReturnWriter<'_>, code: u32) -> Result { 521 writer.write_code(code)?; 522 writer.write_payload(&self.ptr)?; 523 writer.write_payload(&self.cookie)?; 524 Ok(()) 525 } 526 527 pub(crate) fn submit_oneway( 528 &self, 529 transaction: DLArc<Transaction>, 530 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>, 531 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> { 532 if guard.is_dead { 533 return Err((BinderError::new_dead(), transaction)); 534 } 535 536 let inner = self.inner.access_mut(guard); 537 if inner.has_oneway_transaction { 538 inner.oneway_todo.push_back(transaction); 539 } else { 540 inner.has_oneway_transaction = true; 541 guard.push_work(&self.owner, transaction)?; 542 } 543 Ok(()) 544 } 545 546 pub(crate) fn release(&self) { 547 let mut guard = self.owner.inner.lock(); 548 while let Some(work) = self.inner.access_mut(&mut guard).oneway_todo.pop_front() { 549 drop(guard); 550 work.into_arc().cancel(); 551 guard = self.owner.inner.lock(); 552 } 553 554 while let Some(death) = self.inner.access_mut(&mut guard).death_list.pop_front() { 555 drop(guard); 556 death.into_arc().set_dead(); 557 guard = self.owner.inner.lock(); 558 } 559 } 560 561 pub(crate) fn pending_oneway_finished(&self) { 562 let mut guard = self.owner.inner.lock(); 563 if guard.is_dead { 564 // Cleanup will happen in `Process::deferred_release`. 565 return; 566 } 567 568 let inner = self.inner.access_mut(&mut guard); 569 570 let transaction = inner.oneway_todo.pop_front(); 571 inner.has_oneway_transaction = transaction.is_some(); 572 if let Some(transaction) = transaction { 573 match guard.push_work(&self.owner, transaction) { 574 Ok(()) => {} 575 Err((_err, work)) => { 576 // Process is dead. 577 // This shouldn't happen due to the `is_dead` check, but if it does, just drop 578 // the transaction and return. 579 drop(guard); 580 drop(work); 581 } 582 } 583 } 584 } 585 586 /// Finds an outdated transaction that the given transaction can replace. 587 /// 588 /// If one is found, it is removed from the list and returned. 589 pub(crate) fn take_outdated_transaction( 590 &self, 591 new: &Transaction, 592 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>, 593 ) -> Option<DLArc<Transaction>> { 594 let inner = self.inner.access_mut(guard); 595 let mut cursor = inner.oneway_todo.cursor_front(); 596 while let Some(next) = cursor.peek_next() { 597 if new.can_replace(&next) { 598 return Some(next.remove()); 599 } 600 cursor.move_next(); 601 } 602 None 603 } 604 605 /// This is split into a separate function since it's called by both `Node::do_work` and 606 /// `NodeWrapper::do_work`. 607 fn do_work_locked( 608 &self, 609 writer: &mut BinderReturnWriter<'_>, 610 mut guard: Guard<'_, ProcessInner, SpinLockBackend>, 611 ) -> Result<bool> { 612 let inner = self.inner.access_mut(&mut guard); 613 let strong = inner.strong.count > 0; 614 let has_strong = inner.strong.has_count; 615 let weak = strong || inner.weak.count > 0; 616 let has_weak = inner.weak.has_count; 617 618 if weak && !has_weak { 619 inner.weak.has_count = true; 620 inner.active_inc_refs += 1; 621 } 622 623 if strong && !has_strong { 624 inner.strong.has_count = true; 625 inner.active_inc_refs += 1; 626 } 627 628 let no_active_inc_refs = inner.active_inc_refs == 0; 629 let should_drop_weak = no_active_inc_refs && (!weak && has_weak); 630 let should_drop_strong = no_active_inc_refs && (!strong && has_strong); 631 if should_drop_weak { 632 inner.weak.has_count = false; 633 } 634 if should_drop_strong { 635 inner.strong.has_count = false; 636 } 637 if no_active_inc_refs && !weak { 638 // Remove the node if there are no references to it. 639 guard.remove_node(self.ptr); 640 } 641 drop(guard); 642 643 if weak && !has_weak { 644 self.write(writer, BR_INCREFS)?; 645 } 646 if strong && !has_strong { 647 self.write(writer, BR_ACQUIRE)?; 648 } 649 if should_drop_strong { 650 self.write(writer, BR_RELEASE)?; 651 } 652 if should_drop_weak { 653 self.write(writer, BR_DECREFS)?; 654 } 655 656 Ok(true) 657 } 658 659 pub(crate) fn add_freeze_listener( 660 &self, 661 process: &Arc<Process>, 662 // If the vector needs to be resized, it's done via this argument. 663 vec_alloc: &mut KVVec<Arc<Process>>, 664 ) -> Result<Result<(), usize>> { 665 let mut guard = self.owner.inner.lock(); 666 // Do not check for `guard.dead`. The `dead` flag that matters here is the owner of the 667 // listener, not the target. 668 let inner = self.inner.access_mut(&mut guard); 669 let len = inner.freeze_list.len(); 670 if len == inner.freeze_list.capacity() { 671 if len >= vec_alloc.capacity() { 672 // Request the caller to reallocate. 673 return Ok(Err((1 + len).next_power_of_two())); 674 } 675 mem::swap(&mut inner.freeze_list, vec_alloc); 676 for elem in vec_alloc.drain_all() { 677 inner.freeze_list.push_within_capacity(elem)?; 678 } 679 } 680 inner.freeze_list.push_within_capacity(process.clone())?; 681 Ok(Ok(())) 682 } 683 684 pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec<Arc<Process>> { 685 let mut guard = self.owner.inner.lock(); 686 let inner = self.inner.access_mut(&mut guard); 687 let len = inner.freeze_list.len(); 688 inner 689 .freeze_list 690 .retain(|proc| !core::ptr::eq::<Process>(&**proc, p)); 691 if len == inner.freeze_list.len() { 692 pr_warn_ratelimited!( 693 "Could not remove freeze listener for {}\n", 694 p.pid_in_current_ns() 695 ); 696 } 697 // If the vector is empty it needs to be freed. However, we can't free it here because that 698 // might sleep, so return it to the caller. 699 if inner.freeze_list.is_empty() { 700 return mem::take(&mut inner.freeze_list); 701 } 702 KVVec::new() 703 } 704 705 pub(crate) fn freeze_list<'a>(&'a self, guard: &'a ProcessInner) -> &'a [Arc<Process>] { 706 &self.inner.access(guard).freeze_list 707 } 708 } 709 710 impl DeliverToRead for Node { 711 fn do_work( 712 self: DArc<Self>, 713 _thread: &Thread, 714 writer: &mut BinderReturnWriter<'_>, 715 ) -> Result<bool> { 716 let mut owner_inner = self.owner.inner.lock(); 717 let inner = self.inner.access_mut(&mut owner_inner); 718 719 assert!(inner.delivery_state.has_pushed_node); 720 if inner.delivery_state.has_pushed_wrapper { 721 // If the wrapper is scheduled, then we are either a normal push or weak zero2one 722 // increment, and the wrapper is a strong zero2one increment, so the wrapper always 723 // takes precedence over us. 724 assert!(inner.delivery_state.has_strong_zero2one); 725 inner.delivery_state.has_pushed_node = false; 726 inner.delivery_state.has_weak_zero2one = false; 727 return Ok(true); 728 } 729 730 inner.delivery_state.has_pushed_node = false; 731 inner.delivery_state.has_weak_zero2one = false; 732 inner.delivery_state.has_strong_zero2one = false; 733 734 self.do_work_locked(writer, owner_inner) 735 } 736 737 fn cancel(self: DArc<Self>) { 738 let _drop_outside_lock; 739 let mut owner_inner = self.owner.inner.lock(); 740 741 // We only do something on BINDER_THREAD_EXIT, not process exit. 742 if owner_inner.is_dead { 743 return; 744 } 745 746 // If BINDER_THREAD_EXIT is invoked on a thread with a pending node refcount update, we 747 // should move ourselves to ensure the refcount update is still delivered. 748 if let Some(node) = ListArc::try_from_arc_borrow(self.as_arc_borrow()) { 749 _drop_outside_lock = owner_inner.push_work(&self.owner, node); 750 } 751 } 752 753 fn should_sync_wakeup(&self) -> bool { 754 false 755 } 756 757 #[inline(never)] 758 fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result { 759 seq_print!( 760 m, 761 "{}node work {}: u{:016x} c{:016x}\n", 762 prefix, 763 self.debug_id, 764 self.ptr, 765 self.cookie, 766 ); 767 Ok(()) 768 } 769 } 770 771 /// Represents something that holds one or more ref-counts to a `Node`. 772 /// 773 /// Whenever process A holds a refcount to a node owned by a different process B, then process A 774 /// will store a `NodeRef` that refers to the `Node` in process B. When process A releases the 775 /// refcount, we destroy the NodeRef, which decrements the ref-count in process A. 776 /// 777 /// This type is also used for some other cases. For example, a transaction allocation holds a 778 /// refcount on the target node, and this is implemented by storing a `NodeRef` in the allocation 779 /// so that the destructor of the allocation will drop a refcount of the `Node`. 780 pub(crate) struct NodeRef { 781 pub(crate) node: DArc<Node>, 782 /// How many times does this NodeRef hold a refcount on the Node? 783 strong_node_count: usize, 784 weak_node_count: usize, 785 /// How many times does userspace hold a refcount on this NodeRef? 786 strong_count: usize, 787 weak_count: usize, 788 } 789 790 impl NodeRef { 791 pub(crate) fn new(node: DArc<Node>, strong_count: usize, weak_count: usize) -> Self { 792 Self { 793 node, 794 strong_node_count: strong_count, 795 weak_node_count: weak_count, 796 strong_count, 797 weak_count, 798 } 799 } 800 801 pub(crate) fn absorb(&mut self, mut other: Self) { 802 assert!( 803 Arc::ptr_eq(&self.node, &other.node), 804 "absorb called with differing nodes" 805 ); 806 self.strong_node_count += other.strong_node_count; 807 self.weak_node_count += other.weak_node_count; 808 self.strong_count += other.strong_count; 809 self.weak_count += other.weak_count; 810 other.strong_count = 0; 811 other.weak_count = 0; 812 other.strong_node_count = 0; 813 other.weak_node_count = 0; 814 815 if self.strong_node_count >= 2 || self.weak_node_count >= 2 { 816 let mut guard = self.node.owner.inner.lock(); 817 let inner = self.node.inner.access_mut(&mut guard); 818 819 if self.strong_node_count >= 2 { 820 inner.strong.count -= self.strong_node_count - 1; 821 self.strong_node_count = 1; 822 assert_ne!(inner.strong.count, 0); 823 } 824 if self.weak_node_count >= 2 { 825 inner.weak.count -= self.weak_node_count - 1; 826 self.weak_node_count = 1; 827 assert_ne!(inner.weak.count, 0); 828 } 829 } 830 } 831 832 pub(crate) fn get_count(&self) -> (usize, usize) { 833 (self.strong_count, self.weak_count) 834 } 835 836 pub(crate) fn clone(&self, strong: bool) -> Result<NodeRef> { 837 if strong && self.strong_count == 0 { 838 binder_debug!(UserError, "tried to use weak ref as strong ref"); 839 return Err(EINVAL); 840 } 841 Ok(self 842 .node 843 .owner 844 .inner 845 .lock() 846 .new_node_ref(self.node.clone(), strong, None)) 847 } 848 849 /// Updates (increments or decrements) the number of references held against the node. If the 850 /// count being updated transitions from 0 to 1 or from 1 to 0, the node is notified by having 851 /// its `update_refcount` function called. 852 /// 853 /// Returns whether `self` should be removed (when both counts are zero). 854 pub(crate) fn update(&mut self, inc: bool, strong: bool) -> bool { 855 if strong && self.strong_count == 0 { 856 return false; 857 } 858 let (count, node_count, other_count) = if strong { 859 ( 860 &mut self.strong_count, 861 &mut self.strong_node_count, 862 self.weak_count, 863 ) 864 } else { 865 ( 866 &mut self.weak_count, 867 &mut self.weak_node_count, 868 self.strong_count, 869 ) 870 }; 871 if inc { 872 if *count == 0 { 873 *node_count = 1; 874 self.node.update_refcount(true, 1, strong); 875 } 876 *count += 1; 877 } else { 878 if *count == 0 { 879 binder_debug!( 880 UserError, 881 "performed invalid {} decrement on ref", 882 if strong { "strong" } else { "weak" } 883 ); 884 return false; 885 } 886 *count -= 1; 887 if *count == 0 { 888 self.node.update_refcount(false, *node_count, strong); 889 *node_count = 0; 890 return other_count == 0; 891 } 892 } 893 false 894 } 895 } 896 897 impl Drop for NodeRef { 898 // This destructor is called conditionally from `Allocation::drop`. That branch is often 899 // mispredicted. Inlining this method call reduces the cost of those branch mispredictions. 900 #[inline(always)] 901 fn drop(&mut self) { 902 if self.strong_node_count > 0 { 903 self.node 904 .update_refcount(false, self.strong_node_count, true); 905 } 906 if self.weak_node_count > 0 { 907 self.node 908 .update_refcount(false, self.weak_node_count, false); 909 } 910 } 911 } 912 913 struct NodeDeathInner { 914 dead: bool, 915 cleared: bool, 916 notification_done: bool, 917 /// Indicates whether the normal flow was interrupted by removing the handle. In this case, we 918 /// need behave as if the death notification didn't exist (i.e., we don't deliver anything to 919 /// the user. 920 aborted: bool, 921 } 922 923 /// Used to deliver notifications when a process dies. 924 /// 925 /// A process can request to be notified when a process dies using `BC_REQUEST_DEATH_NOTIFICATION`. 926 /// This will make the driver send a `BR_DEAD_BINDER` to userspace when the process dies (or 927 /// immediately if it is already dead). Userspace is supposed to respond with `BC_DEAD_BINDER_DONE` 928 /// once it has processed the notification. 929 /// 930 /// Userspace can unregister from death notifications using the `BC_CLEAR_DEATH_NOTIFICATION` 931 /// command. In this case, the kernel will respond with `BR_CLEAR_DEATH_NOTIFICATION_DONE` once the 932 /// notification has been removed. Note that if the remote process dies before the kernel has 933 /// responded with `BR_CLEAR_DEATH_NOTIFICATION_DONE`, then the kernel will still send a 934 /// `BR_DEAD_BINDER`, which userspace must be able to process. In this case, the kernel will wait 935 /// for the `BC_DEAD_BINDER_DONE` command before it sends `BR_CLEAR_DEATH_NOTIFICATION_DONE`. 936 /// 937 /// Note that even if the kernel sends a `BR_DEAD_BINDER`, this does not remove the death 938 /// notification. Userspace must still remove it manually using `BC_CLEAR_DEATH_NOTIFICATION`. 939 /// 940 /// If a process uses `BC_RELEASE` to destroy its last refcount on a node that has an active death 941 /// registration, then the death registration is immediately deleted (we implement this using the 942 /// `aborted` field). However, userspace is not supposed to delete a `NodeRef` without first 943 /// deregistering death notifications, so this codepath is not executed under normal circumstances. 944 #[pin_data] 945 pub(crate) struct NodeDeath { 946 node: DArc<Node>, 947 process: Arc<Process>, 948 pub(crate) cookie: u64, 949 #[pin] 950 links_track: AtomicTracker<0>, 951 /// Used by the owner `Node` to store a list of registered death notifications. 952 /// 953 /// # Invariants 954 /// 955 /// Only ever used with the `death_list` list of `self.node`. 956 #[pin] 957 death_links: ListLinks<1>, 958 /// Used by the process to keep track of the death notifications for which we have sent a 959 /// `BR_DEAD_BINDER` but not yet received a `BC_DEAD_BINDER_DONE`. 960 /// 961 /// # Invariants 962 /// 963 /// Only ever used with the `delivered_deaths` list of `self.process`. 964 #[pin] 965 delivered_links: ListLinks<2>, 966 #[pin] 967 delivered_links_track: AtomicTracker<2>, 968 #[pin] 969 inner: SpinLock<NodeDeathInner>, 970 } 971 972 impl NodeDeath { 973 /// Constructs a new node death notification object. 974 pub(crate) fn new( 975 node: DArc<Node>, 976 process: Arc<Process>, 977 cookie: u64, 978 ) -> impl PinInit<DTRWrap<Self>> { 979 DTRWrap::new(pin_init!( 980 Self { 981 node, 982 process, 983 cookie, 984 links_track <- AtomicTracker::new(), 985 death_links <- ListLinks::new(), 986 delivered_links <- ListLinks::new(), 987 delivered_links_track <- AtomicTracker::new(), 988 inner <- kernel::new_spinlock!(NodeDeathInner { 989 dead: false, 990 cleared: false, 991 notification_done: false, 992 aborted: false, 993 }, "NodeDeath::inner"), 994 } 995 )) 996 } 997 998 /// Sets the cleared flag to `true`. 999 /// 1000 /// It removes `self` from the node's death notification list if needed. 1001 /// 1002 /// Returns whether it needs to be queued. 1003 pub(crate) fn set_cleared(self: &DArc<Self>, abort: bool) -> bool { 1004 let (needs_removal, needs_queueing) = { 1005 // Update state and determine if we need to queue a work item. We only need to do it 1006 // when the node is not dead or if the user already completed the death notification. 1007 let mut inner = self.inner.lock(); 1008 if abort { 1009 inner.aborted = true; 1010 } 1011 if inner.cleared { 1012 // Already cleared. 1013 return false; 1014 } 1015 inner.cleared = true; 1016 (!inner.dead, !inner.dead || inner.notification_done) 1017 }; 1018 1019 // Remove death notification from node. 1020 if needs_removal { 1021 let mut owner_inner = self.node.owner.inner.lock(); 1022 let node_inner = self.node.inner.access_mut(&mut owner_inner); 1023 // SAFETY: A `NodeDeath` is never inserted into the death list of any node other than 1024 // its owner, so it is either in this death list or in no death list. 1025 unsafe { node_inner.death_list.remove(self) }; 1026 } 1027 needs_queueing 1028 } 1029 1030 /// Sets the 'notification done' flag to `true`. 1031 pub(crate) fn set_notification_done(self: DArc<Self>, thread: &Thread) { 1032 let needs_queueing = { 1033 let mut inner = self.inner.lock(); 1034 inner.notification_done = true; 1035 inner.cleared 1036 }; 1037 if needs_queueing { 1038 if let Some(death) = ListArc::try_from_arc_or_drop(self) { 1039 let _ = thread.push_work_if_looper(death); 1040 } 1041 } 1042 } 1043 1044 /// Sets the 'dead' flag to `true` and queues work item if needed. 1045 pub(crate) fn set_dead(self: DArc<Self>) { 1046 let needs_queueing = { 1047 let mut inner = self.inner.lock(); 1048 if inner.cleared { 1049 false 1050 } else { 1051 inner.dead = true; 1052 true 1053 } 1054 }; 1055 if needs_queueing { 1056 // Push the death notification to the target process. There is nothing else to do if 1057 // it's already dead. 1058 if let Some(death) = ListArc::try_from_arc_or_drop(self) { 1059 let process = death.process.clone(); 1060 let _ = process.push_work(death); 1061 } 1062 } 1063 } 1064 } 1065 1066 kernel::list::impl_list_arc_safe! { 1067 impl ListArcSafe<0> for NodeDeath { 1068 tracked_by links_track: AtomicTracker; 1069 } 1070 } 1071 1072 kernel::list::impl_list_arc_safe! { 1073 impl ListArcSafe<1> for DTRWrap<NodeDeath> { untracked; } 1074 } 1075 kernel::list::impl_list_item! { 1076 impl ListItem<1> for DTRWrap<NodeDeath> { 1077 using ListLinks { self.wrapped.death_links }; 1078 } 1079 } 1080 1081 kernel::list::impl_list_arc_safe! { 1082 impl ListArcSafe<2> for DTRWrap<NodeDeath> { 1083 tracked_by wrapped: NodeDeath; 1084 } 1085 } 1086 kernel::list::impl_list_arc_safe! { 1087 impl ListArcSafe<2> for NodeDeath { 1088 tracked_by delivered_links_track: AtomicTracker<2>; 1089 } 1090 } 1091 kernel::list::impl_list_item! { 1092 impl ListItem<2> for DTRWrap<NodeDeath> { 1093 using ListLinks { self.wrapped.delivered_links }; 1094 } 1095 } 1096 1097 impl DeliverToRead for NodeDeath { 1098 fn do_work( 1099 self: DArc<Self>, 1100 _thread: &Thread, 1101 writer: &mut BinderReturnWriter<'_>, 1102 ) -> Result<bool> { 1103 let done = { 1104 let inner = self.inner.lock(); 1105 if inner.aborted { 1106 return Ok(true); 1107 } 1108 inner.cleared && (!inner.dead || inner.notification_done) 1109 }; 1110 1111 let cookie = self.cookie; 1112 let cmd = if done { 1113 BR_CLEAR_DEATH_NOTIFICATION_DONE 1114 } else { 1115 let process = self.process.clone(); 1116 let mut process_inner = process.inner.lock(); 1117 let inner = self.inner.lock(); 1118 if inner.aborted { 1119 return Ok(true); 1120 } 1121 // We're still holding the inner lock, so it cannot be aborted while we insert it into 1122 // the delivered list. 1123 process_inner.death_delivered(self.clone()); 1124 binder_debug!( 1125 DeathNotification, 1126 "sending death notification, cookie {:016x}", 1127 cookie 1128 ); 1129 BR_DEAD_BINDER 1130 }; 1131 1132 writer.write_code(cmd)?; 1133 writer.write_payload(&cookie)?; 1134 // DEAD_BINDER notifications can cause transactions, so stop processing work items when we 1135 // get to a death notification. 1136 Ok(cmd != BR_DEAD_BINDER) 1137 } 1138 1139 fn cancel(self: DArc<Self>) { 1140 binder_debug!( 1141 pid = self.process.task.pid(), 1142 DeadTransaction, 1143 "undelivered death notification, {:016x}", 1144 self.cookie 1145 ); 1146 } 1147 1148 fn should_sync_wakeup(&self) -> bool { 1149 false 1150 } 1151 1152 #[inline(never)] 1153 fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result { 1154 let inner = self.inner.lock(); 1155 1156 let dead_binder = inner.dead && !inner.notification_done; 1157 1158 if dead_binder { 1159 if inner.cleared { 1160 seq_print!(m, "{}has cleared dead binder\n", prefix); 1161 } else { 1162 seq_print!(m, "{}has dead binder\n", prefix); 1163 } 1164 } else { 1165 seq_print!(m, "{}has cleared death notification\n", prefix); 1166 } 1167 1168 Ok(()) 1169 } 1170 } 1171