1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 //! This module defines the `Thread` type, which represents a userspace thread that is using 6 //! binder. 7 //! 8 //! The `Process` object stores all of the threads in an rb tree. 9 10 use kernel::{ 11 bindings, 12 bits::bit_u32, 13 fs::LocalFile, 14 list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, 15 prelude::*, 16 security, 17 seq_file::SeqFile, 18 seq_print, 19 sync::atomic::{ordering::Relaxed, Atomic}, 20 sync::{aref::ARef, Arc, CondVar, SpinLock}, 21 task::Task, 22 uaccess::{UserPtr, UserSlice, UserSliceReader}, 23 uapi, 24 }; 25 26 use crate::{ 27 allocation::{Allocation, AllocationView, BinderObject, BinderObjectRef, NewAllocation}, 28 defs::*, 29 error::BinderResult, 30 process::{GetWorkOrRegister, Process}, 31 ptr_align, 32 stats::GLOBAL_STATS, 33 trace::{trace_transaction_alloc_buf, trace_transaction_buffer_release}, 34 transaction::{Transaction, TransactionFlag, TransactionFlags, TransactionInfo}, 35 BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverCode, DeliverToRead, 36 }; 37 38 use core::mem::size_of; 39 40 fn is_aligned(value: usize, to: usize) -> bool { 41 value % to == 0 42 } 43 44 /// Stores the layout of the scatter-gather entries. This is used during the `translate_objects` 45 /// call and is discarded when it returns. 46 struct ScatterGatherState { 47 /// A struct that tracks the amount of unused buffer space. 48 unused_buffer_space: UnusedBufferSpace, 49 /// Scatter-gather entries to copy. 50 sg_entries: KVec<ScatterGatherEntry>, 51 /// Indexes into `sg_entries` corresponding to the last binder_buffer_object that 52 /// was processed and all of its ancestors. The array is in sorted order. 53 ancestors: KVec<usize>, 54 } 55 56 /// This entry specifies an additional buffer that should be copied using the scatter-gather 57 /// mechanism. 58 struct ScatterGatherEntry { 59 /// The index in the offset array of the BINDER_TYPE_PTR that this entry originates from. 60 obj_index: usize, 61 /// Offset in target buffer. 62 offset: usize, 63 /// User address in source buffer. 64 sender_uaddr: usize, 65 /// Number of bytes to copy. 66 length: usize, 67 /// The minimum offset of the next fixup in this buffer. 68 fixup_min_offset: usize, 69 /// The offsets within this buffer that contain pointers which should be translated. 70 pointer_fixups: KVec<PointerFixupEntry>, 71 } 72 73 /// This entry specifies that a fixup should happen at `target_offset` of the 74 /// buffer. 75 enum PointerFixupEntry { 76 /// A fixup for a `binder_buffer_object`. 77 Fixup { 78 /// The translated pointer to write. 79 pointer_value: u64, 80 /// The offset at which the value should be written. The offset is relative 81 /// to the original buffer. 82 target_offset: usize, 83 }, 84 /// A skip for a `binder_fd_array_object`. 85 Skip { 86 /// The number of bytes to skip. 87 skip: usize, 88 /// The offset at which the skip should happen. The offset is relative 89 /// to the original buffer. 90 target_offset: usize, 91 }, 92 } 93 94 /// Return type of `apply_and_validate_fixup_in_parent`. 95 struct ParentFixupInfo { 96 /// The index of the parent buffer in `sg_entries`. 97 parent_sg_index: usize, 98 /// The number of ancestors of the buffer. 99 /// 100 /// The buffer is considered an ancestor of itself, so this is always at 101 /// least one. 102 num_ancestors: usize, 103 /// New value of `fixup_min_offset` if this fixup is applied. 104 new_min_offset: usize, 105 /// The offset of the fixup in the target buffer. 106 target_offset: usize, 107 } 108 109 impl ScatterGatherState { 110 /// Called when a `binder_buffer_object` or `binder_fd_array_object` tries 111 /// to access a region in its parent buffer. These accesses have various 112 /// restrictions, which this method verifies. 113 /// 114 /// The `parent_offset` and `length` arguments describe the offset and 115 /// length of the access in the parent buffer. 116 /// 117 /// # Detailed restrictions 118 /// 119 /// Obviously the fixup must be in-bounds for the parent buffer. 120 /// 121 /// For safety reasons, we only allow fixups inside a buffer to happen 122 /// at increasing offsets; additionally, we only allow fixup on the last 123 /// buffer object that was verified, or one of its parents. 124 /// 125 /// Example of what is allowed: 126 /// 127 /// A 128 /// B (parent = A, offset = 0) 129 /// C (parent = A, offset = 16) 130 /// D (parent = C, offset = 0) 131 /// E (parent = A, offset = 32) // min_offset is 16 (C.parent_offset) 132 /// 133 /// Examples of what is not allowed: 134 /// 135 /// Decreasing offsets within the same parent: 136 /// A 137 /// C (parent = A, offset = 16) 138 /// B (parent = A, offset = 0) // decreasing offset within A 139 /// 140 /// Arcerring to a parent that wasn't the last object or any of its parents: 141 /// A 142 /// B (parent = A, offset = 0) 143 /// C (parent = A, offset = 0) 144 /// C (parent = A, offset = 16) 145 /// D (parent = B, offset = 0) // B is not A or any of A's parents 146 fn validate_parent_fixup( 147 &self, 148 parent: usize, 149 parent_offset: usize, 150 length: usize, 151 ) -> Result<ParentFixupInfo> { 152 // Using `position` would also be correct, but `rposition` avoids 153 // quadratic running times. 154 let ancestors_i = self 155 .ancestors 156 .iter() 157 .copied() 158 .rposition(|sg_idx| self.sg_entries[sg_idx].obj_index == parent) 159 .ok_or(EINVAL)?; 160 let sg_idx = self.ancestors[ancestors_i]; 161 let sg_entry = match self.sg_entries.get(sg_idx) { 162 Some(sg_entry) => sg_entry, 163 None => { 164 pr_err_ratelimited!( 165 "self.ancestors[{}] is {}, but self.sg_entries.len() is {}\n", 166 ancestors_i, 167 sg_idx, 168 self.sg_entries.len() 169 ); 170 return Err(EINVAL); 171 } 172 }; 173 if sg_entry.fixup_min_offset > parent_offset { 174 pr_warn_ratelimited!( 175 "validate_parent_fixup: fixup_min_offset={}, parent_offset={}\n", 176 sg_entry.fixup_min_offset, 177 parent_offset 178 ); 179 return Err(EINVAL); 180 } 181 let new_min_offset = parent_offset.checked_add(length).ok_or(EINVAL)?; 182 if new_min_offset > sg_entry.length { 183 pr_warn_ratelimited!( 184 "validate_parent_fixup: new_min_offset={}, sg_entry.length={}\n", 185 new_min_offset, 186 sg_entry.length 187 ); 188 return Err(EINVAL); 189 } 190 let target_offset = sg_entry.offset.checked_add(parent_offset).ok_or(EINVAL)?; 191 // The `ancestors_i + 1` operation can't overflow since the output of the addition is at 192 // most `self.ancestors.len()`, which also fits in a usize. 193 Ok(ParentFixupInfo { 194 parent_sg_index: sg_idx, 195 num_ancestors: ancestors_i + 1, 196 new_min_offset, 197 target_offset, 198 }) 199 } 200 } 201 202 /// Keeps track of how much unused buffer space is left. The initial amount is the number of bytes 203 /// requested by the user using the `buffers_size` field of `binder_transaction_data_sg`. Each time 204 /// we translate an object of type `BINDER_TYPE_PTR`, some of the unused buffer space is consumed. 205 struct UnusedBufferSpace { 206 /// The start of the remaining space. 207 offset: usize, 208 /// The end of the remaining space. 209 limit: usize, 210 } 211 impl UnusedBufferSpace { 212 /// Claim the next `size` bytes from the unused buffer space. The offset for the claimed chunk 213 /// into the buffer is returned. 214 fn claim_next(&mut self, size: usize) -> Result<usize> { 215 // We require every chunk to be aligned. 216 let size = ptr_align(size).ok_or(EINVAL)?; 217 let new_offset = self.offset.checked_add(size).ok_or(EINVAL)?; 218 219 if new_offset <= self.limit { 220 let offset = self.offset; 221 self.offset = new_offset; 222 Ok(offset) 223 } else { 224 Err(EINVAL) 225 } 226 } 227 } 228 229 #[must_use] 230 pub(crate) enum PushWorkRes { 231 Ok, 232 OkNotifyPoll, 233 FailedDead(DLArc<dyn DeliverToRead>), 234 } 235 236 impl PushWorkRes { 237 fn is_ok(&self) -> bool { 238 match self { 239 PushWorkRes::Ok => true, 240 PushWorkRes::OkNotifyPoll => true, 241 PushWorkRes::FailedDead(_) => false, 242 } 243 } 244 } 245 246 /// The fields of `Thread` protected by the spinlock. 247 struct InnerThread { 248 /// Determines the looper state of the thread. It is a bit-wise combination of the constants 249 /// prefixed with `LOOPER_`. 250 looper_flags: LooperFlags, 251 252 /// Determines whether the looper should return. 253 looper_need_return: bool, 254 255 /// Determines if thread is dead. 256 is_dead: bool, 257 258 /// Work item used to deliver error codes to the thread that started a transaction. Stored here 259 /// so that it can be reused. 260 reply_work: DArc<ThreadError>, 261 262 /// Work item used to deliver error codes to the current thread. Stored here so that it can be 263 /// reused. 264 return_work: DArc<ThreadError>, 265 266 /// Determines whether the work list below should be processed. When set to false, `work_list` 267 /// is treated as if it were empty. 268 process_work_list: bool, 269 /// List of work items to deliver to userspace. 270 work_list: List<DTRWrap<dyn DeliverToRead>>, 271 current_transaction: Option<DArc<Transaction>>, 272 273 /// Extended error information for this thread. 274 extended_error: ExtendedError, 275 } 276 277 kernel::impl_flags!( 278 /// Represents multiple looper flags. 279 #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] 280 pub struct LooperFlags(u32); 281 282 /// Represents a single looper flag. 283 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 284 pub enum LooperFlag { 285 Registered = bit_u32(0), 286 Entered = bit_u32(1), 287 Exited = bit_u32(2), 288 Invalid = bit_u32(3), 289 Waiting = bit_u32(4), 290 WaitingProc = bit_u32(5), 291 Poll = bit_u32(6), 292 } 293 ); 294 295 impl InnerThread { 296 fn new(pid: i32) -> Result<Self> { 297 fn next_err_id() -> u32 { 298 static EE_ID: Atomic<u32> = Atomic::new(0); 299 EE_ID.fetch_add(1, Relaxed) 300 } 301 302 Ok(Self { 303 looper_flags: LooperFlags::default(), 304 looper_need_return: false, 305 is_dead: false, 306 process_work_list: false, 307 reply_work: ThreadError::try_new(pid)?, 308 return_work: ThreadError::try_new(pid)?, 309 work_list: List::new(), 310 current_transaction: None, 311 extended_error: ExtendedError::new(next_err_id(), BR_OK, 0), 312 }) 313 } 314 315 fn pop_work(&mut self) -> Option<DLArc<dyn DeliverToRead>> { 316 if !self.process_work_list { 317 return None; 318 } 319 320 let ret = self.work_list.pop_front(); 321 self.process_work_list = !self.work_list.is_empty(); 322 ret 323 } 324 325 fn push_work(&mut self, work: DLArc<dyn DeliverToRead>) -> PushWorkRes { 326 if self.is_dead { 327 return PushWorkRes::FailedDead(work); 328 } 329 self.work_list.push_back(work); 330 self.process_work_list = true; 331 if self.looper_flags.contains(LooperFlag::Poll) { 332 PushWorkRes::OkNotifyPoll 333 } else { 334 PushWorkRes::Ok 335 } 336 } 337 338 fn push_reply_work(&mut self, code: u32) -> PushWorkRes { 339 if let Ok(work) = ListArc::try_from_arc(self.reply_work.clone()) { 340 work.set_error_code(code); 341 self.push_work(work) 342 } else { 343 pr_warn_ratelimited!("Thread reply work is already in use.\n"); 344 PushWorkRes::Ok 345 } 346 } 347 348 fn push_return_work(&mut self, reply: u32) -> PushWorkRes { 349 if let Ok(work) = ListArc::try_from_arc(self.return_work.clone()) { 350 work.set_error_code(reply); 351 self.push_work(work) 352 } else { 353 pr_warn_ratelimited!("Thread return work is already in use.\n"); 354 PushWorkRes::Ok 355 } 356 } 357 358 /// Used to push work items that do not need to be processed immediately and can wait until the 359 /// thread gets another work item. 360 fn push_work_deferred(&mut self, work: DLArc<dyn DeliverToRead>) { 361 self.work_list.push_back(work); 362 } 363 364 /// Fetches the transaction this thread can reply to. If the thread has a pending transaction 365 /// (that it could respond to) but it has also issued a transaction, it must first wait for the 366 /// previously-issued transaction to complete. 367 /// 368 /// The `thread` parameter should be the thread containing this `ThreadInner`. 369 fn pop_transaction_to_reply(&mut self, thread: &Thread) -> Result<DArc<Transaction>> { 370 let transaction = self.current_transaction.take().ok_or(EINVAL)?; 371 if core::ptr::eq(thread, transaction.from.as_ref()) { 372 self.current_transaction = Some(transaction); 373 return Err(EINVAL); 374 } 375 // Find a new current transaction for this thread. 376 self.current_transaction = transaction.find_from(thread).cloned(); 377 Ok(transaction) 378 } 379 380 fn pop_transaction_replied(&mut self, transaction: &DArc<Transaction>) -> bool { 381 match self.current_transaction.take() { 382 None => false, 383 Some(old) => { 384 if !Arc::ptr_eq(transaction, &old) { 385 self.current_transaction = Some(old); 386 return false; 387 } 388 self.current_transaction = old.clone_next(); 389 true 390 } 391 } 392 } 393 394 fn looper_enter(&mut self) { 395 self.looper_flags |= LooperFlag::Entered; 396 if self.looper_flags.contains(LooperFlag::Registered) { 397 self.looper_flags |= LooperFlag::Invalid; 398 } 399 } 400 401 fn looper_register(&mut self, valid: bool) { 402 self.looper_flags |= LooperFlag::Registered; 403 if !valid || self.looper_flags.contains(LooperFlag::Entered) { 404 self.looper_flags |= LooperFlag::Invalid; 405 } 406 } 407 408 fn looper_exit(&mut self) { 409 self.looper_flags |= LooperFlag::Exited; 410 } 411 412 /// Determines whether the thread is part of a pool, i.e., if it is a looper. 413 fn is_looper(&self) -> bool { 414 self.looper_flags 415 .contains_any(LooperFlag::Entered | LooperFlag::Registered) 416 } 417 418 /// Determines whether the thread should attempt to fetch work items from the process queue. 419 /// This is generally case when the thread is registered as a looper and not part of a 420 /// transaction stack. But if there is local work, we want to return to userspace before we 421 /// deliver any remote work. 422 fn should_use_process_work_queue(&self) -> bool { 423 self.current_transaction.is_none() && !self.process_work_list && self.is_looper() 424 } 425 426 fn poll(&mut self) -> u32 { 427 self.looper_flags |= LooperFlag::Poll; 428 if self.process_work_list || self.looper_need_return { 429 bindings::POLLIN 430 } else { 431 0 432 } 433 } 434 } 435 436 /// This represents a thread that's used with binder. 437 #[pin_data] 438 pub(crate) struct Thread { 439 pub(crate) id: i32, 440 pub(crate) process: Arc<Process>, 441 pub(crate) task: ARef<Task>, 442 #[pin] 443 inner: SpinLock<InnerThread>, 444 #[pin] 445 work_condvar: CondVar, 446 /// Used to insert this thread into the process' `ready_threads` list. 447 /// 448 /// INVARIANT: May never be used for any other list than the `self.process.ready_threads`. 449 #[pin] 450 links: ListLinks, 451 #[pin] 452 links_track: AtomicTracker, 453 } 454 455 kernel::list::impl_list_arc_safe! { 456 impl ListArcSafe<0> for Thread { 457 tracked_by links_track: AtomicTracker; 458 } 459 } 460 kernel::list::impl_list_item! { 461 impl ListItem<0> for Thread { 462 using ListLinks { self.links }; 463 } 464 } 465 466 impl Thread { 467 pub(crate) fn new(id: i32, process: Arc<Process>) -> Result<Arc<Self>> { 468 let inner = InnerThread::new(process.task.pid())?; 469 470 Arc::pin_init( 471 try_pin_init!(Thread { 472 id, 473 process, 474 task: ARef::from(&**kernel::current!()), 475 inner <- kernel::new_spinlock!(inner, "Thread::inner"), 476 work_condvar <- kernel::new_condvar!("Thread::work_condvar"), 477 links <- ListLinks::new(), 478 links_track <- AtomicTracker::new(), 479 }), 480 GFP_KERNEL, 481 ) 482 } 483 484 #[inline(never)] 485 pub(crate) fn debug_print(self: &Arc<Self>, m: &SeqFile, print_all: bool) -> Result { 486 let inner = self.inner.lock(); 487 488 if print_all || inner.current_transaction.is_some() || !inner.work_list.is_empty() { 489 seq_print!( 490 m, 491 " thread {}: l {:02x} need_return {}\n", 492 self.id, 493 u32::from(inner.looper_flags), 494 inner.looper_need_return, 495 ); 496 } 497 498 let mut t_opt = inner.current_transaction.as_ref(); 499 while let Some(t) = t_opt { 500 if Arc::ptr_eq(&t.from, self) { 501 t.debug_print_inner(m, " outgoing transaction "); 502 t_opt = t.from_parent.as_ref(); 503 } else if Arc::ptr_eq(&t.to, &self.process) { 504 t.debug_print_inner(m, " incoming transaction "); 505 t_opt = t.find_from(self); 506 } else { 507 t.debug_print_inner(m, " bad transaction "); 508 t_opt = None; 509 } 510 } 511 512 for work in &inner.work_list { 513 work.debug_print(m, " ", " pending transaction ")?; 514 } 515 Ok(()) 516 } 517 518 pub(crate) fn clear_extended_error(&self, debug_id: usize) { 519 self.inner.lock().extended_error = ExtendedError::new(debug_id as u32, BR_OK, 0); 520 } 521 522 pub(crate) fn get_extended_error(&self, data: UserSlice) -> Result { 523 let mut writer = data.writer(); 524 let mut inner = self.inner.lock(); 525 let ee = inner.extended_error; 526 inner.extended_error = ExtendedError::new(0, BR_OK, 0); 527 drop(inner); 528 writer.write(&ee)?; 529 Ok(()) 530 } 531 532 pub(crate) fn set_current_transaction(&self, transaction: DArc<Transaction>) { 533 self.inner.lock().current_transaction = Some(transaction); 534 } 535 536 pub(crate) fn has_current_transaction(&self) -> bool { 537 self.inner.lock().current_transaction.is_some() 538 } 539 540 /// Attempts to fetch a work item from the thread-local queue. The behaviour if the queue is 541 /// empty depends on `wait`: if it is true, the function waits for some work to be queued (or a 542 /// signal); otherwise it returns indicating that none is available. 543 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 544 // mangled symbol names. 545 #[export_name = "rust_binder_waitlcl"] 546 fn get_work_local(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> { 547 { 548 let mut inner = self.inner.lock(); 549 if inner.looper_need_return { 550 return Ok(inner.pop_work()); 551 } 552 } 553 554 // Try once if the caller does not want to wait. 555 if !wait { 556 return self.inner.lock().pop_work().ok_or(EAGAIN).map(Some); 557 } 558 559 // Loop waiting only on the local queue (i.e., not registering with the process queue). 560 let mut inner = self.inner.lock(); 561 loop { 562 if let Some(work) = inner.pop_work() { 563 return Ok(Some(work)); 564 } 565 566 inner.looper_flags |= LooperFlag::Waiting; 567 let signal_pending = self.work_condvar.wait_interruptible_freezable(&mut inner); 568 inner.looper_flags &= !LooperFlag::Waiting; 569 570 if signal_pending { 571 return Err(EINTR); 572 } 573 if inner.looper_need_return { 574 return Ok(None); 575 } 576 } 577 } 578 579 /// Attempts to fetch a work item from the thread-local queue, falling back to the process-wide 580 /// queue if none is available locally. 581 /// 582 /// This must only be called when the thread is not participating in a transaction chain. If it 583 /// is, the local version (`get_work_local`) should be used instead. 584 // #[export_name] is a temporary workaround so that ps output does not become unreadable from 585 // mangled symbol names. 586 #[export_name = "rust_binder_wait"] 587 fn get_work(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> { 588 let thread_has_deferred_work; 589 590 // Try to get work from the thread's work queue, using only a local lock. 591 { 592 let mut inner = self.inner.lock(); 593 594 // The process_work_list boolean is used to make us go to sleep even if there is work 595 // in the thread todo-list, but it doesn't apply to the process todo-list. Furthermore, 596 // work in the thread todo-list must still be delivered before the process list. 597 // 598 // Thus, in some scenarios we must return the thread work now even if we were requested 599 // to wait. Adjust `process_work_list` to `true` accordingly. 600 inner.process_work_list |= inner.looper_need_return; 601 inner.process_work_list |= !wait; 602 603 if let Some(work) = inner.pop_work() { 604 return Ok(Some(work)); 605 } 606 if inner.looper_need_return { 607 drop(inner); 608 return Ok(self.process.get_work()); 609 } 610 611 // Note that if the thread list is empty, then the call to `pop_work()` has changed 612 // `process_work_list` back to `false` even if we set it to `true` above. 613 thread_has_deferred_work = !inner.work_list.is_empty(); 614 } 615 616 // If the caller doesn't want to wait, try to grab work from the process queue. 617 // 618 // We know nothing will have been queued directly to the thread queue because it is not in 619 // a transaction and it is not in the process' ready list. We also know the thread list has 620 // no deferred work due to the `inner.process_work_list |= !wait` call above. 621 if !wait { 622 return self.process.get_work().ok_or(EAGAIN).map(Some); 623 } 624 625 // Get work from the process queue. If none is available, atomically register as ready. 626 let reg = match self 627 .process 628 .get_work_or_register(self, thread_has_deferred_work) 629 { 630 GetWorkOrRegister::Work(work) => return Ok(Some(work)), 631 GetWorkOrRegister::Register(reg) => reg, 632 }; 633 634 let mut inner = self.inner.lock(); 635 loop { 636 if let Some(work) = inner.pop_work() { 637 return Ok(Some(work)); 638 } 639 640 inner.looper_flags |= LooperFlag::Waiting | LooperFlag::WaitingProc; 641 let signal_pending = self.work_condvar.wait_interruptible_freezable(&mut inner); 642 inner.looper_flags &= !(LooperFlag::Waiting | LooperFlag::WaitingProc); 643 644 if signal_pending || inner.looper_need_return { 645 // We need to return now. 646 // 647 // We need to pull the thread off the list of ready threads (by dropping `reg`), 648 // then check the state again after it's off the list to ensure that something was 649 // not queued in the meantime. If something has been queued (or if there is 650 // deferred work), we just return it (instead of the error). 651 drop(inner); 652 drop(reg); 653 654 inner = self.inner.lock(); 655 inner.process_work_list = true; 656 let res = match inner.pop_work() { 657 Some(work) => Ok(Some(work)), 658 None if signal_pending => Err(EINTR), 659 None => Ok(None), 660 }; 661 return res; 662 } 663 } 664 } 665 666 /// Push the provided work item to be delivered to user space via this thread. 667 /// 668 /// Returns whether the item was successfully pushed. This can only fail if the thread is dead. 669 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> PushWorkRes { 670 let sync = work.should_sync_wakeup(); 671 self.push_work_inner(work, sync) 672 } 673 674 pub(crate) fn push_work_inner( 675 &self, 676 work: DLArc<dyn DeliverToRead>, 677 sync: bool, 678 ) -> PushWorkRes { 679 let res = self.inner.lock().push_work(work); 680 681 if res.is_ok() { 682 if sync { 683 self.work_condvar.notify_sync(); 684 } else { 685 self.work_condvar.notify_one(); 686 } 687 } 688 689 res 690 } 691 692 /// Attempts to push to given work item to the thread if it's a looper thread (i.e., if it's 693 /// part of a thread pool) and is alive. Otherwise, push the work item to the process instead. 694 pub(crate) fn push_work_if_looper(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { 695 let mut inner = self.inner.lock(); 696 if inner.is_looper() && !inner.is_dead { 697 // Not notifying work_condvar: Reply to current thread. 698 let res = inner.push_work(work); 699 drop(inner); 700 if matches!(res, PushWorkRes::OkNotifyPoll) { 701 self.process.notify_poll(false); 702 } 703 Ok(()) 704 } else { 705 drop(inner); 706 self.process.push_work(work) 707 } 708 } 709 710 pub(crate) fn push_work_deferred(&self, work: DLArc<dyn DeliverToRead>) { 711 self.inner.lock().push_work_deferred(work); 712 } 713 714 pub(crate) fn push_return_work(&self, reply: u32) { 715 // Not notifying work_condvar: Reply to current thread. 716 let res = self.inner.lock().push_return_work(reply); 717 if matches!(res, PushWorkRes::OkNotifyPoll) { 718 self.process.notify_poll(false); 719 } 720 } 721 722 pub(crate) fn pop_work_even_if_deferred(&self) -> Option<DLArc<dyn DeliverToRead>> { 723 let mut thread_inner = self.inner.lock(); 724 thread_inner.process_work_list = true; 725 thread_inner.pop_work() 726 } 727 728 fn translate_object( 729 &self, 730 obj_index: usize, 731 offset: usize, 732 object: BinderObjectRef<'_>, 733 view: &mut AllocationView<'_>, 734 allow_fds: bool, 735 sg_state: &mut ScatterGatherState, 736 ) -> BinderResult { 737 match object { 738 BinderObjectRef::Binder(obj) => { 739 let strong = obj.hdr.type_ == BINDER_TYPE_BINDER; 740 // SAFETY: `binder` is a `binder_uintptr_t`; any bit pattern is a valid 741 // representation. 742 let ptr = unsafe { obj.__bindgen_anon_1.binder }; 743 let cookie = obj.cookie; 744 let flags = obj.flags; 745 let node = self 746 .process 747 .as_arc_borrow() 748 .get_node(ptr, cookie, flags, strong, self)?; 749 security::binder_transfer_binder(&self.process.cred, &view.alloc.process.cred)?; 750 view.transfer_binder_object(offset, obj, strong, node)?; 751 } 752 BinderObjectRef::Handle(obj) => { 753 let strong = obj.hdr.type_ == BINDER_TYPE_HANDLE; 754 // SAFETY: `handle` is a `u32`; any bit pattern is a valid representation. 755 let handle = unsafe { obj.__bindgen_anon_1.handle }; 756 let node = self.process.get_node_from_handle(handle, strong)?; 757 security::binder_transfer_binder(&self.process.cred, &view.alloc.process.cred)?; 758 view.transfer_binder_object(offset, obj, strong, node)?; 759 } 760 BinderObjectRef::Fd(obj) => { 761 if !allow_fds { 762 return Err(EPERM.into()); 763 } 764 765 // SAFETY: `fd` is a `u32`; any bit pattern is a valid representation. 766 let fd = unsafe { obj.__bindgen_anon_1.fd }; 767 let file = LocalFile::fget(fd)?; 768 // SAFETY: The binder driver never calls `fdget_pos` and this code runs from an 769 // ioctl, so there are no active calls to `fdget_pos` on this thread. 770 let file = unsafe { LocalFile::assume_no_fdget_pos(file) }; 771 security::binder_transfer_file( 772 &self.process.cred, 773 &view.alloc.process.cred, 774 &file, 775 )?; 776 777 let mut obj_write = BinderFdObject::default(); 778 obj_write.hdr.type_ = BINDER_TYPE_FD; 779 // This will be overwritten with the actual fd when the transaction is received. 780 obj_write.__bindgen_anon_1.fd = u32::MAX; 781 obj_write.cookie = obj.cookie; 782 view.write::<BinderFdObject>(offset, &obj_write)?; 783 784 const FD_FIELD_OFFSET: usize = 785 core::mem::offset_of!(uapi::binder_fd_object, __bindgen_anon_1.fd); 786 787 let field_offset = offset + FD_FIELD_OFFSET; 788 crate::trace::trace_transaction_fd_send(view.alloc.debug_id, fd, field_offset); 789 790 view.alloc.info_add_fd(file, field_offset, false)?; 791 } 792 BinderObjectRef::Ptr(obj) => { 793 let obj_length = obj.length.try_into().map_err(|_| EINVAL)?; 794 let alloc_offset = match sg_state.unused_buffer_space.claim_next(obj_length) { 795 Ok(alloc_offset) => alloc_offset, 796 Err(err) => { 797 binder_debug!( 798 UserError, 799 "failed to claim space for a BINDER_TYPE_PTR (offset: {}, limit: {}, size: {})", 800 sg_state.unused_buffer_space.offset, 801 sg_state.unused_buffer_space.limit, 802 obj_length 803 ); 804 return Err(err.into()); 805 } 806 }; 807 808 let sg_state_idx = sg_state.sg_entries.len(); 809 sg_state.sg_entries.push( 810 ScatterGatherEntry { 811 obj_index, 812 offset: alloc_offset, 813 sender_uaddr: obj.buffer as usize, 814 length: obj_length, 815 pointer_fixups: KVec::new(), 816 fixup_min_offset: 0, 817 }, 818 GFP_KERNEL, 819 )?; 820 821 let buffer_ptr_in_user_space = (view.alloc.ptr + alloc_offset) as u64; 822 823 if obj.flags & uapi::BINDER_BUFFER_FLAG_HAS_PARENT == 0 { 824 sg_state.ancestors.clear(); 825 sg_state.ancestors.push(sg_state_idx, GFP_KERNEL)?; 826 } else { 827 // Another buffer also has a pointer to this buffer, and we need to fixup that 828 // pointer too. 829 830 let parent_index = usize::try_from(obj.parent).map_err(|_| EINVAL)?; 831 let parent_offset = usize::try_from(obj.parent_offset).map_err(|_| EINVAL)?; 832 833 let info = sg_state.validate_parent_fixup( 834 parent_index, 835 parent_offset, 836 size_of::<u64>(), 837 )?; 838 839 sg_state.ancestors.truncate(info.num_ancestors); 840 sg_state.ancestors.push(sg_state_idx, GFP_KERNEL)?; 841 842 let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { 843 Some(parent_entry) => parent_entry, 844 None => { 845 pr_err_ratelimited!( 846 "validate_parent_fixup returned index out of bounds for sg.entries\n" 847 ); 848 return Err(EINVAL.into()); 849 } 850 }; 851 852 parent_entry.fixup_min_offset = info.new_min_offset; 853 parent_entry.pointer_fixups.push( 854 PointerFixupEntry::Fixup { 855 pointer_value: buffer_ptr_in_user_space, 856 target_offset: info.target_offset, 857 }, 858 GFP_KERNEL, 859 )?; 860 } 861 862 let mut obj_write = BinderBufferObject::default(); 863 obj_write.hdr.type_ = BINDER_TYPE_PTR; 864 obj_write.flags = obj.flags; 865 obj_write.buffer = buffer_ptr_in_user_space; 866 obj_write.length = obj.length; 867 obj_write.parent = obj.parent; 868 obj_write.parent_offset = obj.parent_offset; 869 view.write::<BinderBufferObject>(offset, &obj_write)?; 870 } 871 BinderObjectRef::Fda(obj) => { 872 if !allow_fds { 873 return Err(EPERM.into()); 874 } 875 let parent_index = usize::try_from(obj.parent).map_err(|_| EINVAL)?; 876 let parent_offset = usize::try_from(obj.parent_offset).map_err(|_| EINVAL)?; 877 let num_fds = usize::try_from(obj.num_fds).map_err(|_| EINVAL)?; 878 let fds_len = num_fds.checked_mul(size_of::<u32>()).ok_or(EINVAL)?; 879 880 if !is_aligned(parent_offset, size_of::<u32>()) { 881 binder_debug!(UserError, "FDA parent offset not aligned correctly"); 882 return Err(EINVAL.into()); 883 } 884 885 let info = sg_state.validate_parent_fixup(parent_index, parent_offset, fds_len)?; 886 view.alloc.info_add_fd_reserve(num_fds)?; 887 888 sg_state.ancestors.truncate(info.num_ancestors); 889 let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { 890 Some(parent_entry) => parent_entry, 891 None => { 892 pr_err_ratelimited!( 893 "validate_parent_fixup returned index out of bounds for sg.entries\n" 894 ); 895 return Err(EINVAL.into()); 896 } 897 }; 898 899 if !is_aligned(parent_entry.sender_uaddr, size_of::<u32>()) { 900 binder_debug!(UserError, "FDA parent buffer not aligned correctly"); 901 return Err(EINVAL.into()); 902 } 903 904 parent_entry.fixup_min_offset = info.new_min_offset; 905 parent_entry 906 .pointer_fixups 907 .push( 908 PointerFixupEntry::Skip { 909 skip: fds_len, 910 target_offset: info.target_offset, 911 }, 912 GFP_KERNEL, 913 ) 914 .map_err(|_| ENOMEM)?; 915 916 let fda_uaddr = parent_entry 917 .sender_uaddr 918 .checked_add(parent_offset) 919 .ok_or(EINVAL)?; 920 921 let mut fda_bytes = KVec::new(); 922 UserSlice::new(UserPtr::from_addr(fda_uaddr as usize), fds_len) 923 .read_all(&mut fda_bytes, GFP_KERNEL)?; 924 925 if fds_len != fda_bytes.len() { 926 pr_err_ratelimited!( 927 "UserSlice::read_all returned wrong length in BINDER_TYPE_FDA\n" 928 ); 929 return Err(EINVAL.into()); 930 } 931 932 for i in (0..fds_len).step_by(size_of::<u32>()) { 933 let fd = { 934 let mut fd_bytes = [0u8; size_of::<u32>()]; 935 fd_bytes.copy_from_slice(&fda_bytes[i..i + size_of::<u32>()]); 936 u32::from_ne_bytes(fd_bytes) 937 }; 938 939 let file = LocalFile::fget(fd)?; 940 // SAFETY: The binder driver never calls `fdget_pos` and this code runs from an 941 // ioctl, so there are no active calls to `fdget_pos` on this thread. 942 let file = unsafe { LocalFile::assume_no_fdget_pos(file) }; 943 security::binder_transfer_file( 944 &self.process.cred, 945 &view.alloc.process.cred, 946 &file, 947 )?; 948 949 // The `validate_parent_fixup` call ensuers that this addition will not 950 // overflow. 951 view.alloc.info_add_fd(file, info.target_offset + i, true)?; 952 } 953 drop(fda_bytes); 954 955 let mut obj_write = BinderFdArrayObject::default(); 956 obj_write.hdr.type_ = BINDER_TYPE_FDA; 957 obj_write.num_fds = obj.num_fds; 958 obj_write.parent = obj.parent; 959 obj_write.parent_offset = obj.parent_offset; 960 view.write::<BinderFdArrayObject>(offset, &obj_write)?; 961 } 962 } 963 Ok(()) 964 } 965 966 fn apply_sg(&self, alloc: &mut Allocation, sg_state: &mut ScatterGatherState) -> BinderResult { 967 for sg_entry in &mut sg_state.sg_entries { 968 let mut end_of_previous_fixup = sg_entry.offset; 969 let offset_end = sg_entry.offset.checked_add(sg_entry.length).ok_or(EINVAL)?; 970 971 let mut reader = 972 UserSlice::new(UserPtr::from_addr(sg_entry.sender_uaddr), sg_entry.length).reader(); 973 for fixup in &mut sg_entry.pointer_fixups { 974 let (fixup_len, fixup_offset) = match fixup { 975 PointerFixupEntry::Fixup { target_offset, .. } => { 976 (size_of::<u64>(), *target_offset) 977 } 978 PointerFixupEntry::Skip { 979 skip, 980 target_offset, 981 } => (*skip, *target_offset), 982 }; 983 984 let target_offset_end = fixup_offset.checked_add(fixup_len).ok_or(EINVAL)?; 985 if fixup_offset < end_of_previous_fixup || offset_end < target_offset_end { 986 binder_debug!( 987 UserError, 988 "fixups oob {fixup_offset} {end_of_previous_fixup} {offset_end} {target_offset_end}" 989 ); 990 return Err(EINVAL.into()); 991 } 992 993 let copy_off = end_of_previous_fixup; 994 let copy_len = fixup_offset - end_of_previous_fixup; 995 if let Err(err) = alloc.copy_into(&mut reader, copy_off, copy_len) { 996 binder_debug!(UserError, "failed copying into alloc: {err:?}"); 997 return Err(err.into()); 998 } 999 if let PointerFixupEntry::Fixup { pointer_value, .. } = fixup { 1000 let res = alloc.write::<u64>(fixup_offset, pointer_value); 1001 if let Err(err) = res { 1002 binder_debug!(UserError, "failed copying ptr into alloc: {err:?}"); 1003 return Err(err.into()); 1004 } 1005 } 1006 if let Err(err) = reader.skip(fixup_len) { 1007 binder_debug!( 1008 UserError, 1009 "failed skipping {fixup_len} from reader: {err:?}" 1010 ); 1011 return Err(err.into()); 1012 } 1013 end_of_previous_fixup = target_offset_end; 1014 } 1015 let copy_off = end_of_previous_fixup; 1016 let copy_len = offset_end - end_of_previous_fixup; 1017 if let Err(err) = alloc.copy_into(&mut reader, copy_off, copy_len) { 1018 binder_debug!(UserError, "failed copying remainder into alloc: {err:?}"); 1019 return Err(err.into()); 1020 } 1021 } 1022 Ok(()) 1023 } 1024 1025 /// This method copies the payload of a transaction into the target process. 1026 /// 1027 /// The resulting payload will have several different components, which will be stored next to 1028 /// each other in the allocation. Furthermore, various objects can be embedded in the payload, 1029 /// and those objects have to be translated so that they make sense to the target transaction. 1030 pub(crate) fn copy_transaction_data( 1031 &self, 1032 to_process: Arc<Process>, 1033 info: &mut TransactionInfo, 1034 debug_id: usize, 1035 allow_fds: bool, 1036 txn_security_ctx_offset: Option<&mut usize>, 1037 ) -> BinderResult<NewAllocation> { 1038 let mut secctx = if let Some(offset) = txn_security_ctx_offset { 1039 let secid = self.process.cred.get_secid(); 1040 let ctx = match security::SecurityCtx::from_secid(secid) { 1041 Ok(ctx) => ctx, 1042 Err(err) => { 1043 pr_warn_ratelimited!( 1044 "Failed to get security ctx for id {}: {:?}\n", 1045 secid, 1046 err 1047 ); 1048 return Err(err.into()); 1049 } 1050 }; 1051 Some((offset, ctx)) 1052 } else { 1053 None 1054 }; 1055 1056 let data_size = info.data_size; 1057 let aligned_data_size = ptr_align(data_size).ok_or(EINVAL)?; 1058 let offsets_size = info.offsets_size; 1059 let buffers_size = info.buffers_size; 1060 let aligned_secctx_size = match secctx.as_ref() { 1061 Some((_offset, ctx)) => ptr_align(ctx.len()).ok_or(EINVAL)?, 1062 None => 0, 1063 }; 1064 1065 if !is_aligned(offsets_size, size_of::<u64>()) { 1066 return Err(EINVAL.into()); 1067 } 1068 if !is_aligned(buffers_size, size_of::<u64>()) { 1069 return Err(EINVAL.into()); 1070 } 1071 1072 // This guarantees that at least `sizeof(usize)` bytes will be allocated. 1073 let len = usize::max( 1074 aligned_data_size 1075 .checked_add(offsets_size) 1076 .and_then(|sum| sum.checked_add(buffers_size)) 1077 .and_then(|sum| sum.checked_add(aligned_secctx_size)) 1078 .ok_or(ENOMEM)?, 1079 size_of::<u64>(), 1080 ); 1081 let secctx_off = aligned_data_size + offsets_size + buffers_size; 1082 let mut alloc = to_process.buffer_alloc(debug_id, len, info)?; 1083 1084 trace_transaction_alloc_buf(debug_id, data_size, offsets_size, buffers_size); 1085 1086 let mut buffer_reader = UserSlice::new(info.data_ptr, data_size).reader(); 1087 let mut end_of_previous_object = 0; 1088 let mut sg_state = None; 1089 1090 // Copy offsets if there are any. 1091 if offsets_size > 0 { 1092 let mut offsets_reader = UserSlice::new(info.offsets_ptr, offsets_size).reader(); 1093 1094 let offsets_start = aligned_data_size; 1095 let offsets_end = aligned_data_size + offsets_size; 1096 1097 // This state is used for BINDER_TYPE_PTR objects. 1098 let sg_state = sg_state.insert(ScatterGatherState { 1099 unused_buffer_space: UnusedBufferSpace { 1100 offset: offsets_end, 1101 limit: offsets_end + buffers_size, 1102 }, 1103 sg_entries: KVec::new(), 1104 ancestors: KVec::new(), 1105 }); 1106 1107 // Traverse the objects specified. 1108 let mut view = AllocationView::new(&mut alloc, data_size); 1109 for (index, index_offset) in (offsets_start..offsets_end) 1110 .step_by(size_of::<u64>()) 1111 .enumerate() 1112 { 1113 let offset = offsets_reader.read::<u64>()?; 1114 view.alloc.write(index_offset, &offset)?; 1115 let offset: usize = offset.try_into().map_err(|_| EINVAL)?; 1116 1117 if offset < end_of_previous_object || !is_aligned(offset, size_of::<u32>()) { 1118 binder_debug!(UserError, "got transaction with invalid offset"); 1119 return Err(EINVAL.into()); 1120 } 1121 1122 // Copy data between two objects. 1123 if end_of_previous_object < offset { 1124 view.copy_into( 1125 &mut buffer_reader, 1126 end_of_previous_object, 1127 offset - end_of_previous_object, 1128 )?; 1129 } 1130 1131 let mut object = BinderObject::read_from(&mut buffer_reader)?; 1132 1133 match self.translate_object( 1134 index, 1135 offset, 1136 object.as_ref(), 1137 &mut view, 1138 allow_fds, 1139 sg_state, 1140 ) { 1141 Ok(()) => end_of_previous_object = offset + object.size(), 1142 Err(err) => { 1143 binder_debug!(UserError, "error while translating object: {err:?}"); 1144 return Err(err); 1145 } 1146 } 1147 1148 // Update the indexes containing objects to clean up. 1149 let offset_after_object = index_offset + size_of::<u64>(); 1150 view.alloc 1151 .set_info_offsets(offsets_start..offset_after_object); 1152 } 1153 } 1154 1155 // Copy remaining raw data. 1156 alloc.copy_into( 1157 &mut buffer_reader, 1158 end_of_previous_object, 1159 data_size - end_of_previous_object, 1160 )?; 1161 1162 if let Some(sg_state) = sg_state.as_mut() { 1163 self.apply_sg(&mut alloc, sg_state)?; 1164 } 1165 1166 if let Some((off_out, secctx)) = secctx.as_mut() { 1167 if let Err(err) = alloc.write(secctx_off, secctx.as_bytes()) { 1168 binder_debug!(UserError, "failed to write security context: {err:?}"); 1169 return Err(err.into()); 1170 } 1171 **off_out = secctx_off; 1172 } 1173 Ok(alloc) 1174 } 1175 1176 fn unwind_transaction_stack(self: &Arc<Self>) { 1177 let mut thread = self.clone(); 1178 while let Ok(transaction) = { 1179 let mut inner = thread.inner.lock(); 1180 inner.pop_transaction_to_reply(thread.as_ref()) 1181 } { 1182 binder_debug!( 1183 DeadTransaction, 1184 "release transaction {} in, still active", 1185 transaction.debug_id 1186 ); 1187 1188 let reply = Err(BR_DEAD_REPLY); 1189 if !transaction 1190 .from 1191 .deliver_single_reply(reply, &transaction, None) 1192 { 1193 break; 1194 } 1195 1196 thread = transaction.from.clone(); 1197 } 1198 } 1199 1200 pub(crate) fn deliver_reply( 1201 &self, 1202 reply: Result<DLArc<Transaction>, u32>, 1203 transaction: &DArc<Transaction>, 1204 extended_error: Option<ExtendedError>, 1205 ) { 1206 if self.deliver_single_reply(reply, transaction, extended_error) { 1207 transaction.from.unwind_transaction_stack(); 1208 } 1209 } 1210 1211 /// Delivers a reply to the thread that started a transaction. The reply can either be a 1212 /// reply-transaction or an error code to be delivered instead. 1213 /// 1214 /// Returns whether the thread is dead. If it is, the caller is expected to unwind the 1215 /// transaction stack by completing transactions for threads that are dead. 1216 fn deliver_single_reply( 1217 &self, 1218 reply: Result<DLArc<Transaction>, u32>, 1219 transaction: &DArc<Transaction>, 1220 extended_error: Option<ExtendedError>, 1221 ) -> bool { 1222 if let Ok(transaction) = &reply { 1223 crate::trace::trace_transaction(true, transaction, Some(&self.task)); 1224 transaction.set_outstanding(&mut self.process.inner.lock()); 1225 } 1226 1227 let ret = { 1228 let mut inner = self.inner.lock(); 1229 if !inner.pop_transaction_replied(transaction) { 1230 return false; 1231 } 1232 1233 if inner.is_dead { 1234 return true; 1235 } 1236 1237 if let Some(ee) = extended_error { 1238 if inner.extended_error.command == BR_OK { 1239 inner.extended_error = ee; 1240 } 1241 } 1242 1243 match reply { 1244 Ok(work) => inner.push_work(work), 1245 Err(code) => inner.push_reply_work(code), 1246 } 1247 }; 1248 1249 // Notify the thread now that we've released the inner lock. 1250 self.work_condvar.notify_sync(); 1251 if matches!(ret, PushWorkRes::OkNotifyPoll) { 1252 self.process.notify_poll(true); 1253 } 1254 false 1255 } 1256 1257 /// Determines if the given transaction is the current transaction for this thread. 1258 fn is_current_transaction(&self, transaction: &DArc<Transaction>) -> bool { 1259 let inner = self.inner.lock(); 1260 match &inner.current_transaction { 1261 None => false, 1262 Some(current) => Arc::ptr_eq(current, transaction), 1263 } 1264 } 1265 1266 /// Determines the current top of the transaction stack. It fails if the top is in another 1267 /// thread (i.e., this thread belongs to a stack but it has called another thread). The top is 1268 /// [`None`] if the thread is not currently participating in a transaction stack. 1269 fn top_of_transaction_stack(&self) -> Result<Option<DArc<Transaction>>> { 1270 let inner = self.inner.lock(); 1271 if let Some(cur) = &inner.current_transaction { 1272 if core::ptr::eq(self, cur.from.as_ref()) { 1273 pr_warn_ratelimited!("got new transaction with bad transaction stack\n"); 1274 return Err(EINVAL); 1275 } 1276 Ok(Some(cur.clone())) 1277 } else { 1278 Ok(None) 1279 } 1280 } 1281 1282 // No inlining avoids allocating stack space for `BinderTransactionData` for the entire 1283 // duration of `transaction()`. 1284 #[inline(never)] 1285 fn read_transaction_info( 1286 &self, 1287 cmd: u32, 1288 reader: &mut UserSliceReader, 1289 info: &mut TransactionInfo, 1290 ) -> Result { 1291 let td = match cmd { 1292 BC_TRANSACTION | BC_REPLY => { 1293 reader.read::<BinderTransactionData>()?.with_buffers_size(0) 1294 } 1295 BC_TRANSACTION_SG | BC_REPLY_SG => reader.read::<BinderTransactionDataSg>()?, 1296 _ => return Err(EINVAL), 1297 }; 1298 1299 // SAFETY: Above `read` call initializes all bytes, so this union read is ok. 1300 let trd_data_ptr = unsafe { &td.transaction_data.data.ptr }; 1301 1302 info.is_reply = matches!(cmd, BC_REPLY | BC_REPLY_SG); 1303 info.from_pid = self.process.task.pid(); 1304 info.from_tid = self.id; 1305 info.code = td.transaction_data.code; 1306 info.flags = TransactionFlags::from_bits(td.transaction_data.flags); 1307 info.data_ptr = UserPtr::from_addr(trd_data_ptr.buffer as usize); 1308 info.data_size = td.transaction_data.data_size as usize; 1309 info.offsets_ptr = UserPtr::from_addr(trd_data_ptr.offsets as usize); 1310 info.offsets_size = td.transaction_data.offsets_size as usize; 1311 info.buffers_size = td.buffers_size as usize; 1312 // SAFETY: Above `read` call initializes all bytes, so this union read is ok. 1313 info.target_handle = unsafe { td.transaction_data.target.handle }; 1314 1315 info.debug_id = super::next_debug_id(); 1316 1317 Ok(()) 1318 } 1319 1320 #[inline(never)] 1321 fn transaction(self: &Arc<Self>, cmd: u32, reader: &mut UserSliceReader) -> Result { 1322 let mut info = TransactionInfo::zeroed(); 1323 self.read_transaction_info(cmd, reader, &mut info)?; 1324 1325 self.clear_extended_error(info.debug_id); 1326 1327 let ret = if info.is_reply { 1328 self.reply_inner(&mut info) 1329 } else if info.is_oneway() { 1330 self.oneway_transaction_inner(&mut info) 1331 } else { 1332 self.transaction_inner(&mut info) 1333 }; 1334 1335 // This runs when return work is passed to the caller. This is not 1336 // always the same as the transaction failing, as reply errors are 1337 // delivered to the remote process. 1338 if let Err(err) = ret { 1339 self.push_return_work(err.reply); 1340 if err.reply != BR_TRANSACTION_COMPLETE { 1341 info.reply = err.reply; 1342 info.error_line = Some(err.line); 1343 if let Some(source) = err.source { 1344 info.errno = Some(source); 1345 1346 self.inner.lock().extended_error = 1347 ExtendedError::new(info.debug_id as u32, err.reply, source.to_errno()); 1348 } 1349 } 1350 } 1351 1352 info.write_log(&self.process.ctx); 1353 1354 if info.oneway_spam_suspect { 1355 // If this is both a oneway spam suspect and a failure, we report it twice. This is 1356 // useful in case the transaction failed with BR_TRANSACTION_PENDING_FROZEN. 1357 info.report_netlink(BR_ONEWAY_SPAM_SUSPECT, &self.process.ctx); 1358 } 1359 // This runs when the transaction failed. 1360 if info.reply != 0 { 1361 info.report_netlink(info.reply, &self.process.ctx); 1362 if let Some(errno) = info.errno { 1363 let error_line: &dyn kernel::fmt::Display = match &info.error_line { 1364 Some(line) => line, 1365 None => &0, 1366 }; 1367 binder_debug!( 1368 FailedTransaction, 1369 "transaction {} to {}:{} failed {:?}, code {} size {}-{} line {}", 1370 if info.is_reply { 1371 "reply" 1372 } else if info.is_oneway() { 1373 "async" 1374 } else { 1375 "call" 1376 }, 1377 info.to_pid, 1378 info.to_tid, 1379 errno, 1380 info.code, 1381 info.data_size, 1382 info.offsets_size, 1383 error_line 1384 ); 1385 } 1386 } 1387 1388 Ok(()) 1389 } 1390 1391 fn transaction_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { 1392 let node_ref = self.process.get_transaction_node(info.target_handle)?; 1393 info.to_pid = node_ref.node.owner.task.pid(); 1394 info.to_node_debug_id = node_ref.node.debug_id; 1395 security::binder_transaction(&self.process.cred, &node_ref.node.owner.cred)?; 1396 // TODO: We need to ensure that there isn't a pending transaction in the work queue. How 1397 // could this happen? 1398 let top = self.top_of_transaction_stack()?; 1399 let list_completion = DTRWrap::arc_try_new(DeliverCode::new( 1400 BR_TRANSACTION_COMPLETE, 1401 self.process.task.pid(), 1402 ))?; 1403 let completion = list_completion.clone_arc(); 1404 let transaction = Transaction::new(node_ref, top, self, info)?; 1405 1406 // Check that the transaction stack hasn't changed while the lock was released, then update 1407 // it with the new transaction. 1408 { 1409 let mut inner = self.inner.lock(); 1410 if !transaction.is_stacked_on(&inner.current_transaction) { 1411 binder_debug!(UserError, "got new transaction with bad transaction stack"); 1412 return Err(EINVAL.into()); 1413 } 1414 inner.current_transaction = Some(transaction.clone_arc()); 1415 // We push the completion as a deferred work so that we wait for the reply before 1416 // returning to userland. 1417 inner.push_work_deferred(list_completion); 1418 } 1419 1420 if let Err(e) = transaction.submit(info) { 1421 completion.skip(); 1422 // Define `transaction` first to drop it after `inner`. 1423 let transaction; 1424 let mut inner = self.inner.lock(); 1425 transaction = inner.current_transaction.take().unwrap(); 1426 inner.current_transaction = transaction.clone_next(); 1427 Err(e) 1428 } else { 1429 Ok(()) 1430 } 1431 } 1432 1433 fn reply_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { 1434 let orig = match self.inner.lock().pop_transaction_to_reply(self) { 1435 Ok(orig) => orig, 1436 Err(err) => { 1437 binder_debug!(UserError, "got reply transaction with no transaction stack"); 1438 return Err(err.into()); 1439 } 1440 }; 1441 if !orig.from.is_current_transaction(&orig) { 1442 binder_debug!( 1443 UserError, 1444 "got reply transaction with bad transaction stack" 1445 ); 1446 return Err(EINVAL.into()); 1447 } 1448 1449 info.to_tid = orig.from.id; 1450 info.to_pid = orig.from.process.task.pid(); 1451 1452 // We need to complete the transaction even if we cannot complete building the reply. 1453 let out = (|| -> BinderResult<_> { 1454 let completion = DTRWrap::arc_try_new(DeliverCode::new( 1455 BR_TRANSACTION_COMPLETE, 1456 self.process.task.pid(), 1457 ))?; 1458 let process = orig.from.process.clone(); 1459 let allow_fds = orig.flags.contains(TransactionFlag::AcceptFds); 1460 let reply = Transaction::new_reply(self, process, info, allow_fds)?; 1461 if info.flags.contains(TransactionFlag::DeferComplete) { 1462 // The flag is set. Perform a deferred push so that `read` can wait for the 1463 // next incoming transaction without a userspace roundtrip. 1464 self.inner.lock().push_work_deferred(completion); 1465 } else { 1466 // Not notifying work_condvar: Reply to current thread. 1467 let res = self.inner.lock().push_work(completion); 1468 if matches!(res, PushWorkRes::OkNotifyPoll) { 1469 self.process.notify_poll(false); 1470 } 1471 } 1472 orig.from.deliver_reply(Ok(reply), &orig, None); 1473 Ok(()) 1474 })() 1475 .map_err(|mut err| { 1476 // At this point we only return `BR_TRANSACTION_COMPLETE` to the caller, and we must let 1477 // the sender know that the transaction has completed (with an error in this case). 1478 1479 let param = err.source.as_ref().map_or(0, |e| e.to_errno()); 1480 let ee = ExtendedError::new(info.debug_id as u32, err.reply, param); 1481 orig.from 1482 .deliver_reply(Err(BR_FAILED_REPLY), &orig, Some(ee)); 1483 info.reply = BR_FAILED_REPLY; 1484 info.errno = err.source; 1485 info.error_line = Some(err.line); 1486 err.reply = BR_TRANSACTION_COMPLETE; 1487 err 1488 }); 1489 1490 out 1491 } 1492 1493 fn oneway_transaction_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { 1494 let node_ref = self.process.get_transaction_node(info.target_handle)?; 1495 info.to_pid = node_ref.node.owner.task.pid(); 1496 info.to_node_debug_id = node_ref.node.debug_id; 1497 security::binder_transaction(&self.process.cred, &node_ref.node.owner.cred)?; 1498 let transaction = Transaction::new(node_ref, None, self, info)?; 1499 let code = if self.process.is_oneway_spam_detection_enabled() && info.oneway_spam_suspect { 1500 BR_ONEWAY_SPAM_SUSPECT 1501 } else { 1502 BR_TRANSACTION_COMPLETE 1503 }; 1504 let list_completion = 1505 DTRWrap::arc_try_new(DeliverCode::new(code, self.process.task.pid()))?; 1506 let completion = list_completion.clone_arc(); 1507 // Not notifying work_condvar: Reply to current thread. 1508 let res = self.inner.lock().push_work(list_completion); 1509 match transaction.submit(info) { 1510 Ok(()) => { 1511 if matches!(res, PushWorkRes::OkNotifyPoll) { 1512 self.process.notify_poll(false); 1513 } 1514 Ok(()) 1515 } 1516 Err(err) => { 1517 completion.skip(); 1518 Err(err) 1519 } 1520 } 1521 } 1522 1523 fn write(self: &Arc<Self>, req: &mut BinderWriteRead) -> Result { 1524 let write_start = req.write_buffer.wrapping_add(req.write_consumed); 1525 let write_len = req.write_size.saturating_sub(req.write_consumed); 1526 let mut reader = 1527 UserSlice::new(UserPtr::from_addr(write_start as usize), write_len as usize).reader(); 1528 1529 while reader.len() >= size_of::<u32>() && self.inner.lock().return_work.is_unused() { 1530 let before = reader.len(); 1531 let cmd = reader.read::<u32>()?; 1532 crate::trace::trace_command(cmd); 1533 GLOBAL_STATS.inc_bc(cmd); 1534 self.process.stats.inc_bc(cmd); 1535 match cmd { 1536 BC_TRANSACTION | BC_TRANSACTION_SG | BC_REPLY | BC_REPLY_SG => { 1537 self.transaction(cmd, &mut reader)?; 1538 } 1539 BC_FREE_BUFFER => { 1540 let buffer = self.process.buffer_get(reader.read()?); 1541 if let Some(buffer) = buffer { 1542 if buffer.looper_need_return_on_free() { 1543 self.inner.lock().looper_need_return = true; 1544 } 1545 trace_transaction_buffer_release(buffer.debug_id); 1546 drop(buffer); 1547 } 1548 } 1549 BC_INCREFS => { 1550 self.process 1551 .as_arc_borrow() 1552 .update_ref(reader.read()?, true, false)? 1553 } 1554 BC_ACQUIRE => { 1555 self.process 1556 .as_arc_borrow() 1557 .update_ref(reader.read()?, true, true)? 1558 } 1559 BC_RELEASE => { 1560 self.process 1561 .as_arc_borrow() 1562 .update_ref(reader.read()?, false, true)? 1563 } 1564 BC_DECREFS => { 1565 self.process 1566 .as_arc_borrow() 1567 .update_ref(reader.read()?, false, false)? 1568 } 1569 BC_INCREFS_DONE => self.process.inc_ref_done(&mut reader, false)?, 1570 BC_ACQUIRE_DONE => self.process.inc_ref_done(&mut reader, true)?, 1571 BC_REQUEST_DEATH_NOTIFICATION => self.process.request_death(&mut reader, self)?, 1572 BC_CLEAR_DEATH_NOTIFICATION => self.process.clear_death(&mut reader, self)?, 1573 BC_DEAD_BINDER_DONE => self.process.dead_binder_done(reader.read()?, self), 1574 BC_REGISTER_LOOPER => { 1575 let valid = self.process.register_thread(); 1576 self.inner.lock().looper_register(valid); 1577 } 1578 BC_ENTER_LOOPER => self.inner.lock().looper_enter(), 1579 BC_EXIT_LOOPER => self.inner.lock().looper_exit(), 1580 BC_REQUEST_FREEZE_NOTIFICATION => self.process.request_freeze_notif(&mut reader)?, 1581 BC_CLEAR_FREEZE_NOTIFICATION => self.process.clear_freeze_notif(&mut reader)?, 1582 BC_FREEZE_NOTIFICATION_DONE => self.process.freeze_notif_done(&mut reader)?, 1583 1584 // Fail if given an unknown error code. 1585 // BC_ATTEMPT_ACQUIRE and BC_ACQUIRE_RESULT are no longer supported. 1586 _ => return Err(EINVAL), 1587 } 1588 // Update the number of write bytes consumed. 1589 req.write_consumed += (before - reader.len()) as u64; 1590 } 1591 1592 Ok(()) 1593 } 1594 1595 fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> Result { 1596 let read_start = req.read_buffer.wrapping_add(req.read_consumed); 1597 let read_len = req.read_size.saturating_sub(req.read_consumed); 1598 let mut writer = BinderReturnWriter::new( 1599 UserSlice::new(UserPtr::from_addr(read_start as usize), read_len as usize).writer(), 1600 self, 1601 ); 1602 let (in_pool, has_transaction, thread_todo, use_proc_queue) = { 1603 let inner = self.inner.lock(); 1604 ( 1605 inner.is_looper(), 1606 inner.current_transaction.is_some(), 1607 !inner.work_list.is_empty(), 1608 inner.should_use_process_work_queue(), 1609 ) 1610 }; 1611 1612 crate::trace::trace_wait_for_work(use_proc_queue, has_transaction, thread_todo); 1613 1614 let getter = if use_proc_queue { 1615 Self::get_work 1616 } else { 1617 Self::get_work_local 1618 }; 1619 1620 // Reserve some room at the beginning of the read buffer so that we can send a 1621 // BR_SPAWN_LOOPER if we need to. 1622 let mut has_noop_placeholder = false; 1623 if req.read_consumed == 0 { 1624 if let Err(err) = writer.write_code(BR_NOOP) { 1625 pr_warn_ratelimited!("Failure when writing BR_NOOP at beginning of buffer.\n"); 1626 return Err(err); 1627 } 1628 has_noop_placeholder = true; 1629 } 1630 1631 // Loop doing work while there is room in the buffer. 1632 let initial_len = writer.len(); 1633 while writer.len() >= size_of::<uapi::binder_transaction_data_secctx>() + 4 { 1634 match getter(self, wait && initial_len == writer.len()) { 1635 Ok(Some(work)) => match work.into_arc().do_work(self, &mut writer) { 1636 Ok(true) => {} 1637 Ok(false) => break, 1638 Err(err) => { 1639 return Err(err); 1640 } 1641 }, 1642 Ok(None) => { 1643 break; 1644 } 1645 Err(err) => { 1646 // Propagate the error if we haven't written anything else. 1647 if err != EINTR && err != EAGAIN { 1648 pr_warn_ratelimited!("Failure in work getter: {:?}\n", err); 1649 } 1650 if initial_len == writer.len() { 1651 return Err(err); 1652 } else { 1653 break; 1654 } 1655 } 1656 } 1657 } 1658 1659 req.read_consumed += read_len - writer.len() as u64; 1660 1661 // Write BR_SPAWN_LOOPER if the process needs more threads for its pool. 1662 if has_noop_placeholder && in_pool && self.process.needs_thread() { 1663 let mut writer = UserSlice::new( 1664 UserPtr::from_addr(req.read_buffer as usize), 1665 req.read_size as usize, 1666 ) 1667 .writer(); 1668 writer.write(&BR_SPAWN_LOOPER)?; 1669 } 1670 Ok(()) 1671 } 1672 1673 pub(crate) fn write_read(self: &Arc<Self>, data: UserSlice, wait: bool) -> Result { 1674 let (mut reader, mut writer) = data.reader_writer(); 1675 let mut req = reader.read::<BinderWriteRead>()?; 1676 1677 // Go through the write buffer. 1678 let mut ret = Ok(()); 1679 if req.write_size > 0 { 1680 ret = self.write(&mut req); 1681 crate::trace::trace_write_done(ret); 1682 if let Err(err) = ret { 1683 pr_warn_ratelimited!( 1684 "Write failure {:?} in pid:{}\n", 1685 err, 1686 self.process.pid_in_current_ns() 1687 ); 1688 req.read_consumed = 0; 1689 writer.write(&req)?; 1690 self.inner.lock().looper_need_return = false; 1691 return ret; 1692 } 1693 } 1694 1695 // Go through the work queue. 1696 if req.read_size > 0 { 1697 ret = self.read(&mut req, wait); 1698 crate::trace::trace_read_done(ret); 1699 if ret.is_err() && ret != Err(EINTR) { 1700 pr_warn_ratelimited!( 1701 "Read failure {:?} in pid:{}\n", 1702 ret, 1703 self.process.pid_in_current_ns() 1704 ); 1705 } 1706 } 1707 1708 // Write the request back so that the consumed fields are visible to the caller. 1709 writer.write(&req)?; 1710 1711 self.inner.lock().looper_need_return = false; 1712 1713 ret 1714 } 1715 1716 pub(crate) fn poll(&self) -> Result<(bool, u32)> { 1717 let mut inner = self.inner.lock(); 1718 Ok((inner.should_use_process_work_queue(), inner.poll())) 1719 } 1720 1721 /// Make the call to `get_work` or `get_work_local` return immediately, if any. 1722 pub(crate) fn exit_looper(&self) { 1723 let mut inner = self.inner.lock(); 1724 let should_notify = inner.looper_flags.contains(LooperFlag::Waiting); 1725 if should_notify { 1726 inner.looper_need_return = true; 1727 } 1728 drop(inner); 1729 1730 if should_notify { 1731 self.work_condvar.notify_one(); 1732 } 1733 } 1734 1735 pub(crate) fn release(self: &Arc<Self>) { 1736 self.inner.lock().is_dead = true; 1737 1738 self.unwind_transaction_stack(); 1739 1740 // Cancel all pending work items. 1741 while let Some(work) = self.pop_work_even_if_deferred() { 1742 work.into_arc().cancel(); 1743 } 1744 } 1745 } 1746 1747 #[pin_data] 1748 struct ThreadError { 1749 error_code: Atomic<u32>, 1750 pid: i32, 1751 #[pin] 1752 links_track: AtomicTracker, 1753 } 1754 1755 impl ThreadError { 1756 fn try_new(pid: i32) -> Result<DArc<Self>> { 1757 DTRWrap::arc_pin_init(pin_init!(Self { 1758 error_code: Atomic::new(BR_OK), 1759 pid, 1760 links_track <- AtomicTracker::new(), 1761 })) 1762 .map(ListArc::into_arc) 1763 } 1764 1765 fn set_error_code(&self, code: u32) { 1766 self.error_code.store(code, Relaxed); 1767 } 1768 1769 fn is_unused(&self) -> bool { 1770 self.error_code.load(Relaxed) == BR_OK 1771 } 1772 } 1773 1774 impl DeliverToRead for ThreadError { 1775 fn do_work( 1776 self: DArc<Self>, 1777 _thread: &Thread, 1778 writer: &mut BinderReturnWriter<'_>, 1779 ) -> Result<bool> { 1780 let code = self.error_code.load(Relaxed); 1781 self.error_code.store(BR_OK, Relaxed); 1782 writer.write_code(code)?; 1783 Ok(true) 1784 } 1785 1786 fn cancel(self: DArc<Self>) { 1787 let code = self.error_code.load(Relaxed); 1788 if code != BR_OK { 1789 binder_debug!( 1790 pid = self.pid, 1791 DeadTransaction, 1792 "undelivered TRANSACTION_ERROR: {code}" 1793 ); 1794 } 1795 } 1796 1797 fn should_sync_wakeup(&self) -> bool { 1798 false 1799 } 1800 1801 fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result { 1802 seq_print!( 1803 m, 1804 "{}transaction error: {}\n", 1805 prefix, 1806 self.error_code.load(Relaxed) 1807 ); 1808 Ok(()) 1809 } 1810 } 1811 1812 kernel::list::impl_list_arc_safe! { 1813 impl ListArcSafe<0> for ThreadError { 1814 tracked_by links_track: AtomicTracker; 1815 } 1816 } 1817