1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2025 Google LLC. 4 5 use core::mem::{size_of, size_of_val, MaybeUninit}; 6 use core::ops::Range; 7 8 use kernel::{ 9 bindings, 10 fs::file::{File, FileDescriptorReservation}, 11 prelude::*, 12 sync::{aref::ARef, Arc}, 13 transmute::{AsBytes, FromBytes}, 14 uaccess::UserSliceReader, 15 uapi, 16 }; 17 18 use crate::{ 19 deferred_close::DeferredFdCloser, 20 defs::*, 21 node::{Node, NodeRef}, 22 process::Process, 23 trace::trace_transaction_failed_buffer_release, 24 DArc, 25 }; 26 27 #[derive(Default)] 28 pub(crate) struct AllocationInfo { 29 /// Range within the allocation where we can find the offsets to the object descriptors. 30 pub(crate) offsets: Option<Range<usize>>, 31 /// The target node of the transaction this allocation is associated to. 32 /// Not set for replies. 33 pub(crate) target_node: Option<NodeRef>, 34 /// When this allocation is dropped, call `pending_oneway_finished` on the node. 35 /// 36 /// This is used to serialize oneway transaction on the same node. Binder guarantees that 37 /// oneway transactions to the same node are delivered sequentially in the order they are sent. 38 pub(crate) oneway_node: Option<DArc<Node>>, 39 /// Zero the data in the buffer on free. 40 pub(crate) clear_on_free: bool, 41 /// List of files embedded in this transaction. 42 file_list: FileList, 43 } 44 45 /// Represents an allocation that the kernel is currently using. 46 /// 47 /// When allocations are idle, the range allocator holds the data related to them. 48 /// 49 /// # Invariants 50 /// 51 /// This allocation corresponds to an allocation in the range allocator, so the relevant pages are 52 /// marked in use in the page range. 53 pub(crate) struct Allocation { 54 pub(crate) offset: usize, 55 size: usize, 56 pub(crate) ptr: usize, 57 pub(crate) process: Arc<Process>, 58 allocation_info: Option<AllocationInfo>, 59 free_on_drop: bool, 60 #[allow(dead_code)] 61 pub(crate) debug_id: usize, 62 } 63 64 impl Allocation { 65 pub(crate) fn new( 66 process: Arc<Process>, 67 debug_id: usize, 68 offset: usize, 69 size: usize, 70 ptr: usize, 71 ) -> Self { 72 Self { 73 process, 74 offset, 75 size, 76 ptr, 77 debug_id, 78 allocation_info: None, 79 free_on_drop: true, 80 } 81 } 82 83 fn size_check(&self, offset: usize, size: usize) -> Result { 84 let overflow_fail = offset.checked_add(size).is_none(); 85 let cmp_size_fail = offset.wrapping_add(size) > self.size; 86 if overflow_fail || cmp_size_fail { 87 return Err(EFAULT); 88 } 89 Ok(()) 90 } 91 92 pub(crate) fn copy_into( 93 &self, 94 reader: &mut UserSliceReader, 95 offset: usize, 96 size: usize, 97 ) -> Result { 98 self.size_check(offset, size)?; 99 100 // SAFETY: While this object exists, the range allocator will keep the range allocated, and 101 // in turn, the pages will be marked as in use. 102 unsafe { 103 self.process 104 .pages 105 .copy_from_user_slice(reader, self.offset + offset, size) 106 } 107 } 108 109 pub(crate) fn read<T: FromBytes>(&self, offset: usize) -> Result<T> { 110 self.size_check(offset, size_of::<T>())?; 111 112 // SAFETY: While this object exists, the range allocator will keep the range allocated, and 113 // in turn, the pages will be marked as in use. 114 unsafe { self.process.pages.read(self.offset + offset) } 115 } 116 117 pub(crate) fn write<T: ?Sized>(&self, offset: usize, obj: &T) -> Result { 118 self.size_check(offset, size_of_val::<T>(obj))?; 119 120 // SAFETY: While this object exists, the range allocator will keep the range allocated, and 121 // in turn, the pages will be marked as in use. 122 unsafe { self.process.pages.write(self.offset + offset, obj) } 123 } 124 125 pub(crate) fn fill_zero(&self) -> Result { 126 // SAFETY: While this object exists, the range allocator will keep the range allocated, and 127 // in turn, the pages will be marked as in use. 128 unsafe { self.process.pages.fill_zero(self.offset, self.size) } 129 } 130 131 pub(crate) fn keep_alive(mut self) { 132 self.process 133 .buffer_make_freeable(self.offset, self.allocation_info.take()); 134 self.free_on_drop = false; 135 } 136 137 pub(crate) fn set_info(&mut self, info: AllocationInfo) { 138 self.allocation_info = Some(info); 139 } 140 141 pub(crate) fn get_or_init_info(&mut self) -> &mut AllocationInfo { 142 self.allocation_info.get_or_insert_with(Default::default) 143 } 144 145 pub(crate) fn set_info_offsets(&mut self, offsets: Range<usize>) { 146 self.get_or_init_info().offsets = Some(offsets); 147 } 148 149 pub(crate) fn set_info_oneway_node(&mut self, oneway_node: DArc<Node>) { 150 self.get_or_init_info().oneway_node = Some(oneway_node); 151 } 152 153 pub(crate) fn set_info_clear_on_drop(&mut self) { 154 self.get_or_init_info().clear_on_free = true; 155 } 156 157 pub(crate) fn set_info_target_node(&mut self, target_node: NodeRef) { 158 self.get_or_init_info().target_node = Some(target_node); 159 } 160 161 pub(crate) fn take_oneway_node(&mut self) -> Option<DArc<Node>> { 162 if let Some(info) = self.allocation_info.as_mut() { 163 info.oneway_node.take() 164 } else { 165 None 166 } 167 } 168 169 /// Reserve enough space to push at least `num_fds` fds. 170 pub(crate) fn info_add_fd_reserve(&mut self, num_fds: usize) -> Result { 171 self.get_or_init_info() 172 .file_list 173 .files_to_translate 174 .reserve(num_fds, GFP_KERNEL)?; 175 176 Ok(()) 177 } 178 179 pub(crate) fn info_add_fd( 180 &mut self, 181 file: ARef<File>, 182 buffer_offset: usize, 183 close_on_free: bool, 184 ) -> Result { 185 self.get_or_init_info().file_list.files_to_translate.push( 186 FileEntry { 187 file, 188 buffer_offset, 189 close_on_free, 190 }, 191 GFP_KERNEL, 192 )?; 193 194 Ok(()) 195 } 196 197 pub(crate) fn set_info_close_on_free(&mut self, cof: FdsCloseOnFree) { 198 self.get_or_init_info().file_list.close_on_free = cof.0; 199 } 200 201 pub(crate) fn translate_fds(&mut self) -> Result<TranslatedFds> { 202 let file_list = match self.allocation_info.as_mut() { 203 Some(info) => &mut info.file_list, 204 None => return Ok(TranslatedFds::new()), 205 }; 206 207 let files = core::mem::take(&mut file_list.files_to_translate); 208 209 let num_close_on_free = files.iter().filter(|entry| entry.close_on_free).count(); 210 let mut close_on_free = KVec::with_capacity(num_close_on_free, GFP_KERNEL)?; 211 212 let mut reservations = KVVec::with_capacity(files.len(), GFP_KERNEL)?; 213 for file_info in files { 214 let res = FileDescriptorReservation::get_unused_fd_flags(bindings::O_CLOEXEC)?; 215 let fd = res.reserved_fd(); 216 self.write::<u32>(file_info.buffer_offset, &fd)?; 217 crate::trace::trace_transaction_fd_recv(self.debug_id, fd, file_info.buffer_offset); 218 219 reservations.push( 220 Reservation { 221 res, 222 file: file_info.file, 223 }, 224 GFP_KERNEL, 225 )?; 226 if file_info.close_on_free { 227 close_on_free.push(fd, GFP_KERNEL)?; 228 } 229 } 230 231 Ok(TranslatedFds { 232 reservations, 233 close_on_free: FdsCloseOnFree(close_on_free), 234 }) 235 } 236 237 /// Should the looper return to userspace when freeing this allocation? 238 pub(crate) fn looper_need_return_on_free(&self) -> bool { 239 // Closing fds involves pushing task_work for execution when we return to userspace. Hence, 240 // we should return to userspace asap if we are closing fds. 241 match self.allocation_info { 242 Some(ref info) => !info.file_list.close_on_free.is_empty(), 243 None => false, 244 } 245 } 246 } 247 248 impl Drop for Allocation { 249 fn drop(&mut self) { 250 if !self.free_on_drop { 251 return; 252 } 253 254 if let Some(mut info) = self.allocation_info.take() { 255 if let Some(oneway_node) = info.oneway_node.as_ref() { 256 oneway_node.pending_oneway_finished(); 257 } 258 259 info.target_node = None; 260 261 if let Some(offsets) = info.offsets.clone() { 262 let view = AllocationView::new(self, offsets.start); 263 for i in offsets.step_by(size_of::<u64>()) { 264 if view.cleanup_object(i).is_err() { 265 pr_warn_ratelimited!("Error cleaning up object at offset {}\n", i) 266 } 267 } 268 } 269 270 if self.process.task == kernel::current!().group_leader() { 271 for &fd in &info.file_list.close_on_free { 272 let closer = match DeferredFdCloser::new(GFP_KERNEL) { 273 Ok(closer) => closer, 274 Err(kernel::alloc::AllocError) => { 275 // Ignore allocation failures. 276 break; 277 } 278 }; 279 280 // Here, we ignore errors. The operation can fail if the fd is not valid, or if 281 // the method is called from a kthread. However, this is always called from a 282 // syscall, so the latter case cannot happen, and we don't care about the first 283 // case. 284 let _ = closer.close_fd(fd); 285 } 286 } 287 288 if info.clear_on_free { 289 if let Err(e) = self.fill_zero() { 290 pr_warn_ratelimited!("Failed to clear data on free: {:?}\n", e); 291 } 292 } 293 } 294 295 self.process.buffer_raw_free(self.ptr); 296 } 297 } 298 299 /// A wrapper around `Allocation` that is being created. 300 /// 301 /// If the allocation is destroyed while wrapped in this wrapper, then the allocation will be 302 /// considered to be part of a failed transaction. Successful transactions avoid that by calling 303 /// `success`, which skips the destructor. 304 #[repr(transparent)] 305 pub(crate) struct NewAllocation(pub(crate) Allocation); 306 307 impl NewAllocation { 308 pub(crate) fn success(self) -> Allocation { 309 // This skips the destructor. 310 // 311 // SAFETY: This type is `#[repr(transparent)]`, so the layout matches. 312 unsafe { core::mem::transmute(self) } 313 } 314 } 315 316 impl core::ops::Deref for NewAllocation { 317 type Target = Allocation; 318 fn deref(&self) -> &Allocation { 319 &self.0 320 } 321 } 322 323 impl core::ops::DerefMut for NewAllocation { 324 fn deref_mut(&mut self) -> &mut Allocation { 325 &mut self.0 326 } 327 } 328 329 impl Drop for NewAllocation { 330 fn drop(&mut self) { 331 trace_transaction_failed_buffer_release(self.debug_id); 332 } 333 } 334 335 /// A view into the beginning of an allocation. 336 /// 337 /// All attempts to read or write outside of the view will fail. To intentionally access outside of 338 /// this view, use the `alloc` field of this struct directly. 339 pub(crate) struct AllocationView<'a> { 340 pub(crate) alloc: &'a mut Allocation, 341 limit: usize, 342 } 343 344 impl<'a> AllocationView<'a> { 345 pub(crate) fn new(alloc: &'a mut Allocation, limit: usize) -> Self { 346 AllocationView { alloc, limit } 347 } 348 349 pub(crate) fn read<T: FromBytes>(&self, offset: usize) -> Result<T> { 350 if offset.checked_add(size_of::<T>()).ok_or(EINVAL)? > self.limit { 351 return Err(EINVAL); 352 } 353 self.alloc.read(offset) 354 } 355 356 pub(crate) fn write<T: AsBytes>(&self, offset: usize, obj: &T) -> Result { 357 if offset.checked_add(size_of::<T>()).ok_or(EINVAL)? > self.limit { 358 return Err(EINVAL); 359 } 360 self.alloc.write(offset, obj) 361 } 362 363 pub(crate) fn copy_into( 364 &self, 365 reader: &mut UserSliceReader, 366 offset: usize, 367 size: usize, 368 ) -> Result { 369 if offset.checked_add(size).ok_or(EINVAL)? > self.limit { 370 return Err(EINVAL); 371 } 372 self.alloc.copy_into(reader, offset, size) 373 } 374 375 pub(crate) fn transfer_binder_object( 376 &self, 377 offset: usize, 378 obj: &uapi::flat_binder_object, 379 strong: bool, 380 node_ref: NodeRef, 381 ) -> Result { 382 let mut newobj = FlatBinderObject::default(); 383 let node = node_ref.node.clone(); 384 if Arc::ptr_eq(&node_ref.node.owner, &self.alloc.process) { 385 // The receiving process is the owner of the node, so send it a binder object (instead 386 // of a handle). 387 let (ptr, cookie) = node.get_id(); 388 newobj.hdr.type_ = if strong { 389 BINDER_TYPE_BINDER 390 } else { 391 BINDER_TYPE_WEAK_BINDER 392 }; 393 newobj.flags = obj.flags; 394 newobj.__bindgen_anon_1.binder = ptr as uapi::binder_uintptr_t; 395 newobj.cookie = cookie as uapi::binder_uintptr_t; 396 self.write(offset, &newobj)?; 397 // Increment the user ref count on the node. It will be decremented as part of the 398 // destruction of the buffer, when we see a binder or weak-binder object. 399 node.update_refcount(true, 1, strong); 400 } else { 401 // The receiving process is different from the owner, so we need to insert a handle to 402 // the binder object. 403 let handle = self 404 .alloc 405 .process 406 .as_arc_borrow() 407 .insert_or_update_handle(node_ref, false)?; 408 newobj.hdr.type_ = if strong { 409 BINDER_TYPE_HANDLE 410 } else { 411 BINDER_TYPE_WEAK_HANDLE 412 }; 413 newobj.flags = obj.flags; 414 newobj.__bindgen_anon_1.handle = handle; 415 if self.write(offset, &newobj).is_err() { 416 // Decrement ref count on the handle we just created. 417 let _ = self 418 .alloc 419 .process 420 .as_arc_borrow() 421 .update_ref(handle, false, strong); 422 return Err(EINVAL); 423 } 424 } 425 426 Ok(()) 427 } 428 429 fn cleanup_object(&self, index_offset: usize) -> Result { 430 let offset = self.alloc.read::<u64>(index_offset)?; 431 let offset: usize = offset.try_into().map_err(|_| EINVAL)?; 432 let header = self.read::<BinderObjectHeader>(offset)?; 433 match header.type_ { 434 BINDER_TYPE_WEAK_BINDER | BINDER_TYPE_BINDER => { 435 let obj = self.read::<FlatBinderObject>(offset)?; 436 let strong = header.type_ == BINDER_TYPE_BINDER; 437 // SAFETY: The type is `BINDER_TYPE_{WEAK_}BINDER`, so the `binder` field is 438 // populated. 439 let ptr = unsafe { obj.__bindgen_anon_1.binder }; 440 let cookie = obj.cookie; 441 self.alloc.process.update_node(ptr, cookie, strong); 442 Ok(()) 443 } 444 BINDER_TYPE_WEAK_HANDLE | BINDER_TYPE_HANDLE => { 445 let obj = self.read::<FlatBinderObject>(offset)?; 446 let strong = header.type_ == BINDER_TYPE_HANDLE; 447 // SAFETY: The type is `BINDER_TYPE_{WEAK_}HANDLE`, so the `handle` field is 448 // populated. 449 let handle = unsafe { obj.__bindgen_anon_1.handle }; 450 self.alloc 451 .process 452 .as_arc_borrow() 453 .update_ref(handle, false, strong) 454 } 455 _ => Ok(()), 456 } 457 } 458 } 459 460 /// A binder object as it is serialized. 461 /// 462 /// # Invariants 463 /// 464 /// All bytes must be initialized, and the value of `self.hdr.type_` must be one of the allowed 465 /// types. 466 #[repr(C)] 467 pub(crate) union BinderObject { 468 hdr: uapi::binder_object_header, 469 fbo: uapi::flat_binder_object, 470 fdo: uapi::binder_fd_object, 471 bbo: uapi::binder_buffer_object, 472 fdao: uapi::binder_fd_array_object, 473 } 474 475 /// A view into a `BinderObject` that can be used in a match statement. 476 pub(crate) enum BinderObjectRef<'a> { 477 Binder(&'a mut uapi::flat_binder_object), 478 Handle(&'a mut uapi::flat_binder_object), 479 Fd(&'a mut uapi::binder_fd_object), 480 Ptr(&'a mut uapi::binder_buffer_object), 481 Fda(&'a mut uapi::binder_fd_array_object), 482 } 483 484 impl BinderObject { 485 pub(crate) fn read_from(reader: &mut UserSliceReader) -> Result<BinderObject> { 486 let object = Self::read_from_inner(|slice| { 487 let read_len = usize::min(slice.len(), reader.len()); 488 reader.clone_reader().read_slice(&mut slice[..read_len])?; 489 Ok(()) 490 })?; 491 492 // If we used a object type smaller than the largest object size, then we've read more 493 // bytes than we needed to. However, we used `.clone_reader()` to avoid advancing the 494 // original reader. Now, we call `skip` so that the caller's reader is advanced by the 495 // right amount. 496 // 497 // The `skip` call fails if the reader doesn't have `size` bytes available. This could 498 // happen if the type header corresponds to an object type that is larger than the rest of 499 // the reader. 500 // 501 // Any extra bytes beyond the size of the object are inaccessible after this call, so 502 // reading them again from the `reader` later does not result in TOCTOU bugs. 503 reader.skip(object.size())?; 504 505 Ok(object) 506 } 507 508 /// Use the provided reader closure to construct a `BinderObject`. 509 /// 510 /// The closure should write the bytes for the object into the provided slice. 511 pub(crate) fn read_from_inner<R>(reader: R) -> Result<BinderObject> 512 where 513 R: FnOnce(&mut [u8; size_of::<BinderObject>()]) -> Result, 514 { 515 let mut obj = MaybeUninit::<BinderObject>::zeroed(); 516 517 // SAFETY: The lengths of `BinderObject` and `[u8; size_of::<BinderObject>()]` are equal, 518 // and the byte array has an alignment requirement of one, so the pointer cast is okay. 519 // Additionally, `obj` was initialized to zeros, so the byte array will not be 520 // uninitialized. 521 (reader)(unsafe { &mut *obj.as_mut_ptr().cast() })?; 522 523 // SAFETY: The entire object is initialized, so accessing this field is safe. 524 let type_ = unsafe { obj.assume_init_ref().hdr.type_ }; 525 if Self::type_to_size(type_).is_none() { 526 // The value of `obj.hdr_type_` was invalid. 527 return Err(EINVAL); 528 } 529 530 // SAFETY: All bytes are initialized (since we zeroed them at the start) and we checked 531 // that `self.hdr.type_` is one of the allowed types, so the type invariants are satisfied. 532 unsafe { Ok(obj.assume_init()) } 533 } 534 535 pub(crate) fn as_ref(&mut self) -> BinderObjectRef<'_> { 536 use BinderObjectRef::*; 537 // SAFETY: The constructor ensures that all bytes of `self` are initialized, and all 538 // variants of this union accept all initialized bit patterns. 539 unsafe { 540 match self.hdr.type_ { 541 BINDER_TYPE_WEAK_BINDER | BINDER_TYPE_BINDER => Binder(&mut self.fbo), 542 BINDER_TYPE_WEAK_HANDLE | BINDER_TYPE_HANDLE => Handle(&mut self.fbo), 543 BINDER_TYPE_FD => Fd(&mut self.fdo), 544 BINDER_TYPE_PTR => Ptr(&mut self.bbo), 545 BINDER_TYPE_FDA => Fda(&mut self.fdao), 546 // SAFETY: By the type invariant, the value of `self.hdr.type_` cannot have any 547 // other value than the ones checked above. 548 _ => core::hint::unreachable_unchecked(), 549 } 550 } 551 } 552 553 pub(crate) fn size(&self) -> usize { 554 // SAFETY: The entire object is initialized, so accessing this field is safe. 555 let type_ = unsafe { self.hdr.type_ }; 556 557 // SAFETY: The type invariants guarantee that the type field is correct. 558 unsafe { Self::type_to_size(type_).unwrap_unchecked() } 559 } 560 561 fn type_to_size(type_: u32) -> Option<usize> { 562 match type_ { 563 BINDER_TYPE_WEAK_BINDER => Some(size_of::<uapi::flat_binder_object>()), 564 BINDER_TYPE_BINDER => Some(size_of::<uapi::flat_binder_object>()), 565 BINDER_TYPE_WEAK_HANDLE => Some(size_of::<uapi::flat_binder_object>()), 566 BINDER_TYPE_HANDLE => Some(size_of::<uapi::flat_binder_object>()), 567 BINDER_TYPE_FD => Some(size_of::<uapi::binder_fd_object>()), 568 BINDER_TYPE_PTR => Some(size_of::<uapi::binder_buffer_object>()), 569 BINDER_TYPE_FDA => Some(size_of::<uapi::binder_fd_array_object>()), 570 _ => None, 571 } 572 } 573 } 574 575 #[derive(Default)] 576 struct FileList { 577 files_to_translate: KVVec<FileEntry>, 578 close_on_free: KVec<u32>, 579 } 580 581 struct FileEntry { 582 /// The file for which a descriptor will be created in the recipient process. 583 file: ARef<File>, 584 /// The offset in the buffer where the file descriptor is stored. 585 buffer_offset: usize, 586 /// Whether this fd should be closed when the allocation is freed. 587 close_on_free: bool, 588 } 589 590 pub(crate) struct TranslatedFds { 591 reservations: KVVec<Reservation>, 592 /// If commit is called, then these fds should be closed. (If commit is not called, then they 593 /// shouldn't be closed.) 594 close_on_free: FdsCloseOnFree, 595 } 596 597 struct Reservation { 598 res: FileDescriptorReservation, 599 file: ARef<File>, 600 } 601 602 impl TranslatedFds { 603 pub(crate) fn new() -> Self { 604 Self { 605 reservations: KVVec::new(), 606 close_on_free: FdsCloseOnFree(KVec::new()), 607 } 608 } 609 610 pub(crate) fn commit(self) -> FdsCloseOnFree { 611 for entry in self.reservations { 612 entry.res.fd_install(entry.file); 613 } 614 615 self.close_on_free 616 } 617 } 618 619 pub(crate) struct FdsCloseOnFree(KVec<u32>); 620