1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::{ 6 mem, 7 sync::atomic::{ 8 fence, 9 Ordering, // 10 }, 11 }; 12 13 use kernel::{ 14 device, 15 dma::{ 16 Coherent, 17 CoherentBox, 18 DmaAddress, // 19 }, 20 io::{ 21 io_project, 22 poll::read_poll_timeout, 23 Io, // 24 }, 25 new_mutex, 26 prelude::*, 27 ptr, 28 sync::{ 29 aref::ARef, 30 Mutex, // 31 }, 32 time::Delta, 33 transmute::{ 34 AsBytes, 35 FromBytes, // 36 }, 37 }; 38 39 use continuation::{ 40 ContinuationRecord, 41 SplitState, // 42 }; 43 44 use pin_init::pin_init_scope; 45 46 use crate::{ 47 driver::Bar0, 48 gsp::{ 49 fw::{ 50 GspMsgElement, 51 MsgFunction, 52 MsgqRxHeader, 53 MsgqTxHeader, 54 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 55 }, 56 PteArray, 57 GSP_PAGE_SHIFT, 58 GSP_PAGE_SIZE, // 59 }, 60 num, 61 regs, 62 sbuffer::SBufferIter, // 63 }; 64 65 /// Marker type representing the absence of a reply for a command. Commands using this as their 66 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 67 pub(crate) struct NoReply; 68 69 /// Trait implemented by types representing a command to send to the GSP. 70 /// 71 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 72 /// a given command. 73 /// 74 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 75 /// into the space reserved for it in the command queue buffer. 76 /// 77 /// Some commands may be followed by a variable-length payload. For these, the 78 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 79 /// defined as well. 80 pub(crate) trait CommandToGsp { 81 /// Function identifying this command to the GSP. 82 const FUNCTION: MsgFunction; 83 84 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 85 type Command: FromBytes + AsBytes; 86 87 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 88 /// have a reply. 89 type Reply; 90 91 /// Error type returned by [`CommandToGsp::init`]. 92 type InitError; 93 94 /// In-place command initializer responsible for filling the command in the command queue 95 /// buffer. 96 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 97 98 /// Size of the variable-length payload following the command structure generated by 99 /// [`CommandToGsp::init`]. 100 /// 101 /// Most commands don't have a variable-length payload, so this is zero by default. 102 fn variable_payload_len(&self) -> usize { 103 0 104 } 105 106 /// Method initializing the variable-length payload. 107 /// 108 /// The command buffer is circular, which means that we may need to jump back to its beginning 109 /// while in the middle of a command. For this reason, the variable-length payload is 110 /// initialized using a [`SBufferIter`]. 111 /// 112 /// This method will receive a buffer of the length returned by 113 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 114 /// unwritten space will lead to an error. 115 /// 116 /// Most commands don't have a variable-length payload, so this does nothing by default. 117 fn init_variable_payload( 118 &self, 119 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 120 ) -> Result { 121 Ok(()) 122 } 123 124 /// Total size of the command (including its variable-length payload) without the 125 /// [`GspMsgElement`] header. 126 fn size(&self) -> usize { 127 size_of::<Self::Command>() + self.variable_payload_len() 128 } 129 } 130 131 /// Trait representing messages received from the GSP. 132 /// 133 /// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. 134 pub(crate) trait MessageFromGsp: Sized { 135 /// Function identifying this message from the GSP. 136 const FUNCTION: MsgFunction; 137 138 /// Error type returned by [`MessageFromGsp::read`]. 139 type InitError; 140 141 /// Type containing the raw message to be read from the message queue. 142 type Message: FromBytes; 143 144 /// Method reading the message from the message queue and returning it. 145 /// 146 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 147 /// it. 148 fn read( 149 msg: &Self::Message, 150 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 151 ) -> Result<Self, Self::InitError>; 152 } 153 154 /// Number of GSP pages making the [`Msgq`]. 155 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 156 157 /// Circular buffer of a [`Msgq`]. 158 /// 159 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 160 /// messages. 161 #[repr(C, align(0x1000))] 162 #[derive(Debug)] 163 struct MsgqData { 164 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 165 } 166 167 // Annoyingly we are forced to use a literal to specify the alignment of 168 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 169 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 170 171 /// Unidirectional message queue. 172 /// 173 /// Contains the data for a message queue, that either the driver or GSP writes to. 174 /// 175 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 176 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 177 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 178 #[repr(C)] 179 // There is no struct defined for this in the open-gpu-kernel-source headers. 180 // Instead it is defined by code in `GspMsgQueuesInit()`. 181 struct Msgq { 182 /// Header for sending messages, including the write pointer. 183 tx: MsgqTxHeader, 184 /// Header for receiving messages, including the read pointer. 185 rx: MsgqRxHeader, 186 /// The message queue proper. 187 msgq: MsgqData, 188 } 189 190 /// Structure shared between the driver and the GSP and containing the command and message queues. 191 #[repr(C)] 192 struct GspMem { 193 /// Self-mapping page table entries. 194 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 195 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 196 /// write and read pointers that the CPU updates. This means that the read pointer here is an 197 /// index into the GSP queue. 198 /// 199 /// This member is read-only for the GSP. 200 cpuq: Msgq, 201 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 202 /// write and read pointers that the GSP updates. This means that the read pointer here is an 203 /// index into the CPU queue. 204 /// 205 /// This member is read-only for the driver. 206 gspq: Msgq, 207 } 208 209 impl GspMem { 210 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 211 } 212 213 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 214 // that is not a problem because they are not used outside the kernel. 215 unsafe impl AsBytes for GspMem {} 216 217 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 218 // that is not a problem because they are not used outside the kernel. 219 unsafe impl FromBytes for GspMem {} 220 221 /// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`]. 222 /// 223 /// This provides the low-level functionality to communicate with the GSP, including allocation of 224 /// queue space to write messages to and management of read/write pointers. 225 /// 226 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 227 /// 228 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 229 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 230 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 231 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 232 struct DmaGspMem(Coherent<GspMem>); 233 234 impl DmaGspMem { 235 /// Allocate a new instance and map it for `dev`. 236 fn new(dev: &device::Device<device::Bound>) -> Result<Self> { 237 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 238 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 239 240 let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?; 241 gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); 242 gsp_mem.cpuq.rx = MsgqRxHeader::new(); 243 244 let gsp_mem: Coherent<_> = gsp_mem.into(); 245 PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?; 246 247 Ok(Self(gsp_mem)) 248 } 249 250 /// Returns the region of the CPU message queue that the driver is currently allowed to write 251 /// to. 252 /// 253 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 254 /// that case the second slice will have a non-zero length. 255 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 256 let tx = self.cpu_write_ptr(); 257 let rx = self.gsp_read_ptr(); 258 259 // Pointer to the first entry of the CPU message queue. 260 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]); 261 262 let (tail_end, wrap_end) = if rx == 0 { 263 // The write area is non-wrapping, and stops at the second-to-last entry of the command 264 // queue (to leave the last one empty). 265 (MSGQ_NUM_PAGES - 1, 0) 266 } else if rx <= tx { 267 // The write area wraps and continues until `rx - 1`. 268 (MSGQ_NUM_PAGES, rx - 1) 269 } else { 270 // The write area doesn't wrap and stops at `rx - 1`. 271 (rx - 1, 0) 272 }; 273 274 // SAFETY: 275 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 276 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, 277 // thus the created slices are valid. 278 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`, 279 // inclusive, belongs to the driver for writing and is not accessed concurrently by 280 // the GSP. 281 // - The caller holds a reference to `self` for as long as the returned slices are live, 282 // meaning the CPU write pointer cannot be advanced and thus that the returned area 283 // remains exclusive to the CPU for the duration of the slices. 284 // - The created slices point to non-overlapping sub-ranges of `data` in all 285 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly 286 // less than `tx` where the first slice starts; in the other cases the second slice is 287 // empty), so creating two `&mut` references from them does not violate aliasing rules. 288 unsafe { 289 ( 290 core::slice::from_raw_parts_mut( 291 data.add(num::u32_as_usize(tx)), 292 num::u32_as_usize(tail_end - tx), 293 ), 294 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)), 295 ) 296 } 297 } 298 299 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 300 /// to write to, in bytes. 301 fn driver_write_area_size(&self) -> usize { 302 let tx = self.cpu_write_ptr(); 303 let rx = self.gsp_read_ptr(); 304 305 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 306 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 307 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 308 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 309 num::u32_as_usize(slots) * GSP_PAGE_SIZE 310 } 311 312 /// Returns the region of the GSP message queue that the driver is currently allowed to read 313 /// from. 314 /// 315 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 316 /// that case the second slice will have a non-zero length. 317 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 318 let tx = self.gsp_write_ptr(); 319 let rx = self.cpu_read_ptr(); 320 321 // Pointer to the first entry of the GSP message queue. 322 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]); 323 324 let (tail_end, wrap_end) = if rx <= tx { 325 // Read area is non-wrapping and stops right before `tx`. 326 (tx, 0) 327 } else { 328 // Read area is wrapping and stops right before `tx`. 329 (MSGQ_NUM_PAGES, tx) 330 }; 331 332 // SAFETY: 333 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 334 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`, 335 // thus the created slices are valid. 336 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`, 337 // inclusive, belongs to the driver for reading and is not accessed concurrently by 338 // the GSP. 339 // - The caller holds a reference to `self` for as long as the returned slices are live, 340 // meaning the CPU read pointer cannot be advanced and thus that the returned area 341 // remains exclusive to the CPU for the duration of the slices. 342 unsafe { 343 ( 344 core::slice::from_raw_parts( 345 data.add(num::u32_as_usize(rx)), 346 num::u32_as_usize(tail_end - rx), 347 ), 348 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)), 349 ) 350 } 351 } 352 353 /// Allocates a region on the command queue that is large enough to send a command of `size` 354 /// bytes, waiting for space to become available based on the provided timeout. 355 /// 356 /// This returns a [`GspCommand`] ready to be written to by the caller. 357 /// 358 /// # Errors 359 /// 360 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 361 /// - `ETIMEDOUT` if space does not become available within the timeout. 362 /// - `EIO` if the command header is not properly aligned. 363 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 364 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 365 return Err(EMSGSIZE); 366 } 367 read_poll_timeout( 368 || Ok(self.driver_write_area_size()), 369 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 370 Delta::from_micros(1), 371 timeout, 372 )?; 373 374 // Get the current writable area as an array of bytes. 375 let (slice_1, slice_2) = { 376 let (slice_1, slice_2) = self.driver_write_area(); 377 378 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 379 }; 380 381 // Extract area for the `GspMsgElement`. 382 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 383 384 // Create the contents area. 385 let (slice_1, slice_2) = if slice_1.len() > size { 386 // Contents fits entirely in `slice_1`. 387 (&mut slice_1[..size], &mut slice_2[0..0]) 388 } else { 389 // Need all of `slice_1` and some of `slice_2`. 390 let slice_2_len = size - slice_1.len(); 391 (slice_1, &mut slice_2[..slice_2_len]) 392 }; 393 394 Ok(GspCommand { 395 header, 396 contents: (slice_1, slice_2), 397 }) 398 } 399 400 // Returns the index of the memory page the GSP will write the next message to. 401 // 402 // # Invariants 403 // 404 // - The returned value is within `0..MSGQ_NUM_PAGES`. 405 fn gsp_write_ptr(&self) -> u32 { 406 MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES 407 } 408 409 // Returns the index of the memory page the GSP will read the next command from. 410 // 411 // # Invariants 412 // 413 // - The returned value is within `0..MSGQ_NUM_PAGES`. 414 fn gsp_read_ptr(&self) -> u32 { 415 MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES 416 } 417 418 // Returns the index of the memory page the CPU can read the next message from. 419 // 420 // # Invariants 421 // 422 // - The returned value is within `0..MSGQ_NUM_PAGES`. 423 fn cpu_read_ptr(&self) -> u32 { 424 MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES 425 } 426 427 // Informs the GSP that it can send `elem_count` new pages into the message queue. 428 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 429 let rx = io_project!(self.0, .cpuq.rx); 430 let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 431 432 // Ensure read pointer is properly ordered. 433 fence(Ordering::SeqCst); 434 435 MsgqRxHeader::set_read_ptr(rx, rptr) 436 } 437 438 // Returns the index of the memory page the CPU can write the next command to. 439 // 440 // # Invariants 441 // 442 // - The returned value is within `0..MSGQ_NUM_PAGES`. 443 fn cpu_write_ptr(&self) -> u32 { 444 MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES 445 } 446 447 // Informs the GSP that it can process `elem_count` new pages from the command queue. 448 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 449 let tx = io_project!(self.0, .cpuq.tx); 450 let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 451 MsgqTxHeader::set_write_ptr(tx, wptr); 452 453 // Ensure all command data is visible before triggering the GSP read. 454 fence(Ordering::SeqCst); 455 } 456 } 457 458 /// A command ready to be sent on the command queue. 459 /// 460 /// This is the type returned by [`DmaGspMem::allocate_command`]. 461 struct GspCommand<'a> { 462 // Writable reference to the header of the command. 463 header: &'a mut GspMsgElement, 464 // Writable slices to the contents of the command. The second slice is zero unless the command 465 // loops over the command queue. 466 contents: (&'a mut [u8], &'a mut [u8]), 467 } 468 469 /// A message ready to be processed from the message queue. 470 /// 471 /// This is the type returned by [`Cmdq::wait_for_msg`]. 472 struct GspMessage<'a> { 473 // Reference to the header of the message. 474 header: &'a GspMsgElement, 475 // Slices to the contents of the message. The second slice is zero unless the message loops 476 // over the message queue. 477 contents: (&'a [u8], &'a [u8]), 478 } 479 480 /// GSP command queue. 481 /// 482 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 483 /// area. 484 #[pin_data] 485 pub(crate) struct Cmdq { 486 /// Inner mutex-protected state. 487 #[pin] 488 inner: Mutex<CmdqInner>, 489 /// DMA handle of the command queue's shared memory region. 490 pub(super) dma_handle: DmaAddress, 491 } 492 493 impl Cmdq { 494 /// Offset of the data after the PTEs. 495 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 496 497 /// Offset of command queue ring buffer. 498 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 499 + core::mem::offset_of!(Msgq, msgq) 500 - Self::POST_PTE_OFFSET; 501 502 /// Offset of message queue ring buffer. 503 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 504 + core::mem::offset_of!(Msgq, msgq) 505 - Self::POST_PTE_OFFSET; 506 507 /// Number of page table entries for the GSP shared region. 508 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 509 510 /// Default timeout for receiving a message from the GSP. 511 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 512 513 /// Creates a new command queue for `dev`. 514 pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { 515 pin_init_scope(move || { 516 let gsp_mem = DmaGspMem::new(dev)?; 517 518 Ok(try_pin_init!(Self { 519 dma_handle: gsp_mem.0.dma_handle(), 520 inner <- new_mutex!(CmdqInner { 521 dev: dev.into(), 522 gsp_mem, 523 seq: 0, 524 }), 525 })) 526 }) 527 } 528 529 /// Computes the checksum for the message pointed to by `it`. 530 /// 531 /// A message is made of several parts, so `it` is an iterator over byte slices representing 532 /// these parts. 533 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 534 let sum64 = it 535 .enumerate() 536 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 537 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 538 539 ((sum64 >> 32) as u32) ^ (sum64 as u32) 540 } 541 542 /// Notifies the GSP that we have updated the command queue pointers. 543 fn notify_gsp(bar: Bar0<'_>) { 544 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); 545 } 546 547 /// Sends `command` to the GSP and waits for the reply. 548 /// 549 /// Messages with non-matching function codes are silently consumed until the expected reply 550 /// arrives. 551 /// 552 /// The queue is locked for the entire send+receive cycle to ensure that no other command can 553 /// be interleaved. 554 /// 555 /// # Errors 556 /// 557 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is 558 /// not received within the timeout. 559 /// - `EIO` if the variable payload requested by the command has not been entirely 560 /// written to by its [`CommandToGsp::init_variable_payload`] method. 561 /// 562 /// Error codes returned by the command and reply initializers are propagated as-is. 563 pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply> 564 where 565 M: CommandToGsp, 566 M::Reply: MessageFromGsp, 567 Error: From<M::InitError>, 568 Error: From<<M::Reply as MessageFromGsp>::InitError>, 569 { 570 let mut inner = self.inner.lock(); 571 inner.send_command(bar, command)?; 572 573 loop { 574 match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) { 575 Ok(reply) => break Ok(reply), 576 Err(ERANGE) => continue, 577 Err(e) => break Err(e), 578 } 579 } 580 } 581 582 /// Sends `command` to the GSP without waiting for a reply. 583 /// 584 /// # Errors 585 /// 586 /// - `ETIMEDOUT` if space does not become available within the timeout. 587 /// - `EIO` if the variable payload requested by the command has not been entirely 588 /// written to by its [`CommandToGsp::init_variable_payload`] method. 589 /// 590 /// Error codes returned by the command initializers are propagated as-is. 591 pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result 592 where 593 M: CommandToGsp<Reply = NoReply>, 594 Error: From<M::InitError>, 595 { 596 self.inner.lock().send_command(bar, command) 597 } 598 599 /// Receive a message from the GSP. 600 /// 601 /// See [`CmdqInner::receive_msg`] for details. 602 pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M> 603 where 604 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 605 Error: From<M::InitError>, 606 { 607 self.inner.lock().receive_msg(timeout) 608 } 609 } 610 611 /// Inner mutex protected state of [`Cmdq`]. 612 struct CmdqInner { 613 /// Device this command queue belongs to. 614 dev: ARef<device::Device>, 615 /// Current command sequence number. 616 seq: u32, 617 /// Memory area shared with the GSP for communicating commands and messages. 618 gsp_mem: DmaGspMem, 619 } 620 621 impl CmdqInner { 622 /// Timeout for waiting for space on the command queue. 623 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 624 625 /// Sends `command` to the GSP, without splitting it. 626 /// 627 /// # Errors 628 /// 629 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 630 /// - `ETIMEDOUT` if space does not become available within the timeout. 631 /// - `EIO` if the variable payload requested by the command has not been entirely 632 /// written to by its [`CommandToGsp::init_variable_payload`] method. 633 /// 634 /// Error codes returned by the command initializers are propagated as-is. 635 fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 636 where 637 M: CommandToGsp, 638 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 639 Error: From<M::InitError>, 640 { 641 let size_in_bytes = command.size(); 642 let dst = self 643 .gsp_mem 644 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 645 646 // Extract area for the command itself. The GSP message header and the command header 647 // together are guaranteed to fit entirely into a single page, so it's ok to only look 648 // at `dst.contents.0` here. 649 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 650 651 // Fill the header and command in-place. 652 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 653 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 654 // fails. 655 unsafe { 656 pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; 657 pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; 658 } 659 660 // Fill the variable-length payload, which may be empty. 661 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 662 command.init_variable_payload(&mut sbuffer)?; 663 664 if !sbuffer.is_empty() { 665 return Err(EIO); 666 } 667 drop(sbuffer); 668 669 // Compute checksum now that the whole message is ready. 670 dst.header 671 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 672 dst.header.as_bytes(), 673 dst.contents.0, 674 dst.contents.1, 675 ]))); 676 677 dev_dbg!( 678 &self.dev, 679 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 680 self.seq, 681 M::FUNCTION, 682 dst.header.length(), 683 ); 684 685 // All set - update the write pointer and inform the GSP of the new command. 686 let elem_count = dst.header.element_count(); 687 self.seq += 1; 688 self.gsp_mem.advance_cpu_write_ptr(elem_count); 689 Cmdq::notify_gsp(bar); 690 691 Ok(()) 692 } 693 694 /// Sends `command` to the GSP. 695 /// 696 /// The command may be split into multiple messages if it is large. 697 /// 698 /// # Errors 699 /// 700 /// - `ETIMEDOUT` if space does not become available within the timeout. 701 /// - `EIO` if the variable payload requested by the command has not been entirely 702 /// written to by its [`CommandToGsp::init_variable_payload`] method. 703 /// 704 /// Error codes returned by the command initializers are propagated as-is. 705 fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 706 where 707 M: CommandToGsp, 708 Error: From<M::InitError>, 709 { 710 match SplitState::new(command)? { 711 SplitState::Single(command) => self.send_single_command(bar, command), 712 SplitState::Split(command, mut continuations) => { 713 self.send_single_command(bar, command)?; 714 715 while let Some(continuation) = continuations.next() { 716 // Turbofish needed because the compiler cannot infer M here. 717 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?; 718 } 719 720 Ok(()) 721 } 722 } 723 } 724 725 /// Wait for a message to become available on the message queue. 726 /// 727 /// This works purely at the transport layer and does not interpret or validate the message 728 /// beyond the advertised length in its [`GspMsgElement`]. 729 /// 730 /// This method returns: 731 /// 732 /// - A reference to the [`GspMsgElement`] of the message, 733 /// - Two byte slices with the contents of the message. The second slice is empty unless the 734 /// message loops across the message queue. 735 /// 736 /// # Errors 737 /// 738 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 739 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 740 /// message queue. 741 /// 742 /// Error codes returned by the message constructor are propagated as-is. 743 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 744 // Wait for a message to arrive from the GSP. 745 let (slice_1, slice_2) = read_poll_timeout( 746 || Ok(self.gsp_mem.driver_read_area()), 747 |driver_area| !driver_area.0.is_empty(), 748 Delta::from_millis(1), 749 timeout, 750 ) 751 .map(|(slice_1, slice_2)| (slice_1.as_flattened(), slice_2.as_flattened()))?; 752 753 // Extract the `GspMsgElement`. 754 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 755 756 dev_dbg!( 757 &self.dev, 758 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 759 header.sequence(), 760 header.function(), 761 header.length(), 762 ); 763 764 let payload_length = header.payload_length(); 765 766 // Check that the driver read area is large enough for the message. 767 if slice_1.len() + slice_2.len() < payload_length { 768 return Err(EIO); 769 } 770 771 // Cut the message slices down to the actual length of the message. 772 let (slice_1, slice_2) = if slice_1.len() > payload_length { 773 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 774 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 775 } else { 776 ( 777 slice_1, 778 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 779 // large as `payload_length`. 780 slice_2.split_at(payload_length - slice_1.len()).0, 781 ) 782 }; 783 784 // Validate checksum. 785 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 786 header.as_bytes(), 787 slice_1, 788 slice_2, 789 ])) != 0 790 { 791 dev_err!( 792 &self.dev, 793 "GSP RPC: receive: Call {} - bad checksum\n", 794 header.sequence() 795 ); 796 return Err(EIO); 797 } 798 799 Ok(GspMessage { 800 header, 801 contents: (slice_1, slice_2), 802 }) 803 } 804 805 /// Receive a message from the GSP. 806 /// 807 /// The expected message type is specified using the `M` generic parameter. If the pending 808 /// message has a different function code, `ERANGE` is returned and the message is consumed. 809 /// 810 /// The read pointer is always advanced past the message, regardless of whether it matched. 811 /// 812 /// # Errors 813 /// 814 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 815 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 816 /// message queue. 817 /// - `EINVAL` if the function code of the message was not recognized. 818 /// - `ERANGE` if the message had a recognized but non-matching function code. 819 /// 820 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 821 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 822 where 823 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 824 Error: From<M::InitError>, 825 { 826 let message = self.wait_for_msg(timeout)?; 827 let function = message.header.function().map_err(|_| EINVAL)?; 828 829 // Extract the message. Store the result as we want to advance the read pointer even in 830 // case of failure. 831 let result = if function == M::FUNCTION { 832 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 833 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 834 835 M::read(cmd, &mut sbuffer) 836 .map_err(|e| e.into()) 837 .inspect(|_| { 838 if !sbuffer.is_empty() { 839 dev_warn!( 840 &self.dev, 841 "GSP message {:?} has unprocessed data\n", 842 function 843 ); 844 } 845 }) 846 } else { 847 Err(ERANGE) 848 }; 849 850 // Advance the read pointer past this message. 851 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 852 message.header.length().div_ceil(GSP_PAGE_SIZE), 853 )?); 854 855 result 856 } 857 } 858