1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::mem; 6 7 use kernel::{ 8 device, 9 dma::{ 10 CoherentAllocation, 11 DmaAddress, // 12 }, 13 dma_write, 14 io::poll::read_poll_timeout, 15 prelude::*, 16 sync::aref::ARef, 17 time::Delta, 18 transmute::{ 19 AsBytes, 20 FromBytes, // 21 }, 22 }; 23 24 use continuation::{ 25 ContinuationRecord, 26 SplitState, // 27 }; 28 29 use crate::{ 30 driver::Bar0, 31 gsp::{ 32 fw::{ 33 GspMsgElement, 34 MsgFunction, 35 MsgqRxHeader, 36 MsgqTxHeader, 37 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 38 }, 39 PteArray, 40 GSP_PAGE_SHIFT, 41 GSP_PAGE_SIZE, // 42 }, 43 num, 44 regs, 45 sbuffer::SBufferIter, // 46 }; 47 48 /// Marker type representing the absence of a reply for a command. Commands using this as their 49 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 50 pub(crate) struct NoReply; 51 52 /// Trait implemented by types representing a command to send to the GSP. 53 /// 54 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 55 /// a given command. 56 /// 57 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 58 /// into the space reserved for it in the command queue buffer. 59 /// 60 /// Some commands may be followed by a variable-length payload. For these, the 61 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 62 /// defined as well. 63 pub(crate) trait CommandToGsp { 64 /// Function identifying this command to the GSP. 65 const FUNCTION: MsgFunction; 66 67 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 68 type Command: FromBytes + AsBytes; 69 70 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 71 /// have a reply. 72 type Reply; 73 74 /// Error type returned by [`CommandToGsp::init`]. 75 type InitError; 76 77 /// In-place command initializer responsible for filling the command in the command queue 78 /// buffer. 79 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 80 81 /// Size of the variable-length payload following the command structure generated by 82 /// [`CommandToGsp::init`]. 83 /// 84 /// Most commands don't have a variable-length payload, so this is zero by default. 85 fn variable_payload_len(&self) -> usize { 86 0 87 } 88 89 /// Method initializing the variable-length payload. 90 /// 91 /// The command buffer is circular, which means that we may need to jump back to its beginning 92 /// while in the middle of a command. For this reason, the variable-length payload is 93 /// initialized using a [`SBufferIter`]. 94 /// 95 /// This method will receive a buffer of the length returned by 96 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 97 /// unwritten space will lead to an error. 98 /// 99 /// Most commands don't have a variable-length payload, so this does nothing by default. 100 fn init_variable_payload( 101 &self, 102 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 103 ) -> Result { 104 Ok(()) 105 } 106 107 /// Total size of the command (including its variable-length payload) without the 108 /// [`GspMsgElement`] header. 109 fn size(&self) -> usize { 110 size_of::<Self::Command>() + self.variable_payload_len() 111 } 112 } 113 114 /// Trait representing messages received from the GSP. 115 /// 116 /// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. 117 pub(crate) trait MessageFromGsp: Sized { 118 /// Function identifying this message from the GSP. 119 const FUNCTION: MsgFunction; 120 121 /// Error type returned by [`MessageFromGsp::read`]. 122 type InitError; 123 124 /// Type containing the raw message to be read from the message queue. 125 type Message: FromBytes; 126 127 /// Method reading the message from the message queue and returning it. 128 /// 129 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 130 /// it. 131 fn read( 132 msg: &Self::Message, 133 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 134 ) -> Result<Self, Self::InitError>; 135 } 136 137 /// Number of GSP pages making the [`Msgq`]. 138 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 139 140 /// Circular buffer of a [`Msgq`]. 141 /// 142 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 143 /// messages. 144 #[repr(C, align(0x1000))] 145 #[derive(Debug)] 146 struct MsgqData { 147 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 148 } 149 150 // Annoyingly we are forced to use a literal to specify the alignment of 151 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 152 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 153 154 /// Unidirectional message queue. 155 /// 156 /// Contains the data for a message queue, that either the driver or GSP writes to. 157 /// 158 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 159 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 160 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 161 #[repr(C)] 162 // There is no struct defined for this in the open-gpu-kernel-source headers. 163 // Instead it is defined by code in `GspMsgQueuesInit()`. 164 // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 165 pub(super) struct Msgq { 166 /// Header for sending messages, including the write pointer. 167 pub(super) tx: MsgqTxHeader, 168 /// Header for receiving messages, including the read pointer. 169 pub(super) rx: MsgqRxHeader, 170 /// The message queue proper. 171 msgq: MsgqData, 172 } 173 174 /// Structure shared between the driver and the GSP and containing the command and message queues. 175 #[repr(C)] 176 // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 177 pub(super) struct GspMem { 178 /// Self-mapping page table entries. 179 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 180 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 181 /// write and read pointers that the CPU updates. This means that the read pointer here is an 182 /// index into the GSP queue. 183 /// 184 /// This member is read-only for the GSP. 185 pub(super) cpuq: Msgq, 186 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 187 /// write and read pointers that the GSP updates. This means that the read pointer here is an 188 /// index into the CPU queue. 189 /// 190 /// This member is read-only for the driver. 191 pub(super) gspq: Msgq, 192 } 193 194 impl GspMem { 195 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 196 } 197 198 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 199 // that is not a problem because they are not used outside the kernel. 200 unsafe impl AsBytes for GspMem {} 201 202 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 203 // that is not a problem because they are not used outside the kernel. 204 unsafe impl FromBytes for GspMem {} 205 206 /// Wrapper around [`GspMem`] to share it with the GPU using a [`CoherentAllocation`]. 207 /// 208 /// This provides the low-level functionality to communicate with the GSP, including allocation of 209 /// queue space to write messages to and management of read/write pointers. 210 /// 211 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 212 /// 213 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 214 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 215 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 216 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 217 struct DmaGspMem(CoherentAllocation<GspMem>); 218 219 impl DmaGspMem { 220 /// Allocate a new instance and map it for `dev`. 221 fn new(dev: &device::Device<device::Bound>) -> Result<Self> { 222 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 223 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 224 225 let gsp_mem = 226 CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; 227 228 let start = gsp_mem.dma_handle(); 229 // Write values one by one to avoid an on-stack instance of `PteArray`. 230 for i in 0..GspMem::PTE_ARRAY_SIZE { 231 dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?); 232 } 233 234 dma_write!( 235 gsp_mem, 236 [0]?.cpuq.tx, 237 MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) 238 ); 239 dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new()); 240 241 Ok(Self(gsp_mem)) 242 } 243 244 /// Returns the region of the CPU message queue that the driver is currently allowed to write 245 /// to. 246 /// 247 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 248 /// that case the second slice will have a non-zero length. 249 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 250 let tx = self.cpu_write_ptr() as usize; 251 let rx = self.gsp_read_ptr() as usize; 252 253 // SAFETY: 254 // - The `CoherentAllocation` contains exactly one object. 255 // - We will only access the driver-owned part of the shared memory. 256 // - Per the safety statement of the function, no concurrent access will be performed. 257 let gsp_mem = &mut unsafe { self.0.as_slice_mut(0, 1) }.unwrap()[0]; 258 // PANIC: per the invariant of `cpu_write_ptr`, `tx` is `< MSGQ_NUM_PAGES`. 259 let (before_tx, after_tx) = gsp_mem.cpuq.msgq.data.split_at_mut(tx); 260 261 // The area starting at `tx` and ending at `rx - 2` modulo MSGQ_NUM_PAGES, inclusive, 262 // belongs to the driver for writing. 263 264 if rx == 0 { 265 // Since `rx` is zero, leave an empty slot at end of the buffer. 266 let last = after_tx.len() - 1; 267 (&mut after_tx[..last], &mut []) 268 } else if rx <= tx { 269 // The area is discontiguous and we leave an empty slot before `rx`. 270 // PANIC: 271 // - The index `rx - 1` is non-negative because `rx != 0` in this branch. 272 // - The index does not exceed `before_tx.len()` (which equals `tx`) because 273 // `rx <= tx` in this branch. 274 (after_tx, &mut before_tx[..(rx - 1)]) 275 } else { 276 // The area is contiguous and we leave an empty slot before `rx`. 277 // PANIC: 278 // - The index `rx - tx - 1` is non-negative because `rx > tx` in this branch. 279 // - The index does not exceed `after_tx.len()` (which is `MSGQ_NUM_PAGES - tx`) 280 // because `rx < MSGQ_NUM_PAGES` by the `gsp_read_ptr` invariant. 281 (&mut after_tx[..(rx - tx - 1)], &mut []) 282 } 283 } 284 285 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 286 /// to write to, in bytes. 287 fn driver_write_area_size(&self) -> usize { 288 let tx = self.cpu_write_ptr(); 289 let rx = self.gsp_read_ptr(); 290 291 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 292 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 293 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 294 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 295 num::u32_as_usize(slots) * GSP_PAGE_SIZE 296 } 297 298 /// Returns the region of the GSP message queue that the driver is currently allowed to read 299 /// from. 300 /// 301 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 302 /// that case the second slice will have a non-zero length. 303 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 304 let tx = self.gsp_write_ptr() as usize; 305 let rx = self.cpu_read_ptr() as usize; 306 307 // SAFETY: 308 // - The `CoherentAllocation` contains exactly one object. 309 // - We will only access the driver-owned part of the shared memory. 310 // - Per the safety statement of the function, no concurrent access will be performed. 311 let gsp_mem = &unsafe { self.0.as_slice(0, 1) }.unwrap()[0]; 312 let data = &gsp_mem.gspq.msgq.data; 313 314 // The area starting at `rx` and ending at `tx - 1` modulo MSGQ_NUM_PAGES, inclusive, 315 // belongs to the driver for reading. 316 // PANIC: 317 // - per the invariant of `cpu_read_ptr`, `rx < MSGQ_NUM_PAGES` 318 // - per the invariant of `gsp_write_ptr`, `tx < MSGQ_NUM_PAGES` 319 if rx <= tx { 320 // The area is contiguous. 321 (&data[rx..tx], &[]) 322 } else { 323 // The area is discontiguous. 324 (&data[rx..], &data[..tx]) 325 } 326 } 327 328 /// Allocates a region on the command queue that is large enough to send a command of `size` 329 /// bytes, waiting for space to become available based on the provided timeout. 330 /// 331 /// This returns a [`GspCommand`] ready to be written to by the caller. 332 /// 333 /// # Errors 334 /// 335 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 336 /// - `ETIMEDOUT` if space does not become available within the timeout. 337 /// - `EIO` if the command header is not properly aligned. 338 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 339 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 340 return Err(EMSGSIZE); 341 } 342 read_poll_timeout( 343 || Ok(self.driver_write_area_size()), 344 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 345 Delta::from_micros(1), 346 timeout, 347 )?; 348 349 // Get the current writable area as an array of bytes. 350 let (slice_1, slice_2) = { 351 let (slice_1, slice_2) = self.driver_write_area(); 352 353 #[allow(clippy::incompatible_msrv)] 354 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 355 }; 356 357 // Extract area for the `GspMsgElement`. 358 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 359 360 // Create the contents area. 361 let (slice_1, slice_2) = if slice_1.len() > size { 362 // Contents fits entirely in `slice_1`. 363 (&mut slice_1[..size], &mut slice_2[0..0]) 364 } else { 365 // Need all of `slice_1` and some of `slice_2`. 366 let slice_2_len = size - slice_1.len(); 367 (slice_1, &mut slice_2[..slice_2_len]) 368 }; 369 370 Ok(GspCommand { 371 header, 372 contents: (slice_1, slice_2), 373 }) 374 } 375 376 // Returns the index of the memory page the GSP will write the next message to. 377 // 378 // # Invariants 379 // 380 // - The returned value is within `0..MSGQ_NUM_PAGES`. 381 fn gsp_write_ptr(&self) -> u32 { 382 super::fw::gsp_mem::gsp_write_ptr(&self.0) 383 } 384 385 // Returns the index of the memory page the GSP will read the next command from. 386 // 387 // # Invariants 388 // 389 // - The returned value is within `0..MSGQ_NUM_PAGES`. 390 fn gsp_read_ptr(&self) -> u32 { 391 super::fw::gsp_mem::gsp_read_ptr(&self.0) 392 } 393 394 // Returns the index of the memory page the CPU can read the next message from. 395 // 396 // # Invariants 397 // 398 // - The returned value is within `0..MSGQ_NUM_PAGES`. 399 fn cpu_read_ptr(&self) -> u32 { 400 super::fw::gsp_mem::cpu_read_ptr(&self.0) 401 } 402 403 // Informs the GSP that it can send `elem_count` new pages into the message queue. 404 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 405 super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) 406 } 407 408 // Returns the index of the memory page the CPU can write the next command to. 409 // 410 // # Invariants 411 // 412 // - The returned value is within `0..MSGQ_NUM_PAGES`. 413 fn cpu_write_ptr(&self) -> u32 { 414 super::fw::gsp_mem::cpu_write_ptr(&self.0) 415 } 416 417 // Informs the GSP that it can process `elem_count` new pages from the command queue. 418 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 419 super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) 420 } 421 } 422 423 /// A command ready to be sent on the command queue. 424 /// 425 /// This is the type returned by [`DmaGspMem::allocate_command`]. 426 struct GspCommand<'a> { 427 // Writable reference to the header of the command. 428 header: &'a mut GspMsgElement, 429 // Writable slices to the contents of the command. The second slice is zero unless the command 430 // loops over the command queue. 431 contents: (&'a mut [u8], &'a mut [u8]), 432 } 433 434 /// A message ready to be processed from the message queue. 435 /// 436 /// This is the type returned by [`Cmdq::wait_for_msg`]. 437 struct GspMessage<'a> { 438 // Reference to the header of the message. 439 header: &'a GspMsgElement, 440 // Slices to the contents of the message. The second slice is zero unless the message loops 441 // over the message queue. 442 contents: (&'a [u8], &'a [u8]), 443 } 444 445 /// GSP command queue. 446 /// 447 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 448 /// area. 449 pub(crate) struct Cmdq { 450 /// Device this command queue belongs to. 451 dev: ARef<device::Device>, 452 /// Current command sequence number. 453 seq: u32, 454 /// Memory area shared with the GSP for communicating commands and messages. 455 gsp_mem: DmaGspMem, 456 } 457 458 impl Cmdq { 459 /// Offset of the data after the PTEs. 460 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 461 462 /// Offset of command queue ring buffer. 463 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 464 + core::mem::offset_of!(Msgq, msgq) 465 - Self::POST_PTE_OFFSET; 466 467 /// Offset of message queue ring buffer. 468 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 469 + core::mem::offset_of!(Msgq, msgq) 470 - Self::POST_PTE_OFFSET; 471 472 /// Number of page table entries for the GSP shared region. 473 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 474 475 /// Timeout for waiting for space on the command queue. 476 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 477 478 /// Default timeout for receiving a message from the GSP. 479 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 480 481 /// Creates a new command queue for `dev`. 482 pub(crate) fn new(dev: &device::Device<device::Bound>) -> Result<Cmdq> { 483 let gsp_mem = DmaGspMem::new(dev)?; 484 485 Ok(Cmdq { 486 dev: dev.into(), 487 seq: 0, 488 gsp_mem, 489 }) 490 } 491 492 /// Computes the checksum for the message pointed to by `it`. 493 /// 494 /// A message is made of several parts, so `it` is an iterator over byte slices representing 495 /// these parts. 496 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 497 let sum64 = it 498 .enumerate() 499 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 500 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 501 502 ((sum64 >> 32) as u32) ^ (sum64 as u32) 503 } 504 505 /// Notifies the GSP that we have updated the command queue pointers. 506 fn notify_gsp(bar: &Bar0) { 507 regs::NV_PGSP_QUEUE_HEAD::default() 508 .set_address(0) 509 .write(bar); 510 } 511 512 /// Sends `command` to the GSP, without splitting it. 513 /// 514 /// # Errors 515 /// 516 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 517 /// - `ETIMEDOUT` if space does not become available within the timeout. 518 /// - `EIO` if the variable payload requested by the command has not been entirely 519 /// written to by its [`CommandToGsp::init_variable_payload`] method. 520 /// 521 /// Error codes returned by the command initializers are propagated as-is. 522 fn send_single_command<M>(&mut self, bar: &Bar0, command: M) -> Result 523 where 524 M: CommandToGsp, 525 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 526 Error: From<M::InitError>, 527 { 528 let size_in_bytes = command.size(); 529 let dst = self 530 .gsp_mem 531 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 532 533 // Extract area for the command itself. The GSP message header and the command header 534 // together are guaranteed to fit entirely into a single page, so it's ok to only look 535 // at `dst.contents.0` here. 536 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 537 538 // Fill the header and command in-place. 539 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 540 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 541 // fails. 542 unsafe { 543 msg_element.__init(core::ptr::from_mut(dst.header))?; 544 command.init().__init(core::ptr::from_mut(cmd))?; 545 } 546 547 // Fill the variable-length payload, which may be empty. 548 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 549 command.init_variable_payload(&mut sbuffer)?; 550 551 if !sbuffer.is_empty() { 552 return Err(EIO); 553 } 554 drop(sbuffer); 555 556 // Compute checksum now that the whole message is ready. 557 dst.header 558 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 559 dst.header.as_bytes(), 560 dst.contents.0, 561 dst.contents.1, 562 ]))); 563 564 dev_dbg!( 565 &self.dev, 566 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 567 self.seq, 568 M::FUNCTION, 569 dst.header.length(), 570 ); 571 572 // All set - update the write pointer and inform the GSP of the new command. 573 let elem_count = dst.header.element_count(); 574 self.seq += 1; 575 self.gsp_mem.advance_cpu_write_ptr(elem_count); 576 Cmdq::notify_gsp(bar); 577 578 Ok(()) 579 } 580 581 /// Sends `command` to the GSP. 582 /// 583 /// The command may be split into multiple messages if it is large. 584 /// 585 /// # Errors 586 /// 587 /// - `ETIMEDOUT` if space does not become available within the timeout. 588 /// - `EIO` if the variable payload requested by the command has not been entirely 589 /// written to by its [`CommandToGsp::init_variable_payload`] method. 590 /// 591 /// Error codes returned by the command initializers are propagated as-is. 592 fn send_command_internal<M>(&mut self, bar: &Bar0, command: M) -> Result 593 where 594 M: CommandToGsp, 595 Error: From<M::InitError>, 596 { 597 match SplitState::new(command)? { 598 SplitState::Single(command) => self.send_single_command(bar, command), 599 SplitState::Split(command, mut continuations) => { 600 self.send_single_command(bar, command)?; 601 602 while let Some(continuation) = continuations.next() { 603 // Turbofish needed because the compiler cannot infer M here. 604 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?; 605 } 606 607 Ok(()) 608 } 609 } 610 } 611 612 /// Sends `command` to the GSP and waits for the reply. 613 /// 614 /// Messages with non-matching function codes are silently consumed until the expected reply 615 /// arrives. 616 /// 617 /// # Errors 618 /// 619 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is 620 /// not received within the timeout. 621 /// - `EIO` if the variable payload requested by the command has not been entirely 622 /// written to by its [`CommandToGsp::init_variable_payload`] method. 623 /// 624 /// Error codes returned by the command and reply initializers are propagated as-is. 625 pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result<M::Reply> 626 where 627 M: CommandToGsp, 628 M::Reply: MessageFromGsp, 629 Error: From<M::InitError>, 630 Error: From<<M::Reply as MessageFromGsp>::InitError>, 631 { 632 self.send_command_internal(bar, command)?; 633 634 loop { 635 match self.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) { 636 Ok(reply) => break Ok(reply), 637 Err(ERANGE) => continue, 638 Err(e) => break Err(e), 639 } 640 } 641 } 642 643 /// Sends `command` to the GSP without waiting for a reply. 644 /// 645 /// # Errors 646 /// 647 /// - `ETIMEDOUT` if space does not become available within the timeout. 648 /// - `EIO` if the variable payload requested by the command has not been entirely 649 /// written to by its [`CommandToGsp::init_variable_payload`] method. 650 /// 651 /// Error codes returned by the command initializers are propagated as-is. 652 pub(crate) fn send_command_no_wait<M>(&mut self, bar: &Bar0, command: M) -> Result 653 where 654 M: CommandToGsp<Reply = NoReply>, 655 Error: From<M::InitError>, 656 { 657 self.send_command_internal(bar, command) 658 } 659 660 /// Wait for a message to become available on the message queue. 661 /// 662 /// This works purely at the transport layer and does not interpret or validate the message 663 /// beyond the advertised length in its [`GspMsgElement`]. 664 /// 665 /// This method returns: 666 /// 667 /// - A reference to the [`GspMsgElement`] of the message, 668 /// - Two byte slices with the contents of the message. The second slice is empty unless the 669 /// message loops across the message queue. 670 /// 671 /// # Errors 672 /// 673 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 674 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 675 /// message queue. 676 /// 677 /// Error codes returned by the message constructor are propagated as-is. 678 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 679 // Wait for a message to arrive from the GSP. 680 let (slice_1, slice_2) = read_poll_timeout( 681 || Ok(self.gsp_mem.driver_read_area()), 682 |driver_area| !driver_area.0.is_empty(), 683 Delta::from_millis(1), 684 timeout, 685 ) 686 .map(|(slice_1, slice_2)| { 687 #[allow(clippy::incompatible_msrv)] 688 (slice_1.as_flattened(), slice_2.as_flattened()) 689 })?; 690 691 // Extract the `GspMsgElement`. 692 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 693 694 dev_dbg!( 695 self.dev, 696 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 697 header.sequence(), 698 header.function(), 699 header.length(), 700 ); 701 702 let payload_length = header.payload_length(); 703 704 // Check that the driver read area is large enough for the message. 705 if slice_1.len() + slice_2.len() < payload_length { 706 return Err(EIO); 707 } 708 709 // Cut the message slices down to the actual length of the message. 710 let (slice_1, slice_2) = if slice_1.len() > payload_length { 711 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 712 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 713 } else { 714 ( 715 slice_1, 716 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 717 // large as `payload_length`. 718 slice_2.split_at(payload_length - slice_1.len()).0, 719 ) 720 }; 721 722 // Validate checksum. 723 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 724 header.as_bytes(), 725 slice_1, 726 slice_2, 727 ])) != 0 728 { 729 dev_err!( 730 self.dev, 731 "GSP RPC: receive: Call {} - bad checksum\n", 732 header.sequence() 733 ); 734 return Err(EIO); 735 } 736 737 Ok(GspMessage { 738 header, 739 contents: (slice_1, slice_2), 740 }) 741 } 742 743 /// Receive a message from the GSP. 744 /// 745 /// The expected message type is specified using the `M` generic parameter. If the pending 746 /// message has a different function code, `ERANGE` is returned and the message is consumed. 747 /// 748 /// The read pointer is always advanced past the message, regardless of whether it matched. 749 /// 750 /// # Errors 751 /// 752 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 753 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 754 /// message queue. 755 /// - `EINVAL` if the function code of the message was not recognized. 756 /// - `ERANGE` if the message had a recognized but non-matching function code. 757 /// 758 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 759 pub(crate) fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 760 where 761 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 762 Error: From<M::InitError>, 763 { 764 let message = self.wait_for_msg(timeout)?; 765 let function = message.header.function().map_err(|_| EINVAL)?; 766 767 // Extract the message. Store the result as we want to advance the read pointer even in 768 // case of failure. 769 let result = if function == M::FUNCTION { 770 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 771 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 772 773 M::read(cmd, &mut sbuffer) 774 .map_err(|e| e.into()) 775 .inspect(|_| { 776 if !sbuffer.is_empty() { 777 dev_warn!( 778 &self.dev, 779 "GSP message {:?} has unprocessed data\n", 780 function 781 ); 782 } 783 }) 784 } else { 785 Err(ERANGE) 786 }; 787 788 // Advance the read pointer past this message. 789 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 790 message.header.length().div_ceil(GSP_PAGE_SIZE), 791 )?); 792 793 result 794 } 795 796 /// Returns the DMA handle of the command queue's shared memory region. 797 pub(crate) fn dma_handle(&self) -> DmaAddress { 798 self.gsp_mem.0.dma_handle() 799 } 800 } 801