1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::mem; 6 7 use kernel::{ 8 device, 9 dma::{ 10 Coherent, 11 DmaAddress, // 12 }, 13 dma_write, 14 io::{ 15 poll::read_poll_timeout, 16 Io, // 17 }, 18 new_mutex, 19 prelude::*, 20 ptr, 21 sync::{ 22 aref::ARef, 23 Mutex, // 24 }, 25 time::Delta, 26 transmute::{ 27 AsBytes, 28 FromBytes, // 29 }, 30 }; 31 32 use continuation::{ 33 ContinuationRecord, 34 SplitState, // 35 }; 36 37 use pin_init::pin_init_scope; 38 39 use crate::{ 40 driver::Bar0, 41 gsp::{ 42 fw::{ 43 GspMsgElement, 44 MsgFunction, 45 MsgqRxHeader, 46 MsgqTxHeader, 47 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 48 }, 49 PteArray, 50 GSP_PAGE_SHIFT, 51 GSP_PAGE_SIZE, // 52 }, 53 num, 54 regs, 55 sbuffer::SBufferIter, // 56 }; 57 58 /// Marker type representing the absence of a reply for a command. Commands using this as their 59 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 60 pub(crate) struct NoReply; 61 62 /// Trait implemented by types representing a command to send to the GSP. 63 /// 64 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 65 /// a given command. 66 /// 67 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 68 /// into the space reserved for it in the command queue buffer. 69 /// 70 /// Some commands may be followed by a variable-length payload. For these, the 71 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 72 /// defined as well. 73 pub(crate) trait CommandToGsp { 74 /// Function identifying this command to the GSP. 75 const FUNCTION: MsgFunction; 76 77 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 78 type Command: FromBytes + AsBytes; 79 80 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 81 /// have a reply. 82 type Reply; 83 84 /// Error type returned by [`CommandToGsp::init`]. 85 type InitError; 86 87 /// In-place command initializer responsible for filling the command in the command queue 88 /// buffer. 89 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 90 91 /// Size of the variable-length payload following the command structure generated by 92 /// [`CommandToGsp::init`]. 93 /// 94 /// Most commands don't have a variable-length payload, so this is zero by default. 95 fn variable_payload_len(&self) -> usize { 96 0 97 } 98 99 /// Method initializing the variable-length payload. 100 /// 101 /// The command buffer is circular, which means that we may need to jump back to its beginning 102 /// while in the middle of a command. For this reason, the variable-length payload is 103 /// initialized using a [`SBufferIter`]. 104 /// 105 /// This method will receive a buffer of the length returned by 106 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 107 /// unwritten space will lead to an error. 108 /// 109 /// Most commands don't have a variable-length payload, so this does nothing by default. 110 fn init_variable_payload( 111 &self, 112 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 113 ) -> Result { 114 Ok(()) 115 } 116 117 /// Total size of the command (including its variable-length payload) without the 118 /// [`GspMsgElement`] header. 119 fn size(&self) -> usize { 120 size_of::<Self::Command>() + self.variable_payload_len() 121 } 122 } 123 124 /// Trait representing messages received from the GSP. 125 /// 126 /// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. 127 pub(crate) trait MessageFromGsp: Sized { 128 /// Function identifying this message from the GSP. 129 const FUNCTION: MsgFunction; 130 131 /// Error type returned by [`MessageFromGsp::read`]. 132 type InitError; 133 134 /// Type containing the raw message to be read from the message queue. 135 type Message: FromBytes; 136 137 /// Method reading the message from the message queue and returning it. 138 /// 139 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 140 /// it. 141 fn read( 142 msg: &Self::Message, 143 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 144 ) -> Result<Self, Self::InitError>; 145 } 146 147 /// Number of GSP pages making the [`Msgq`]. 148 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 149 150 /// Circular buffer of a [`Msgq`]. 151 /// 152 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 153 /// messages. 154 #[repr(C, align(0x1000))] 155 #[derive(Debug)] 156 struct MsgqData { 157 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 158 } 159 160 // Annoyingly we are forced to use a literal to specify the alignment of 161 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 162 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 163 164 /// Unidirectional message queue. 165 /// 166 /// Contains the data for a message queue, that either the driver or GSP writes to. 167 /// 168 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 169 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 170 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 171 #[repr(C)] 172 // There is no struct defined for this in the open-gpu-kernel-source headers. 173 // Instead it is defined by code in `GspMsgQueuesInit()`. 174 // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 175 pub(super) struct Msgq { 176 /// Header for sending messages, including the write pointer. 177 pub(super) tx: MsgqTxHeader, 178 /// Header for receiving messages, including the read pointer. 179 pub(super) rx: MsgqRxHeader, 180 /// The message queue proper. 181 msgq: MsgqData, 182 } 183 184 /// Structure shared between the driver and the GSP and containing the command and message queues. 185 #[repr(C)] 186 // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 187 pub(super) struct GspMem { 188 /// Self-mapping page table entries. 189 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 190 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 191 /// write and read pointers that the CPU updates. This means that the read pointer here is an 192 /// index into the GSP queue. 193 /// 194 /// This member is read-only for the GSP. 195 pub(super) cpuq: Msgq, 196 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 197 /// write and read pointers that the GSP updates. This means that the read pointer here is an 198 /// index into the CPU queue. 199 /// 200 /// This member is read-only for the driver. 201 pub(super) gspq: Msgq, 202 } 203 204 impl GspMem { 205 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 206 } 207 208 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 209 // that is not a problem because they are not used outside the kernel. 210 unsafe impl AsBytes for GspMem {} 211 212 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 213 // that is not a problem because they are not used outside the kernel. 214 unsafe impl FromBytes for GspMem {} 215 216 /// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`]. 217 /// 218 /// This provides the low-level functionality to communicate with the GSP, including allocation of 219 /// queue space to write messages to and management of read/write pointers. 220 /// 221 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 222 /// 223 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 224 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 225 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 226 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 227 struct DmaGspMem(Coherent<GspMem>); 228 229 impl DmaGspMem { 230 /// Allocate a new instance and map it for `dev`. 231 fn new(dev: &device::Device<device::Bound>) -> Result<Self> { 232 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 233 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 234 235 let gsp_mem = Coherent::<GspMem>::zeroed(dev, GFP_KERNEL)?; 236 237 let start = gsp_mem.dma_handle(); 238 // Write values one by one to avoid an on-stack instance of `PteArray`. 239 for i in 0..GspMem::PTE_ARRAY_SIZE { 240 dma_write!(gsp_mem, .ptes.0[i], PteArray::<0>::entry(start, i)?); 241 } 242 243 dma_write!( 244 gsp_mem, 245 .cpuq.tx, 246 MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) 247 ); 248 dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new()); 249 250 Ok(Self(gsp_mem)) 251 } 252 253 /// Returns the region of the CPU message queue that the driver is currently allowed to write 254 /// to. 255 /// 256 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 257 /// that case the second slice will have a non-zero length. 258 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 259 let tx = self.cpu_write_ptr(); 260 let rx = self.gsp_read_ptr(); 261 262 // Pointer to the first entry of the CPU message queue. 263 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[0]); 264 265 let (tail_end, wrap_end) = if rx == 0 { 266 // The write area is non-wrapping, and stops at the second-to-last entry of the command 267 // queue (to leave the last one empty). 268 (MSGQ_NUM_PAGES - 1, 0) 269 } else if rx <= tx { 270 // The write area wraps and continues until `rx - 1`. 271 (MSGQ_NUM_PAGES, rx - 1) 272 } else { 273 // The write area doesn't wrap and stops at `rx - 1`. 274 (rx - 1, 0) 275 }; 276 277 // SAFETY: 278 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 279 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, 280 // thus the created slices are valid. 281 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`, 282 // inclusive, belongs to the driver for writing and is not accessed concurrently by 283 // the GSP. 284 // - The caller holds a reference to `self` for as long as the returned slices are live, 285 // meaning the CPU write pointer cannot be advanced and thus that the returned area 286 // remains exclusive to the CPU for the duration of the slices. 287 // - The created slices point to non-overlapping sub-ranges of `data` in all 288 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly 289 // less than `tx` where the first slice starts; in the other cases the second slice is 290 // empty), so creating two `&mut` references from them does not violate aliasing rules. 291 unsafe { 292 ( 293 core::slice::from_raw_parts_mut( 294 data.add(num::u32_as_usize(tx)), 295 num::u32_as_usize(tail_end - tx), 296 ), 297 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)), 298 ) 299 } 300 } 301 302 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 303 /// to write to, in bytes. 304 fn driver_write_area_size(&self) -> usize { 305 let tx = self.cpu_write_ptr(); 306 let rx = self.gsp_read_ptr(); 307 308 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 309 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 310 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 311 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 312 num::u32_as_usize(slots) * GSP_PAGE_SIZE 313 } 314 315 /// Returns the region of the GSP message queue that the driver is currently allowed to read 316 /// from. 317 /// 318 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 319 /// that case the second slice will have a non-zero length. 320 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 321 let tx = self.gsp_write_ptr(); 322 let rx = self.cpu_read_ptr(); 323 324 // Pointer to the first entry of the GSP message queue. 325 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[0]); 326 327 let (tail_end, wrap_end) = if rx <= tx { 328 // Read area is non-wrapping and stops right before `tx`. 329 (tx, 0) 330 } else { 331 // Read area is wrapping and stops right before `tx`. 332 (MSGQ_NUM_PAGES, tx) 333 }; 334 335 // SAFETY: 336 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 337 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`, 338 // thus the created slices are valid. 339 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`, 340 // inclusive, belongs to the driver for reading and is not accessed concurrently by 341 // the GSP. 342 // - The caller holds a reference to `self` for as long as the returned slices are live, 343 // meaning the CPU read pointer cannot be advanced and thus that the returned area 344 // remains exclusive to the CPU for the duration of the slices. 345 unsafe { 346 ( 347 core::slice::from_raw_parts( 348 data.add(num::u32_as_usize(rx)), 349 num::u32_as_usize(tail_end - rx), 350 ), 351 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)), 352 ) 353 } 354 } 355 356 /// Allocates a region on the command queue that is large enough to send a command of `size` 357 /// bytes, waiting for space to become available based on the provided timeout. 358 /// 359 /// This returns a [`GspCommand`] ready to be written to by the caller. 360 /// 361 /// # Errors 362 /// 363 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 364 /// - `ETIMEDOUT` if space does not become available within the timeout. 365 /// - `EIO` if the command header is not properly aligned. 366 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 367 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 368 return Err(EMSGSIZE); 369 } 370 read_poll_timeout( 371 || Ok(self.driver_write_area_size()), 372 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 373 Delta::from_micros(1), 374 timeout, 375 )?; 376 377 // Get the current writable area as an array of bytes. 378 let (slice_1, slice_2) = { 379 let (slice_1, slice_2) = self.driver_write_area(); 380 381 #[allow(clippy::incompatible_msrv)] 382 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 383 }; 384 385 // Extract area for the `GspMsgElement`. 386 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 387 388 // Create the contents area. 389 let (slice_1, slice_2) = if slice_1.len() > size { 390 // Contents fits entirely in `slice_1`. 391 (&mut slice_1[..size], &mut slice_2[0..0]) 392 } else { 393 // Need all of `slice_1` and some of `slice_2`. 394 let slice_2_len = size - slice_1.len(); 395 (slice_1, &mut slice_2[..slice_2_len]) 396 }; 397 398 Ok(GspCommand { 399 header, 400 contents: (slice_1, slice_2), 401 }) 402 } 403 404 // Returns the index of the memory page the GSP will write the next message to. 405 // 406 // # Invariants 407 // 408 // - The returned value is within `0..MSGQ_NUM_PAGES`. 409 fn gsp_write_ptr(&self) -> u32 { 410 super::fw::gsp_mem::gsp_write_ptr(&self.0) 411 } 412 413 // Returns the index of the memory page the GSP will read the next command from. 414 // 415 // # Invariants 416 // 417 // - The returned value is within `0..MSGQ_NUM_PAGES`. 418 fn gsp_read_ptr(&self) -> u32 { 419 super::fw::gsp_mem::gsp_read_ptr(&self.0) 420 } 421 422 // Returns the index of the memory page the CPU can read the next message from. 423 // 424 // # Invariants 425 // 426 // - The returned value is within `0..MSGQ_NUM_PAGES`. 427 fn cpu_read_ptr(&self) -> u32 { 428 super::fw::gsp_mem::cpu_read_ptr(&self.0) 429 } 430 431 // Informs the GSP that it can send `elem_count` new pages into the message queue. 432 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 433 super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) 434 } 435 436 // Returns the index of the memory page the CPU can write the next command to. 437 // 438 // # Invariants 439 // 440 // - The returned value is within `0..MSGQ_NUM_PAGES`. 441 fn cpu_write_ptr(&self) -> u32 { 442 super::fw::gsp_mem::cpu_write_ptr(&self.0) 443 } 444 445 // Informs the GSP that it can process `elem_count` new pages from the command queue. 446 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 447 super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) 448 } 449 } 450 451 /// A command ready to be sent on the command queue. 452 /// 453 /// This is the type returned by [`DmaGspMem::allocate_command`]. 454 struct GspCommand<'a> { 455 // Writable reference to the header of the command. 456 header: &'a mut GspMsgElement, 457 // Writable slices to the contents of the command. The second slice is zero unless the command 458 // loops over the command queue. 459 contents: (&'a mut [u8], &'a mut [u8]), 460 } 461 462 /// A message ready to be processed from the message queue. 463 /// 464 /// This is the type returned by [`Cmdq::wait_for_msg`]. 465 struct GspMessage<'a> { 466 // Reference to the header of the message. 467 header: &'a GspMsgElement, 468 // Slices to the contents of the message. The second slice is zero unless the message loops 469 // over the message queue. 470 contents: (&'a [u8], &'a [u8]), 471 } 472 473 /// GSP command queue. 474 /// 475 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 476 /// area. 477 #[pin_data] 478 pub(crate) struct Cmdq { 479 /// Inner mutex-protected state. 480 #[pin] 481 inner: Mutex<CmdqInner>, 482 /// DMA handle of the command queue's shared memory region. 483 pub(super) dma_handle: DmaAddress, 484 } 485 486 impl Cmdq { 487 /// Offset of the data after the PTEs. 488 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 489 490 /// Offset of command queue ring buffer. 491 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 492 + core::mem::offset_of!(Msgq, msgq) 493 - Self::POST_PTE_OFFSET; 494 495 /// Offset of message queue ring buffer. 496 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 497 + core::mem::offset_of!(Msgq, msgq) 498 - Self::POST_PTE_OFFSET; 499 500 /// Number of page table entries for the GSP shared region. 501 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 502 503 /// Default timeout for receiving a message from the GSP. 504 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 505 506 /// Creates a new command queue for `dev`. 507 pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { 508 pin_init_scope(move || { 509 let gsp_mem = DmaGspMem::new(dev)?; 510 511 Ok(try_pin_init!(Self { 512 dma_handle: gsp_mem.0.dma_handle(), 513 inner <- new_mutex!(CmdqInner { 514 dev: dev.into(), 515 gsp_mem, 516 seq: 0, 517 }), 518 })) 519 }) 520 } 521 522 /// Computes the checksum for the message pointed to by `it`. 523 /// 524 /// A message is made of several parts, so `it` is an iterator over byte slices representing 525 /// these parts. 526 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 527 let sum64 = it 528 .enumerate() 529 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 530 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 531 532 ((sum64 >> 32) as u32) ^ (sum64 as u32) 533 } 534 535 /// Notifies the GSP that we have updated the command queue pointers. 536 fn notify_gsp(bar: &Bar0) { 537 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); 538 } 539 540 /// Sends `command` to the GSP and waits for the reply. 541 /// 542 /// Messages with non-matching function codes are silently consumed until the expected reply 543 /// arrives. 544 /// 545 /// The queue is locked for the entire send+receive cycle to ensure that no other command can 546 /// be interleaved. 547 /// 548 /// # Errors 549 /// 550 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is 551 /// not received within the timeout. 552 /// - `EIO` if the variable payload requested by the command has not been entirely 553 /// written to by its [`CommandToGsp::init_variable_payload`] method. 554 /// 555 /// Error codes returned by the command and reply initializers are propagated as-is. 556 pub(crate) fn send_command<M>(&self, bar: &Bar0, command: M) -> Result<M::Reply> 557 where 558 M: CommandToGsp, 559 M::Reply: MessageFromGsp, 560 Error: From<M::InitError>, 561 Error: From<<M::Reply as MessageFromGsp>::InitError>, 562 { 563 let mut inner = self.inner.lock(); 564 inner.send_command(bar, command)?; 565 566 loop { 567 match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) { 568 Ok(reply) => break Ok(reply), 569 Err(ERANGE) => continue, 570 Err(e) => break Err(e), 571 } 572 } 573 } 574 575 /// Sends `command` to the GSP without waiting for a reply. 576 /// 577 /// # Errors 578 /// 579 /// - `ETIMEDOUT` if space does not become available within the timeout. 580 /// - `EIO` if the variable payload requested by the command has not been entirely 581 /// written to by its [`CommandToGsp::init_variable_payload`] method. 582 /// 583 /// Error codes returned by the command initializers are propagated as-is. 584 pub(crate) fn send_command_no_wait<M>(&self, bar: &Bar0, command: M) -> Result 585 where 586 M: CommandToGsp<Reply = NoReply>, 587 Error: From<M::InitError>, 588 { 589 self.inner.lock().send_command(bar, command) 590 } 591 592 /// Receive a message from the GSP. 593 /// 594 /// See [`CmdqInner::receive_msg`] for details. 595 pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M> 596 where 597 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 598 Error: From<M::InitError>, 599 { 600 self.inner.lock().receive_msg(timeout) 601 } 602 } 603 604 /// Inner mutex protected state of [`Cmdq`]. 605 struct CmdqInner { 606 /// Device this command queue belongs to. 607 dev: ARef<device::Device>, 608 /// Current command sequence number. 609 seq: u32, 610 /// Memory area shared with the GSP for communicating commands and messages. 611 gsp_mem: DmaGspMem, 612 } 613 614 impl CmdqInner { 615 /// Timeout for waiting for space on the command queue. 616 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 617 618 /// Sends `command` to the GSP, without splitting it. 619 /// 620 /// # Errors 621 /// 622 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 623 /// - `ETIMEDOUT` if space does not become available within the timeout. 624 /// - `EIO` if the variable payload requested by the command has not been entirely 625 /// written to by its [`CommandToGsp::init_variable_payload`] method. 626 /// 627 /// Error codes returned by the command initializers are propagated as-is. 628 fn send_single_command<M>(&mut self, bar: &Bar0, command: M) -> Result 629 where 630 M: CommandToGsp, 631 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 632 Error: From<M::InitError>, 633 { 634 let size_in_bytes = command.size(); 635 let dst = self 636 .gsp_mem 637 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 638 639 // Extract area for the command itself. The GSP message header and the command header 640 // together are guaranteed to fit entirely into a single page, so it's ok to only look 641 // at `dst.contents.0` here. 642 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 643 644 // Fill the header and command in-place. 645 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 646 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 647 // fails. 648 unsafe { 649 msg_element.__init(core::ptr::from_mut(dst.header))?; 650 command.init().__init(core::ptr::from_mut(cmd))?; 651 } 652 653 // Fill the variable-length payload, which may be empty. 654 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 655 command.init_variable_payload(&mut sbuffer)?; 656 657 if !sbuffer.is_empty() { 658 return Err(EIO); 659 } 660 drop(sbuffer); 661 662 // Compute checksum now that the whole message is ready. 663 dst.header 664 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 665 dst.header.as_bytes(), 666 dst.contents.0, 667 dst.contents.1, 668 ]))); 669 670 dev_dbg!( 671 &self.dev, 672 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 673 self.seq, 674 M::FUNCTION, 675 dst.header.length(), 676 ); 677 678 // All set - update the write pointer and inform the GSP of the new command. 679 let elem_count = dst.header.element_count(); 680 self.seq += 1; 681 self.gsp_mem.advance_cpu_write_ptr(elem_count); 682 Cmdq::notify_gsp(bar); 683 684 Ok(()) 685 } 686 687 /// Sends `command` to the GSP. 688 /// 689 /// The command may be split into multiple messages if it is large. 690 /// 691 /// # Errors 692 /// 693 /// - `ETIMEDOUT` if space does not become available within the timeout. 694 /// - `EIO` if the variable payload requested by the command has not been entirely 695 /// written to by its [`CommandToGsp::init_variable_payload`] method. 696 /// 697 /// Error codes returned by the command initializers are propagated as-is. 698 fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result 699 where 700 M: CommandToGsp, 701 Error: From<M::InitError>, 702 { 703 match SplitState::new(command)? { 704 SplitState::Single(command) => self.send_single_command(bar, command), 705 SplitState::Split(command, mut continuations) => { 706 self.send_single_command(bar, command)?; 707 708 while let Some(continuation) = continuations.next() { 709 // Turbofish needed because the compiler cannot infer M here. 710 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?; 711 } 712 713 Ok(()) 714 } 715 } 716 } 717 718 /// Wait for a message to become available on the message queue. 719 /// 720 /// This works purely at the transport layer and does not interpret or validate the message 721 /// beyond the advertised length in its [`GspMsgElement`]. 722 /// 723 /// This method returns: 724 /// 725 /// - A reference to the [`GspMsgElement`] of the message, 726 /// - Two byte slices with the contents of the message. The second slice is empty unless the 727 /// message loops across the message queue. 728 /// 729 /// # Errors 730 /// 731 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 732 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 733 /// message queue. 734 /// 735 /// Error codes returned by the message constructor are propagated as-is. 736 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 737 // Wait for a message to arrive from the GSP. 738 let (slice_1, slice_2) = read_poll_timeout( 739 || Ok(self.gsp_mem.driver_read_area()), 740 |driver_area| !driver_area.0.is_empty(), 741 Delta::from_millis(1), 742 timeout, 743 ) 744 .map(|(slice_1, slice_2)| { 745 #[allow(clippy::incompatible_msrv)] 746 (slice_1.as_flattened(), slice_2.as_flattened()) 747 })?; 748 749 // Extract the `GspMsgElement`. 750 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 751 752 dev_dbg!( 753 &self.dev, 754 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 755 header.sequence(), 756 header.function(), 757 header.length(), 758 ); 759 760 let payload_length = header.payload_length(); 761 762 // Check that the driver read area is large enough for the message. 763 if slice_1.len() + slice_2.len() < payload_length { 764 return Err(EIO); 765 } 766 767 // Cut the message slices down to the actual length of the message. 768 let (slice_1, slice_2) = if slice_1.len() > payload_length { 769 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 770 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 771 } else { 772 ( 773 slice_1, 774 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 775 // large as `payload_length`. 776 slice_2.split_at(payload_length - slice_1.len()).0, 777 ) 778 }; 779 780 // Validate checksum. 781 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 782 header.as_bytes(), 783 slice_1, 784 slice_2, 785 ])) != 0 786 { 787 dev_err!( 788 &self.dev, 789 "GSP RPC: receive: Call {} - bad checksum\n", 790 header.sequence() 791 ); 792 return Err(EIO); 793 } 794 795 Ok(GspMessage { 796 header, 797 contents: (slice_1, slice_2), 798 }) 799 } 800 801 /// Receive a message from the GSP. 802 /// 803 /// The expected message type is specified using the `M` generic parameter. If the pending 804 /// message has a different function code, `ERANGE` is returned and the message is consumed. 805 /// 806 /// The read pointer is always advanced past the message, regardless of whether it matched. 807 /// 808 /// # Errors 809 /// 810 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 811 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 812 /// message queue. 813 /// - `EINVAL` if the function code of the message was not recognized. 814 /// - `ERANGE` if the message had a recognized but non-matching function code. 815 /// 816 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 817 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 818 where 819 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 820 Error: From<M::InitError>, 821 { 822 let message = self.wait_for_msg(timeout)?; 823 let function = message.header.function().map_err(|_| EINVAL)?; 824 825 // Extract the message. Store the result as we want to advance the read pointer even in 826 // case of failure. 827 let result = if function == M::FUNCTION { 828 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 829 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 830 831 M::read(cmd, &mut sbuffer) 832 .map_err(|e| e.into()) 833 .inspect(|_| { 834 if !sbuffer.is_empty() { 835 dev_warn!( 836 &self.dev, 837 "GSP message {:?} has unprocessed data\n", 838 function 839 ); 840 } 841 }) 842 } else { 843 Err(ERANGE) 844 }; 845 846 // Advance the read pointer past this message. 847 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 848 message.header.length().div_ceil(GSP_PAGE_SIZE), 849 )?); 850 851 result 852 } 853 } 854