1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::{ 6 mem, 7 sync::atomic::{ 8 fence, 9 Ordering, // 10 }, 11 }; 12 13 use kernel::{ 14 device, 15 dma::{ 16 Coherent, 17 CoherentBox, 18 DmaAddress, // 19 }, 20 io::{ 21 io_project, 22 poll::read_poll_timeout, 23 Io, // 24 }, 25 new_mutex, 26 prelude::*, 27 ptr, 28 sync::{ 29 aref::ARef, 30 Mutex, // 31 }, 32 time::Delta, 33 transmute::{ 34 AsBytes, 35 FromBytes, // 36 }, 37 }; 38 39 use continuation::{ 40 ContinuationRecord, 41 SplitState, // 42 }; 43 44 use pin_init::pin_init_scope; 45 46 use crate::{ 47 driver::Bar0, 48 gsp::{ 49 fw::{ 50 GspMsgElement, 51 MsgFunction, 52 MsgqRxHeader, 53 MsgqTxHeader, 54 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 55 }, 56 PteArray, 57 GSP_PAGE_SHIFT, 58 GSP_PAGE_SIZE, // 59 }, 60 num, 61 sbuffer::SBufferIter, // 62 }; 63 64 use super::regs; 65 66 /// Marker type representing the absence of a reply for a command. Commands using this as their 67 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 68 pub(crate) struct NoReply; 69 70 /// Trait implemented by types representing a command to send to the GSP. 71 /// 72 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 73 /// a given command. 74 /// 75 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 76 /// into the space reserved for it in the command queue buffer. 77 /// 78 /// Some commands may be followed by a variable-length payload. For these, the 79 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 80 /// defined as well. 81 pub(crate) trait CommandToGsp { 82 /// Function identifying this command to the GSP. 83 const FUNCTION: MsgFunction; 84 85 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 86 type Command: FromBytes + AsBytes; 87 88 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 89 /// have a reply. 90 type Reply; 91 92 /// Error type returned by [`CommandToGsp::init`]. 93 type InitError; 94 95 /// In-place command initializer responsible for filling the command in the command queue 96 /// buffer. 97 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 98 99 /// Size of the variable-length payload following the command structure generated by 100 /// [`CommandToGsp::init`]. 101 /// 102 /// Most commands don't have a variable-length payload, so this is zero by default. 103 fn variable_payload_len(&self) -> usize { 104 0 105 } 106 107 /// Method initializing the variable-length payload. 108 /// 109 /// The command buffer is circular, which means that we may need to jump back to its beginning 110 /// while in the middle of a command. For this reason, the variable-length payload is 111 /// initialized using a [`SBufferIter`]. 112 /// 113 /// This method will receive a buffer of the length returned by 114 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 115 /// unwritten space will lead to an error. 116 /// 117 /// Most commands don't have a variable-length payload, so this does nothing by default. 118 fn init_variable_payload( 119 &self, 120 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 121 ) -> Result { 122 Ok(()) 123 } 124 125 /// Total size of the command (including its variable-length payload) without the 126 /// [`GspMsgElement`] header. 127 fn size(&self) -> usize { 128 size_of::<Self::Command>() + self.variable_payload_len() 129 } 130 } 131 132 /// Trait representing messages received from the GSP. 133 /// 134 /// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. 135 pub(crate) trait MessageFromGsp: Sized { 136 /// Function identifying this message from the GSP. 137 const FUNCTION: MsgFunction; 138 139 /// Error type returned by [`MessageFromGsp::read`]. 140 type InitError; 141 142 /// Type containing the raw message to be read from the message queue. 143 type Message: FromBytes; 144 145 /// Method reading the message from the message queue and returning it. 146 /// 147 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 148 /// it. 149 fn read( 150 msg: &Self::Message, 151 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 152 ) -> Result<Self, Self::InitError>; 153 } 154 155 /// Number of GSP pages making the [`Msgq`]. 156 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 157 158 /// Circular buffer of a [`Msgq`]. 159 /// 160 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 161 /// messages. 162 #[repr(C, align(0x1000))] 163 #[derive(Debug)] 164 struct MsgqData { 165 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 166 } 167 168 // Annoyingly we are forced to use a literal to specify the alignment of 169 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 170 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 171 172 /// Unidirectional message queue. 173 /// 174 /// Contains the data for a message queue, that either the driver or GSP writes to. 175 /// 176 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 177 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 178 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 179 #[repr(C)] 180 // There is no struct defined for this in the open-gpu-kernel-source headers. 181 // Instead it is defined by code in `GspMsgQueuesInit()`. 182 struct Msgq { 183 /// Header for sending messages, including the write pointer. 184 tx: MsgqTxHeader, 185 /// Header for receiving messages, including the read pointer. 186 rx: MsgqRxHeader, 187 /// The message queue proper. 188 msgq: MsgqData, 189 } 190 191 /// Structure shared between the driver and the GSP and containing the command and message queues. 192 #[repr(C)] 193 struct GspMem { 194 /// Self-mapping page table entries. 195 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 196 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 197 /// write and read pointers that the CPU updates. This means that the read pointer here is an 198 /// index into the GSP queue. 199 /// 200 /// This member is read-only for the GSP. 201 cpuq: Msgq, 202 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 203 /// write and read pointers that the GSP updates. This means that the read pointer here is an 204 /// index into the CPU queue. 205 /// 206 /// This member is read-only for the driver. 207 gspq: Msgq, 208 } 209 210 impl GspMem { 211 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 212 } 213 214 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 215 // that is not a problem because they are not used outside the kernel. 216 unsafe impl AsBytes for GspMem {} 217 218 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 219 // that is not a problem because they are not used outside the kernel. 220 unsafe impl FromBytes for GspMem {} 221 222 /// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`]. 223 /// 224 /// This provides the low-level functionality to communicate with the GSP, including allocation of 225 /// queue space to write messages to and management of read/write pointers. 226 /// 227 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 228 /// 229 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 230 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 231 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 232 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 233 struct DmaGspMem(Coherent<GspMem>); 234 235 impl DmaGspMem { 236 /// Allocate a new instance and map it for `dev`. 237 fn new(dev: &device::Device<device::Bound>) -> Result<Self> { 238 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 239 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 240 241 let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?; 242 gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); 243 gsp_mem.cpuq.rx = MsgqRxHeader::new(); 244 245 let gsp_mem: Coherent<_> = gsp_mem.into(); 246 PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; 247 248 Ok(Self(gsp_mem)) 249 } 250 251 /// Returns the region of the CPU message queue that the driver is currently allowed to write 252 /// to. 253 /// 254 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 255 /// that case the second slice will have a non-zero length. 256 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 257 let tx = self.cpu_write_ptr(); 258 let rx = self.gsp_read_ptr(); 259 260 // Pointer to the first entry of the CPU message queue. 261 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]); 262 263 let (tail_end, wrap_end) = if rx == 0 { 264 // The write area is non-wrapping, and stops at the second-to-last entry of the command 265 // queue (to leave the last one empty). 266 (MSGQ_NUM_PAGES - 1, 0) 267 } else if rx <= tx { 268 // The write area wraps and continues until `rx - 1`. 269 (MSGQ_NUM_PAGES, rx - 1) 270 } else { 271 // The write area doesn't wrap and stops at `rx - 1`. 272 (rx - 1, 0) 273 }; 274 275 // SAFETY: 276 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 277 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, 278 // thus the created slices are valid. 279 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`, 280 // inclusive, belongs to the driver for writing and is not accessed concurrently by 281 // the GSP. 282 // - The caller holds a reference to `self` for as long as the returned slices are live, 283 // meaning the CPU write pointer cannot be advanced and thus that the returned area 284 // remains exclusive to the CPU for the duration of the slices. 285 // - The created slices point to non-overlapping sub-ranges of `data` in all 286 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly 287 // less than `tx` where the first slice starts; in the other cases the second slice is 288 // empty), so creating two `&mut` references from them does not violate aliasing rules. 289 unsafe { 290 ( 291 core::slice::from_raw_parts_mut( 292 data.add(num::u32_as_usize(tx)), 293 num::u32_as_usize(tail_end - tx), 294 ), 295 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)), 296 ) 297 } 298 } 299 300 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 301 /// to write to, in bytes. 302 fn driver_write_area_size(&self) -> usize { 303 let tx = self.cpu_write_ptr(); 304 let rx = self.gsp_read_ptr(); 305 306 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 307 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 308 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 309 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 310 num::u32_as_usize(slots) * GSP_PAGE_SIZE 311 } 312 313 /// Returns the region of the GSP message queue that the driver is currently allowed to read 314 /// from. 315 /// 316 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 317 /// that case the second slice will have a non-zero length. 318 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 319 let tx = self.gsp_write_ptr(); 320 let rx = self.cpu_read_ptr(); 321 322 // Pointer to the first entry of the GSP message queue. 323 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]); 324 325 let (tail_end, wrap_end) = if rx <= tx { 326 // Read area is non-wrapping and stops right before `tx`. 327 (tx, 0) 328 } else { 329 // Read area is wrapping and stops right before `tx`. 330 (MSGQ_NUM_PAGES, tx) 331 }; 332 333 // SAFETY: 334 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 335 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`, 336 // thus the created slices are valid. 337 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`, 338 // inclusive, belongs to the driver for reading and is not accessed concurrently by 339 // the GSP. 340 // - The caller holds a reference to `self` for as long as the returned slices are live, 341 // meaning the CPU read pointer cannot be advanced and thus that the returned area 342 // remains exclusive to the CPU for the duration of the slices. 343 unsafe { 344 ( 345 core::slice::from_raw_parts( 346 data.add(num::u32_as_usize(rx)), 347 num::u32_as_usize(tail_end - rx), 348 ), 349 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)), 350 ) 351 } 352 } 353 354 /// Allocates a region on the command queue that is large enough to send a command of `size` 355 /// bytes, waiting for space to become available based on the provided timeout. 356 /// 357 /// This returns a [`GspCommand`] ready to be written to by the caller. 358 /// 359 /// # Errors 360 /// 361 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 362 /// - `ETIMEDOUT` if space does not become available within the timeout. 363 /// - `EIO` if the command header is not properly aligned. 364 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 365 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 366 return Err(EMSGSIZE); 367 } 368 read_poll_timeout( 369 || Ok(self.driver_write_area_size()), 370 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 371 Delta::from_micros(1), 372 timeout, 373 )?; 374 375 // Get the current writable area as an array of bytes. 376 let (slice_1, slice_2) = { 377 let (slice_1, slice_2) = self.driver_write_area(); 378 379 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 380 }; 381 382 // Extract area for the `GspMsgElement`. 383 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 384 385 // Create the contents area. 386 let (slice_1, slice_2) = if slice_1.len() > size { 387 // Contents fits entirely in `slice_1`. 388 (&mut slice_1[..size], &mut slice_2[0..0]) 389 } else { 390 // Need all of `slice_1` and some of `slice_2`. 391 let slice_2_len = size - slice_1.len(); 392 (slice_1, &mut slice_2[..slice_2_len]) 393 }; 394 395 Ok(GspCommand { 396 header, 397 contents: (slice_1, slice_2), 398 }) 399 } 400 401 // Returns the index of the memory page the GSP will write the next message to. 402 // 403 // # Invariants 404 // 405 // - The returned value is within `0..MSGQ_NUM_PAGES`. 406 fn gsp_write_ptr(&self) -> u32 { 407 MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES 408 } 409 410 // Returns the index of the memory page the GSP will read the next command from. 411 // 412 // # Invariants 413 // 414 // - The returned value is within `0..MSGQ_NUM_PAGES`. 415 fn gsp_read_ptr(&self) -> u32 { 416 MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES 417 } 418 419 // Returns the index of the memory page the CPU can read the next message from. 420 // 421 // # Invariants 422 // 423 // - The returned value is within `0..MSGQ_NUM_PAGES`. 424 fn cpu_read_ptr(&self) -> u32 { 425 MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES 426 } 427 428 // Informs the GSP that it can send `elem_count` new pages into the message queue. 429 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 430 let rx = io_project!(self.0, .cpuq.rx); 431 let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 432 433 // Ensure read pointer is properly ordered. 434 fence(Ordering::SeqCst); 435 436 MsgqRxHeader::set_read_ptr(rx, rptr) 437 } 438 439 // Returns the index of the memory page the CPU can write the next command to. 440 // 441 // # Invariants 442 // 443 // - The returned value is within `0..MSGQ_NUM_PAGES`. 444 fn cpu_write_ptr(&self) -> u32 { 445 MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES 446 } 447 448 // Informs the GSP that it can process `elem_count` new pages from the command queue. 449 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 450 let tx = io_project!(self.0, .cpuq.tx); 451 let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 452 MsgqTxHeader::set_write_ptr(tx, wptr); 453 454 // Ensure all command data is visible before triggering the GSP read. 455 fence(Ordering::SeqCst); 456 } 457 } 458 459 /// A command ready to be sent on the command queue. 460 /// 461 /// This is the type returned by [`DmaGspMem::allocate_command`]. 462 struct GspCommand<'a> { 463 // Writable reference to the header of the command. 464 header: &'a mut GspMsgElement, 465 // Writable slices to the contents of the command. The second slice is zero unless the command 466 // loops over the command queue. 467 contents: (&'a mut [u8], &'a mut [u8]), 468 } 469 470 /// A message ready to be processed from the message queue. 471 /// 472 /// This is the type returned by [`Cmdq::wait_for_msg`]. 473 struct GspMessage<'a> { 474 // Reference to the header of the message. 475 header: &'a GspMsgElement, 476 // Slices to the contents of the message. The second slice is zero unless the message loops 477 // over the message queue. 478 contents: (&'a [u8], &'a [u8]), 479 } 480 481 /// GSP command queue. 482 /// 483 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 484 /// area. 485 #[pin_data] 486 pub(crate) struct Cmdq { 487 /// Inner mutex-protected state. 488 #[pin] 489 inner: Mutex<CmdqInner>, 490 /// DMA address of the command queue's shared memory region. 491 pub(super) dma_addr: DmaAddress, 492 } 493 494 impl Cmdq { 495 /// Offset of the data after the PTEs. 496 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 497 498 /// Offset of command queue ring buffer. 499 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 500 + core::mem::offset_of!(Msgq, msgq) 501 - Self::POST_PTE_OFFSET; 502 503 /// Offset of message queue ring buffer. 504 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 505 + core::mem::offset_of!(Msgq, msgq) 506 - Self::POST_PTE_OFFSET; 507 508 /// Number of page table entries for the GSP shared region. 509 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 510 511 /// Default timeout for receiving a message from the GSP. 512 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 513 514 /// Creates a new command queue for `dev`. 515 pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { 516 pin_init_scope(move || { 517 let gsp_mem = DmaGspMem::new(dev)?; 518 519 Ok(try_pin_init!(Self { 520 dma_addr: gsp_mem.0.dma_address(), 521 inner <- new_mutex!(CmdqInner { 522 dev: dev.into(), 523 gsp_mem, 524 seq: 0, 525 }), 526 })) 527 }) 528 } 529 530 /// Computes the checksum for the message pointed to by `it`. 531 /// 532 /// A message is made of several parts, so `it` is an iterator over byte slices representing 533 /// these parts. 534 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 535 let sum64 = it 536 .enumerate() 537 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 538 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 539 540 ((sum64 >> 32) as u32) ^ (sum64 as u32) 541 } 542 543 /// Notifies the GSP that we have updated the command queue pointers. 544 fn notify_gsp(bar: Bar0<'_>) { 545 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); 546 } 547 548 /// Sends `command` to the GSP and waits for the reply. 549 /// 550 /// Messages with non-matching function codes are silently consumed until the expected reply 551 /// arrives. 552 /// 553 /// The queue is locked for the entire send+receive cycle to ensure that no other command can 554 /// be interleaved. 555 /// 556 /// # Errors 557 /// 558 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is 559 /// not received within the timeout. 560 /// - `EIO` if the variable payload requested by the command has not been entirely 561 /// written to by its [`CommandToGsp::init_variable_payload`] method. 562 /// 563 /// Error codes returned by the command and reply initializers are propagated as-is. 564 pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply> 565 where 566 M: CommandToGsp, 567 M::Reply: MessageFromGsp, 568 Error: From<M::InitError>, 569 Error: From<<M::Reply as MessageFromGsp>::InitError>, 570 { 571 let mut inner = self.inner.lock(); 572 inner.send_command(bar, command)?; 573 574 loop { 575 match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) { 576 Ok(reply) => break Ok(reply), 577 Err(ERANGE) => continue, 578 Err(e) => break Err(e), 579 } 580 } 581 } 582 583 /// Sends `command` to the GSP without waiting for a reply. 584 /// 585 /// # Errors 586 /// 587 /// - `ETIMEDOUT` if space does not become available within the timeout. 588 /// - `EIO` if the variable payload requested by the command has not been entirely 589 /// written to by its [`CommandToGsp::init_variable_payload`] method. 590 /// 591 /// Error codes returned by the command initializers are propagated as-is. 592 pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result 593 where 594 M: CommandToGsp<Reply = NoReply>, 595 Error: From<M::InitError>, 596 { 597 self.inner.lock().send_command(bar, command) 598 } 599 600 /// Receive a message from the GSP. 601 /// 602 /// See [`CmdqInner::receive_msg`] for details. 603 pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M> 604 where 605 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 606 Error: From<M::InitError>, 607 { 608 self.inner.lock().receive_msg(timeout) 609 } 610 } 611 612 /// Inner mutex protected state of [`Cmdq`]. 613 struct CmdqInner { 614 /// Device this command queue belongs to. 615 dev: ARef<device::Device>, 616 /// Current command sequence number. 617 seq: u32, 618 /// Memory area shared with the GSP for communicating commands and messages. 619 gsp_mem: DmaGspMem, 620 } 621 622 impl CmdqInner { 623 /// Timeout for waiting for space on the command queue. 624 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 625 626 /// Sends `command` to the GSP, without splitting it. 627 /// 628 /// # Errors 629 /// 630 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 631 /// - `ETIMEDOUT` if space does not become available within the timeout. 632 /// - `EIO` if the variable payload requested by the command has not been entirely 633 /// written to by its [`CommandToGsp::init_variable_payload`] method. 634 /// 635 /// Error codes returned by the command initializers are propagated as-is. 636 fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 637 where 638 M: CommandToGsp, 639 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 640 Error: From<M::InitError>, 641 { 642 let size_in_bytes = command.size(); 643 let dst = self 644 .gsp_mem 645 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 646 647 // Extract area for the command itself. The GSP message header and the command header 648 // together are guaranteed to fit entirely into a single page, so it's ok to only look 649 // at `dst.contents.0` here. 650 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 651 652 // Fill the header and command in-place. 653 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 654 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 655 // fails. 656 unsafe { 657 pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; 658 pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; 659 } 660 661 // Fill the variable-length payload, which may be empty. 662 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 663 command.init_variable_payload(&mut sbuffer)?; 664 665 if !sbuffer.is_empty() { 666 return Err(EIO); 667 } 668 drop(sbuffer); 669 670 // Compute checksum now that the whole message is ready. 671 dst.header 672 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 673 dst.header.as_bytes(), 674 dst.contents.0, 675 dst.contents.1, 676 ]))); 677 678 dev_dbg!( 679 &self.dev, 680 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 681 self.seq, 682 M::FUNCTION, 683 dst.header.length(), 684 ); 685 686 // All set - update the write pointer and inform the GSP of the new command. 687 let elem_count = dst.header.element_count(); 688 self.seq += 1; 689 self.gsp_mem.advance_cpu_write_ptr(elem_count); 690 Cmdq::notify_gsp(bar); 691 692 Ok(()) 693 } 694 695 /// Sends `command` to the GSP. 696 /// 697 /// The command may be split into multiple messages if it is large. 698 /// 699 /// # Errors 700 /// 701 /// - `ETIMEDOUT` if space does not become available within the timeout. 702 /// - `EIO` if the variable payload requested by the command has not been entirely 703 /// written to by its [`CommandToGsp::init_variable_payload`] method. 704 /// 705 /// Error codes returned by the command initializers are propagated as-is. 706 fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 707 where 708 M: CommandToGsp, 709 Error: From<M::InitError>, 710 { 711 match SplitState::new(command)? { 712 SplitState::Single(command) => self.send_single_command(bar, command), 713 SplitState::Split(command, mut continuations) => { 714 self.send_single_command(bar, command)?; 715 716 while let Some(continuation) = continuations.next() { 717 // Turbofish needed because the compiler cannot infer M here. 718 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?; 719 } 720 721 Ok(()) 722 } 723 } 724 } 725 726 /// Wait for a message to become available on the message queue. 727 /// 728 /// This works purely at the transport layer and does not interpret or validate the message 729 /// beyond the advertised length in its [`GspMsgElement`]. 730 /// 731 /// This method returns: 732 /// 733 /// - A reference to the [`GspMsgElement`] of the message, 734 /// - Two byte slices with the contents of the message. The second slice is empty unless the 735 /// message loops across the message queue. 736 /// 737 /// # Errors 738 /// 739 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 740 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 741 /// message queue. 742 /// 743 /// Error codes returned by the message constructor are propagated as-is. 744 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 745 // Wait for a message to arrive from the GSP. 746 let (slice_1, slice_2) = read_poll_timeout( 747 || Ok(self.gsp_mem.driver_read_area()), 748 |driver_area| !driver_area.0.is_empty(), 749 Delta::from_millis(1), 750 timeout, 751 ) 752 .map(|(slice_1, slice_2)| (slice_1.as_flattened(), slice_2.as_flattened()))?; 753 754 // Extract the `GspMsgElement`. 755 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 756 757 dev_dbg!( 758 &self.dev, 759 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 760 header.sequence(), 761 header.function(), 762 header.length(), 763 ); 764 765 let payload_length = header.payload_length(); 766 767 // Check that the driver read area is large enough for the message. 768 if slice_1.len() + slice_2.len() < payload_length { 769 return Err(EIO); 770 } 771 772 // Cut the message slices down to the actual length of the message. 773 let (slice_1, slice_2) = if slice_1.len() > payload_length { 774 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 775 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 776 } else { 777 ( 778 slice_1, 779 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 780 // large as `payload_length`. 781 slice_2.split_at(payload_length - slice_1.len()).0, 782 ) 783 }; 784 785 // Validate checksum. 786 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 787 header.as_bytes(), 788 slice_1, 789 slice_2, 790 ])) != 0 791 { 792 dev_err!( 793 &self.dev, 794 "GSP RPC: receive: Call {} - bad checksum\n", 795 header.sequence() 796 ); 797 return Err(EIO); 798 } 799 800 Ok(GspMessage { 801 header, 802 contents: (slice_1, slice_2), 803 }) 804 } 805 806 /// Receive a message from the GSP. 807 /// 808 /// The expected message type is specified using the `M` generic parameter. If the pending 809 /// message has a different function code, `ERANGE` is returned and the message is consumed. 810 /// 811 /// The read pointer is always advanced past the message, regardless of whether it matched. 812 /// 813 /// # Errors 814 /// 815 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 816 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 817 /// message queue. 818 /// - `EINVAL` if the function code of the message was not recognized. 819 /// - `ERANGE` if the message had a recognized but non-matching function code. 820 /// 821 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 822 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 823 where 824 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 825 Error: From<M::InitError>, 826 { 827 let message = self.wait_for_msg(timeout)?; 828 let function = message.header.function().map_err(|_| EINVAL)?; 829 830 // Extract the message. Store the result as we want to advance the read pointer even in 831 // case of failure. 832 let result = if function == M::FUNCTION { 833 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 834 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 835 836 M::read(cmd, &mut sbuffer) 837 .map_err(|e| e.into()) 838 .inspect(|_| { 839 if !sbuffer.is_empty() { 840 dev_warn!( 841 &self.dev, 842 "GSP message {:?} has unprocessed data\n", 843 function 844 ); 845 } 846 }) 847 } else { 848 Err(ERANGE) 849 }; 850 851 // Advance the read pointer past this message. 852 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 853 message.header.length().div_ceil(GSP_PAGE_SIZE), 854 )?); 855 856 result 857 } 858 } 859