1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::{ 6 mem, 7 sync::atomic::{ 8 fence, 9 Ordering, // 10 }, 11 }; 12 13 use kernel::{ 14 device, 15 dma::{ 16 Coherent, 17 CoherentBox, 18 DmaAddress, // 19 }, 20 io::{ 21 io_project, 22 poll::read_poll_timeout, 23 Io, // 24 }, 25 new_mutex, 26 prelude::*, 27 ptr, 28 sync::Mutex, 29 time::Delta, 30 transmute::{ 31 AsBytes, 32 FromBytes, // 33 }, 34 }; 35 36 use continuation::{ 37 ContinuationRecord, 38 SplitState, // 39 }; 40 41 use pin_init::pin_init_scope; 42 43 use crate::{ 44 driver::Bar0, 45 gsp::{ 46 fw::{ 47 GspMsgElement, 48 MsgFunction, 49 MsgqRxHeader, 50 MsgqTxHeader, 51 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 52 }, 53 PteArray, 54 GSP_PAGE_SHIFT, 55 GSP_PAGE_SIZE, // 56 }, 57 num, 58 sbuffer::SBufferIter, // 59 }; 60 61 use super::regs; 62 63 /// Marker type representing the absence of a reply for a command. Commands using this as their 64 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 65 pub(crate) struct NoReply; 66 67 /// Trait implemented by types representing a command to send to the GSP. 68 /// 69 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 70 /// a given command. 71 /// 72 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 73 /// into the space reserved for it in the command queue buffer. 74 /// 75 /// Some commands may be followed by a variable-length payload. For these, the 76 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 77 /// defined as well. 78 pub(crate) trait CommandToGsp { 79 /// Function identifying this command to the GSP. 80 const FUNCTION: MsgFunction; 81 82 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 83 type Command: FromBytes + AsBytes; 84 85 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 86 /// have a reply. 87 type Reply; 88 89 /// Error type returned by [`CommandToGsp::init`]. 90 type InitError; 91 92 /// In-place command initializer responsible for filling the command in the command queue 93 /// buffer. 94 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 95 96 /// Size of the variable-length payload following the command structure generated by 97 /// [`CommandToGsp::init`]. 98 /// 99 /// Most commands don't have a variable-length payload, so this is zero by default. 100 fn variable_payload_len(&self) -> usize { 101 0 102 } 103 104 /// Method initializing the variable-length payload. 105 /// 106 /// The command buffer is circular, which means that we may need to jump back to its beginning 107 /// while in the middle of a command. For this reason, the variable-length payload is 108 /// initialized using a [`SBufferIter`]. 109 /// 110 /// This method will receive a buffer of the length returned by 111 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 112 /// unwritten space will lead to an error. 113 /// 114 /// Most commands don't have a variable-length payload, so this does nothing by default. 115 fn init_variable_payload( 116 &self, 117 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 118 ) -> Result { 119 Ok(()) 120 } 121 122 /// Total size of the command (including its variable-length payload) without the 123 /// [`GspMsgElement`] header. 124 fn size(&self) -> usize { 125 size_of::<Self::Command>() + self.variable_payload_len() 126 } 127 } 128 129 /// Trait representing messages received from the GSP. 130 /// 131 /// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. 132 pub(crate) trait MessageFromGsp: Sized { 133 /// Function identifying this message from the GSP. 134 const FUNCTION: MsgFunction; 135 136 /// Error type returned by [`MessageFromGsp::read`]. 137 type InitError; 138 139 /// Type containing the raw message to be read from the message queue. 140 type Message: FromBytes; 141 142 /// Method reading the message from the message queue and returning it. 143 /// 144 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 145 /// it. 146 fn read( 147 msg: &Self::Message, 148 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 149 ) -> Result<Self, Self::InitError>; 150 } 151 152 /// Number of GSP pages making the [`Msgq`]. 153 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 154 155 /// Circular buffer of a [`Msgq`]. 156 /// 157 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 158 /// messages. 159 #[repr(C, align(0x1000))] 160 #[derive(Debug)] 161 struct MsgqData { 162 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 163 } 164 165 // Annoyingly we are forced to use a literal to specify the alignment of 166 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 167 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 168 169 /// Unidirectional message queue. 170 /// 171 /// Contains the data for a message queue, that either the driver or GSP writes to. 172 /// 173 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 174 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 175 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 176 #[repr(C)] 177 // There is no struct defined for this in the open-gpu-kernel-source headers. 178 // Instead it is defined by code in `GspMsgQueuesInit()`. 179 struct Msgq { 180 /// Header for sending messages, including the write pointer. 181 tx: MsgqTxHeader, 182 /// Header for receiving messages, including the read pointer. 183 rx: MsgqRxHeader, 184 /// The message queue proper. 185 msgq: MsgqData, 186 } 187 188 /// Structure shared between the driver and the GSP and containing the command and message queues. 189 #[repr(C)] 190 struct GspMem { 191 /// Self-mapping page table entries. 192 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 193 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 194 /// write and read pointers that the CPU updates. This means that the read pointer here is an 195 /// index into the GSP queue. 196 /// 197 /// This member is read-only for the GSP. 198 cpuq: Msgq, 199 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 200 /// write and read pointers that the GSP updates. This means that the read pointer here is an 201 /// index into the CPU queue. 202 /// 203 /// This member is read-only for the driver. 204 gspq: Msgq, 205 } 206 207 impl GspMem { 208 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 209 } 210 211 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 212 // that is not a problem because they are not used outside the kernel. 213 unsafe impl AsBytes for GspMem {} 214 215 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 216 // that is not a problem because they are not used outside the kernel. 217 unsafe impl FromBytes for GspMem {} 218 219 /// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`]. 220 /// 221 /// This provides the low-level functionality to communicate with the GSP, including allocation of 222 /// queue space to write messages to and management of read/write pointers. 223 /// 224 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 225 /// 226 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 227 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 228 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 229 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 230 struct DmaGspMem<'a>(Coherent<'a, GspMem>); 231 232 impl<'a> DmaGspMem<'a> { 233 /// Allocate a new instance and map it for `dev`. 234 fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> { 235 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 236 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 237 238 let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?; 239 gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); 240 gsp_mem.cpuq.rx = MsgqRxHeader::new(); 241 242 let gsp_mem: Coherent<'_, _> = gsp_mem.into(); 243 PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; 244 245 Ok(Self(gsp_mem)) 246 } 247 248 /// Returns the region of the CPU message queue that the driver is currently allowed to write 249 /// to. 250 /// 251 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 252 /// that case the second slice will have a non-zero length. 253 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 254 let tx = self.cpu_write_ptr(); 255 let rx = self.gsp_read_ptr(); 256 257 // Pointer to the first entry of the CPU message queue. 258 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]); 259 260 let (tail_end, wrap_end) = if rx == 0 { 261 // The write area is non-wrapping, and stops at the second-to-last entry of the command 262 // queue (to leave the last one empty). 263 (MSGQ_NUM_PAGES - 1, 0) 264 } else if rx <= tx { 265 // The write area wraps and continues until `rx - 1`. 266 (MSGQ_NUM_PAGES, rx - 1) 267 } else { 268 // The write area doesn't wrap and stops at `rx - 1`. 269 (rx - 1, 0) 270 }; 271 272 // SAFETY: 273 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 274 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, 275 // thus the created slices are valid. 276 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`, 277 // inclusive, belongs to the driver for writing and is not accessed concurrently by 278 // the GSP. 279 // - The caller holds a reference to `self` for as long as the returned slices are live, 280 // meaning the CPU write pointer cannot be advanced and thus that the returned area 281 // remains exclusive to the CPU for the duration of the slices. 282 // - The created slices point to non-overlapping sub-ranges of `data` in all 283 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly 284 // less than `tx` where the first slice starts; in the other cases the second slice is 285 // empty), so creating two `&mut` references from them does not violate aliasing rules. 286 unsafe { 287 ( 288 core::slice::from_raw_parts_mut( 289 data.add(num::u32_as_usize(tx)), 290 num::u32_as_usize(tail_end - tx), 291 ), 292 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)), 293 ) 294 } 295 } 296 297 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 298 /// to write to, in bytes. 299 fn driver_write_area_size(&self) -> usize { 300 let tx = self.cpu_write_ptr(); 301 let rx = self.gsp_read_ptr(); 302 303 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 304 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 305 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 306 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 307 num::u32_as_usize(slots) * GSP_PAGE_SIZE 308 } 309 310 /// Returns the region of the GSP message queue that the driver is currently allowed to read 311 /// from. 312 /// 313 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 314 /// that case the second slice will have a non-zero length. 315 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 316 let tx = self.gsp_write_ptr(); 317 let rx = self.cpu_read_ptr(); 318 319 // Pointer to the first entry of the GSP message queue. 320 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]); 321 322 let (tail_end, wrap_end) = if rx <= tx { 323 // Read area is non-wrapping and stops right before `tx`. 324 (tx, 0) 325 } else { 326 // Read area is wrapping and stops right before `tx`. 327 (MSGQ_NUM_PAGES, tx) 328 }; 329 330 // SAFETY: 331 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 332 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`, 333 // thus the created slices are valid. 334 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`, 335 // inclusive, belongs to the driver for reading and is not accessed concurrently by 336 // the GSP. 337 // - The caller holds a reference to `self` for as long as the returned slices are live, 338 // meaning the CPU read pointer cannot be advanced and thus that the returned area 339 // remains exclusive to the CPU for the duration of the slices. 340 unsafe { 341 ( 342 core::slice::from_raw_parts( 343 data.add(num::u32_as_usize(rx)), 344 num::u32_as_usize(tail_end - rx), 345 ), 346 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)), 347 ) 348 } 349 } 350 351 /// Allocates a region on the command queue that is large enough to send a command of `size` 352 /// bytes, waiting for space to become available based on the provided timeout. 353 /// 354 /// This returns a [`GspCommand`] ready to be written to by the caller. 355 /// 356 /// # Errors 357 /// 358 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 359 /// - `ETIMEDOUT` if space does not become available within the timeout. 360 /// - `EIO` if the command header is not properly aligned. 361 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 362 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 363 return Err(EMSGSIZE); 364 } 365 read_poll_timeout( 366 || Ok(self.driver_write_area_size()), 367 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 368 Delta::from_micros(1), 369 timeout, 370 )?; 371 372 // Get the current writable area as an array of bytes. 373 let (slice_1, slice_2) = { 374 let (slice_1, slice_2) = self.driver_write_area(); 375 376 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 377 }; 378 379 // Extract area for the `GspMsgElement`. 380 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 381 382 // Create the contents area. 383 let (slice_1, slice_2) = if slice_1.len() > size { 384 // Contents fits entirely in `slice_1`. 385 (&mut slice_1[..size], &mut slice_2[0..0]) 386 } else { 387 // Need all of `slice_1` and some of `slice_2`. 388 let slice_2_len = size - slice_1.len(); 389 (slice_1, &mut slice_2[..slice_2_len]) 390 }; 391 392 Ok(GspCommand { 393 header, 394 contents: (slice_1, slice_2), 395 }) 396 } 397 398 // Returns the index of the memory page the GSP will write the next message to. 399 // 400 // # Invariants 401 // 402 // - The returned value is within `0..MSGQ_NUM_PAGES`. 403 fn gsp_write_ptr(&self) -> u32 { 404 MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES 405 } 406 407 // Returns the index of the memory page the GSP will read the next command from. 408 // 409 // # Invariants 410 // 411 // - The returned value is within `0..MSGQ_NUM_PAGES`. 412 fn gsp_read_ptr(&self) -> u32 { 413 MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES 414 } 415 416 // Returns the index of the memory page the CPU can read the next message from. 417 // 418 // # Invariants 419 // 420 // - The returned value is within `0..MSGQ_NUM_PAGES`. 421 fn cpu_read_ptr(&self) -> u32 { 422 MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES 423 } 424 425 // Informs the GSP that it can send `elem_count` new pages into the message queue. 426 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 427 let rx = io_project!(self.0, .cpuq.rx); 428 let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 429 430 // Ensure read pointer is properly ordered. 431 fence(Ordering::SeqCst); 432 433 MsgqRxHeader::set_read_ptr(rx, rptr) 434 } 435 436 // Returns the index of the memory page the CPU can write the next command to. 437 // 438 // # Invariants 439 // 440 // - The returned value is within `0..MSGQ_NUM_PAGES`. 441 fn cpu_write_ptr(&self) -> u32 { 442 MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES 443 } 444 445 // Informs the GSP that it can process `elem_count` new pages from the command queue. 446 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 447 let tx = io_project!(self.0, .cpuq.tx); 448 let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 449 MsgqTxHeader::set_write_ptr(tx, wptr); 450 451 // Ensure all command data is visible before triggering the GSP read. 452 fence(Ordering::SeqCst); 453 } 454 } 455 456 /// A command ready to be sent on the command queue. 457 /// 458 /// This is the type returned by [`DmaGspMem::allocate_command`]. 459 struct GspCommand<'a> { 460 // Writable reference to the header of the command. 461 header: &'a mut GspMsgElement, 462 // Writable slices to the contents of the command. The second slice is zero unless the command 463 // loops over the command queue. 464 contents: (&'a mut [u8], &'a mut [u8]), 465 } 466 467 /// A message ready to be processed from the message queue. 468 /// 469 /// This is the type returned by [`Cmdq::wait_for_msg`]. 470 struct GspMessage<'a> { 471 // Reference to the header of the message. 472 header: &'a GspMsgElement, 473 // Slices to the contents of the message. The second slice is zero unless the message loops 474 // over the message queue. 475 contents: (&'a [u8], &'a [u8]), 476 } 477 478 /// GSP command queue. 479 /// 480 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 481 /// area. 482 #[pin_data] 483 pub(crate) struct Cmdq<'cmdq> { 484 /// Inner mutex-protected state. 485 #[pin] 486 inner: Mutex<CmdqInner<'cmdq>>, 487 /// DMA address of the command queue's shared memory region. 488 pub(super) dma_addr: DmaAddress, 489 } 490 491 impl<'cmdq> Cmdq<'cmdq> { 492 /// Offset of the data after the PTEs. 493 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 494 495 /// Offset of command queue ring buffer. 496 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 497 + core::mem::offset_of!(Msgq, msgq) 498 - Self::POST_PTE_OFFSET; 499 500 /// Offset of message queue ring buffer. 501 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 502 + core::mem::offset_of!(Msgq, msgq) 503 - Self::POST_PTE_OFFSET; 504 505 /// Number of page table entries for the GSP shared region. 506 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 507 508 /// Default timeout for receiving a message from the GSP. 509 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 510 511 /// Creates a new command queue for `dev`. 512 pub(crate) fn new( 513 dev: &'cmdq device::Device<device::Bound>, 514 ) -> impl PinInit<Self, Error> + 'cmdq { 515 pin_init_scope(move || { 516 let gsp_mem = DmaGspMem::new(dev)?; 517 518 Ok(try_pin_init!(Self { 519 dma_addr: gsp_mem.0.dma_address(), 520 inner <- new_mutex!(CmdqInner { 521 dev, 522 gsp_mem, 523 seq: 0, 524 }), 525 })) 526 }) 527 } 528 529 /// Computes the checksum for the message pointed to by `it`. 530 /// 531 /// A message is made of several parts, so `it` is an iterator over byte slices representing 532 /// these parts. 533 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 534 let sum64 = it 535 .enumerate() 536 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 537 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 538 539 ((sum64 >> 32) as u32) ^ (sum64 as u32) 540 } 541 542 /// Notifies the GSP that we have updated the command queue pointers. 543 fn notify_gsp(bar: Bar0<'_>) { 544 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); 545 } 546 547 /// Sends `command` to the GSP and waits for the reply. 548 /// 549 /// Messages with non-matching function codes are silently consumed until the expected reply 550 /// arrives. 551 /// 552 /// The queue is locked for the entire send+receive cycle to ensure that no other command can 553 /// be interleaved. 554 /// 555 /// # Errors 556 /// 557 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is 558 /// not received within the timeout. 559 /// - `EIO` if the variable payload requested by the command has not been entirely 560 /// written to by its [`CommandToGsp::init_variable_payload`] method. 561 /// 562 /// Error codes returned by the command and reply initializers are propagated as-is. 563 pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply> 564 where 565 M: CommandToGsp, 566 M::Reply: MessageFromGsp, 567 Error: From<M::InitError>, 568 Error: From<<M::Reply as MessageFromGsp>::InitError>, 569 { 570 let mut inner = self.inner.lock(); 571 inner.send_command(bar, command)?; 572 573 loop { 574 match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) { 575 Ok(reply) => break Ok(reply), 576 Err(ERANGE) => continue, 577 Err(e) => break Err(e), 578 } 579 } 580 } 581 582 /// Sends `command` to the GSP without waiting for a reply. 583 /// 584 /// # Errors 585 /// 586 /// - `ETIMEDOUT` if space does not become available within the timeout. 587 /// - `EIO` if the variable payload requested by the command has not been entirely 588 /// written to by its [`CommandToGsp::init_variable_payload`] method. 589 /// 590 /// Error codes returned by the command initializers are propagated as-is. 591 pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result 592 where 593 M: CommandToGsp<Reply = NoReply>, 594 Error: From<M::InitError>, 595 { 596 self.inner.lock().send_command(bar, command) 597 } 598 599 /// Receive a message from the GSP. 600 /// 601 /// See [`CmdqInner::receive_msg`] for details. 602 pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M> 603 where 604 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 605 Error: From<M::InitError>, 606 { 607 self.inner.lock().receive_msg(timeout) 608 } 609 } 610 611 /// Inner mutex protected state of [`Cmdq`]. 612 struct CmdqInner<'a> { 613 /// Device this command queue belongs to. 614 dev: &'a device::Device, 615 /// Current command sequence number. 616 seq: u32, 617 /// Memory area shared with the GSP for communicating commands and messages. 618 gsp_mem: DmaGspMem<'a>, 619 } 620 621 impl CmdqInner<'_> { 622 /// Timeout for waiting for space on the command queue. 623 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 624 625 /// Sends `command` to the GSP, without splitting it. 626 /// 627 /// # Errors 628 /// 629 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 630 /// - `ETIMEDOUT` if space does not become available within the timeout. 631 /// - `EIO` if the variable payload requested by the command has not been entirely 632 /// written to by its [`CommandToGsp::init_variable_payload`] method. 633 /// 634 /// Error codes returned by the command initializers are propagated as-is. 635 fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 636 where 637 M: CommandToGsp, 638 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 639 Error: From<M::InitError>, 640 { 641 let size_in_bytes = command.size(); 642 let dst = self 643 .gsp_mem 644 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 645 646 // Extract area for the command itself. The GSP message header and the command header 647 // together are guaranteed to fit entirely into a single page, so it's ok to only look 648 // at `dst.contents.0` here. 649 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 650 651 // Fill the header and command in-place. 652 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 653 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 654 // fails. 655 unsafe { 656 pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; 657 pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; 658 } 659 660 // Fill the variable-length payload, which may be empty. 661 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 662 command.init_variable_payload(&mut sbuffer)?; 663 664 if !sbuffer.is_empty() { 665 return Err(EIO); 666 } 667 drop(sbuffer); 668 669 // Compute checksum now that the whole message is ready. 670 dst.header 671 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 672 dst.header.as_bytes(), 673 dst.contents.0, 674 dst.contents.1, 675 ]))); 676 677 dev_dbg!( 678 &self.dev, 679 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 680 self.seq, 681 M::FUNCTION, 682 dst.header.length(), 683 ); 684 685 // All set - update the write pointer and inform the GSP of the new command. 686 let elem_count = dst.header.element_count(); 687 self.seq += 1; 688 self.gsp_mem.advance_cpu_write_ptr(elem_count); 689 Cmdq::notify_gsp(bar); 690 691 Ok(()) 692 } 693 694 /// Sends `command` to the GSP. 695 /// 696 /// The command may be split into multiple messages if it is large. 697 /// 698 /// # Errors 699 /// 700 /// - `ETIMEDOUT` if space does not become available within the timeout. 701 /// - `EIO` if the variable payload requested by the command has not been entirely 702 /// written to by its [`CommandToGsp::init_variable_payload`] method. 703 /// 704 /// Error codes returned by the command initializers are propagated as-is. 705 fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result 706 where 707 M: CommandToGsp, 708 Error: From<M::InitError>, 709 { 710 match SplitState::new(command)? { 711 SplitState::Single(command) => self.send_single_command(bar, command), 712 SplitState::Split(command, mut continuations) => { 713 self.send_single_command(bar, command)?; 714 715 while let Some(continuation) = continuations.next() { 716 // Turbofish needed because the compiler cannot infer M here. 717 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?; 718 } 719 720 Ok(()) 721 } 722 } 723 } 724 725 /// Wait for a message to become available on the message queue. 726 /// 727 /// This works purely at the transport layer and does not interpret or validate the message 728 /// beyond the advertised length in its [`GspMsgElement`]. 729 /// 730 /// This method returns: 731 /// 732 /// - A reference to the [`GspMsgElement`] of the message, 733 /// - Two byte slices with the contents of the message. The second slice is empty unless the 734 /// message loops across the message queue. 735 /// 736 /// # Errors 737 /// 738 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 739 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 740 /// message queue. 741 /// 742 /// Error codes returned by the message constructor are propagated as-is. 743 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 744 // Wait for a message to arrive from the GSP. 745 let (slice_1, slice_2) = read_poll_timeout( 746 || Ok(self.gsp_mem.driver_read_area()), 747 |driver_area| !driver_area.0.is_empty(), 748 Delta::from_millis(1), 749 timeout, 750 ) 751 .map(|(slice_1, slice_2)| (slice_1.as_flattened(), slice_2.as_flattened()))?; 752 753 // Extract the `GspMsgElement`. 754 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 755 756 dev_dbg!( 757 &self.dev, 758 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 759 header.sequence(), 760 header.function(), 761 header.length(), 762 ); 763 764 let payload_length = header.payload_length(); 765 766 // Check that the driver read area is large enough for the message. 767 if slice_1.len() + slice_2.len() < payload_length { 768 return Err(EIO); 769 } 770 771 // Cut the message slices down to the actual length of the message. 772 let (slice_1, slice_2) = if slice_1.len() > payload_length { 773 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 774 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 775 } else { 776 ( 777 slice_1, 778 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 779 // large as `payload_length`. 780 slice_2.split_at(payload_length - slice_1.len()).0, 781 ) 782 }; 783 784 // Validate checksum. 785 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 786 header.as_bytes(), 787 slice_1, 788 slice_2, 789 ])) != 0 790 { 791 dev_err!( 792 &self.dev, 793 "GSP RPC: receive: Call {} - bad checksum\n", 794 header.sequence() 795 ); 796 return Err(EIO); 797 } 798 799 Ok(GspMessage { 800 header, 801 contents: (slice_1, slice_2), 802 }) 803 } 804 805 /// Receive a message from the GSP. 806 /// 807 /// The expected message type is specified using the `M` generic parameter. If the pending 808 /// message has a different function code, `ERANGE` is returned and the message is consumed. 809 /// 810 /// The read pointer is always advanced past the message, regardless of whether it matched. 811 /// 812 /// # Errors 813 /// 814 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 815 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 816 /// message queue. 817 /// - `EINVAL` if the function code of the message was not recognized. 818 /// - `ERANGE` if the message had a recognized but non-matching function code. 819 /// 820 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 821 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 822 where 823 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 824 Error: From<M::InitError>, 825 { 826 let message = self.wait_for_msg(timeout)?; 827 let function = message.header.function().map_err(|_| EINVAL)?; 828 829 // Extract the message. Store the result as we want to advance the read pointer even in 830 // case of failure. 831 let result = if function == M::FUNCTION { 832 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 833 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 834 835 M::read(cmd, &mut sbuffer) 836 .map_err(|e| e.into()) 837 .inspect(|_| { 838 if !sbuffer.is_empty() { 839 dev_warn!( 840 &self.dev, 841 "GSP message {:?} has unprocessed data\n", 842 function 843 ); 844 } 845 }) 846 } else { 847 Err(ERANGE) 848 }; 849 850 // Advance the read pointer past this message. 851 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 852 message.header.length().div_ceil(GSP_PAGE_SIZE), 853 )?); 854 855 result 856 } 857 } 858