1 // SPDX-License-Identifier: GPL-2.0 2 3 mod continuation; 4 5 use core::mem; 6 7 use kernel::{ 8 device, 9 dma::{ 10 Coherent, 11 CoherentBox, 12 DmaAddress, // 13 }, 14 io::{ 15 io_project, 16 poll::read_poll_timeout, 17 Io, // 18 }, 19 new_mutex, 20 prelude::*, 21 ptr, 22 sync::{ 23 barrier::{ 24 dma_mb, 25 Full, 26 Read, 27 Write, // 28 }, 29 Mutex, // 30 }, 31 time::{ 32 Delta, 33 Instant, 34 Monotonic, // 35 }, 36 transmute::{ 37 AsBytes, 38 FromBytes, // 39 }, 40 }; 41 42 use continuation::{ 43 ContinuationRecord, 44 SplitState, // 45 }; 46 47 use pin_init::pin_init_scope; 48 49 use crate::{ 50 driver::Bar0, 51 gsp::{ 52 fw::{ 53 GspMsgElement, 54 MsgFunction, 55 MsgqRxHeader, 56 MsgqTxHeader, 57 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, // 58 }, 59 PteArray, 60 GSP_PAGE_SHIFT, 61 GSP_PAGE_SIZE, // 62 }, 63 num, 64 sbuffer::SBufferIter, // 65 }; 66 67 use super::regs; 68 69 /// Marker type representing the absence of a reply for a command. Commands using this as their 70 /// reply type are sent using [`Cmdq::send_command_no_wait`]. 71 pub(crate) struct NoReply; 72 73 /// Trait implemented by types representing a command to send to the GSP. 74 /// 75 /// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send 76 /// a given command. 77 /// 78 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly 79 /// into the space reserved for it in the command queue buffer. 80 /// 81 /// Some commands may be followed by a variable-length payload. For these, the 82 /// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be 83 /// defined as well. 84 pub(crate) trait CommandToGsp { 85 /// Function identifying this command to the GSP. 86 const FUNCTION: MsgFunction; 87 88 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer. 89 type Command: FromBytes + AsBytes; 90 91 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't 92 /// have a reply. 93 type Reply; 94 95 /// Error type returned by [`CommandToGsp::init`]. 96 type InitError; 97 98 /// In-place command initializer responsible for filling the command in the command queue 99 /// buffer. 100 fn init(&self) -> impl Init<Self::Command, Self::InitError>; 101 102 /// Size of the variable-length payload following the command structure generated by 103 /// [`CommandToGsp::init`]. 104 /// 105 /// Most commands don't have a variable-length payload, so this is zero by default. 106 fn variable_payload_len(&self) -> usize { 107 0 108 } 109 110 /// Method initializing the variable-length payload. 111 /// 112 /// The command buffer is circular, which means that we may need to jump back to its beginning 113 /// while in the middle of a command. For this reason, the variable-length payload is 114 /// initialized using a [`SBufferIter`]. 115 /// 116 /// This method will receive a buffer of the length returned by 117 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving 118 /// unwritten space will lead to an error. 119 /// 120 /// Most commands don't have a variable-length payload, so this does nothing by default. 121 fn init_variable_payload( 122 &self, 123 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, 124 ) -> Result { 125 Ok(()) 126 } 127 128 /// Total size of the command (including its variable-length payload) without the 129 /// [`GspMsgElement`] header. 130 fn size(&self) -> usize { 131 size_of::<Self::Command>() + self.variable_payload_len() 132 } 133 } 134 135 /// Trait representing messages received from the GSP. 136 /// 137 /// A reply that [`Cmdq::send_command`] waits for, or an event that [`Cmdq::await_msg`] waits for. 138 /// The receiver matches a message's function code against [`Self::FUNCTION`] and decodes the 139 /// message with [`Self::read`]. 140 pub(crate) trait MessageFromGsp: Sized { 141 /// Function identifying this message from the GSP. 142 const FUNCTION: MsgFunction; 143 144 /// Error type returned by [`MessageFromGsp::read`]. 145 type InitError; 146 147 /// Type containing the raw message to be read from the message queue. 148 type Message: FromBytes; 149 150 /// Method reading the message from the message queue and returning it. 151 /// 152 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns 153 /// it. 154 fn read( 155 msg: &Self::Message, 156 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>, 157 ) -> Result<Self, Self::InitError>; 158 } 159 160 /// Number of GSP pages making the [`Msgq`]. 161 pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f; 162 163 /// Circular buffer of a [`Msgq`]. 164 /// 165 /// This area of memory is to be shared between the driver and the GSP to exchange commands or 166 /// messages. 167 #[repr(C, align(0x1000))] 168 #[derive(Debug)] 169 struct MsgqData { 170 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)], 171 } 172 173 // Annoyingly we are forced to use a literal to specify the alignment of 174 // `MsgqData`, so check that it corresponds to the actual GSP page size here. 175 static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); 176 177 /// Unidirectional message queue. 178 /// 179 /// Contains the data for a message queue, that either the driver or GSP writes to. 180 /// 181 /// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the 182 /// read pointer of `rx` actually refers to the `Msgq` owned by the other side. 183 /// This design ensures that only the driver or GSP ever writes to a given instance of this struct. 184 #[repr(C)] 185 // There is no struct defined for this in the open-gpu-kernel-source headers. 186 // Instead it is defined by code in `GspMsgQueuesInit()`. 187 struct Msgq { 188 /// Header for sending messages, including the write pointer. 189 tx: MsgqTxHeader, 190 /// Header for receiving messages, including the read pointer. 191 rx: MsgqRxHeader, 192 /// The message queue proper. 193 msgq: MsgqData, 194 } 195 196 /// Structure shared between the driver and the GSP and containing the command and message queues. 197 #[repr(C)] 198 struct GspMem { 199 /// Self-mapping page table entries. 200 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 201 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 202 /// write and read pointers that the CPU updates. This means that the read pointer here is an 203 /// index into the GSP queue. 204 /// 205 /// This member is read-only for the GSP. 206 cpuq: Msgq, 207 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 208 /// write and read pointers that the GSP updates. This means that the read pointer here is an 209 /// index into the CPU queue. 210 /// 211 /// This member is read-only for the driver. 212 gspq: Msgq, 213 } 214 215 impl GspMem { 216 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 217 } 218 219 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but 220 // that is not a problem because they are not used outside the kernel. 221 unsafe impl AsBytes for GspMem {} 222 223 // SAFETY: These structs don't meet the no-padding requirements of FromBytes but 224 // that is not a problem because they are not used outside the kernel. 225 unsafe impl FromBytes for GspMem {} 226 227 /// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`]. 228 /// 229 /// This provides the low-level functionality to communicate with the GSP, including allocation of 230 /// queue space to write messages to and management of read/write pointers. 231 /// 232 /// This is shared with the GSP, with clear ownership rules regarding the command queues: 233 /// 234 /// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write 235 /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. 236 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read 237 /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. 238 struct DmaGspMem<'a>(Coherent<'a, GspMem>); 239 240 impl<'a> DmaGspMem<'a> { 241 /// Allocate a new instance and map it for `dev`. 242 fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> { 243 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); 244 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); 245 246 let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?; 247 gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); 248 gsp_mem.cpuq.rx = MsgqRxHeader::new(); 249 250 let gsp_mem: Coherent<'_, _> = gsp_mem.into(); 251 PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; 252 253 Ok(Self(gsp_mem)) 254 } 255 256 /// Returns the region of the CPU message queue that the driver is currently allowed to write 257 /// to. 258 /// 259 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 260 /// that case the second slice will have a non-zero length. 261 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) { 262 let tx = self.cpu_write_ptr(); 263 let rx = self.gsp_read_ptr(); 264 265 // Pointer to the first entry of the CPU message queue. 266 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]); 267 268 let (tail_end, wrap_end) = if rx == 0 { 269 // The write area is non-wrapping, and stops at the second-to-last entry of the command 270 // queue (to leave the last one empty). 271 (MSGQ_NUM_PAGES - 1, 0) 272 } else if rx <= tx { 273 // The write area wraps and continues until `rx - 1`. 274 (MSGQ_NUM_PAGES, rx - 1) 275 } else { 276 // The write area doesn't wrap and stops at `rx - 1`. 277 (rx - 1, 0) 278 }; 279 280 // SAFETY: 281 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 282 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, 283 // thus the created slices are valid. 284 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`, 285 // inclusive, belongs to the driver for writing and is not accessed concurrently by 286 // the GSP. 287 // - The caller holds a reference to `self` for as long as the returned slices are live, 288 // meaning the CPU write pointer cannot be advanced and thus that the returned area 289 // remains exclusive to the CPU for the duration of the slices. 290 // - The created slices point to non-overlapping sub-ranges of `data` in all 291 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly 292 // less than `tx` where the first slice starts; in the other cases the second slice is 293 // empty), so creating two `&mut` references from them does not violate aliasing rules. 294 unsafe { 295 ( 296 core::slice::from_raw_parts_mut( 297 data.add(num::u32_as_usize(tx)), 298 num::u32_as_usize(tail_end - tx), 299 ), 300 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)), 301 ) 302 } 303 } 304 305 /// Returns the size of the region of the CPU message queue that the driver is currently allowed 306 /// to write to, in bytes. 307 fn driver_write_area_size(&self) -> usize { 308 let tx = self.cpu_write_ptr(); 309 let rx = self.gsp_read_ptr(); 310 311 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and 312 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES - 313 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`. 314 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES; 315 num::u32_as_usize(slots) * GSP_PAGE_SIZE 316 } 317 318 /// Returns the region of the GSP message queue that the driver is currently allowed to read 319 /// from. 320 /// 321 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In 322 /// that case the second slice will have a non-zero length. 323 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) { 324 let tx = self.gsp_write_ptr(); 325 let rx = self.cpu_read_ptr(); 326 327 // Pointer to the first entry of the GSP message queue. 328 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]); 329 330 let (tail_end, wrap_end) = if rx <= tx { 331 // Read area is non-wrapping and stops right before `tx`. 332 (tx, 0) 333 } else { 334 // Read area is wrapping and stops right before `tx`. 335 (MSGQ_NUM_PAGES, tx) 336 }; 337 338 // SAFETY: 339 // - `data` was created from a valid pointer, and `rx` and `tx` are in the 340 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`, 341 // thus the created slices are valid. 342 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`, 343 // inclusive, belongs to the driver for reading and is not accessed concurrently by 344 // the GSP. 345 // - The caller holds a reference to `self` for as long as the returned slices are live, 346 // meaning the CPU read pointer cannot be advanced and thus that the returned area 347 // remains exclusive to the CPU for the duration of the slices. 348 unsafe { 349 ( 350 core::slice::from_raw_parts( 351 data.add(num::u32_as_usize(rx)), 352 num::u32_as_usize(tail_end - rx), 353 ), 354 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)), 355 ) 356 } 357 } 358 359 /// Allocates a region on the command queue that is large enough to send a command of `size` 360 /// bytes, waiting for space to become available based on the provided timeout. 361 /// 362 /// This returns a [`GspCommand`] ready to be written to by the caller. 363 /// 364 /// # Errors 365 /// 366 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`]. 367 /// - `ETIMEDOUT` if space does not become available within the timeout. 368 /// - `EIO` if the command header is not properly aligned. 369 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> { 370 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX { 371 return Err(EMSGSIZE); 372 } 373 read_poll_timeout( 374 || Ok(self.driver_write_area_size()), 375 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size, 376 Delta::from_micros(1), 377 timeout, 378 )?; 379 380 // Get the current writable area as an array of bytes. 381 let (slice_1, slice_2) = { 382 let (slice_1, slice_2) = self.driver_write_area(); 383 384 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut()) 385 }; 386 387 // Extract area for the `GspMsgElement`. 388 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?; 389 390 // Create the contents area. 391 let (slice_1, slice_2) = if slice_1.len() > size { 392 // Contents fits entirely in `slice_1`. 393 (&mut slice_1[..size], &mut slice_2[0..0]) 394 } else { 395 // Need all of `slice_1` and some of `slice_2`. 396 let slice_2_len = size - slice_1.len(); 397 (slice_1, &mut slice_2[..slice_2_len]) 398 }; 399 400 Ok(GspCommand { 401 header, 402 contents: (slice_1, slice_2), 403 }) 404 } 405 406 // Returns the index of the memory page the GSP will write the next message to. 407 // 408 // # Invariants 409 // 410 // - The returned value is within `0..MSGQ_NUM_PAGES`. 411 fn gsp_write_ptr(&self) -> u32 { 412 let ptr = MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES; 413 414 // ORDERING: LOAD->LOAD ordering needed to order `gsp_write_ptr` read before data read. 415 dma_mb(Read); 416 417 ptr 418 } 419 420 // Returns the index of the memory page the GSP will read the next command from. 421 // 422 // # Invariants 423 // 424 // - The returned value is within `0..MSGQ_NUM_PAGES`. 425 fn gsp_read_ptr(&self) -> u32 { 426 let ptr = MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES; 427 428 // ORDERING: LOAD->STORE ordering needed to order `gsp_read_ptr` read before data write. 429 dma_mb(Full); 430 431 ptr 432 } 433 434 // Returns the index of the memory page the CPU can read the next message from. 435 // 436 // # Invariants 437 // 438 // - The returned value is within `0..MSGQ_NUM_PAGES`. 439 fn cpu_read_ptr(&self) -> u32 { 440 MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES 441 } 442 443 // Informs the GSP that it can send `elem_count` new pages into the message queue. 444 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 445 // ORDERING: LOAD->STORE ordering needed to order `cpu_read_ptr` write after data read. 446 dma_mb(Full); 447 448 let rx = io_project!(self.0, .cpuq.rx); 449 let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 450 MsgqRxHeader::set_read_ptr(rx, rptr) 451 } 452 453 // Returns the index of the memory page the CPU can write the next command to. 454 // 455 // # Invariants 456 // 457 // - The returned value is within `0..MSGQ_NUM_PAGES`. 458 fn cpu_write_ptr(&self) -> u32 { 459 MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES 460 } 461 462 // Informs the GSP that it can process `elem_count` new pages from the command queue. 463 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 464 // ORDERING: STORE->STORE ordering needed to order `cpu_write_ptr` write after data write. 465 dma_mb(Write); 466 467 let tx = io_project!(self.0, .cpuq.tx); 468 let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; 469 MsgqTxHeader::set_write_ptr(tx, wptr); 470 } 471 } 472 473 /// A command ready to be sent on the command queue. 474 /// 475 /// This is the type returned by [`DmaGspMem::allocate_command`]. 476 struct GspCommand<'a> { 477 // Writable reference to the header of the command. 478 header: &'a mut GspMsgElement, 479 // Writable slices to the contents of the command. The second slice is zero unless the command 480 // loops over the command queue. 481 contents: (&'a mut [u8], &'a mut [u8]), 482 } 483 484 /// A message ready to be processed from the message queue. 485 /// 486 /// This is the type returned by [`CmdqInner::wait_for_msg`]. 487 struct GspMessage<'a> { 488 // Reference to the header of the message. 489 header: &'a GspMsgElement, 490 // Slices to the contents of the message. The second slice is zero unless the message loops 491 // over the message queue. 492 contents: (&'a [u8], &'a [u8]), 493 } 494 495 /// GSP command queue. 496 /// 497 /// Provides the ability to send commands and receive messages from the GSP using a shared memory 498 /// area. 499 #[pin_data] 500 pub(crate) struct Cmdq<'cmdq> { 501 /// Inner mutex-protected state. 502 #[pin] 503 inner: Mutex<CmdqInner<'cmdq>>, 504 /// DMA address of the command queue's shared memory region. 505 pub(super) dma_addr: DmaAddress, 506 } 507 508 impl<'cmdq> Cmdq<'cmdq> { 509 /// Offset of the data after the PTEs. 510 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); 511 512 /// Offset of command queue ring buffer. 513 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq) 514 + core::mem::offset_of!(Msgq, msgq) 515 - Self::POST_PTE_OFFSET; 516 517 /// Offset of message queue ring buffer. 518 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq) 519 + core::mem::offset_of!(Msgq, msgq) 520 - Self::POST_PTE_OFFSET; 521 522 /// Number of page table entries for the GSP shared region. 523 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT; 524 525 /// Default timeout for receiving a message from the GSP. 526 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); 527 528 /// Creates a new command queue for `dev`. 529 pub(crate) fn new( 530 dev: &'cmdq device::Device<device::Bound>, 531 bar: Bar0<'cmdq>, 532 ) -> impl PinInit<Self, Error> + 'cmdq { 533 pin_init_scope(move || { 534 let gsp_mem = DmaGspMem::new(dev)?; 535 536 Ok(try_pin_init!(Self { 537 dma_addr: gsp_mem.0.dma_address(), 538 inner <- new_mutex!(CmdqInner { 539 dev, 540 bar, 541 gsp_mem, 542 seq: 0, 543 }), 544 })) 545 }) 546 } 547 548 /// Computes the checksum for the message pointed to by `it`. 549 /// 550 /// A message is made of several parts, so `it` is an iterator over byte slices representing 551 /// these parts. 552 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 { 553 let sum64 = it 554 .enumerate() 555 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte)) 556 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol)); 557 558 ((sum64 >> 32) as u32) ^ (sum64 as u32) 559 } 560 561 /// Notifies the GSP that we have updated the command queue pointers. 562 fn notify_gsp(bar: Bar0<'_>) { 563 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); 564 } 565 566 /// Sends `command` to the GSP and waits for the reply. 567 /// 568 /// Events that arrive before the reply are logged and consumed. 569 /// 570 /// The queue is locked for the entire send+receive cycle to ensure that no other command can 571 /// be interleaved. 572 /// 573 /// # Errors 574 /// 575 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply does 576 /// not arrive within [`Self::RECEIVE_TIMEOUT`] of the send, however many events arrive 577 /// while waiting. 578 /// - `EIO` if the variable payload requested by the command has not been entirely 579 /// written to by its [`CommandToGsp::init_variable_payload`] method. 580 /// 581 /// Error codes returned by the command and reply initializers are propagated as-is. 582 pub(crate) fn send_command<M>(&self, command: M) -> Result<M::Reply> 583 where 584 M: CommandToGsp, 585 M::Reply: MessageFromGsp, 586 Error: From<M::InitError>, 587 Error: From<<M::Reply as MessageFromGsp>::InitError>, 588 { 589 let mut inner = self.inner.lock(); 590 inner.send_command(command)?; 591 592 inner.await_msg() 593 } 594 595 /// Sends `command` to the GSP without waiting for a reply. 596 /// 597 /// # Errors 598 /// 599 /// - `ETIMEDOUT` if space does not become available within the timeout. 600 /// - `EIO` if the variable payload requested by the command has not been entirely 601 /// written to by its [`CommandToGsp::init_variable_payload`] method. 602 /// 603 /// Error codes returned by the command initializers are propagated as-is. 604 pub(crate) fn send_command_no_wait<M>(&self, command: M) -> Result 605 where 606 M: CommandToGsp<Reply = NoReply>, 607 Error: From<M::InitError>, 608 { 609 self.inner.lock().send_command(command) 610 } 611 612 /// Waits for an unsolicited GSP event of type `M`. Events that arrive before it are logged and 613 /// consumed. 614 /// 615 /// The queue mutex is held for the whole wait, up to [`Self::RECEIVE_TIMEOUT`], so no other 616 /// caller can send a command or consume an event meanwhile. 617 /// 618 /// # Errors 619 /// 620 /// - `ETIMEDOUT` if the event does not arrive within [`Self::RECEIVE_TIMEOUT`] of the call, 621 /// however many other events arrive while waiting. 622 /// - `EIO` if a message fails framing or checksum validation. 623 /// 624 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 625 pub(crate) fn await_msg<M: MessageFromGsp>(&self) -> Result<M> 626 where 627 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 628 Error: From<M::InitError>, 629 { 630 self.inner.lock().await_msg() 631 } 632 633 /// Logs and consumes every message the GSP has already posted, and returns without waiting for 634 /// more. 635 /// 636 /// No caller is waiting for a reply while this holds the queue mutex, so every message is 637 /// logged as an event. See "Draining the GSP-to-CPU queue" in 638 /// `Documentation/gpu/nova/core/interrupts.rst`. 639 /// 640 /// # Errors 641 /// 642 /// `EIO` if a message fails framing or checksum validation. 643 pub(crate) fn drain(&self) -> Result { 644 self.inner.lock().drain() 645 } 646 } 647 648 /// Inner mutex protected state of [`Cmdq`]. 649 struct CmdqInner<'a> { 650 /// Device this command queue belongs to. 651 dev: &'a device::Device, 652 /// MMIO mapping of PCI BAR0, for writing the GSP doorbell. 653 bar: Bar0<'a>, 654 /// Current command sequence number. 655 seq: u32, 656 /// Memory area shared with the GSP for communicating commands and messages. 657 gsp_mem: DmaGspMem<'a>, 658 } 659 660 impl CmdqInner<'_> { 661 /// Timeout for waiting for space on the command queue. 662 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); 663 664 /// Sends `command` to the GSP, without splitting it. 665 /// 666 /// # Errors 667 /// 668 /// - `EMSGSIZE` if the command exceeds the maximum queue element size. 669 /// - `ETIMEDOUT` if space does not become available within the timeout. 670 /// - `EIO` if the variable payload requested by the command has not been entirely 671 /// written to by its [`CommandToGsp::init_variable_payload`] method. 672 /// 673 /// Error codes returned by the command initializers are propagated as-is. 674 fn send_single_command<M>(&mut self, command: M) -> Result 675 where 676 M: CommandToGsp, 677 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 678 Error: From<M::InitError>, 679 { 680 let size_in_bytes = command.size(); 681 let dst = self 682 .gsp_mem 683 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?; 684 685 // Extract area for the command itself. The GSP message header and the command header 686 // together are guaranteed to fit entirely into a single page, so it's ok to only look 687 // at `dst.contents.0` here. 688 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?; 689 690 // Fill the header and command in-place. 691 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION); 692 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer 693 // fails. 694 unsafe { 695 pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; 696 pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; 697 } 698 699 // Fill the variable-length payload, which may be empty. 700 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]); 701 command.init_variable_payload(&mut sbuffer)?; 702 703 if !sbuffer.is_empty() { 704 return Err(EIO); 705 } 706 drop(sbuffer); 707 708 // Compute checksum now that the whole message is ready. 709 dst.header 710 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([ 711 dst.header.as_bytes(), 712 dst.contents.0, 713 dst.contents.1, 714 ]))); 715 716 dev_dbg!( 717 &self.dev, 718 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n", 719 self.seq, 720 M::FUNCTION, 721 dst.header.length(), 722 ); 723 724 // All set - update the write pointer and inform the GSP of the new command. 725 let elem_count = dst.header.element_count(); 726 self.seq += 1; 727 self.gsp_mem.advance_cpu_write_ptr(elem_count); 728 Cmdq::notify_gsp(self.bar); 729 730 Ok(()) 731 } 732 733 /// Sends `command` to the GSP. 734 /// 735 /// The command may be split into multiple messages if it is large. 736 /// 737 /// # Errors 738 /// 739 /// - `ETIMEDOUT` if space does not become available within the timeout. 740 /// - `EIO` if the variable payload requested by the command has not been entirely 741 /// written to by its [`CommandToGsp::init_variable_payload`] method. 742 /// 743 /// Error codes returned by the command initializers are propagated as-is. 744 fn send_command<M>(&mut self, command: M) -> Result 745 where 746 M: CommandToGsp, 747 Error: From<M::InitError>, 748 { 749 match SplitState::new(command)? { 750 SplitState::Single(command) => self.send_single_command(command), 751 SplitState::Split(command, mut continuations) => { 752 self.send_single_command(command)?; 753 754 while let Some(continuation) = continuations.next() { 755 // Turbofish needed because the compiler cannot infer M here. 756 self.send_single_command::<ContinuationRecord<'_>>(continuation)?; 757 } 758 759 Ok(()) 760 } 761 } 762 } 763 764 /// Wait for a message to become available on the message queue. 765 /// 766 /// This works purely at the transport layer and does not interpret or validate the message 767 /// beyond the advertised length in its [`GspMsgElement`]. 768 /// 769 /// This method returns: 770 /// 771 /// - A reference to the [`GspMsgElement`] of the message, 772 /// - Two byte slices with the contents of the message. The second slice is empty unless the 773 /// message loops across the message queue. 774 /// 775 /// # Errors 776 /// 777 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 778 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 779 /// message queue. 780 /// 781 /// Error codes returned by the message constructor are propagated as-is. 782 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> { 783 // Wait for a message to arrive from the GSP. 784 let (slice_1, slice_2) = read_poll_timeout( 785 || Ok(self.gsp_mem.driver_read_area()), 786 |driver_area| !driver_area.0.is_empty(), 787 Delta::from_millis(1), 788 timeout, 789 ) 790 .map(|(slice_1, slice_2)| (slice_1.as_flattened(), slice_2.as_flattened()))?; 791 792 // Extract the `GspMsgElement`. 793 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?; 794 795 dev_dbg!( 796 &self.dev, 797 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n", 798 header.sequence(), 799 header.function(), 800 header.length(), 801 ); 802 803 let payload_length = header.payload_length(); 804 805 // Check that the driver read area is large enough for the message. 806 if slice_1.len() + slice_2.len() < payload_length { 807 return Err(EIO); 808 } 809 810 // Cut the message slices down to the actual length of the message. 811 let (slice_1, slice_2) = if slice_1.len() > payload_length { 812 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`. 813 (slice_1.split_at(payload_length).0, &slice_2[0..0]) 814 } else { 815 ( 816 slice_1, 817 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as 818 // large as `payload_length`. 819 slice_2.split_at(payload_length - slice_1.len()).0, 820 ) 821 }; 822 823 // Validate checksum. 824 if Cmdq::calculate_checksum(SBufferIter::new_reader([ 825 header.as_bytes(), 826 slice_1, 827 slice_2, 828 ])) != 0 829 { 830 dev_err!( 831 &self.dev, 832 "GSP RPC: receive: Call {} - bad checksum\n", 833 header.sequence() 834 ); 835 return Err(EIO); 836 } 837 838 Ok(GspMessage { 839 header, 840 contents: (slice_1, slice_2), 841 }) 842 } 843 844 /// Receive a message from the GSP. 845 /// 846 /// A message whose function code is `M::FUNCTION` is decoded and returned. Any other message 847 /// is logged as an event. 848 /// 849 /// The read pointer is always advanced past the message, regardless of whether it matched. 850 /// 851 /// # Errors 852 /// 853 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. 854 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the 855 /// message queue. 856 /// - `ENOMSG` if the message was not the awaited reply. 857 /// 858 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 859 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M> 860 where 861 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 862 Error: From<M::InitError>, 863 { 864 let message = self.wait_for_msg(timeout)?; 865 let function = message.header.function(); 866 let seq = message.header.sequence(); 867 868 // An early return here would leave the read pointer on this message. 869 let result = if matches!(function, Ok(f) if f == M::FUNCTION) { 870 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; 871 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); 872 873 M::read(cmd, &mut sbuffer) 874 .map_err(|e| e.into()) 875 .inspect(|_| { 876 if !sbuffer.is_empty() { 877 dev_warn!( 878 &self.dev, 879 "GSP message {:?} has unprocessed data\n", 880 M::FUNCTION 881 ); 882 } 883 }) 884 } else { 885 self.log_event(function, seq); 886 887 Err(ENOMSG) 888 }; 889 890 // Advance the read pointer past this message. 891 self.gsp_mem.advance_cpu_read_ptr(u32::try_from( 892 message.header.length().div_ceil(GSP_PAGE_SIZE), 893 )?); 894 895 result 896 } 897 898 /// Receives a message of type `M`, waiting up to [`Cmdq::RECEIVE_TIMEOUT`] from the call. 899 /// 900 /// Any other message that arrives first is logged as an event and does not extend the 901 /// deadline. 902 /// 903 /// # Errors 904 /// 905 /// - `ETIMEDOUT` if no message of type `M` arrives before the deadline, however many other 906 /// messages arrive while waiting. 907 /// - `EIO` if a message fails framing or checksum validation (see [`Self::wait_for_msg`]). 908 /// 909 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. 910 fn await_msg<M: MessageFromGsp>(&mut self) -> Result<M> 911 where 912 // This allows all error types, including `Infallible`, to be used for `M::InitError`. 913 Error: From<M::InitError>, 914 { 915 let deadline = Instant::<Monotonic>::now() + Cmdq::RECEIVE_TIMEOUT; 916 loop { 917 let remaining = deadline - Instant::<Monotonic>::now(); 918 if remaining.is_negative() { 919 break Err(ETIMEDOUT); 920 } 921 match self.receive_msg::<M>(remaining) { 922 Ok(msg) => break Ok(msg), 923 Err(ENOMSG) => continue, 924 Err(e) => break Err(e), 925 } 926 } 927 } 928 929 /// Logs an event, meaning a message that no caller was waiting for. 930 /// 931 /// An OS error or robust-channel record is logged at error level and an unknown function code 932 /// at warning level. Every other event is recorded only by the receive trace in 933 /// [`Self::wait_for_msg`]. 934 fn log_event(&self, function: Result<MsgFunction, u32>, seq: u32) { 935 match function { 936 Ok(MsgFunction::OsErrorLog) => { 937 dev_err!(&self.dev, "GSP reported an OS error (seq {})\n", seq); 938 } 939 Ok(MsgFunction::RcTriggered) => { 940 dev_err!( 941 &self.dev, 942 "GSP triggered robust-channel recovery (seq {})\n", 943 seq 944 ); 945 } 946 // Nothing to do for the remaining known function codes. 947 Ok(_) => {} 948 Err(raw) => { 949 dev_warn!( 950 &self.dev, 951 "unknown GSP message function {:#x} (seq {})\n", 952 raw, 953 seq 954 ); 955 } 956 } 957 } 958 959 /// Logs and consumes every message the queue holds. 960 /// 961 /// # Errors 962 /// 963 /// `EIO` if a message fails framing or checksum validation, or if a message's page count 964 /// overflows a `u32`. 965 fn drain(&mut self) -> Result { 966 while !self.gsp_mem.driver_read_area().0.is_empty() { 967 // A message is available, so this returns without waiting. 968 let msg = self.wait_for_msg(Delta::ZERO)?; 969 970 let pages = 971 u32::try_from(msg.header.length().div_ceil(GSP_PAGE_SIZE)).map_err(|_| { 972 dev_err!(&self.dev, "GSP drain: message length overflow\n"); 973 EIO 974 })?; 975 let function = msg.header.function(); 976 let seq = msg.header.sequence(); 977 978 self.gsp_mem.advance_cpu_read_ptr(pages); 979 self.log_event(function, seq); 980 } 981 982 Ok(()) 983 } 984 } 985