1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Data Object Exchange 4 * PCIe r6.0, sec 6.30 DOE 5 * 6 * Copyright (C) 2021 Huawei 7 * Jonathan Cameron <Jonathan.Cameron@huawei.com> 8 * 9 * Copyright (C) 2022 Intel Corporation 10 * Ira Weiny <ira.weiny@intel.com> 11 */ 12 13 #define dev_fmt(fmt) "DOE: " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/delay.h> 17 #include <linux/jiffies.h> 18 #include <linux/mutex.h> 19 #include <linux/pci.h> 20 #include <linux/pci-doe.h> 21 #include <linux/workqueue.h> 22 23 #define PCI_DOE_PROTOCOL_DISCOVERY 0 24 25 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */ 26 #define PCI_DOE_TIMEOUT HZ 27 #define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128) 28 29 #define PCI_DOE_FLAG_CANCEL 0 30 #define PCI_DOE_FLAG_DEAD 1 31 32 /* Max data object length is 2^18 dwords */ 33 #define PCI_DOE_MAX_LENGTH (1 << 18) 34 35 /** 36 * struct pci_doe_mb - State for a single DOE mailbox 37 * 38 * This state is used to manage a single DOE mailbox capability. All fields 39 * should be considered opaque to the consumers and the structure passed into 40 * the helpers below after being created by devm_pci_doe_create() 41 * 42 * @pdev: PCI device this mailbox belongs to 43 * @cap_offset: Capability offset 44 * @prots: Array of protocols supported (encoded as long values) 45 * @wq: Wait queue for work item 46 * @work_queue: Queue of pci_doe_work items 47 * @flags: Bit array of PCI_DOE_FLAG_* flags 48 */ 49 struct pci_doe_mb { 50 struct pci_dev *pdev; 51 u16 cap_offset; 52 struct xarray prots; 53 54 wait_queue_head_t wq; 55 struct workqueue_struct *work_queue; 56 unsigned long flags; 57 }; 58 59 static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout) 60 { 61 if (wait_event_timeout(doe_mb->wq, 62 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags), 63 timeout)) 64 return -EIO; 65 return 0; 66 } 67 68 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val) 69 { 70 struct pci_dev *pdev = doe_mb->pdev; 71 int offset = doe_mb->cap_offset; 72 73 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val); 74 } 75 76 static int pci_doe_abort(struct pci_doe_mb *doe_mb) 77 { 78 struct pci_dev *pdev = doe_mb->pdev; 79 int offset = doe_mb->cap_offset; 80 unsigned long timeout_jiffies; 81 82 pci_dbg(pdev, "[%x] Issuing Abort\n", offset); 83 84 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; 85 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT); 86 87 do { 88 int rc; 89 u32 val; 90 91 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); 92 if (rc) 93 return rc; 94 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 95 96 /* Abort success! */ 97 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) && 98 !FIELD_GET(PCI_DOE_STATUS_BUSY, val)) 99 return 0; 100 101 } while (!time_after(jiffies, timeout_jiffies)); 102 103 /* Abort has timed out and the MB is dead */ 104 pci_err(pdev, "[%x] ABORT timed out\n", offset); 105 return -EIO; 106 } 107 108 static int pci_doe_send_req(struct pci_doe_mb *doe_mb, 109 struct pci_doe_task *task) 110 { 111 struct pci_dev *pdev = doe_mb->pdev; 112 int offset = doe_mb->cap_offset; 113 size_t length; 114 u32 val; 115 int i; 116 117 /* 118 * Check the DOE busy bit is not set. If it is set, this could indicate 119 * someone other than Linux (e.g. firmware) is using the mailbox. Note 120 * it is expected that firmware and OS will negotiate access rights via 121 * an, as yet to be defined, method. 122 */ 123 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 124 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val)) 125 return -EBUSY; 126 127 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) 128 return -EIO; 129 130 /* Length is 2 DW of header + length of payload in DW */ 131 length = 2 + task->request_pl_sz / sizeof(u32); 132 if (length > PCI_DOE_MAX_LENGTH) 133 return -EIO; 134 if (length == PCI_DOE_MAX_LENGTH) 135 length = 0; 136 137 /* Write DOE Header */ 138 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) | 139 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type); 140 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val); 141 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, 142 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, 143 length)); 144 for (i = 0; i < task->request_pl_sz / sizeof(u32); i++) 145 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, 146 task->request_pl[i]); 147 148 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO); 149 150 return 0; 151 } 152 153 static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb) 154 { 155 struct pci_dev *pdev = doe_mb->pdev; 156 int offset = doe_mb->cap_offset; 157 u32 val; 158 159 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 160 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) 161 return true; 162 return false; 163 } 164 165 static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) 166 { 167 struct pci_dev *pdev = doe_mb->pdev; 168 int offset = doe_mb->cap_offset; 169 size_t length, payload_length; 170 u32 val; 171 int i; 172 173 /* Read the first dword to get the protocol */ 174 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 175 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) || 176 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) { 177 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n", 178 doe_mb->cap_offset, task->prot.vid, task->prot.type, 179 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val), 180 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val)); 181 return -EIO; 182 } 183 184 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 185 /* Read the second dword to get the length */ 186 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 187 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 188 189 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val); 190 /* A value of 0x0 indicates max data object length */ 191 if (!length) 192 length = PCI_DOE_MAX_LENGTH; 193 if (length < 2) 194 return -EIO; 195 196 /* First 2 dwords have already been read */ 197 length -= 2; 198 payload_length = min(length, task->response_pl_sz / sizeof(u32)); 199 /* Read the rest of the response payload */ 200 for (i = 0; i < payload_length; i++) { 201 pci_read_config_dword(pdev, offset + PCI_DOE_READ, 202 &task->response_pl[i]); 203 /* Prior to the last ack, ensure Data Object Ready */ 204 if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb)) 205 return -EIO; 206 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 207 } 208 209 /* Flush excess length */ 210 for (; i < length; i++) { 211 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 212 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 213 } 214 215 /* Final error check to pick up on any since Data Object Ready */ 216 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 217 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) 218 return -EIO; 219 220 return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32); 221 } 222 223 static void signal_task_complete(struct pci_doe_task *task, int rv) 224 { 225 task->rv = rv; 226 task->complete(task); 227 } 228 229 static void signal_task_abort(struct pci_doe_task *task, int rv) 230 { 231 struct pci_doe_mb *doe_mb = task->doe_mb; 232 struct pci_dev *pdev = doe_mb->pdev; 233 234 if (pci_doe_abort(doe_mb)) { 235 /* 236 * If the device can't process an abort; set the mailbox dead 237 * - no more submissions 238 */ 239 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n", 240 doe_mb->cap_offset); 241 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); 242 } 243 signal_task_complete(task, rv); 244 } 245 246 static void doe_statemachine_work(struct work_struct *work) 247 { 248 struct pci_doe_task *task = container_of(work, struct pci_doe_task, 249 work); 250 struct pci_doe_mb *doe_mb = task->doe_mb; 251 struct pci_dev *pdev = doe_mb->pdev; 252 int offset = doe_mb->cap_offset; 253 unsigned long timeout_jiffies; 254 u32 val; 255 int rc; 256 257 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) { 258 signal_task_complete(task, -EIO); 259 return; 260 } 261 262 /* Send request */ 263 rc = pci_doe_send_req(doe_mb, task); 264 if (rc) { 265 /* 266 * The specification does not provide any guidance on how to 267 * resolve conflicting requests from other entities. 268 * Furthermore, it is likely that busy will not be detected 269 * most of the time. Flag any detection of status busy with an 270 * error. 271 */ 272 if (rc == -EBUSY) 273 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n", 274 offset); 275 signal_task_abort(task, rc); 276 return; 277 } 278 279 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; 280 /* Poll for response */ 281 retry_resp: 282 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 283 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) { 284 signal_task_abort(task, -EIO); 285 return; 286 } 287 288 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) { 289 if (time_after(jiffies, timeout_jiffies)) { 290 signal_task_abort(task, -EIO); 291 return; 292 } 293 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); 294 if (rc) { 295 signal_task_abort(task, rc); 296 return; 297 } 298 goto retry_resp; 299 } 300 301 rc = pci_doe_recv_resp(doe_mb, task); 302 if (rc < 0) { 303 signal_task_abort(task, rc); 304 return; 305 } 306 307 signal_task_complete(task, rc); 308 } 309 310 static void pci_doe_task_complete(struct pci_doe_task *task) 311 { 312 complete(task->private); 313 } 314 315 static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid, 316 u8 *protocol) 317 { 318 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX, 319 *index); 320 u32 response_pl; 321 DECLARE_COMPLETION_ONSTACK(c); 322 struct pci_doe_task task = { 323 .prot.vid = PCI_VENDOR_ID_PCI_SIG, 324 .prot.type = PCI_DOE_PROTOCOL_DISCOVERY, 325 .request_pl = &request_pl, 326 .request_pl_sz = sizeof(request_pl), 327 .response_pl = &response_pl, 328 .response_pl_sz = sizeof(response_pl), 329 .complete = pci_doe_task_complete, 330 .private = &c, 331 }; 332 int rc; 333 334 rc = pci_doe_submit_task(doe_mb, &task); 335 if (rc < 0) 336 return rc; 337 338 wait_for_completion(&c); 339 340 if (task.rv != sizeof(response_pl)) 341 return -EIO; 342 343 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl); 344 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL, 345 response_pl); 346 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX, 347 response_pl); 348 349 return 0; 350 } 351 352 static void *pci_doe_xa_prot_entry(u16 vid, u8 prot) 353 { 354 return xa_mk_value((vid << 8) | prot); 355 } 356 357 static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb) 358 { 359 u8 index = 0; 360 u8 xa_idx = 0; 361 362 do { 363 int rc; 364 u16 vid; 365 u8 prot; 366 367 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot); 368 if (rc) 369 return rc; 370 371 pci_dbg(doe_mb->pdev, 372 "[%x] Found protocol %d vid: %x prot: %x\n", 373 doe_mb->cap_offset, xa_idx, vid, prot); 374 375 rc = xa_insert(&doe_mb->prots, xa_idx++, 376 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL); 377 if (rc) 378 return rc; 379 } while (index); 380 381 return 0; 382 } 383 384 static void pci_doe_xa_destroy(void *mb) 385 { 386 struct pci_doe_mb *doe_mb = mb; 387 388 xa_destroy(&doe_mb->prots); 389 } 390 391 static void pci_doe_destroy_workqueue(void *mb) 392 { 393 struct pci_doe_mb *doe_mb = mb; 394 395 destroy_workqueue(doe_mb->work_queue); 396 } 397 398 static void pci_doe_flush_mb(void *mb) 399 { 400 struct pci_doe_mb *doe_mb = mb; 401 402 /* Stop all pending work items from starting */ 403 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); 404 405 /* Cancel an in progress work item, if necessary */ 406 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags); 407 wake_up(&doe_mb->wq); 408 409 /* Flush all work items */ 410 flush_workqueue(doe_mb->work_queue); 411 } 412 413 /** 414 * pcim_doe_create_mb() - Create a DOE mailbox object 415 * 416 * @pdev: PCI device to create the DOE mailbox for 417 * @cap_offset: Offset of the DOE mailbox 418 * 419 * Create a single mailbox object to manage the mailbox protocol at the 420 * cap_offset specified. 421 * 422 * RETURNS: created mailbox object on success 423 * ERR_PTR(-errno) on failure 424 */ 425 struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset) 426 { 427 struct pci_doe_mb *doe_mb; 428 struct device *dev = &pdev->dev; 429 int rc; 430 431 doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL); 432 if (!doe_mb) 433 return ERR_PTR(-ENOMEM); 434 435 doe_mb->pdev = pdev; 436 doe_mb->cap_offset = cap_offset; 437 init_waitqueue_head(&doe_mb->wq); 438 439 xa_init(&doe_mb->prots); 440 rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb); 441 if (rc) 442 return ERR_PTR(rc); 443 444 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0, 445 dev_driver_string(&pdev->dev), 446 pci_name(pdev), 447 doe_mb->cap_offset); 448 if (!doe_mb->work_queue) { 449 pci_err(pdev, "[%x] failed to allocate work queue\n", 450 doe_mb->cap_offset); 451 return ERR_PTR(-ENOMEM); 452 } 453 rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb); 454 if (rc) 455 return ERR_PTR(rc); 456 457 /* Reset the mailbox by issuing an abort */ 458 rc = pci_doe_abort(doe_mb); 459 if (rc) { 460 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n", 461 doe_mb->cap_offset, rc); 462 return ERR_PTR(rc); 463 } 464 465 /* 466 * The state machine and the mailbox should be in sync now; 467 * Set up mailbox flush prior to using the mailbox to query protocols. 468 */ 469 rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb); 470 if (rc) 471 return ERR_PTR(rc); 472 473 rc = pci_doe_cache_protocols(doe_mb); 474 if (rc) { 475 pci_err(pdev, "[%x] failed to cache protocols : %d\n", 476 doe_mb->cap_offset, rc); 477 return ERR_PTR(rc); 478 } 479 480 return doe_mb; 481 } 482 EXPORT_SYMBOL_GPL(pcim_doe_create_mb); 483 484 /** 485 * pci_doe_supports_prot() - Return if the DOE instance supports the given 486 * protocol 487 * @doe_mb: DOE mailbox capability to query 488 * @vid: Protocol Vendor ID 489 * @type: Protocol type 490 * 491 * RETURNS: True if the DOE mailbox supports the protocol specified 492 */ 493 bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) 494 { 495 unsigned long index; 496 void *entry; 497 498 /* The discovery protocol must always be supported */ 499 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY) 500 return true; 501 502 xa_for_each(&doe_mb->prots, index, entry) 503 if (entry == pci_doe_xa_prot_entry(vid, type)) 504 return true; 505 506 return false; 507 } 508 EXPORT_SYMBOL_GPL(pci_doe_supports_prot); 509 510 /** 511 * pci_doe_submit_task() - Submit a task to be processed by the state machine 512 * 513 * @doe_mb: DOE mailbox capability to submit to 514 * @task: task to be queued 515 * 516 * Submit a DOE task (request/response) to the DOE mailbox to be processed. 517 * Returns upon queueing the task object. If the queue is full this function 518 * will sleep until there is room in the queue. 519 * 520 * task->complete will be called when the state machine is done processing this 521 * task. 522 * 523 * Excess data will be discarded. 524 * 525 * RETURNS: 0 when task has been successfully queued, -ERRNO on error 526 */ 527 int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) 528 { 529 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type)) 530 return -EINVAL; 531 532 /* 533 * DOE requests must be a whole number of DW and the response needs to 534 * be big enough for at least 1 DW 535 */ 536 if (task->request_pl_sz % sizeof(u32) || 537 task->response_pl_sz < sizeof(u32)) 538 return -EINVAL; 539 540 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) 541 return -EIO; 542 543 task->doe_mb = doe_mb; 544 INIT_WORK(&task->work, doe_statemachine_work); 545 queue_work(doe_mb->work_queue, &task->work); 546 return 0; 547 } 548 EXPORT_SYMBOL_GPL(pci_doe_submit_task); 549