1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Adjunct processor matrix VFIO device driver callbacks. 4 * 5 * Copyright IBM Corp. 2018 6 * 7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com> 8 * Halil Pasic <pasic@linux.ibm.com> 9 * Pierre Morel <pmorel@linux.ibm.com> 10 */ 11 #include <linux/string.h> 12 #include <linux/vfio.h> 13 #include <linux/device.h> 14 #include <linux/list.h> 15 #include <linux/ctype.h> 16 #include <linux/bitops.h> 17 #include <linux/kvm_host.h> 18 #include <linux/module.h> 19 #include <linux/uuid.h> 20 #include <asm/kvm.h> 21 #include <asm/zcrypt.h> 22 23 #include "vfio_ap_private.h" 24 #include "vfio_ap_debug.h" 25 26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough" 27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device" 28 29 #define AP_QUEUE_ASSIGNED "assigned" 30 #define AP_QUEUE_UNASSIGNED "unassigned" 31 #define AP_QUEUE_IN_USE "in use" 32 33 #define AP_RESET_INTERVAL 20 /* Reset sleep interval (20ms) */ 34 35 static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); 36 static int vfio_ap_mdev_reset_qlist(struct list_head *qlist); 37 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); 38 static const struct vfio_device_ops vfio_ap_matrix_dev_ops; 39 static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); 40 41 /** 42 * get_update_locks_for_kvm: Acquire the locks required to dynamically update a 43 * KVM guest's APCB in the proper order. 44 * 45 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 46 * 47 * The proper locking order is: 48 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 49 * guest's APCB. 50 * 2. kvm->lock: required to update a guest's APCB 51 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 52 * 53 * Note: If @kvm is NULL, the KVM lock will not be taken. 54 */ 55 static inline void get_update_locks_for_kvm(struct kvm *kvm) 56 { 57 mutex_lock(&matrix_dev->guests_lock); 58 if (kvm) 59 mutex_lock(&kvm->lock); 60 mutex_lock(&matrix_dev->mdevs_lock); 61 } 62 63 /** 64 * release_update_locks_for_kvm: Release the locks used to dynamically update a 65 * KVM guest's APCB in the proper order. 66 * 67 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. 68 * 69 * The proper unlocking order is: 70 * 1. matrix_dev->mdevs_lock 71 * 2. kvm->lock 72 * 3. matrix_dev->guests_lock 73 * 74 * Note: If @kvm is NULL, the KVM lock will not be released. 75 */ 76 static inline void release_update_locks_for_kvm(struct kvm *kvm) 77 { 78 mutex_unlock(&matrix_dev->mdevs_lock); 79 if (kvm) 80 mutex_unlock(&kvm->lock); 81 mutex_unlock(&matrix_dev->guests_lock); 82 } 83 84 /** 85 * get_update_locks_for_mdev: Acquire the locks required to dynamically update a 86 * KVM guest's APCB in the proper order. 87 * 88 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 89 * configuration data to use to update a KVM guest's APCB. 90 * 91 * The proper locking order is: 92 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 93 * guest's APCB. 94 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 95 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 96 * 97 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 98 * lock will not be taken. 99 */ 100 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 101 { 102 mutex_lock(&matrix_dev->guests_lock); 103 if (matrix_mdev && matrix_mdev->kvm) 104 mutex_lock(&matrix_mdev->kvm->lock); 105 mutex_lock(&matrix_dev->mdevs_lock); 106 } 107 108 /** 109 * release_update_locks_for_mdev: Release the locks used to dynamically update a 110 * KVM guest's APCB in the proper order. 111 * 112 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP 113 * configuration data to use to update a KVM guest's APCB. 114 * 115 * The proper unlocking order is: 116 * 1. matrix_dev->mdevs_lock 117 * 2. matrix_mdev->kvm->lock 118 * 3. matrix_dev->guests_lock 119 * 120 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM 121 * lock will not be released. 122 */ 123 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) 124 { 125 mutex_unlock(&matrix_dev->mdevs_lock); 126 if (matrix_mdev && matrix_mdev->kvm) 127 mutex_unlock(&matrix_mdev->kvm->lock); 128 mutex_unlock(&matrix_dev->guests_lock); 129 } 130 131 /** 132 * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and 133 * acquire the locks required to update the APCB of 134 * the KVM guest to which the mdev is attached. 135 * 136 * @apqn: the APQN of a queue device. 137 * 138 * The proper locking order is: 139 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM 140 * guest's APCB. 141 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB 142 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev 143 * 144 * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock 145 * will not be taken. 146 * 147 * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn 148 * is not assigned to an ap_matrix_mdev. 149 */ 150 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn) 151 { 152 struct ap_matrix_mdev *matrix_mdev; 153 154 mutex_lock(&matrix_dev->guests_lock); 155 156 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 157 if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) && 158 test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) { 159 if (matrix_mdev->kvm) 160 mutex_lock(&matrix_mdev->kvm->lock); 161 162 mutex_lock(&matrix_dev->mdevs_lock); 163 164 return matrix_mdev; 165 } 166 } 167 168 mutex_lock(&matrix_dev->mdevs_lock); 169 170 return NULL; 171 } 172 173 /** 174 * get_update_locks_for_queue: get the locks required to update the APCB of the 175 * KVM guest to which the matrix mdev linked to a 176 * vfio_ap_queue object is attached. 177 * 178 * @q: a pointer to a vfio_ap_queue object. 179 * 180 * The proper locking order is: 181 * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a 182 * KVM guest's APCB. 183 * 2. q->matrix_mdev->kvm->lock: required to update a guest's APCB 184 * 3. matrix_dev->mdevs_lock: required to access data stored in matrix_mdev 185 * 186 * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock 187 * will not be taken. 188 */ 189 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q) 190 { 191 mutex_lock(&matrix_dev->guests_lock); 192 if (q->matrix_mdev && q->matrix_mdev->kvm) 193 mutex_lock(&q->matrix_mdev->kvm->lock); 194 mutex_lock(&matrix_dev->mdevs_lock); 195 } 196 197 /** 198 * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a 199 * hash table of queues assigned to a matrix mdev 200 * @matrix_mdev: the matrix mdev 201 * @apqn: The APQN of a queue device 202 * 203 * Return: the pointer to the vfio_ap_queue struct representing the queue or 204 * NULL if the queue is not assigned to @matrix_mdev 205 */ 206 static struct vfio_ap_queue *vfio_ap_mdev_get_queue( 207 struct ap_matrix_mdev *matrix_mdev, 208 int apqn) 209 { 210 struct vfio_ap_queue *q; 211 212 hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode, 213 apqn) { 214 if (q && q->apqn == apqn) 215 return q; 216 } 217 218 return NULL; 219 } 220 221 /** 222 * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries 223 * @apqn: The AP Queue number 224 * 225 * Checks the IRQ bit for the status of this APQN using ap_tapq. 226 * Returns if the ap_tapq function succeeded and the bit is clear. 227 * Returns if ap_tapq function failed with invalid, deconfigured or 228 * checkstopped AP. 229 * Otherwise retries up to 5 times after waiting 20ms. 230 */ 231 static void vfio_ap_wait_for_irqclear(int apqn) 232 { 233 struct ap_queue_status status; 234 int retry = 5; 235 236 do { 237 status = ap_tapq(apqn, NULL); 238 switch (status.response_code) { 239 case AP_RESPONSE_NORMAL: 240 case AP_RESPONSE_RESET_IN_PROGRESS: 241 if (!status.irq_enabled) 242 return; 243 fallthrough; 244 case AP_RESPONSE_BUSY: 245 msleep(20); 246 break; 247 case AP_RESPONSE_Q_NOT_AVAIL: 248 case AP_RESPONSE_DECONFIGURED: 249 case AP_RESPONSE_CHECKSTOPPED: 250 default: 251 WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__, 252 status.response_code, apqn); 253 return; 254 } 255 } while (--retry); 256 257 WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n", 258 __func__, status.response_code, apqn); 259 } 260 261 /** 262 * vfio_ap_free_aqic_resources - free vfio_ap_queue resources 263 * @q: The vfio_ap_queue 264 * 265 * Unregisters the ISC in the GIB when the saved ISC not invalid. 266 * Unpins the guest's page holding the NIB when it exists. 267 * Resets the saved_iova and saved_isc to invalid values. 268 */ 269 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) 270 { 271 if (!q) 272 return; 273 if (q->saved_isc != VFIO_AP_ISC_INVALID && 274 !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { 275 kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); 276 q->saved_isc = VFIO_AP_ISC_INVALID; 277 } 278 if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { 279 vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1); 280 q->saved_iova = 0; 281 } 282 } 283 284 /** 285 * vfio_ap_irq_disable - disables and clears an ap_queue interrupt 286 * @q: The vfio_ap_queue 287 * 288 * Uses ap_aqic to disable the interruption and in case of success, reset 289 * in progress or IRQ disable command already proceeded: calls 290 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear 291 * and calls vfio_ap_free_aqic_resources() to free the resources associated 292 * with the AP interrupt handling. 293 * 294 * In the case the AP is busy, or a reset is in progress, 295 * retries after 20ms, up to 5 times. 296 * 297 * Returns if ap_aqic function failed with invalid, deconfigured or 298 * checkstopped AP. 299 * 300 * Return: &struct ap_queue_status 301 */ 302 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q) 303 { 304 union ap_qirq_ctrl aqic_gisa = { .value = 0 }; 305 struct ap_queue_status status; 306 int retries = 5; 307 308 do { 309 status = ap_aqic(q->apqn, aqic_gisa, 0); 310 switch (status.response_code) { 311 case AP_RESPONSE_OTHERWISE_CHANGED: 312 case AP_RESPONSE_NORMAL: 313 vfio_ap_wait_for_irqclear(q->apqn); 314 goto end_free; 315 case AP_RESPONSE_RESET_IN_PROGRESS: 316 case AP_RESPONSE_BUSY: 317 msleep(20); 318 break; 319 case AP_RESPONSE_Q_NOT_AVAIL: 320 case AP_RESPONSE_DECONFIGURED: 321 case AP_RESPONSE_CHECKSTOPPED: 322 case AP_RESPONSE_INVALID_ADDRESS: 323 default: 324 /* All cases in default means AP not operational */ 325 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 326 status.response_code); 327 goto end_free; 328 } 329 } while (retries--); 330 331 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__, 332 status.response_code); 333 end_free: 334 vfio_ap_free_aqic_resources(q); 335 return status; 336 } 337 338 /** 339 * vfio_ap_validate_nib - validate a notification indicator byte (nib) address. 340 * 341 * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction. 342 * @nib: the location for storing the nib address. 343 * 344 * When the PQAP(AQIC) instruction is executed, general register 2 contains the 345 * address of the notification indicator byte (nib) used for IRQ notification. 346 * This function parses and validates the nib from gr2. 347 * 348 * Return: returns zero if the nib address is a valid; otherwise, returns 349 * -EINVAL. 350 */ 351 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib) 352 { 353 *nib = vcpu->run->s.regs.gprs[2]; 354 355 if (!*nib) 356 return -EINVAL; 357 if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT))) 358 return -EINVAL; 359 360 return 0; 361 } 362 363 /** 364 * ensure_nib_shared() - Ensure the address of the NIB is secure and shared 365 * @addr: the physical (absolute) address of the NIB 366 * 367 * This function checks whether the NIB page, which has been pinned with 368 * vfio_pin_pages(), is a shared page belonging to a secure guest. 369 * 370 * It will call uv_pin_shared() on it; if the page was already pinned shared 371 * (i.e. if the NIB belongs to a secure guest and is shared), then 0 372 * (success) is returned. If the NIB was not shared, vfio_pin_pages() had 373 * exported it and now it does not belong to the secure guest anymore. In 374 * that case, an error is returned. 375 * 376 * Context: the NIB (at physical address @addr) has to be pinned with 377 * vfio_pin_pages() before calling this function. 378 * 379 * Return: 0 in case of success, otherwise an error < 0. 380 */ 381 static int ensure_nib_shared(unsigned long addr) 382 { 383 /* 384 * The nib has to be located in shared storage since guest and 385 * host access it. vfio_pin_pages() will do a pin shared and 386 * if that fails (possibly because it's not a shared page) it 387 * calls export. We try to do a second pin shared here so that 388 * the UV gives us an error code if we try to pin a non-shared 389 * page. 390 * 391 * If the page is already pinned shared the UV will return a success. 392 */ 393 return uv_pin_shared(addr); 394 } 395 396 /** 397 * vfio_ap_irq_enable - Enable Interruption for a APQN 398 * 399 * @q: the vfio_ap_queue holding AQIC parameters 400 * @isc: the guest ISC to register with the GIB interface 401 * @vcpu: the vcpu object containing the registers specifying the parameters 402 * passed to the PQAP(AQIC) instruction. 403 * 404 * Pin the NIB saved in *q 405 * Register the guest ISC to GIB interface and retrieve the 406 * host ISC to issue the host side PQAP/AQIC 407 * 408 * status.response_code may be set to AP_RESPONSE_INVALID_ADDRESS in case the 409 * vfio_pin_pages or kvm_s390_gisc_register failed. 410 * 411 * Otherwise return the ap_queue_status returned by the ap_aqic(), 412 * all retry handling will be done by the guest. 413 * 414 * Return: &struct ap_queue_status 415 */ 416 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, 417 int isc, 418 struct kvm_vcpu *vcpu) 419 { 420 union ap_qirq_ctrl aqic_gisa = { .value = 0 }; 421 struct ap_queue_status status = {}; 422 struct kvm_s390_gisa *gisa; 423 struct page *h_page; 424 int nisc; 425 struct kvm *kvm; 426 phys_addr_t h_nib; 427 dma_addr_t nib; 428 int ret; 429 430 /* Verify that the notification indicator byte address is valid */ 431 if (vfio_ap_validate_nib(vcpu, &nib)) { 432 VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n", 433 __func__, &nib, q->apqn); 434 435 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 436 return status; 437 } 438 439 /* The pin will probably be successful even if the NIB was not shared */ 440 ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1, 441 IOMMU_READ | IOMMU_WRITE, &h_page); 442 switch (ret) { 443 case 1: 444 break; 445 default: 446 VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d," 447 "nib=%pad, apqn=%#04x\n", 448 __func__, ret, &nib, q->apqn); 449 450 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 451 return status; 452 } 453 454 kvm = q->matrix_mdev->kvm; 455 gisa = kvm->arch.gisa_int.origin; 456 457 h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK); 458 aqic_gisa.gisc = isc; 459 460 /* NIB in non-shared storage is a rc 6 for PV guests */ 461 if (kvm_s390_pv_cpu_is_protected(vcpu) && 462 ensure_nib_shared(h_nib & PAGE_MASK)) { 463 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); 464 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 465 return status; 466 } 467 468 nisc = kvm_s390_gisc_register(kvm, isc); 469 if (nisc < 0) { 470 VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n", 471 __func__, nisc, isc, q->apqn); 472 473 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); 474 status.response_code = AP_RESPONSE_INVALID_ADDRESS; 475 return status; 476 } 477 478 aqic_gisa.isc = nisc; 479 aqic_gisa.ir = 1; 480 aqic_gisa.gisa = virt_to_phys(gisa) >> 4; 481 482 status = ap_aqic(q->apqn, aqic_gisa, h_nib); 483 switch (status.response_code) { 484 case AP_RESPONSE_NORMAL: 485 /* See if we did clear older IRQ configuration */ 486 vfio_ap_free_aqic_resources(q); 487 q->saved_iova = nib; 488 q->saved_isc = isc; 489 break; 490 case AP_RESPONSE_OTHERWISE_CHANGED: 491 /* We could not modify IRQ settings: clear new configuration */ 492 ret = kvm_s390_gisc_unregister(kvm, isc); 493 if (ret) 494 VFIO_AP_DBF_WARN("%s: kvm_s390_gisc_unregister: rc=%d isc=%d, apqn=%#04x\n", 495 __func__, ret, isc, q->apqn); 496 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); 497 break; 498 default: 499 pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn, 500 status.response_code); 501 vfio_ap_irq_disable(q); 502 break; 503 } 504 505 if (status.response_code != AP_RESPONSE_NORMAL) { 506 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: " 507 "zone=%#x, ir=%#x, gisc=%#x, f=%#x," 508 "gisa=%#x, isc=%#x, apqn=%#04x\n", 509 __func__, status.response_code, 510 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc, 511 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc, 512 q->apqn); 513 } 514 515 return status; 516 } 517 518 /** 519 * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array 520 * of big endian elements that can be passed by 521 * value to an s390dbf sprintf event function to 522 * format a UUID string. 523 * 524 * @guid: the object containing the little endian guid 525 * @uuid: a six-element array of long values that can be passed by value as 526 * arguments for a formatting string specifying a UUID. 527 * 528 * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf 529 * event functions if the memory for the passed string is available as long as 530 * the debug feature exists. Since a mediated device can be removed at any 531 * time, it's name can not be used because %s passes the reference to the string 532 * in memory and the reference will go stale once the device is removed . 533 * 534 * The s390dbf string formatting function allows a maximum of 9 arguments for a 535 * message to be displayed in the 'sprintf' view. In order to use the bytes 536 * comprising the mediated device's UUID to display the mediated device name, 537 * they will have to be converted into an array whose elements can be passed by 538 * value to sprintf. For example: 539 * 540 * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 } 541 * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804 542 * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 } 543 * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx" 544 */ 545 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid) 546 { 547 /* 548 * The input guid is ordered in little endian, so it needs to be 549 * reordered for displaying a UUID as a string. This specifies the 550 * guid indices in proper order. 551 */ 552 uuid[0] = le32_to_cpup((__le32 *)guid); 553 uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]); 554 uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]); 555 uuid[3] = *((__u16 *)&guid->b[8]); 556 uuid[4] = *((__u16 *)&guid->b[10]); 557 uuid[5] = *((__u32 *)&guid->b[12]); 558 } 559 560 /** 561 * handle_pqap - PQAP instruction callback 562 * 563 * @vcpu: The vcpu on which we received the PQAP instruction 564 * 565 * Get the general register contents to initialize internal variables. 566 * REG[0]: APQN 567 * REG[1]: IR and ISC 568 * REG[2]: NIB 569 * 570 * Response.status may be set to following Response Code: 571 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available 572 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured 573 * - AP_RESPONSE_NORMAL (0) : in case of success 574 * Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC. 575 * We take the matrix_dev lock to ensure serialization on queues and 576 * mediated device access. 577 * 578 * Return: 0 if we could handle the request inside KVM. 579 * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault. 580 */ 581 static int handle_pqap(struct kvm_vcpu *vcpu) 582 { 583 uint64_t status; 584 uint16_t apqn; 585 unsigned long uuid[6]; 586 struct vfio_ap_queue *q; 587 struct ap_queue_status qstatus = { 588 .response_code = AP_RESPONSE_Q_NOT_AVAIL, }; 589 struct ap_matrix_mdev *matrix_mdev; 590 591 apqn = vcpu->run->s.regs.gprs[0] & 0xffff; 592 593 /* If we do not use the AIV facility just go to userland */ 594 if (!(vcpu->arch.sie_block->eca & ECA_AIV)) { 595 VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n", 596 __func__, apqn, vcpu->arch.sie_block->eca); 597 598 return -EOPNOTSUPP; 599 } 600 601 mutex_lock(&matrix_dev->mdevs_lock); 602 603 if (!vcpu->kvm->arch.crypto.pqap_hook) { 604 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n", 605 __func__, apqn); 606 607 goto out_unlock; 608 } 609 610 matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook, 611 struct ap_matrix_mdev, pqap_hook); 612 613 /* If the there is no guest using the mdev, there is nothing to do */ 614 if (!matrix_mdev->kvm) { 615 vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid); 616 VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n", 617 __func__, uuid[0], uuid[1], uuid[2], 618 uuid[3], uuid[4], uuid[5], apqn); 619 goto out_unlock; 620 } 621 622 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); 623 if (!q) { 624 VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n", 625 __func__, AP_QID_CARD(apqn), 626 AP_QID_QUEUE(apqn)); 627 goto out_unlock; 628 } 629 630 status = vcpu->run->s.regs.gprs[1]; 631 632 /* If IR bit(16) is set we enable the interrupt */ 633 if ((status >> (63 - 16)) & 0x01) 634 qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu); 635 else 636 qstatus = vfio_ap_irq_disable(q); 637 638 out_unlock: 639 memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus)); 640 vcpu->run->s.regs.gprs[1] >>= 32; 641 mutex_unlock(&matrix_dev->mdevs_lock); 642 return 0; 643 } 644 645 static void vfio_ap_matrix_init(struct ap_config_info *info, 646 struct ap_matrix *matrix) 647 { 648 matrix->apm_max = info->apxa ? info->na : 63; 649 matrix->aqm_max = info->apxa ? info->nd : 15; 650 matrix->adm_max = info->apxa ? info->nd : 15; 651 } 652 653 static void signal_guest_ap_cfg_changed(struct ap_matrix_mdev *matrix_mdev) 654 { 655 if (matrix_mdev->cfg_chg_trigger) 656 eventfd_signal(matrix_mdev->cfg_chg_trigger); 657 } 658 659 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev) 660 { 661 if (matrix_mdev->kvm) { 662 kvm_arch_crypto_set_masks(matrix_mdev->kvm, 663 matrix_mdev->shadow_apcb.apm, 664 matrix_mdev->shadow_apcb.aqm, 665 matrix_mdev->shadow_apcb.adm); 666 667 signal_guest_ap_cfg_changed(matrix_mdev); 668 } 669 } 670 671 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) 672 { 673 DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS); 674 675 bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS); 676 bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm, 677 (unsigned long *)matrix_dev->info.adm, AP_DOMAINS); 678 679 return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, 680 AP_DOMAINS); 681 } 682 683 static bool _queue_passable(struct vfio_ap_queue *q) 684 { 685 if (!q) 686 return false; 687 688 switch (q->reset_status.response_code) { 689 case AP_RESPONSE_NORMAL: 690 case AP_RESPONSE_DECONFIGURED: 691 case AP_RESPONSE_CHECKSTOPPED: 692 return true; 693 default: 694 return false; 695 } 696 } 697 698 /* 699 * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev 700 * to ensure no queue devices are passed through to 701 * the guest that are not bound to the vfio_ap 702 * device driver. 703 * 704 * @matrix_mdev: the matrix mdev whose matrix is to be filtered. 705 * @apm_filtered: a 256-bit bitmap for storing the APIDs filtered from the 706 * guest's AP configuration that are still in the host's AP 707 * configuration. 708 * 709 * Note: If an APQN referencing a queue device that is not bound to the vfio_ap 710 * driver, its APID will be filtered from the guest's APCB. The matrix 711 * structure precludes filtering an individual APQN, so its APID will be 712 * filtered. Consequently, all queues associated with the adapter that 713 * are in the host's AP configuration must be reset. If queues are 714 * subsequently made available again to the guest, they should re-appear 715 * in a reset state 716 * 717 * Return: a boolean value indicating whether the KVM guest's APCB was changed 718 * by the filtering or not. 719 */ 720 static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, 721 unsigned long *apm_filtered) 722 { 723 unsigned long apid, apqi, apqn; 724 DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); 725 DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); 726 727 bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); 728 bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); 729 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 730 bitmap_clear(apm_filtered, 0, AP_DEVICES); 731 732 /* 733 * Copy the adapters, domains and control domains to the shadow_apcb 734 * from the matrix mdev, but only those that are assigned to the host's 735 * AP configuration. 736 */ 737 bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm, 738 (unsigned long *)matrix_dev->info.apm, AP_DEVICES); 739 bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm, 740 (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS); 741 742 for_each_set_bit_inv(apid, matrix_mdev->shadow_apcb.apm, AP_DEVICES) { 743 for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, 744 AP_DOMAINS) { 745 /* 746 * If the APQN is not bound to the vfio_ap device 747 * driver, then we can't assign it to the guest's 748 * AP configuration. The AP architecture won't 749 * allow filtering of a single APQN, so let's filter 750 * the APID since an adapter represents a physical 751 * hardware device. 752 */ 753 apqn = AP_MKQID(apid, apqi); 754 if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) { 755 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 756 757 /* 758 * If the adapter was previously plugged into 759 * the guest, let's let the caller know that 760 * the APID was filtered. 761 */ 762 if (test_bit_inv(apid, prev_shadow_apm)) 763 set_bit_inv(apid, apm_filtered); 764 765 break; 766 } 767 } 768 } 769 770 return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, 771 AP_DEVICES) || 772 !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, 773 AP_DOMAINS); 774 } 775 776 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev) 777 { 778 struct ap_matrix_mdev *matrix_mdev = 779 container_of(vdev, struct ap_matrix_mdev, vdev); 780 781 matrix_mdev->mdev = to_mdev_device(vdev->dev); 782 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); 783 matrix_mdev->pqap_hook = handle_pqap; 784 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); 785 hash_init(matrix_mdev->qtable.queues); 786 787 return 0; 788 } 789 790 static int vfio_ap_mdev_probe(struct mdev_device *mdev) 791 { 792 struct ap_matrix_mdev *matrix_mdev; 793 int ret; 794 795 matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev, 796 &vfio_ap_matrix_dev_ops); 797 if (IS_ERR(matrix_mdev)) 798 return PTR_ERR(matrix_mdev); 799 800 ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); 801 if (ret) 802 goto err_put_vdev; 803 matrix_mdev->req_trigger = NULL; 804 matrix_mdev->cfg_chg_trigger = NULL; 805 dev_set_drvdata(&mdev->dev, matrix_mdev); 806 mutex_lock(&matrix_dev->mdevs_lock); 807 list_add(&matrix_mdev->node, &matrix_dev->mdev_list); 808 mutex_unlock(&matrix_dev->mdevs_lock); 809 return 0; 810 811 err_put_vdev: 812 vfio_put_device(&matrix_mdev->vdev); 813 return ret; 814 } 815 816 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, 817 struct vfio_ap_queue *q) 818 { 819 if (!q || vfio_ap_mdev_get_queue(matrix_mdev, q->apqn)) 820 return; 821 822 q->matrix_mdev = matrix_mdev; 823 hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); 824 } 825 826 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) 827 { 828 struct vfio_ap_queue *q; 829 830 q = vfio_ap_find_queue(apqn); 831 vfio_ap_mdev_link_queue(matrix_mdev, q); 832 } 833 834 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q) 835 { 836 hash_del(&q->mdev_qnode); 837 } 838 839 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q) 840 { 841 q->matrix_mdev = NULL; 842 } 843 844 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) 845 { 846 struct vfio_ap_queue *q; 847 unsigned long apid, apqi; 848 849 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 850 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, 851 AP_DOMAINS) { 852 q = vfio_ap_mdev_get_queue(matrix_mdev, 853 AP_MKQID(apid, apqi)); 854 if (q) 855 q->matrix_mdev = NULL; 856 } 857 } 858 } 859 860 static void vfio_ap_mdev_remove(struct mdev_device *mdev) 861 { 862 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev); 863 864 vfio_unregister_group_dev(&matrix_mdev->vdev); 865 866 mutex_lock(&matrix_dev->guests_lock); 867 mutex_lock(&matrix_dev->mdevs_lock); 868 vfio_ap_mdev_reset_queues(matrix_mdev); 869 vfio_ap_mdev_unlink_fr_queues(matrix_mdev); 870 list_del(&matrix_mdev->node); 871 mutex_unlock(&matrix_dev->mdevs_lock); 872 mutex_unlock(&matrix_dev->guests_lock); 873 vfio_put_device(&matrix_mdev->vdev); 874 } 875 876 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ 877 "already assigned to %s" 878 879 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, 880 unsigned long *apm, 881 unsigned long *aqm) 882 { 883 unsigned long apid, apqi; 884 const struct device *dev = mdev_dev(matrix_mdev->mdev); 885 const char *mdev_name = dev_name(dev); 886 887 for_each_set_bit_inv(apid, apm, AP_DEVICES) 888 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) 889 dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); 890 } 891 892 /** 893 * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs 894 * 895 * @mdev_apm: mask indicating the APIDs of the APQNs to be verified 896 * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified 897 * 898 * Verifies that each APQN derived from the Cartesian product of a bitmap of 899 * AP adapter IDs and AP queue indexes is not configured for any matrix 900 * mediated device. AP queue sharing is not allowed. 901 * 902 * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. 903 */ 904 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, 905 unsigned long *mdev_aqm) 906 { 907 struct ap_matrix_mdev *matrix_mdev; 908 DECLARE_BITMAP(apm, AP_DEVICES); 909 DECLARE_BITMAP(aqm, AP_DOMAINS); 910 911 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 912 /* 913 * If the input apm and aqm are fields of the matrix_mdev 914 * object, then move on to the next matrix_mdev. 915 */ 916 if (mdev_apm == matrix_mdev->matrix.apm && 917 mdev_aqm == matrix_mdev->matrix.aqm) 918 continue; 919 920 memset(apm, 0, sizeof(apm)); 921 memset(aqm, 0, sizeof(aqm)); 922 923 /* 924 * We work on full longs, as we can only exclude the leftover 925 * bits in non-inverse order. The leftover is all zeros. 926 */ 927 if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, 928 AP_DEVICES)) 929 continue; 930 931 if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, 932 AP_DOMAINS)) 933 continue; 934 935 vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); 936 937 return -EADDRINUSE; 938 } 939 940 return 0; 941 } 942 943 /** 944 * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are 945 * not reserved for the default zcrypt driver and 946 * are not assigned to another mdev. 947 * 948 * @matrix_mdev: the mdev to which the APQNs being validated are assigned. 949 * 950 * Return: One of the following values: 951 * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, 952 * most likely -EBUSY indicating the ap_perms_mutex lock is already held. 953 * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the 954 * zcrypt default driver. 955 * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev 956 * o A zero indicating validation succeeded. 957 */ 958 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) 959 { 960 if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm, 961 matrix_mdev->matrix.aqm)) 962 return -EADDRNOTAVAIL; 963 964 return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, 965 matrix_mdev->matrix.aqm); 966 } 967 968 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, 969 unsigned long apid) 970 { 971 unsigned long apqi; 972 973 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) 974 vfio_ap_mdev_link_apqn(matrix_mdev, 975 AP_MKQID(apid, apqi)); 976 } 977 978 static void collect_queues_to_reset(struct ap_matrix_mdev *matrix_mdev, 979 unsigned long apid, 980 struct list_head *qlist) 981 { 982 struct vfio_ap_queue *q; 983 unsigned long apqi; 984 985 for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS) { 986 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); 987 if (q) 988 list_add_tail(&q->reset_qnode, qlist); 989 } 990 } 991 992 static void reset_queues_for_apid(struct ap_matrix_mdev *matrix_mdev, 993 unsigned long apid) 994 { 995 struct list_head qlist; 996 997 INIT_LIST_HEAD(&qlist); 998 collect_queues_to_reset(matrix_mdev, apid, &qlist); 999 vfio_ap_mdev_reset_qlist(&qlist); 1000 } 1001 1002 static int reset_queues_for_apids(struct ap_matrix_mdev *matrix_mdev, 1003 unsigned long *apm_reset) 1004 { 1005 struct list_head qlist; 1006 unsigned long apid; 1007 1008 if (bitmap_empty(apm_reset, AP_DEVICES)) 1009 return 0; 1010 1011 INIT_LIST_HEAD(&qlist); 1012 1013 for_each_set_bit_inv(apid, apm_reset, AP_DEVICES) 1014 collect_queues_to_reset(matrix_mdev, apid, &qlist); 1015 1016 return vfio_ap_mdev_reset_qlist(&qlist); 1017 } 1018 1019 /** 1020 * assign_adapter_store - parses the APID from @buf and sets the 1021 * corresponding bit in the mediated matrix device's APM 1022 * 1023 * @dev: the matrix device 1024 * @attr: the mediated matrix device's assign_adapter attribute 1025 * @buf: a buffer containing the AP adapter number (APID) to 1026 * be assigned 1027 * @count: the number of bytes in @buf 1028 * 1029 * Return: the number of bytes processed if the APID is valid; otherwise, 1030 * returns one of the following errors: 1031 * 1032 * 1. -EINVAL 1033 * The APID is not a valid number 1034 * 1035 * 2. -ENODEV 1036 * The APID exceeds the maximum value configured for the system 1037 * 1038 * 3. -EADDRNOTAVAIL 1039 * An APQN derived from the cross product of the APID being assigned 1040 * and the APQIs previously assigned is not bound to the vfio_ap device 1041 * driver; or, if no APQIs have yet been assigned, the APID is not 1042 * contained in an APQN bound to the vfio_ap device driver. 1043 * 1044 * 4. -EADDRINUSE 1045 * An APQN derived from the cross product of the APID being assigned 1046 * and the APQIs previously assigned is being used by another mediated 1047 * matrix device 1048 * 1049 * 5. -EAGAIN 1050 * A lock required to validate the mdev's AP configuration could not 1051 * be obtained. 1052 */ 1053 static ssize_t assign_adapter_store(struct device *dev, 1054 struct device_attribute *attr, 1055 const char *buf, size_t count) 1056 { 1057 int ret; 1058 unsigned long apid; 1059 DECLARE_BITMAP(apm_filtered, AP_DEVICES); 1060 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1061 1062 mutex_lock(&ap_perms_mutex); 1063 get_update_locks_for_mdev(matrix_mdev); 1064 1065 ret = kstrtoul(buf, 0, &apid); 1066 if (ret) 1067 goto done; 1068 1069 if (apid > matrix_mdev->matrix.apm_max) { 1070 ret = -ENODEV; 1071 goto done; 1072 } 1073 1074 if (test_bit_inv(apid, matrix_mdev->matrix.apm)) { 1075 ret = count; 1076 goto done; 1077 } 1078 1079 set_bit_inv(apid, matrix_mdev->matrix.apm); 1080 1081 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 1082 if (ret) { 1083 clear_bit_inv(apid, matrix_mdev->matrix.apm); 1084 goto done; 1085 } 1086 1087 vfio_ap_mdev_link_adapter(matrix_mdev, apid); 1088 1089 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { 1090 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1091 reset_queues_for_apids(matrix_mdev, apm_filtered); 1092 } 1093 1094 ret = count; 1095 done: 1096 release_update_locks_for_mdev(matrix_mdev); 1097 mutex_unlock(&ap_perms_mutex); 1098 1099 return ret; 1100 } 1101 static DEVICE_ATTR_WO(assign_adapter); 1102 1103 static struct vfio_ap_queue 1104 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev, 1105 unsigned long apid, unsigned long apqi) 1106 { 1107 struct vfio_ap_queue *q = NULL; 1108 1109 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); 1110 /* If the queue is assigned to the matrix mdev, unlink it. */ 1111 if (q) 1112 vfio_ap_unlink_queue_fr_mdev(q); 1113 1114 return q; 1115 } 1116 1117 /** 1118 * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned 1119 * adapter from the matrix mdev to which the 1120 * adapter was assigned. 1121 * @matrix_mdev: the matrix mediated device to which the adapter was assigned. 1122 * @apid: the APID of the unassigned adapter. 1123 * @qlist: list for storing queues associated with unassigned adapter that 1124 * need to be reset. 1125 */ 1126 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, 1127 unsigned long apid, 1128 struct list_head *qlist) 1129 { 1130 unsigned long apqi; 1131 struct vfio_ap_queue *q; 1132 1133 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { 1134 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1135 1136 if (q && qlist) { 1137 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1138 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1139 list_add_tail(&q->reset_qnode, qlist); 1140 } 1141 } 1142 } 1143 1144 static void vfio_ap_mdev_hot_unplug_adapters(struct ap_matrix_mdev *matrix_mdev, 1145 unsigned long *apids) 1146 { 1147 struct vfio_ap_queue *q, *tmpq; 1148 struct list_head qlist; 1149 unsigned long apid; 1150 bool apcb_update = false; 1151 1152 INIT_LIST_HEAD(&qlist); 1153 1154 for_each_set_bit_inv(apid, apids, AP_DEVICES) { 1155 vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist); 1156 1157 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { 1158 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 1159 apcb_update = true; 1160 } 1161 } 1162 1163 /* Only update apcb if needed to avoid impacting guest */ 1164 if (apcb_update) 1165 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1166 1167 vfio_ap_mdev_reset_qlist(&qlist); 1168 1169 list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { 1170 vfio_ap_unlink_mdev_fr_queue(q); 1171 list_del(&q->reset_qnode); 1172 } 1173 } 1174 1175 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, 1176 unsigned long apid) 1177 { 1178 DECLARE_BITMAP(apids, AP_DEVICES); 1179 1180 bitmap_zero(apids, AP_DEVICES); 1181 set_bit_inv(apid, apids); 1182 vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, apids); 1183 } 1184 1185 /** 1186 * unassign_adapter_store - parses the APID from @buf and clears the 1187 * corresponding bit in the mediated matrix device's APM 1188 * 1189 * @dev: the matrix device 1190 * @attr: the mediated matrix device's unassign_adapter attribute 1191 * @buf: a buffer containing the adapter number (APID) to be unassigned 1192 * @count: the number of bytes in @buf 1193 * 1194 * Return: the number of bytes processed if the APID is valid; otherwise, 1195 * returns one of the following errors: 1196 * -EINVAL if the APID is not a number 1197 * -ENODEV if the APID it exceeds the maximum value configured for the 1198 * system 1199 */ 1200 static ssize_t unassign_adapter_store(struct device *dev, 1201 struct device_attribute *attr, 1202 const char *buf, size_t count) 1203 { 1204 int ret; 1205 unsigned long apid; 1206 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1207 1208 get_update_locks_for_mdev(matrix_mdev); 1209 1210 ret = kstrtoul(buf, 0, &apid); 1211 if (ret) 1212 goto done; 1213 1214 if (apid > matrix_mdev->matrix.apm_max) { 1215 ret = -ENODEV; 1216 goto done; 1217 } 1218 1219 if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) { 1220 ret = count; 1221 goto done; 1222 } 1223 1224 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); 1225 vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid); 1226 ret = count; 1227 done: 1228 release_update_locks_for_mdev(matrix_mdev); 1229 return ret; 1230 } 1231 static DEVICE_ATTR_WO(unassign_adapter); 1232 1233 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, 1234 unsigned long apqi) 1235 { 1236 unsigned long apid; 1237 1238 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) 1239 vfio_ap_mdev_link_apqn(matrix_mdev, 1240 AP_MKQID(apid, apqi)); 1241 } 1242 1243 /** 1244 * assign_domain_store - parses the APQI from @buf and sets the 1245 * corresponding bit in the mediated matrix device's AQM 1246 * 1247 * @dev: the matrix device 1248 * @attr: the mediated matrix device's assign_domain attribute 1249 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1250 * be assigned 1251 * @count: the number of bytes in @buf 1252 * 1253 * Return: the number of bytes processed if the APQI is valid; otherwise returns 1254 * one of the following errors: 1255 * 1256 * 1. -EINVAL 1257 * The APQI is not a valid number 1258 * 1259 * 2. -ENODEV 1260 * The APQI exceeds the maximum value configured for the system 1261 * 1262 * 3. -EADDRNOTAVAIL 1263 * An APQN derived from the cross product of the APQI being assigned 1264 * and the APIDs previously assigned is not bound to the vfio_ap device 1265 * driver; or, if no APIDs have yet been assigned, the APQI is not 1266 * contained in an APQN bound to the vfio_ap device driver. 1267 * 1268 * 4. -EADDRINUSE 1269 * An APQN derived from the cross product of the APQI being assigned 1270 * and the APIDs previously assigned is being used by another mediated 1271 * matrix device 1272 * 1273 * 5. -EAGAIN 1274 * The lock required to validate the mdev's AP configuration could not 1275 * be obtained. 1276 */ 1277 static ssize_t assign_domain_store(struct device *dev, 1278 struct device_attribute *attr, 1279 const char *buf, size_t count) 1280 { 1281 int ret; 1282 unsigned long apqi; 1283 DECLARE_BITMAP(apm_filtered, AP_DEVICES); 1284 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1285 1286 mutex_lock(&ap_perms_mutex); 1287 get_update_locks_for_mdev(matrix_mdev); 1288 1289 ret = kstrtoul(buf, 0, &apqi); 1290 if (ret) 1291 goto done; 1292 1293 if (apqi > matrix_mdev->matrix.aqm_max) { 1294 ret = -ENODEV; 1295 goto done; 1296 } 1297 1298 if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { 1299 ret = count; 1300 goto done; 1301 } 1302 1303 set_bit_inv(apqi, matrix_mdev->matrix.aqm); 1304 1305 ret = vfio_ap_mdev_validate_masks(matrix_mdev); 1306 if (ret) { 1307 clear_bit_inv(apqi, matrix_mdev->matrix.aqm); 1308 goto done; 1309 } 1310 1311 vfio_ap_mdev_link_domain(matrix_mdev, apqi); 1312 1313 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { 1314 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1315 reset_queues_for_apids(matrix_mdev, apm_filtered); 1316 } 1317 1318 ret = count; 1319 done: 1320 release_update_locks_for_mdev(matrix_mdev); 1321 mutex_unlock(&ap_perms_mutex); 1322 1323 return ret; 1324 } 1325 static DEVICE_ATTR_WO(assign_domain); 1326 1327 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, 1328 unsigned long apqi, 1329 struct list_head *qlist) 1330 { 1331 unsigned long apid; 1332 struct vfio_ap_queue *q; 1333 1334 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { 1335 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); 1336 1337 if (q && qlist) { 1338 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 1339 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 1340 list_add_tail(&q->reset_qnode, qlist); 1341 } 1342 } 1343 } 1344 1345 static void vfio_ap_mdev_hot_unplug_domains(struct ap_matrix_mdev *matrix_mdev, 1346 unsigned long *apqis) 1347 { 1348 struct vfio_ap_queue *q, *tmpq; 1349 struct list_head qlist; 1350 unsigned long apqi; 1351 bool apcb_update = false; 1352 1353 INIT_LIST_HEAD(&qlist); 1354 1355 for_each_set_bit_inv(apqi, apqis, AP_DOMAINS) { 1356 vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist); 1357 1358 if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 1359 clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); 1360 apcb_update = true; 1361 } 1362 } 1363 1364 /* Only update apcb if needed to avoid impacting guest */ 1365 if (apcb_update) 1366 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1367 1368 vfio_ap_mdev_reset_qlist(&qlist); 1369 1370 list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { 1371 vfio_ap_unlink_mdev_fr_queue(q); 1372 list_del(&q->reset_qnode); 1373 } 1374 } 1375 1376 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, 1377 unsigned long apqi) 1378 { 1379 DECLARE_BITMAP(apqis, AP_DOMAINS); 1380 1381 bitmap_zero(apqis, AP_DEVICES); 1382 set_bit_inv(apqi, apqis); 1383 vfio_ap_mdev_hot_unplug_domains(matrix_mdev, apqis); 1384 } 1385 1386 /** 1387 * unassign_domain_store - parses the APQI from @buf and clears the 1388 * corresponding bit in the mediated matrix device's AQM 1389 * 1390 * @dev: the matrix device 1391 * @attr: the mediated matrix device's unassign_domain attribute 1392 * @buf: a buffer containing the AP queue index (APQI) of the domain to 1393 * be unassigned 1394 * @count: the number of bytes in @buf 1395 * 1396 * Return: the number of bytes processed if the APQI is valid; otherwise, 1397 * returns one of the following errors: 1398 * -EINVAL if the APQI is not a number 1399 * -ENODEV if the APQI exceeds the maximum value configured for the system 1400 */ 1401 static ssize_t unassign_domain_store(struct device *dev, 1402 struct device_attribute *attr, 1403 const char *buf, size_t count) 1404 { 1405 int ret; 1406 unsigned long apqi; 1407 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1408 1409 get_update_locks_for_mdev(matrix_mdev); 1410 1411 ret = kstrtoul(buf, 0, &apqi); 1412 if (ret) 1413 goto done; 1414 1415 if (apqi > matrix_mdev->matrix.aqm_max) { 1416 ret = -ENODEV; 1417 goto done; 1418 } 1419 1420 if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { 1421 ret = count; 1422 goto done; 1423 } 1424 1425 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); 1426 vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi); 1427 ret = count; 1428 1429 done: 1430 release_update_locks_for_mdev(matrix_mdev); 1431 return ret; 1432 } 1433 static DEVICE_ATTR_WO(unassign_domain); 1434 1435 /** 1436 * assign_control_domain_store - parses the domain ID from @buf and sets 1437 * the corresponding bit in the mediated matrix device's ADM 1438 * 1439 * @dev: the matrix device 1440 * @attr: the mediated matrix device's assign_control_domain attribute 1441 * @buf: a buffer containing the domain ID to be assigned 1442 * @count: the number of bytes in @buf 1443 * 1444 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1445 * returns one of the following errors: 1446 * -EINVAL if the ID is not a number 1447 * -ENODEV if the ID exceeds the maximum value configured for the system 1448 */ 1449 static ssize_t assign_control_domain_store(struct device *dev, 1450 struct device_attribute *attr, 1451 const char *buf, size_t count) 1452 { 1453 int ret; 1454 unsigned long id; 1455 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1456 1457 get_update_locks_for_mdev(matrix_mdev); 1458 1459 ret = kstrtoul(buf, 0, &id); 1460 if (ret) 1461 goto done; 1462 1463 if (id > matrix_mdev->matrix.adm_max) { 1464 ret = -ENODEV; 1465 goto done; 1466 } 1467 1468 if (test_bit_inv(id, matrix_mdev->matrix.adm)) { 1469 ret = count; 1470 goto done; 1471 } 1472 1473 /* Set the bit in the ADM (bitmask) corresponding to the AP control 1474 * domain number (id). The bits in the mask, from most significant to 1475 * least significant, correspond to IDs 0 up to the one less than the 1476 * number of control domains that can be assigned. 1477 */ 1478 set_bit_inv(id, matrix_mdev->matrix.adm); 1479 if (vfio_ap_mdev_filter_cdoms(matrix_mdev)) 1480 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1481 1482 ret = count; 1483 done: 1484 release_update_locks_for_mdev(matrix_mdev); 1485 return ret; 1486 } 1487 static DEVICE_ATTR_WO(assign_control_domain); 1488 1489 /** 1490 * unassign_control_domain_store - parses the domain ID from @buf and 1491 * clears the corresponding bit in the mediated matrix device's ADM 1492 * 1493 * @dev: the matrix device 1494 * @attr: the mediated matrix device's unassign_control_domain attribute 1495 * @buf: a buffer containing the domain ID to be unassigned 1496 * @count: the number of bytes in @buf 1497 * 1498 * Return: the number of bytes processed if the domain ID is valid; otherwise, 1499 * returns one of the following errors: 1500 * -EINVAL if the ID is not a number 1501 * -ENODEV if the ID exceeds the maximum value configured for the system 1502 */ 1503 static ssize_t unassign_control_domain_store(struct device *dev, 1504 struct device_attribute *attr, 1505 const char *buf, size_t count) 1506 { 1507 int ret; 1508 unsigned long domid; 1509 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1510 1511 get_update_locks_for_mdev(matrix_mdev); 1512 1513 ret = kstrtoul(buf, 0, &domid); 1514 if (ret) 1515 goto done; 1516 1517 if (domid > matrix_mdev->matrix.adm_max) { 1518 ret = -ENODEV; 1519 goto done; 1520 } 1521 1522 if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) { 1523 ret = count; 1524 goto done; 1525 } 1526 1527 clear_bit_inv(domid, matrix_mdev->matrix.adm); 1528 1529 if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) { 1530 clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm); 1531 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1532 } 1533 1534 ret = count; 1535 done: 1536 release_update_locks_for_mdev(matrix_mdev); 1537 return ret; 1538 } 1539 static DEVICE_ATTR_WO(unassign_control_domain); 1540 1541 static ssize_t control_domains_show(struct device *dev, 1542 struct device_attribute *dev_attr, 1543 char *buf) 1544 { 1545 unsigned long id; 1546 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1547 unsigned long max_domid = matrix_mdev->matrix.adm_max; 1548 int nchars = 0; 1549 1550 mutex_lock(&matrix_dev->mdevs_lock); 1551 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) 1552 nchars += sysfs_emit_at(buf, nchars, "%04lx\n", id); 1553 mutex_unlock(&matrix_dev->mdevs_lock); 1554 1555 return nchars; 1556 } 1557 static DEVICE_ATTR_RO(control_domains); 1558 1559 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) 1560 { 1561 unsigned long apid; 1562 unsigned long apqi; 1563 unsigned long apid1; 1564 unsigned long apqi1; 1565 unsigned long napm_bits = matrix->apm_max + 1; 1566 unsigned long naqm_bits = matrix->aqm_max + 1; 1567 int nchars = 0; 1568 1569 apid1 = find_first_bit_inv(matrix->apm, napm_bits); 1570 apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits); 1571 1572 if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { 1573 for_each_set_bit_inv(apid, matrix->apm, napm_bits) { 1574 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) 1575 nchars += sysfs_emit_at(buf, nchars, "%02lx.%04lx\n", apid, apqi); 1576 } 1577 } else if (apid1 < napm_bits) { 1578 for_each_set_bit_inv(apid, matrix->apm, napm_bits) 1579 nchars += sysfs_emit_at(buf, nchars, "%02lx.\n", apid); 1580 } else if (apqi1 < naqm_bits) { 1581 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) 1582 nchars += sysfs_emit_at(buf, nchars, ".%04lx\n", apqi); 1583 } 1584 1585 return nchars; 1586 } 1587 1588 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, 1589 char *buf) 1590 { 1591 ssize_t nchars; 1592 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1593 1594 mutex_lock(&matrix_dev->mdevs_lock); 1595 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf); 1596 mutex_unlock(&matrix_dev->mdevs_lock); 1597 1598 return nchars; 1599 } 1600 static DEVICE_ATTR_RO(matrix); 1601 1602 static ssize_t guest_matrix_show(struct device *dev, 1603 struct device_attribute *attr, char *buf) 1604 { 1605 ssize_t nchars; 1606 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1607 1608 mutex_lock(&matrix_dev->mdevs_lock); 1609 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf); 1610 mutex_unlock(&matrix_dev->mdevs_lock); 1611 1612 return nchars; 1613 } 1614 static DEVICE_ATTR_RO(guest_matrix); 1615 1616 static ssize_t write_ap_bitmap(unsigned long *bitmap, char *buf, int offset, char sep) 1617 { 1618 return sysfs_emit_at(buf, offset, "0x%016lx%016lx%016lx%016lx%c", 1619 bitmap[0], bitmap[1], bitmap[2], bitmap[3], sep); 1620 } 1621 1622 static ssize_t ap_config_show(struct device *dev, struct device_attribute *attr, 1623 char *buf) 1624 { 1625 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1626 int idx = 0; 1627 1628 idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ','); 1629 idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ','); 1630 idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n'); 1631 1632 return idx; 1633 } 1634 1635 /* Number of characters needed for a complete hex mask representing the bits in .. */ 1636 #define AP_DEVICES_STRLEN (AP_DEVICES / 4 + 3) 1637 #define AP_DOMAINS_STRLEN (AP_DOMAINS / 4 + 3) 1638 #define AP_CONFIG_STRLEN (AP_DEVICES_STRLEN + 2 * AP_DOMAINS_STRLEN) 1639 1640 static int parse_bitmap(char **strbufptr, unsigned long *bitmap, int nbits) 1641 { 1642 char *curmask; 1643 1644 curmask = strsep(strbufptr, ",\n"); 1645 if (!curmask) 1646 return -EINVAL; 1647 1648 bitmap_clear(bitmap, 0, nbits); 1649 return ap_hex2bitmap(curmask, bitmap, nbits); 1650 } 1651 1652 static int ap_matrix_overflow_check(struct ap_matrix_mdev *matrix_mdev) 1653 { 1654 unsigned long bit; 1655 1656 for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) { 1657 if (bit > matrix_mdev->matrix.apm_max) 1658 return -ENODEV; 1659 } 1660 1661 for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) { 1662 if (bit > matrix_mdev->matrix.aqm_max) 1663 return -ENODEV; 1664 } 1665 1666 for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) { 1667 if (bit > matrix_mdev->matrix.adm_max) 1668 return -ENODEV; 1669 } 1670 1671 return 0; 1672 } 1673 1674 static void ap_matrix_copy(struct ap_matrix *dst, struct ap_matrix *src) 1675 { 1676 /* This check works around false positive gcc -Wstringop-overread */ 1677 if (!src) 1678 return; 1679 1680 bitmap_copy(dst->apm, src->apm, AP_DEVICES); 1681 bitmap_copy(dst->aqm, src->aqm, AP_DOMAINS); 1682 bitmap_copy(dst->adm, src->adm, AP_DOMAINS); 1683 } 1684 1685 static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr, 1686 const char *buf, size_t count) 1687 { 1688 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); 1689 struct ap_matrix m_new, m_old, m_added, m_removed; 1690 DECLARE_BITMAP(apm_filtered, AP_DEVICES); 1691 unsigned long newbit; 1692 char *newbuf, *rest; 1693 int rc = count; 1694 bool do_update; 1695 1696 newbuf = kstrndup(buf, AP_CONFIG_STRLEN, GFP_KERNEL); 1697 if (!newbuf) 1698 return -ENOMEM; 1699 rest = newbuf; 1700 1701 mutex_lock(&ap_perms_mutex); 1702 get_update_locks_for_mdev(matrix_mdev); 1703 1704 /* Save old state */ 1705 ap_matrix_copy(&m_old, &matrix_mdev->matrix); 1706 if (parse_bitmap(&rest, m_new.apm, AP_DEVICES) || 1707 parse_bitmap(&rest, m_new.aqm, AP_DOMAINS) || 1708 parse_bitmap(&rest, m_new.adm, AP_DOMAINS)) { 1709 rc = -EINVAL; 1710 goto out; 1711 } 1712 1713 bitmap_andnot(m_removed.apm, m_old.apm, m_new.apm, AP_DEVICES); 1714 bitmap_andnot(m_removed.aqm, m_old.aqm, m_new.aqm, AP_DOMAINS); 1715 bitmap_andnot(m_added.apm, m_new.apm, m_old.apm, AP_DEVICES); 1716 bitmap_andnot(m_added.aqm, m_new.aqm, m_old.aqm, AP_DOMAINS); 1717 1718 /* Need new bitmaps in matrix_mdev for validation */ 1719 ap_matrix_copy(&matrix_mdev->matrix, &m_new); 1720 1721 /* Ensure new state is valid, else undo new state */ 1722 rc = vfio_ap_mdev_validate_masks(matrix_mdev); 1723 if (rc) { 1724 ap_matrix_copy(&matrix_mdev->matrix, &m_old); 1725 goto out; 1726 } 1727 rc = ap_matrix_overflow_check(matrix_mdev); 1728 if (rc) { 1729 ap_matrix_copy(&matrix_mdev->matrix, &m_old); 1730 goto out; 1731 } 1732 rc = count; 1733 1734 /* Need old bitmaps in matrix_mdev for unplug/unlink */ 1735 ap_matrix_copy(&matrix_mdev->matrix, &m_old); 1736 1737 /* Unlink removed adapters/domains */ 1738 vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, m_removed.apm); 1739 vfio_ap_mdev_hot_unplug_domains(matrix_mdev, m_removed.aqm); 1740 1741 /* Need new bitmaps in matrix_mdev for linking new adapters/domains */ 1742 ap_matrix_copy(&matrix_mdev->matrix, &m_new); 1743 1744 /* Link newly added adapters */ 1745 for_each_set_bit_inv(newbit, m_added.apm, AP_DEVICES) 1746 vfio_ap_mdev_link_adapter(matrix_mdev, newbit); 1747 1748 for_each_set_bit_inv(newbit, m_added.aqm, AP_DOMAINS) 1749 vfio_ap_mdev_link_domain(matrix_mdev, newbit); 1750 1751 /* filter resources not bound to vfio-ap */ 1752 do_update = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered); 1753 do_update |= vfio_ap_mdev_filter_cdoms(matrix_mdev); 1754 1755 /* Apply changes to shadow apbc if things changed */ 1756 if (do_update) { 1757 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1758 reset_queues_for_apids(matrix_mdev, apm_filtered); 1759 } 1760 out: 1761 release_update_locks_for_mdev(matrix_mdev); 1762 mutex_unlock(&ap_perms_mutex); 1763 kfree(newbuf); 1764 return rc; 1765 } 1766 static DEVICE_ATTR_RW(ap_config); 1767 1768 static struct attribute *vfio_ap_mdev_attrs[] = { 1769 &dev_attr_assign_adapter.attr, 1770 &dev_attr_unassign_adapter.attr, 1771 &dev_attr_assign_domain.attr, 1772 &dev_attr_unassign_domain.attr, 1773 &dev_attr_assign_control_domain.attr, 1774 &dev_attr_unassign_control_domain.attr, 1775 &dev_attr_ap_config.attr, 1776 &dev_attr_control_domains.attr, 1777 &dev_attr_matrix.attr, 1778 &dev_attr_guest_matrix.attr, 1779 NULL, 1780 }; 1781 1782 static struct attribute_group vfio_ap_mdev_attr_group = { 1783 .attrs = vfio_ap_mdev_attrs 1784 }; 1785 1786 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { 1787 &vfio_ap_mdev_attr_group, 1788 NULL 1789 }; 1790 1791 /** 1792 * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed 1793 * to manage AP resources for the guest whose state is represented by @kvm 1794 * 1795 * @matrix_mdev: a mediated matrix device 1796 * @kvm: reference to KVM instance 1797 * 1798 * Return: 0 if no other mediated matrix device has a reference to @kvm; 1799 * otherwise, returns an -EPERM. 1800 */ 1801 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, 1802 struct kvm *kvm) 1803 { 1804 struct ap_matrix_mdev *m; 1805 1806 if (kvm->arch.crypto.crycbd) { 1807 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1808 kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; 1809 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1810 1811 get_update_locks_for_kvm(kvm); 1812 1813 list_for_each_entry(m, &matrix_dev->mdev_list, node) { 1814 if (m != matrix_mdev && m->kvm == kvm) { 1815 release_update_locks_for_kvm(kvm); 1816 return -EPERM; 1817 } 1818 } 1819 1820 kvm_get_kvm(kvm); 1821 matrix_mdev->kvm = kvm; 1822 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 1823 1824 release_update_locks_for_kvm(kvm); 1825 } 1826 1827 return 0; 1828 } 1829 1830 static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length) 1831 { 1832 struct ap_queue_table *qtable = &matrix_mdev->qtable; 1833 struct vfio_ap_queue *q; 1834 int loop_cursor; 1835 1836 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { 1837 if (q->saved_iova >= iova && q->saved_iova < iova + length) 1838 vfio_ap_irq_disable(q); 1839 } 1840 } 1841 1842 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova, 1843 u64 length) 1844 { 1845 struct ap_matrix_mdev *matrix_mdev = 1846 container_of(vdev, struct ap_matrix_mdev, vdev); 1847 1848 mutex_lock(&matrix_dev->mdevs_lock); 1849 1850 unmap_iova(matrix_mdev, iova, length); 1851 1852 mutex_unlock(&matrix_dev->mdevs_lock); 1853 } 1854 1855 /** 1856 * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed 1857 * by @matrix_mdev. 1858 * 1859 * @matrix_mdev: a matrix mediated device 1860 */ 1861 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) 1862 { 1863 struct kvm *kvm = matrix_mdev->kvm; 1864 1865 if (kvm && kvm->arch.crypto.crycbd) { 1866 down_write(&kvm->arch.crypto.pqap_hook_rwsem); 1867 kvm->arch.crypto.pqap_hook = NULL; 1868 up_write(&kvm->arch.crypto.pqap_hook_rwsem); 1869 1870 get_update_locks_for_kvm(kvm); 1871 1872 kvm_arch_crypto_clear_masks(kvm); 1873 vfio_ap_mdev_reset_queues(matrix_mdev); 1874 kvm_put_kvm(kvm); 1875 matrix_mdev->kvm = NULL; 1876 1877 release_update_locks_for_kvm(kvm); 1878 } 1879 } 1880 1881 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn) 1882 { 1883 struct ap_queue *queue; 1884 struct vfio_ap_queue *q = NULL; 1885 1886 queue = ap_get_qdev(apqn); 1887 if (!queue) 1888 return NULL; 1889 1890 if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver) 1891 q = dev_get_drvdata(&queue->ap_dev.device); 1892 1893 put_device(&queue->ap_dev.device); 1894 1895 return q; 1896 } 1897 1898 static int apq_status_check(int apqn, struct ap_queue_status *status) 1899 { 1900 switch (status->response_code) { 1901 case AP_RESPONSE_NORMAL: 1902 case AP_RESPONSE_DECONFIGURED: 1903 case AP_RESPONSE_CHECKSTOPPED: 1904 return 0; 1905 case AP_RESPONSE_RESET_IN_PROGRESS: 1906 case AP_RESPONSE_BUSY: 1907 return -EBUSY; 1908 case AP_RESPONSE_ASSOC_SECRET_NOT_UNIQUE: 1909 case AP_RESPONSE_ASSOC_FAILED: 1910 /* 1911 * These asynchronous response codes indicate a PQAP(AAPQ) 1912 * instruction to associate a secret with the guest failed. All 1913 * subsequent AP instructions will end with the asynchronous 1914 * response code until the AP queue is reset; so, let's return 1915 * a value indicating a reset needs to be performed again. 1916 */ 1917 return -EAGAIN; 1918 default: 1919 WARN(true, 1920 "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n", 1921 AP_QID_CARD(apqn), AP_QID_QUEUE(apqn), 1922 status->response_code); 1923 return -EIO; 1924 } 1925 } 1926 1927 #define WAIT_MSG "Waited %dms for reset of queue %02x.%04x (%u, %u, %u)" 1928 1929 static void apq_reset_check(struct work_struct *reset_work) 1930 { 1931 int ret = -EBUSY, elapsed = 0; 1932 struct ap_queue_status status; 1933 struct vfio_ap_queue *q; 1934 1935 q = container_of(reset_work, struct vfio_ap_queue, reset_work); 1936 memcpy(&status, &q->reset_status, sizeof(status)); 1937 while (true) { 1938 msleep(AP_RESET_INTERVAL); 1939 elapsed += AP_RESET_INTERVAL; 1940 status = ap_tapq(q->apqn, NULL); 1941 ret = apq_status_check(q->apqn, &status); 1942 if (ret == -EIO) 1943 return; 1944 if (ret == -EBUSY) { 1945 pr_notice_ratelimited(WAIT_MSG, elapsed, 1946 AP_QID_CARD(q->apqn), 1947 AP_QID_QUEUE(q->apqn), 1948 status.response_code, 1949 status.queue_empty, 1950 status.irq_enabled); 1951 } else { 1952 if (q->reset_status.response_code == AP_RESPONSE_RESET_IN_PROGRESS || 1953 q->reset_status.response_code == AP_RESPONSE_BUSY || 1954 q->reset_status.response_code == AP_RESPONSE_STATE_CHANGE_IN_PROGRESS || 1955 ret == -EAGAIN) { 1956 status = ap_zapq(q->apqn, 0); 1957 memcpy(&q->reset_status, &status, sizeof(status)); 1958 continue; 1959 } 1960 if (q->saved_isc != VFIO_AP_ISC_INVALID) 1961 vfio_ap_free_aqic_resources(q); 1962 break; 1963 } 1964 } 1965 } 1966 1967 static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) 1968 { 1969 struct ap_queue_status status; 1970 1971 if (!q) 1972 return; 1973 status = ap_zapq(q->apqn, 0); 1974 memcpy(&q->reset_status, &status, sizeof(status)); 1975 switch (status.response_code) { 1976 case AP_RESPONSE_NORMAL: 1977 case AP_RESPONSE_RESET_IN_PROGRESS: 1978 case AP_RESPONSE_BUSY: 1979 case AP_RESPONSE_STATE_CHANGE_IN_PROGRESS: 1980 /* 1981 * Let's verify whether the ZAPQ completed successfully on a work queue. 1982 */ 1983 queue_work(system_long_wq, &q->reset_work); 1984 break; 1985 case AP_RESPONSE_DECONFIGURED: 1986 case AP_RESPONSE_CHECKSTOPPED: 1987 vfio_ap_free_aqic_resources(q); 1988 break; 1989 default: 1990 WARN(true, 1991 "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n", 1992 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), 1993 status.response_code); 1994 } 1995 } 1996 1997 static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev) 1998 { 1999 int ret = 0, loop_cursor; 2000 struct vfio_ap_queue *q; 2001 2002 hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) 2003 vfio_ap_mdev_reset_queue(q); 2004 2005 hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) { 2006 flush_work(&q->reset_work); 2007 2008 if (q->reset_status.response_code) 2009 ret = -EIO; 2010 } 2011 2012 return ret; 2013 } 2014 2015 static int vfio_ap_mdev_reset_qlist(struct list_head *qlist) 2016 { 2017 int ret = 0; 2018 struct vfio_ap_queue *q; 2019 2020 list_for_each_entry(q, qlist, reset_qnode) 2021 vfio_ap_mdev_reset_queue(q); 2022 2023 list_for_each_entry(q, qlist, reset_qnode) { 2024 flush_work(&q->reset_work); 2025 2026 if (q->reset_status.response_code) 2027 ret = -EIO; 2028 } 2029 2030 return ret; 2031 } 2032 2033 static int vfio_ap_mdev_open_device(struct vfio_device *vdev) 2034 { 2035 struct ap_matrix_mdev *matrix_mdev = 2036 container_of(vdev, struct ap_matrix_mdev, vdev); 2037 2038 if (!vdev->kvm) 2039 return -EINVAL; 2040 2041 return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm); 2042 } 2043 2044 static void vfio_ap_mdev_close_device(struct vfio_device *vdev) 2045 { 2046 struct ap_matrix_mdev *matrix_mdev = 2047 container_of(vdev, struct ap_matrix_mdev, vdev); 2048 2049 vfio_ap_mdev_unset_kvm(matrix_mdev); 2050 } 2051 2052 static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count) 2053 { 2054 struct device *dev = vdev->dev; 2055 struct ap_matrix_mdev *matrix_mdev; 2056 2057 matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); 2058 2059 get_update_locks_for_mdev(matrix_mdev); 2060 2061 if (matrix_mdev->kvm) { 2062 kvm_arch_crypto_clear_masks(matrix_mdev->kvm); 2063 signal_guest_ap_cfg_changed(matrix_mdev); 2064 } 2065 2066 if (matrix_mdev->req_trigger) { 2067 if (!(count % 10)) 2068 dev_notice_ratelimited(dev, 2069 "Relaying device request to user (#%u)\n", 2070 count); 2071 2072 eventfd_signal(matrix_mdev->req_trigger); 2073 } else if (count == 0) { 2074 dev_notice(dev, 2075 "No device request registered, blocked until released by user\n"); 2076 } 2077 2078 release_update_locks_for_mdev(matrix_mdev); 2079 } 2080 2081 static int vfio_ap_mdev_get_device_info(unsigned long arg) 2082 { 2083 unsigned long minsz; 2084 struct vfio_device_info info; 2085 2086 minsz = offsetofend(struct vfio_device_info, num_irqs); 2087 2088 if (copy_from_user(&info, (void __user *)arg, minsz)) 2089 return -EFAULT; 2090 2091 if (info.argsz < minsz) 2092 return -EINVAL; 2093 2094 info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET; 2095 info.num_regions = 0; 2096 info.num_irqs = VFIO_AP_NUM_IRQS; 2097 2098 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 2099 } 2100 2101 static ssize_t vfio_ap_get_irq_info(unsigned long arg) 2102 { 2103 unsigned long minsz; 2104 struct vfio_irq_info info; 2105 2106 minsz = offsetofend(struct vfio_irq_info, count); 2107 2108 if (copy_from_user(&info, (void __user *)arg, minsz)) 2109 return -EFAULT; 2110 2111 if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS) 2112 return -EINVAL; 2113 2114 switch (info.index) { 2115 case VFIO_AP_REQ_IRQ_INDEX: 2116 info.count = 1; 2117 info.flags = VFIO_IRQ_INFO_EVENTFD; 2118 break; 2119 case VFIO_AP_CFG_CHG_IRQ_INDEX: 2120 info.count = 1; 2121 info.flags = VFIO_IRQ_INFO_EVENTFD; 2122 break; 2123 default: 2124 return -EINVAL; 2125 } 2126 2127 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 2128 } 2129 2130 static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg) 2131 { 2132 int ret; 2133 size_t data_size; 2134 unsigned long minsz; 2135 2136 minsz = offsetofend(struct vfio_irq_set, count); 2137 2138 if (copy_from_user(irq_set, (void __user *)arg, minsz)) 2139 return -EFAULT; 2140 2141 ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS, 2142 &data_size); 2143 if (ret) 2144 return ret; 2145 2146 if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 2147 return -EINVAL; 2148 2149 return 0; 2150 } 2151 2152 static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev, 2153 unsigned long arg) 2154 { 2155 s32 fd; 2156 void __user *data; 2157 unsigned long minsz; 2158 struct eventfd_ctx *req_trigger; 2159 2160 minsz = offsetofend(struct vfio_irq_set, count); 2161 data = (void __user *)(arg + minsz); 2162 2163 if (get_user(fd, (s32 __user *)data)) 2164 return -EFAULT; 2165 2166 if (fd == -1) { 2167 if (matrix_mdev->req_trigger) 2168 eventfd_ctx_put(matrix_mdev->req_trigger); 2169 matrix_mdev->req_trigger = NULL; 2170 } else if (fd >= 0) { 2171 req_trigger = eventfd_ctx_fdget(fd); 2172 if (IS_ERR(req_trigger)) 2173 return PTR_ERR(req_trigger); 2174 2175 if (matrix_mdev->req_trigger) 2176 eventfd_ctx_put(matrix_mdev->req_trigger); 2177 2178 matrix_mdev->req_trigger = req_trigger; 2179 } else { 2180 return -EINVAL; 2181 } 2182 2183 return 0; 2184 } 2185 2186 static int vfio_ap_set_cfg_change_irq(struct ap_matrix_mdev *matrix_mdev, unsigned long arg) 2187 { 2188 s32 fd; 2189 void __user *data; 2190 unsigned long minsz; 2191 struct eventfd_ctx *cfg_chg_trigger; 2192 2193 minsz = offsetofend(struct vfio_irq_set, count); 2194 data = (void __user *)(arg + minsz); 2195 2196 if (get_user(fd, (s32 __user *)data)) 2197 return -EFAULT; 2198 2199 if (fd == -1) { 2200 if (matrix_mdev->cfg_chg_trigger) 2201 eventfd_ctx_put(matrix_mdev->cfg_chg_trigger); 2202 matrix_mdev->cfg_chg_trigger = NULL; 2203 } else if (fd >= 0) { 2204 cfg_chg_trigger = eventfd_ctx_fdget(fd); 2205 if (IS_ERR(cfg_chg_trigger)) 2206 return PTR_ERR(cfg_chg_trigger); 2207 2208 if (matrix_mdev->cfg_chg_trigger) 2209 eventfd_ctx_put(matrix_mdev->cfg_chg_trigger); 2210 2211 matrix_mdev->cfg_chg_trigger = cfg_chg_trigger; 2212 } else { 2213 return -EINVAL; 2214 } 2215 2216 return 0; 2217 } 2218 2219 static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev, 2220 unsigned long arg) 2221 { 2222 int ret; 2223 struct vfio_irq_set irq_set; 2224 2225 ret = vfio_ap_irq_set_init(&irq_set, arg); 2226 if (ret) 2227 return ret; 2228 2229 switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 2230 case VFIO_IRQ_SET_DATA_EVENTFD: 2231 switch (irq_set.index) { 2232 case VFIO_AP_REQ_IRQ_INDEX: 2233 return vfio_ap_set_request_irq(matrix_mdev, arg); 2234 case VFIO_AP_CFG_CHG_IRQ_INDEX: 2235 return vfio_ap_set_cfg_change_irq(matrix_mdev, arg); 2236 default: 2237 return -EINVAL; 2238 } 2239 default: 2240 return -EINVAL; 2241 } 2242 } 2243 2244 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, 2245 unsigned int cmd, unsigned long arg) 2246 { 2247 struct ap_matrix_mdev *matrix_mdev = 2248 container_of(vdev, struct ap_matrix_mdev, vdev); 2249 int ret; 2250 2251 mutex_lock(&matrix_dev->mdevs_lock); 2252 switch (cmd) { 2253 case VFIO_DEVICE_GET_INFO: 2254 ret = vfio_ap_mdev_get_device_info(arg); 2255 break; 2256 case VFIO_DEVICE_RESET: 2257 ret = vfio_ap_mdev_reset_queues(matrix_mdev); 2258 break; 2259 case VFIO_DEVICE_GET_IRQ_INFO: 2260 ret = vfio_ap_get_irq_info(arg); 2261 break; 2262 case VFIO_DEVICE_SET_IRQS: 2263 ret = vfio_ap_set_irqs(matrix_mdev, arg); 2264 break; 2265 default: 2266 ret = -EOPNOTSUPP; 2267 break; 2268 } 2269 mutex_unlock(&matrix_dev->mdevs_lock); 2270 2271 return ret; 2272 } 2273 2274 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) 2275 { 2276 struct ap_matrix_mdev *matrix_mdev; 2277 unsigned long apid = AP_QID_CARD(q->apqn); 2278 unsigned long apqi = AP_QID_QUEUE(q->apqn); 2279 2280 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2281 if (test_bit_inv(apid, matrix_mdev->matrix.apm) && 2282 test_bit_inv(apqi, matrix_mdev->matrix.aqm)) 2283 return matrix_mdev; 2284 } 2285 2286 return NULL; 2287 } 2288 2289 static ssize_t status_show(struct device *dev, 2290 struct device_attribute *attr, 2291 char *buf) 2292 { 2293 ssize_t nchars = 0; 2294 struct vfio_ap_queue *q; 2295 unsigned long apid, apqi; 2296 struct ap_matrix_mdev *matrix_mdev; 2297 struct ap_device *apdev = to_ap_dev(dev); 2298 2299 mutex_lock(&matrix_dev->mdevs_lock); 2300 q = dev_get_drvdata(&apdev->device); 2301 matrix_mdev = vfio_ap_mdev_for_queue(q); 2302 2303 /* If the queue is assigned to the matrix mediated device, then 2304 * determine whether it is passed through to a guest; otherwise, 2305 * indicate that it is unassigned. 2306 */ 2307 if (matrix_mdev) { 2308 apid = AP_QID_CARD(q->apqn); 2309 apqi = AP_QID_QUEUE(q->apqn); 2310 /* 2311 * If the queue is passed through to the guest, then indicate 2312 * that it is in use; otherwise, indicate that it is 2313 * merely assigned to a matrix mediated device. 2314 */ 2315 if (matrix_mdev->kvm && 2316 test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 2317 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) 2318 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_IN_USE); 2319 else 2320 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_ASSIGNED); 2321 } else { 2322 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); 2323 } 2324 2325 mutex_unlock(&matrix_dev->mdevs_lock); 2326 2327 return nchars; 2328 } 2329 2330 static DEVICE_ATTR_RO(status); 2331 2332 static struct attribute *vfio_queue_attrs[] = { 2333 &dev_attr_status.attr, 2334 NULL, 2335 }; 2336 2337 static const struct attribute_group vfio_queue_attr_group = { 2338 .attrs = vfio_queue_attrs, 2339 }; 2340 2341 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { 2342 .init = vfio_ap_mdev_init_dev, 2343 .open_device = vfio_ap_mdev_open_device, 2344 .close_device = vfio_ap_mdev_close_device, 2345 .ioctl = vfio_ap_mdev_ioctl, 2346 .dma_unmap = vfio_ap_mdev_dma_unmap, 2347 .bind_iommufd = vfio_iommufd_emulated_bind, 2348 .unbind_iommufd = vfio_iommufd_emulated_unbind, 2349 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 2350 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 2351 .request = vfio_ap_mdev_request 2352 }; 2353 2354 static struct mdev_driver vfio_ap_matrix_driver = { 2355 .device_api = VFIO_DEVICE_API_AP_STRING, 2356 .max_instances = MAX_ZDEV_ENTRIES_EXT, 2357 .driver = { 2358 .name = "vfio_ap_mdev", 2359 .owner = THIS_MODULE, 2360 .mod_name = KBUILD_MODNAME, 2361 .dev_groups = vfio_ap_mdev_attr_groups, 2362 }, 2363 .probe = vfio_ap_mdev_probe, 2364 .remove = vfio_ap_mdev_remove, 2365 }; 2366 2367 int vfio_ap_mdev_register(void) 2368 { 2369 int ret; 2370 2371 ret = mdev_register_driver(&vfio_ap_matrix_driver); 2372 if (ret) 2373 return ret; 2374 2375 matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT; 2376 matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT; 2377 matrix_dev->mdev_types = &matrix_dev->mdev_type; 2378 ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device, 2379 &vfio_ap_matrix_driver, 2380 &matrix_dev->mdev_types, 1); 2381 if (ret) 2382 goto err_driver; 2383 return 0; 2384 2385 err_driver: 2386 mdev_unregister_driver(&vfio_ap_matrix_driver); 2387 return ret; 2388 } 2389 2390 void vfio_ap_mdev_unregister(void) 2391 { 2392 mdev_unregister_parent(&matrix_dev->parent); 2393 mdev_unregister_driver(&vfio_ap_matrix_driver); 2394 } 2395 2396 int vfio_ap_mdev_probe_queue(struct ap_device *apdev) 2397 { 2398 int ret; 2399 struct vfio_ap_queue *q; 2400 DECLARE_BITMAP(apm_filtered, AP_DEVICES); 2401 struct ap_matrix_mdev *matrix_mdev; 2402 2403 ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); 2404 if (ret) 2405 return ret; 2406 2407 q = kzalloc(sizeof(*q), GFP_KERNEL); 2408 if (!q) { 2409 ret = -ENOMEM; 2410 goto err_remove_group; 2411 } 2412 2413 q->apqn = to_ap_queue(&apdev->device)->qid; 2414 q->saved_isc = VFIO_AP_ISC_INVALID; 2415 memset(&q->reset_status, 0, sizeof(q->reset_status)); 2416 INIT_WORK(&q->reset_work, apq_reset_check); 2417 matrix_mdev = get_update_locks_by_apqn(q->apqn); 2418 2419 if (matrix_mdev) { 2420 vfio_ap_mdev_link_queue(matrix_mdev, q); 2421 2422 /* 2423 * If we're in the process of handling the adding of adapters or 2424 * domains to the host's AP configuration, then let the 2425 * vfio_ap device driver's on_scan_complete callback filter the 2426 * matrix and update the guest's AP configuration after all of 2427 * the new queue devices are probed. 2428 */ 2429 if (!bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) || 2430 !bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS)) 2431 goto done; 2432 2433 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { 2434 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2435 reset_queues_for_apids(matrix_mdev, apm_filtered); 2436 } 2437 } 2438 2439 done: 2440 dev_set_drvdata(&apdev->device, q); 2441 release_update_locks_for_mdev(matrix_mdev); 2442 2443 return ret; 2444 2445 err_remove_group: 2446 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); 2447 return ret; 2448 } 2449 2450 void vfio_ap_mdev_remove_queue(struct ap_device *apdev) 2451 { 2452 unsigned long apid, apqi; 2453 struct vfio_ap_queue *q; 2454 struct ap_matrix_mdev *matrix_mdev; 2455 2456 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); 2457 q = dev_get_drvdata(&apdev->device); 2458 get_update_locks_for_queue(q); 2459 matrix_mdev = q->matrix_mdev; 2460 apid = AP_QID_CARD(q->apqn); 2461 apqi = AP_QID_QUEUE(q->apqn); 2462 2463 if (matrix_mdev) { 2464 /* If the queue is assigned to the guest's AP configuration */ 2465 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && 2466 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { 2467 /* 2468 * Since the queues are defined via a matrix of adapters 2469 * and domains, it is not possible to hot unplug a 2470 * single queue; so, let's unplug the adapter. 2471 */ 2472 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); 2473 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2474 reset_queues_for_apid(matrix_mdev, apid); 2475 goto done; 2476 } 2477 } 2478 2479 /* 2480 * If the queue is not in the host's AP configuration, then resetting 2481 * it will fail with response code 01, (APQN not valid); so, let's make 2482 * sure it is in the host's config. 2483 */ 2484 if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) && 2485 test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) { 2486 vfio_ap_mdev_reset_queue(q); 2487 flush_work(&q->reset_work); 2488 } 2489 2490 done: 2491 if (matrix_mdev) 2492 vfio_ap_unlink_queue_fr_mdev(q); 2493 2494 dev_set_drvdata(&apdev->device, NULL); 2495 kfree(q); 2496 release_update_locks_for_mdev(matrix_mdev); 2497 } 2498 2499 /** 2500 * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is 2501 * assigned to a mediated device under the control 2502 * of the vfio_ap device driver. 2503 * 2504 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check. 2505 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check. 2506 * 2507 * Return: 2508 * * -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are 2509 * assigned to a mediated device under the control of the vfio_ap 2510 * device driver. 2511 * * Otherwise, return 0. 2512 */ 2513 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) 2514 { 2515 int ret; 2516 2517 mutex_lock(&matrix_dev->guests_lock); 2518 mutex_lock(&matrix_dev->mdevs_lock); 2519 ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); 2520 mutex_unlock(&matrix_dev->mdevs_lock); 2521 mutex_unlock(&matrix_dev->guests_lock); 2522 2523 return ret; 2524 } 2525 2526 /** 2527 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control 2528 * domains that have been removed from the host's 2529 * AP configuration from a guest. 2530 * 2531 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest. 2532 * @aprem: the adapters that have been removed from the host's AP configuration 2533 * @aqrem: the domains that have been removed from the host's AP configuration 2534 * @cdrem: the control domains that have been removed from the host's AP 2535 * configuration. 2536 */ 2537 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, 2538 unsigned long *aprem, 2539 unsigned long *aqrem, 2540 unsigned long *cdrem) 2541 { 2542 int do_hotplug = 0; 2543 2544 if (!bitmap_empty(aprem, AP_DEVICES)) { 2545 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, 2546 matrix_mdev->shadow_apcb.apm, 2547 aprem, AP_DEVICES); 2548 } 2549 2550 if (!bitmap_empty(aqrem, AP_DOMAINS)) { 2551 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, 2552 matrix_mdev->shadow_apcb.aqm, 2553 aqrem, AP_DEVICES); 2554 } 2555 2556 if (!bitmap_empty(cdrem, AP_DOMAINS)) 2557 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, 2558 matrix_mdev->shadow_apcb.adm, 2559 cdrem, AP_DOMAINS); 2560 2561 if (do_hotplug) 2562 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2563 } 2564 2565 /** 2566 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters, 2567 * domains and control domains that have been removed 2568 * from the host AP configuration and unplugs them 2569 * from those guests. 2570 * 2571 * @ap_remove: bitmap specifying which adapters have been removed from the host 2572 * config. 2573 * @aq_remove: bitmap specifying which domains have been removed from the host 2574 * config. 2575 * @cd_remove: bitmap specifying which control domains have been removed from 2576 * the host config. 2577 */ 2578 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, 2579 unsigned long *aq_remove, 2580 unsigned long *cd_remove) 2581 { 2582 struct ap_matrix_mdev *matrix_mdev; 2583 DECLARE_BITMAP(aprem, AP_DEVICES); 2584 DECLARE_BITMAP(aqrem, AP_DOMAINS); 2585 DECLARE_BITMAP(cdrem, AP_DOMAINS); 2586 int do_remove = 0; 2587 2588 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2589 mutex_lock(&matrix_mdev->kvm->lock); 2590 mutex_lock(&matrix_dev->mdevs_lock); 2591 2592 do_remove |= bitmap_and(aprem, ap_remove, 2593 matrix_mdev->matrix.apm, 2594 AP_DEVICES); 2595 do_remove |= bitmap_and(aqrem, aq_remove, 2596 matrix_mdev->matrix.aqm, 2597 AP_DOMAINS); 2598 do_remove |= bitmap_andnot(cdrem, cd_remove, 2599 matrix_mdev->matrix.adm, 2600 AP_DOMAINS); 2601 2602 if (do_remove) 2603 vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, 2604 cdrem); 2605 2606 mutex_unlock(&matrix_dev->mdevs_lock); 2607 mutex_unlock(&matrix_mdev->kvm->lock); 2608 } 2609 } 2610 2611 /** 2612 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and 2613 * control domains from the host AP configuration 2614 * by unplugging them from the guests that are 2615 * using them. 2616 * @cur_config_info: the current host AP configuration information 2617 * @prev_config_info: the previous host AP configuration information 2618 */ 2619 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info, 2620 struct ap_config_info *prev_config_info) 2621 { 2622 int do_remove; 2623 DECLARE_BITMAP(aprem, AP_DEVICES); 2624 DECLARE_BITMAP(aqrem, AP_DOMAINS); 2625 DECLARE_BITMAP(cdrem, AP_DOMAINS); 2626 2627 do_remove = bitmap_andnot(aprem, 2628 (unsigned long *)prev_config_info->apm, 2629 (unsigned long *)cur_config_info->apm, 2630 AP_DEVICES); 2631 do_remove |= bitmap_andnot(aqrem, 2632 (unsigned long *)prev_config_info->aqm, 2633 (unsigned long *)cur_config_info->aqm, 2634 AP_DEVICES); 2635 do_remove |= bitmap_andnot(cdrem, 2636 (unsigned long *)prev_config_info->adm, 2637 (unsigned long *)cur_config_info->adm, 2638 AP_DEVICES); 2639 2640 if (do_remove) 2641 vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem); 2642 } 2643 2644 /** 2645 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that 2646 * are older than AP type 10 (CEX4). 2647 * @apm: a bitmap of the APIDs to examine 2648 * @aqm: a bitmap of the APQIs of the queues to query for the AP type. 2649 */ 2650 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm) 2651 { 2652 bool apid_cleared; 2653 struct ap_queue_status status; 2654 unsigned long apid, apqi; 2655 struct ap_tapq_hwinfo info; 2656 2657 for_each_set_bit_inv(apid, apm, AP_DEVICES) { 2658 apid_cleared = false; 2659 2660 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { 2661 status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info); 2662 switch (status.response_code) { 2663 /* 2664 * According to the architecture in each case 2665 * below, the queue's info should be filled. 2666 */ 2667 case AP_RESPONSE_NORMAL: 2668 case AP_RESPONSE_RESET_IN_PROGRESS: 2669 case AP_RESPONSE_DECONFIGURED: 2670 case AP_RESPONSE_CHECKSTOPPED: 2671 case AP_RESPONSE_BUSY: 2672 /* 2673 * The vfio_ap device driver only 2674 * supports CEX4 and newer adapters, so 2675 * remove the APID if the adapter is 2676 * older than a CEX4. 2677 */ 2678 if (info.at < AP_DEVICE_TYPE_CEX4) { 2679 clear_bit_inv(apid, apm); 2680 apid_cleared = true; 2681 } 2682 2683 break; 2684 2685 default: 2686 /* 2687 * If we don't know the adapter type, 2688 * clear its APID since it can't be 2689 * determined whether the vfio_ap 2690 * device driver supports it. 2691 */ 2692 clear_bit_inv(apid, apm); 2693 apid_cleared = true; 2694 break; 2695 } 2696 2697 /* 2698 * If we've already cleared the APID from the apm, there 2699 * is no need to continue examining the remainin AP 2700 * queues to determine the type of the adapter. 2701 */ 2702 if (apid_cleared) 2703 continue; 2704 } 2705 } 2706 } 2707 2708 /** 2709 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and 2710 * control domains that have been added to the host's 2711 * AP configuration for each matrix mdev to which they 2712 * are assigned. 2713 * 2714 * @apm_add: a bitmap specifying the adapters that have been added to the AP 2715 * configuration. 2716 * @aqm_add: a bitmap specifying the domains that have been added to the AP 2717 * configuration. 2718 * @adm_add: a bitmap specifying the control domains that have been added to the 2719 * AP configuration. 2720 */ 2721 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, 2722 unsigned long *adm_add) 2723 { 2724 struct ap_matrix_mdev *matrix_mdev; 2725 2726 if (list_empty(&matrix_dev->mdev_list)) 2727 return; 2728 2729 vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); 2730 2731 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2732 bitmap_and(matrix_mdev->apm_add, 2733 matrix_mdev->matrix.apm, apm_add, AP_DEVICES); 2734 bitmap_and(matrix_mdev->aqm_add, 2735 matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); 2736 bitmap_and(matrix_mdev->adm_add, 2737 matrix_mdev->matrix.adm, adm_add, AP_DEVICES); 2738 } 2739 } 2740 2741 /** 2742 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and 2743 * control domains to the host AP configuration 2744 * by updating the bitmaps that specify what adapters, 2745 * domains and control domains have been added so they 2746 * can be hot plugged into the guest when the AP bus 2747 * scan completes (see vfio_ap_on_scan_complete 2748 * function). 2749 * @cur_config_info: the current AP configuration information 2750 * @prev_config_info: the previous AP configuration information 2751 */ 2752 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info, 2753 struct ap_config_info *prev_config_info) 2754 { 2755 bool do_add; 2756 DECLARE_BITMAP(apm_add, AP_DEVICES); 2757 DECLARE_BITMAP(aqm_add, AP_DOMAINS); 2758 DECLARE_BITMAP(adm_add, AP_DOMAINS); 2759 2760 do_add = bitmap_andnot(apm_add, 2761 (unsigned long *)cur_config_info->apm, 2762 (unsigned long *)prev_config_info->apm, 2763 AP_DEVICES); 2764 do_add |= bitmap_andnot(aqm_add, 2765 (unsigned long *)cur_config_info->aqm, 2766 (unsigned long *)prev_config_info->aqm, 2767 AP_DOMAINS); 2768 do_add |= bitmap_andnot(adm_add, 2769 (unsigned long *)cur_config_info->adm, 2770 (unsigned long *)prev_config_info->adm, 2771 AP_DOMAINS); 2772 2773 if (do_add) 2774 vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add); 2775 } 2776 2777 /** 2778 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP 2779 * configuration. 2780 * 2781 * @cur_cfg_info: the current host AP configuration 2782 * @prev_cfg_info: the previous host AP configuration 2783 */ 2784 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, 2785 struct ap_config_info *prev_cfg_info) 2786 { 2787 if (!cur_cfg_info || !prev_cfg_info) 2788 return; 2789 2790 mutex_lock(&matrix_dev->guests_lock); 2791 2792 vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); 2793 vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info); 2794 memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info)); 2795 2796 mutex_unlock(&matrix_dev->guests_lock); 2797 } 2798 2799 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) 2800 { 2801 DECLARE_BITMAP(apm_filtered, AP_DEVICES); 2802 bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; 2803 2804 mutex_lock(&matrix_mdev->kvm->lock); 2805 mutex_lock(&matrix_dev->mdevs_lock); 2806 2807 filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, 2808 matrix_mdev->apm_add, AP_DEVICES); 2809 filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm, 2810 matrix_mdev->aqm_add, AP_DOMAINS); 2811 filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm, 2812 matrix_mdev->adm_add, AP_DOMAINS); 2813 2814 if (filter_adapters || filter_domains) 2815 do_hotplug = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered); 2816 2817 if (filter_cdoms) 2818 do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev); 2819 2820 if (do_hotplug) 2821 vfio_ap_mdev_update_guest_apcb(matrix_mdev); 2822 2823 reset_queues_for_apids(matrix_mdev, apm_filtered); 2824 2825 mutex_unlock(&matrix_dev->mdevs_lock); 2826 mutex_unlock(&matrix_mdev->kvm->lock); 2827 } 2828 2829 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, 2830 struct ap_config_info *old_config_info) 2831 { 2832 struct ap_matrix_mdev *matrix_mdev; 2833 2834 mutex_lock(&matrix_dev->guests_lock); 2835 2836 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { 2837 if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && 2838 bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && 2839 bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) 2840 continue; 2841 2842 vfio_ap_mdev_hot_plug_cfg(matrix_mdev); 2843 bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); 2844 bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); 2845 bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); 2846 } 2847 2848 mutex_unlock(&matrix_dev->guests_lock); 2849 } 2850