1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2026, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/gpu_scheduler.h> 12 #include <linux/types.h> 13 14 #include "aie.h" 15 #include "aie4_host_queue.h" 16 #include "aie4_msg_priv.h" 17 #include "aie4_pci.h" 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_mailbox.h" 21 #include "amdxdna_mailbox_helper.h" 22 #include "amdxdna_pci_drv.h" 23 24 static irqreturn_t cert_comp_isr(int irq, void *p) 25 { 26 struct cert_comp *cert_comp = p; 27 28 wake_up_all(&cert_comp->waitq); 29 return IRQ_HANDLED; 30 } 31 32 static struct cert_comp *aie4_lookup_cert_comp(struct amdxdna_dev_hdl *ndev, u32 msix_idx) 33 { 34 struct amdxdna_dev *xdna = ndev->aie.xdna; 35 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 36 struct cert_comp *cert_comp; 37 int ret; 38 39 guard(mutex)(&ndev->cert_comp_lock); 40 41 cert_comp = xa_load(&ndev->cert_comp_xa, msix_idx); 42 if (cert_comp) { 43 kref_get(&cert_comp->kref); 44 return cert_comp; 45 } 46 47 cert_comp = kzalloc_obj(*cert_comp); 48 if (!cert_comp) 49 return NULL; 50 51 cert_comp->ndev = ndev; 52 cert_comp->msix_idx = msix_idx; 53 init_waitqueue_head(&cert_comp->waitq); 54 kref_init(&cert_comp->kref); 55 56 ret = pci_irq_vector(pdev, cert_comp->msix_idx); 57 if (ret < 0) { 58 XDNA_ERR(xdna, "MSI-X idx %u is invalid, ret:%d", msix_idx, ret); 59 goto free_cert_comp; 60 } 61 cert_comp->irq = ret; 62 63 ret = request_irq(cert_comp->irq, cert_comp_isr, 0, "xdna_hsa", cert_comp); 64 if (ret) { 65 XDNA_ERR(xdna, "request irq %d failed %d", cert_comp->irq, ret); 66 goto free_cert_comp; 67 } 68 69 ret = xa_err(xa_store(&ndev->cert_comp_xa, msix_idx, cert_comp, GFP_KERNEL)); 70 if (ret) { 71 XDNA_ERR(xdna, "store cert_comp for msix idx %d failed %d", msix_idx, ret); 72 goto free_irq; 73 } 74 75 return cert_comp; 76 77 free_irq: 78 free_irq(cert_comp->irq, cert_comp); 79 free_cert_comp: 80 kfree(cert_comp); 81 return NULL; 82 } 83 84 static void cert_comp_release(struct kref *kref) 85 { 86 struct cert_comp *cert_comp = container_of(kref, struct cert_comp, kref); 87 struct amdxdna_dev_hdl *ndev = cert_comp->ndev; 88 89 drm_WARN_ON(&ndev->aie.xdna->ddev, !mutex_is_locked(&ndev->cert_comp_lock)); 90 91 xa_erase(&ndev->cert_comp_xa, cert_comp->msix_idx); 92 free_irq(cert_comp->irq, cert_comp); 93 kfree(cert_comp); 94 } 95 96 static void aie4_put_cert_comp(struct cert_comp *cert_comp) 97 { 98 struct amdxdna_dev_hdl *ndev; 99 100 ndev = cert_comp->ndev; 101 guard(mutex)(&ndev->cert_comp_lock); 102 kref_put(&cert_comp->kref, cert_comp_release); 103 } 104 105 static int aie4_msg_destroy_context(struct amdxdna_dev_hdl *ndev, u32 hw_context_id) 106 { 107 DECLARE_AIE_MSG(aie4_msg_destroy_hw_context, AIE4_MSG_OP_DESTROY_HW_CONTEXT); 108 109 req.hw_context_id = hw_context_id; 110 return aie_send_mgmt_msg_wait(&ndev->aie, &msg); 111 } 112 113 static int aie4_hwctx_create(struct amdxdna_hwctx *hwctx) 114 { 115 DECLARE_AIE_MSG(aie4_msg_create_hw_context, AIE4_MSG_OP_CREATE_HW_CONTEXT); 116 struct amdxdna_client *client = hwctx->client; 117 struct amdxdna_hwctx_priv *priv = hwctx->priv; 118 struct amdxdna_dev *xdna = hwctx->client->xdna; 119 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 120 int ret; 121 122 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 123 124 if (!ndev->partition_id || !hwctx->num_tiles) { 125 XDNA_ERR(xdna, "invalid request partition_id %d, num_tiles %d", 126 ndev->partition_id, hwctx->num_tiles); 127 return -EINVAL; 128 } 129 130 req.partition_id = ndev->partition_id; 131 req.request_num_tiles = hwctx->num_tiles; 132 req.pasid = FIELD_PREP(AIE4_MSG_PASID, client->pasid) | 133 FIELD_PREP(AIE4_MSG_PASID_VLD, 1); 134 req.priority_band = hwctx->qos.priority; 135 136 req.hsa_addr_high = upper_32_bits(amdxdna_gem_dev_addr(priv->umq_bo)); 137 req.hsa_addr_low = lower_32_bits(amdxdna_gem_dev_addr(priv->umq_bo)); 138 139 XDNA_DBG(xdna, "pasid 0x%x, num_tiles %d, hsa[0x%x 0x%x]", 140 req.pasid, req.request_num_tiles, req.hsa_addr_high, req.hsa_addr_low); 141 142 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 143 if (ret) { 144 XDNA_ERR(xdna, "create ctx failed: %d", ret); 145 return ret; 146 } 147 148 XDNA_DBG(xdna, "resp msix: %d, ctx id: %d, doorbell: %d", 149 resp.job_complete_msix_idx, 150 resp.hw_context_id, 151 resp.doorbell_offset); 152 153 /* setup interrupt completion per msix index */ 154 priv->cert_comp = aie4_lookup_cert_comp(ndev, resp.job_complete_msix_idx); 155 if (!priv->cert_comp) { 156 aie4_msg_destroy_context(ndev, resp.hw_context_id); 157 return -EINVAL; 158 } 159 160 priv->hw_ctx_id = resp.hw_context_id; 161 hwctx->doorbell_offset = resp.doorbell_offset; 162 163 return 0; 164 } 165 166 static void aie4_hwctx_destroy(struct amdxdna_hwctx *hwctx) 167 { 168 struct amdxdna_client *client = hwctx->client; 169 struct amdxdna_hwctx_priv *priv = hwctx->priv; 170 struct amdxdna_dev *xdna = client->xdna; 171 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 172 173 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 174 175 aie4_msg_destroy_context(ndev, priv->hw_ctx_id); 176 aie4_put_cert_comp(priv->cert_comp); 177 } 178 179 static void aie4_hwctx_umq_fini(struct amdxdna_hwctx *hwctx) 180 { 181 if (hwctx->priv && hwctx->priv->umq_bo) 182 amdxdna_gem_put_obj(hwctx->priv->umq_bo); 183 } 184 185 static int aie4_hwctx_umq_init(struct amdxdna_hwctx *hwctx) 186 { 187 struct amdxdna_hwctx_priv *priv = hwctx->priv; 188 struct amdxdna_dev *xdna = hwctx->client->xdna; 189 struct amdxdna_gem_obj *umq_bo; 190 struct host_queue_header *qhdr; 191 int ret; 192 193 umq_bo = amdxdna_gem_get_obj(hwctx->client, hwctx->umq_bo_hdl, AMDXDNA_BO_SHARE); 194 if (!umq_bo) { 195 XDNA_ERR(xdna, "cannot find umq_bo handle %d", hwctx->umq_bo_hdl); 196 return -ENOENT; 197 } 198 if (umq_bo->mem.size < sizeof(*qhdr)) { 199 XDNA_ERR(xdna, "umq_bo size is too small"); 200 ret = -EINVAL; 201 goto put_umq_bo; 202 } 203 204 /* get kva address for host queue read index and write index */ 205 qhdr = amdxdna_gem_vmap(umq_bo); 206 if (!qhdr) { 207 ret = -ENOMEM; 208 goto put_umq_bo; 209 } 210 211 priv->umq_bo = umq_bo; 212 priv->umq_read_index = &qhdr->read_index; 213 priv->umq_write_index = &qhdr->write_index; 214 215 return 0; 216 217 put_umq_bo: 218 amdxdna_gem_put_obj(umq_bo); 219 return ret; 220 } 221 222 int aie4_hwctx_init(struct amdxdna_hwctx *hwctx) 223 { 224 struct amdxdna_client *client = hwctx->client; 225 struct amdxdna_dev *xdna = client->xdna; 226 struct amdxdna_hwctx_priv *priv; 227 int ret; 228 229 priv = kzalloc_obj(*priv); 230 if (!priv) 231 return -ENOMEM; 232 hwctx->priv = priv; 233 234 ret = aie4_hwctx_umq_init(hwctx); 235 if (ret) 236 goto free_priv; 237 238 ret = aie4_hwctx_create(hwctx); 239 if (ret) 240 goto umq_fini; 241 242 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 243 return 0; 244 245 umq_fini: 246 aie4_hwctx_umq_fini(hwctx); 247 free_priv: 248 kfree(priv); 249 hwctx->priv = NULL; 250 return ret; 251 } 252 253 void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx) 254 { 255 aie4_hwctx_destroy(hwctx); 256 aie4_hwctx_umq_fini(hwctx); 257 kfree(hwctx->priv); 258 } 259 260 static inline bool valid_queue_index(u64 read, u64 write, u32 capacity) 261 { 262 return (write >= read) && ((write - read) <= capacity); 263 } 264 265 static u64 get_read_index(struct amdxdna_hwctx *hwctx) 266 { 267 u64 wi = READ_ONCE(*hwctx->priv->umq_write_index); 268 u64 ri = READ_ONCE(*hwctx->priv->umq_read_index); 269 struct amdxdna_dev *xdna = hwctx->client->xdna; 270 271 /* 272 * CERT cannot update read index as uint64 atomically. Driver may read 273 * half-updated read index when it has bits in high 32bit. In case read 274 * index is not valid, wait for some time and retry once. It should 275 * allow CERT to complete the read index update. 276 */ 277 if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) { 278 XDNA_WARN(xdna, "Invalid index, ri %llu, wi %llu", ri, wi); 279 usleep_range(100, 200); 280 ri = READ_ONCE(*hwctx->priv->umq_read_index); 281 if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) { 282 XDNA_ERR(xdna, "Invalid index after retry, ri %llu, wi %llu", ri, wi); 283 ri = 0; 284 } 285 } 286 287 return ri; 288 } 289 290 static inline bool check_cmd_done(struct amdxdna_hwctx *hwctx, u64 seq) 291 { 292 u64 read_idx = get_read_index(hwctx); 293 294 return read_idx > seq; 295 } 296 297 int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout) 298 { 299 unsigned long wait_jifs = MAX_SCHEDULE_TIMEOUT; 300 struct amdxdna_hwctx_priv *priv = hwctx->priv; 301 struct cert_comp *cert_comp = priv->cert_comp; 302 long ret; 303 304 if (timeout) 305 wait_jifs = msecs_to_jiffies(timeout); 306 307 ret = wait_event_interruptible_timeout(cert_comp->waitq, 308 (check_cmd_done(hwctx, seq)), 309 wait_jifs); 310 311 if (!ret) 312 ret = -ETIME; 313 314 return ret <= 0 ? ret : 0; 315 } 316 317 int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff) 318 { 319 struct amdxdna_hwctx *hwctx; 320 unsigned long hwctx_id; 321 int idx; 322 323 idx = srcu_read_lock(&client->hwctx_srcu); 324 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 325 if (vm_pgoff == (hwctx->doorbell_offset >> PAGE_SHIFT)) { 326 srcu_read_unlock(&client->hwctx_srcu, idx); 327 return 1; 328 } 329 } 330 srcu_read_unlock(&client->hwctx_srcu, idx); 331 332 return 0; 333 } 334