1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2016-2019 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include <uapi/drm/habanalabs_accel.h> 9 #include "habanalabs.h" 10 11 #include <linux/mm.h> 12 #include <linux/slab.h> 13 #include <linux/uaccess.h> 14 15 #define CB_VA_POOL_SIZE (4UL * SZ_1G) 16 17 static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) 18 { 19 struct hl_device *hdev = ctx->hdev; 20 struct asic_fixed_properties *prop = &hdev->asic_prop; 21 u32 page_size = prop->pmmu.page_size; 22 int rc; 23 24 if (!hdev->supports_cb_mapping) { 25 dev_err_ratelimited(hdev->dev, 26 "Mapping a CB to the device's MMU is not supported\n"); 27 return -EINVAL; 28 } 29 30 if (cb->is_mmu_mapped) 31 return 0; 32 33 cb->roundup_size = roundup(cb->size, page_size); 34 35 cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size); 36 if (!cb->virtual_addr) { 37 dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n"); 38 return -ENOMEM; 39 } 40 41 mutex_lock(&hdev->mmu_lock); 42 43 rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size); 44 if (rc) { 45 dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr); 46 goto err_va_pool_free; 47 } 48 49 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV); 50 if (rc) 51 goto err_mmu_unmap; 52 53 mutex_unlock(&hdev->mmu_lock); 54 55 cb->is_mmu_mapped = true; 56 57 return 0; 58 59 err_mmu_unmap: 60 hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 61 err_va_pool_free: 62 mutex_unlock(&hdev->mmu_lock); 63 gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 64 65 return rc; 66 } 67 68 static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb) 69 { 70 struct hl_device *hdev = ctx->hdev; 71 72 mutex_lock(&hdev->mmu_lock); 73 hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 74 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 75 mutex_unlock(&hdev->mmu_lock); 76 77 gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 78 } 79 80 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb) 81 { 82 if (cb->is_internal) 83 gen_pool_free(hdev->internal_cb_pool, 84 (uintptr_t)cb->kernel_address, cb->size); 85 else 86 hl_asic_dma_free_coherent(hdev, cb->size, cb->kernel_address, cb->bus_address); 87 88 kfree(cb); 89 } 90 91 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb) 92 { 93 if (cb->is_pool) { 94 atomic_set(&cb->is_handle_destroyed, 0); 95 spin_lock(&hdev->cb_pool_lock); 96 list_add(&cb->pool_list, &hdev->cb_pool); 97 spin_unlock(&hdev->cb_pool_lock); 98 } else { 99 cb_fini(hdev, cb); 100 } 101 } 102 103 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size, 104 int ctx_id, bool internal_cb) 105 { 106 struct hl_cb *cb = NULL; 107 u32 cb_offset; 108 void *p; 109 110 /* 111 * We use of GFP_ATOMIC here because this function can be called from 112 * the latency-sensitive code path for command submission. Due to H/W 113 * limitations in some of the ASICs, the kernel must copy the user CB 114 * that is designated for an external queue and actually enqueue 115 * the kernel's copy. Hence, we must never sleep in this code section 116 * and must use GFP_ATOMIC for all memory allocations. 117 */ 118 if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled) 119 cb = kzalloc(sizeof(*cb), GFP_ATOMIC); 120 121 if (!cb) 122 cb = kzalloc(sizeof(*cb), GFP_KERNEL); 123 124 if (!cb) 125 return NULL; 126 127 if (internal_cb) { 128 p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size); 129 if (!p) { 130 kfree(cb); 131 return NULL; 132 } 133 134 cb_offset = p - hdev->internal_cb_pool_virt_addr; 135 cb->is_internal = true; 136 cb->bus_address = hdev->internal_cb_va_base + cb_offset; 137 } else if (ctx_id == HL_KERNEL_ASID_ID) { 138 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_ATOMIC); 139 if (!p) 140 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_KERNEL); 141 } else { 142 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, 143 GFP_USER | __GFP_ZERO); 144 } 145 146 if (!p) { 147 dev_err(hdev->dev, 148 "failed to allocate %d of dma memory for CB\n", 149 cb_size); 150 kfree(cb); 151 return NULL; 152 } 153 154 cb->kernel_address = p; 155 cb->size = cb_size; 156 157 return cb; 158 } 159 160 struct hl_cb_mmap_mem_alloc_args { 161 struct hl_device *hdev; 162 struct hl_ctx *ctx; 163 u32 cb_size; 164 bool internal_cb; 165 bool map_cb; 166 }; 167 168 static void hl_cb_mmap_mem_release(struct hl_mmap_mem_buf *buf) 169 { 170 struct hl_cb *cb = buf->private; 171 172 hl_debugfs_remove_cb(cb); 173 174 if (cb->is_mmu_mapped) 175 cb_unmap_mem(cb->ctx, cb); 176 177 hl_ctx_put(cb->ctx); 178 179 cb_do_release(cb->hdev, cb); 180 } 181 182 static int hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args) 183 { 184 struct hl_cb_mmap_mem_alloc_args *cb_args = args; 185 struct hl_cb *cb; 186 int rc, ctx_id = cb_args->ctx->asid; 187 bool alloc_new_cb = true; 188 189 if (!cb_args->internal_cb) { 190 /* Minimum allocation must be PAGE SIZE */ 191 if (cb_args->cb_size < PAGE_SIZE) 192 cb_args->cb_size = PAGE_SIZE; 193 194 if (ctx_id == HL_KERNEL_ASID_ID && 195 cb_args->cb_size <= cb_args->hdev->asic_prop.cb_pool_cb_size) { 196 197 spin_lock(&cb_args->hdev->cb_pool_lock); 198 if (!list_empty(&cb_args->hdev->cb_pool)) { 199 cb = list_first_entry(&cb_args->hdev->cb_pool, 200 typeof(*cb), pool_list); 201 list_del(&cb->pool_list); 202 spin_unlock(&cb_args->hdev->cb_pool_lock); 203 alloc_new_cb = false; 204 } else { 205 spin_unlock(&cb_args->hdev->cb_pool_lock); 206 dev_dbg(cb_args->hdev->dev, "CB pool is empty\n"); 207 } 208 } 209 } 210 211 if (alloc_new_cb) { 212 cb = hl_cb_alloc(cb_args->hdev, cb_args->cb_size, ctx_id, cb_args->internal_cb); 213 if (!cb) 214 return -ENOMEM; 215 } 216 217 cb->hdev = cb_args->hdev; 218 cb->ctx = cb_args->ctx; 219 cb->buf = buf; 220 cb->buf->mappable_size = cb->size; 221 cb->buf->private = cb; 222 223 hl_ctx_get(cb->ctx); 224 225 if (cb_args->map_cb) { 226 if (ctx_id == HL_KERNEL_ASID_ID) { 227 dev_err(cb_args->hdev->dev, 228 "CB mapping is not supported for kernel context\n"); 229 rc = -EINVAL; 230 goto release_cb; 231 } 232 233 rc = cb_map_mem(cb_args->ctx, cb); 234 if (rc) 235 goto release_cb; 236 } 237 238 hl_debugfs_add_cb(cb); 239 240 return 0; 241 242 release_cb: 243 hl_ctx_put(cb->ctx); 244 cb_do_release(cb_args->hdev, cb); 245 246 return rc; 247 } 248 249 static int hl_cb_mmap(struct hl_mmap_mem_buf *buf, 250 struct vm_area_struct *vma, void *args) 251 { 252 struct hl_cb *cb = buf->private; 253 254 return cb->hdev->asic_funcs->mmap(cb->hdev, vma, cb->kernel_address, 255 cb->bus_address, cb->size); 256 } 257 258 static struct hl_mmap_mem_buf_behavior cb_behavior = { 259 .topic = "CB", 260 .mem_id = HL_MMAP_TYPE_CB, 261 .alloc = hl_cb_mmap_mem_alloc, 262 .release = hl_cb_mmap_mem_release, 263 .mmap = hl_cb_mmap, 264 }; 265 266 int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, 267 struct hl_ctx *ctx, u32 cb_size, bool internal_cb, 268 bool map_cb, u64 *handle) 269 { 270 struct hl_cb_mmap_mem_alloc_args args = { 271 .hdev = hdev, 272 .ctx = ctx, 273 .cb_size = cb_size, 274 .internal_cb = internal_cb, 275 .map_cb = map_cb, 276 }; 277 struct hl_mmap_mem_buf *buf; 278 int ctx_id = ctx->asid; 279 280 if ((hdev->disabled) || (hdev->reset_info.in_reset && (ctx_id != HL_KERNEL_ASID_ID))) { 281 dev_warn_ratelimited(hdev->dev, 282 "Device is disabled or in reset. Can't create new CBs\n"); 283 return -EBUSY; 284 } 285 286 if (cb_size > SZ_2M) { 287 dev_err(hdev->dev, "CB size %d must be less than %d\n", 288 cb_size, SZ_2M); 289 return -EINVAL; 290 } 291 292 buf = hl_mmap_mem_buf_alloc( 293 mmg, &cb_behavior, 294 ctx_id == HL_KERNEL_ASID_ID ? GFP_ATOMIC : GFP_KERNEL, &args); 295 if (!buf) 296 return -ENOMEM; 297 298 *handle = buf->handle; 299 300 return 0; 301 } 302 303 int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle) 304 { 305 struct hl_cb *cb; 306 int rc; 307 308 cb = hl_cb_get(mmg, cb_handle); 309 if (!cb) { 310 dev_dbg(mmg->dev, "CB destroy failed, no CB was found for handle %#llx\n", 311 cb_handle); 312 return -EINVAL; 313 } 314 315 /* Make sure that CB handle isn't destroyed more than once */ 316 rc = atomic_cmpxchg(&cb->is_handle_destroyed, 0, 1); 317 hl_cb_put(cb); 318 if (rc) { 319 dev_dbg(mmg->dev, "CB destroy failed, handle %#llx was already destroyed\n", 320 cb_handle); 321 return -EINVAL; 322 } 323 324 rc = hl_mmap_mem_buf_put_handle(mmg, cb_handle); 325 if (rc < 0) 326 return rc; /* Invalid handle */ 327 328 if (rc == 0) 329 dev_dbg(mmg->dev, "CB 0x%llx is destroyed while still in use\n", cb_handle); 330 331 return 0; 332 } 333 334 static int hl_cb_info(struct hl_mem_mgr *mmg, 335 u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va) 336 { 337 struct hl_cb *cb; 338 int rc = 0; 339 340 cb = hl_cb_get(mmg, handle); 341 if (!cb) { 342 dev_err(mmg->dev, 343 "CB info failed, no match to handle 0x%llx\n", handle); 344 return -EINVAL; 345 } 346 347 if (flags & HL_CB_FLAGS_GET_DEVICE_VA) { 348 if (cb->is_mmu_mapped) { 349 *device_va = cb->virtual_addr; 350 } else { 351 dev_err(mmg->dev, "CB is not mapped to the device's MMU\n"); 352 rc = -EINVAL; 353 goto out; 354 } 355 } else { 356 *usage_cnt = atomic_read(&cb->cs_cnt); 357 } 358 359 out: 360 hl_cb_put(cb); 361 return rc; 362 } 363 364 int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv) 365 { 366 struct hl_fpriv *hpriv = file_priv->driver_priv; 367 struct hl_device *hdev = hpriv->hdev; 368 union hl_cb_args *args = data; 369 u64 handle = 0, device_va = 0; 370 enum hl_device_status status; 371 u32 usage_cnt = 0; 372 int rc; 373 374 if (!hl_device_operational(hdev, &status)) { 375 dev_dbg_ratelimited(hdev->dev, 376 "Device is %s. Can't execute CB IOCTL\n", 377 hdev->status[status]); 378 return -EBUSY; 379 } 380 381 switch (args->in.op) { 382 case HL_CB_OP_CREATE: 383 if (args->in.cb_size > HL_MAX_CB_SIZE) { 384 dev_err(hdev->dev, 385 "User requested CB size %d must be less than %d\n", 386 args->in.cb_size, HL_MAX_CB_SIZE); 387 rc = -EINVAL; 388 } else { 389 rc = hl_cb_create(hdev, &hpriv->mem_mgr, hpriv->ctx, 390 args->in.cb_size, false, 391 !!(args->in.flags & HL_CB_FLAGS_MAP), 392 &handle); 393 } 394 395 memset(args, 0, sizeof(*args)); 396 args->out.cb_handle = handle; 397 break; 398 399 case HL_CB_OP_DESTROY: 400 rc = hl_cb_destroy(&hpriv->mem_mgr, 401 args->in.cb_handle); 402 break; 403 404 case HL_CB_OP_INFO: 405 rc = hl_cb_info(&hpriv->mem_mgr, args->in.cb_handle, 406 args->in.flags, 407 &usage_cnt, 408 &device_va); 409 if (rc) 410 break; 411 412 memset(&args->out, 0, sizeof(args->out)); 413 414 if (args->in.flags & HL_CB_FLAGS_GET_DEVICE_VA) 415 args->out.device_va = device_va; 416 else 417 args->out.usage_cnt = usage_cnt; 418 break; 419 420 default: 421 rc = -EINVAL; 422 break; 423 } 424 425 return rc; 426 } 427 428 struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle) 429 { 430 struct hl_mmap_mem_buf *buf; 431 432 buf = hl_mmap_mem_buf_get(mmg, handle); 433 if (!buf) 434 return NULL; 435 return buf->private; 436 437 } 438 439 void hl_cb_put(struct hl_cb *cb) 440 { 441 hl_mmap_mem_buf_put(cb->buf); 442 } 443 444 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, 445 bool internal_cb) 446 { 447 u64 cb_handle; 448 struct hl_cb *cb; 449 int rc; 450 451 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, cb_size, 452 internal_cb, false, &cb_handle); 453 if (rc) { 454 dev_err(hdev->dev, 455 "Failed to allocate CB for the kernel driver %d\n", rc); 456 return NULL; 457 } 458 459 cb = hl_cb_get(&hdev->kernel_mem_mgr, cb_handle); 460 /* hl_cb_get should never fail here */ 461 if (!cb) { 462 dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", 463 (u32) cb_handle); 464 goto destroy_cb; 465 } 466 467 return cb; 468 469 destroy_cb: 470 hl_cb_destroy(&hdev->kernel_mem_mgr, cb_handle); 471 472 return NULL; 473 } 474 475 int hl_cb_pool_init(struct hl_device *hdev) 476 { 477 struct hl_cb *cb; 478 int i; 479 480 INIT_LIST_HEAD(&hdev->cb_pool); 481 spin_lock_init(&hdev->cb_pool_lock); 482 483 for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) { 484 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size, 485 HL_KERNEL_ASID_ID, false); 486 if (cb) { 487 cb->is_pool = true; 488 list_add(&cb->pool_list, &hdev->cb_pool); 489 } else { 490 hl_cb_pool_fini(hdev); 491 return -ENOMEM; 492 } 493 } 494 495 return 0; 496 } 497 498 int hl_cb_pool_fini(struct hl_device *hdev) 499 { 500 struct hl_cb *cb, *tmp; 501 502 list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) { 503 list_del(&cb->pool_list); 504 cb_fini(hdev, cb); 505 } 506 507 return 0; 508 } 509 510 int hl_cb_va_pool_init(struct hl_ctx *ctx) 511 { 512 struct hl_device *hdev = ctx->hdev; 513 struct asic_fixed_properties *prop = &hdev->asic_prop; 514 int rc; 515 516 if (!hdev->supports_cb_mapping) 517 return 0; 518 519 ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1); 520 if (!ctx->cb_va_pool) { 521 dev_err(hdev->dev, 522 "Failed to create VA gen pool for CB mapping\n"); 523 return -ENOMEM; 524 } 525 526 ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 527 CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 528 if (!ctx->cb_va_pool_base) { 529 rc = -ENOMEM; 530 goto err_pool_destroy; 531 } 532 rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1); 533 if (rc) { 534 dev_err(hdev->dev, 535 "Failed to add memory to VA gen pool for CB mapping\n"); 536 goto err_unreserve_va_block; 537 } 538 539 return 0; 540 541 err_unreserve_va_block: 542 hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 543 err_pool_destroy: 544 gen_pool_destroy(ctx->cb_va_pool); 545 546 return rc; 547 } 548 549 void hl_cb_va_pool_fini(struct hl_ctx *ctx) 550 { 551 struct hl_device *hdev = ctx->hdev; 552 553 if (!hdev->supports_cb_mapping) 554 return; 555 556 gen_pool_destroy(ctx->cb_va_pool); 557 hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 558 } 559