1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4 #include "pvr_cccb.h" 5 #include "pvr_context.h" 6 #include "pvr_device.h" 7 #include "pvr_drv.h" 8 #include "pvr_gem.h" 9 #include "pvr_job.h" 10 #include "pvr_power.h" 11 #include "pvr_rogue_fwif.h" 12 #include "pvr_rogue_fwif_common.h" 13 #include "pvr_rogue_fwif_resetframework.h" 14 #include "pvr_stream.h" 15 #include "pvr_stream_defs.h" 16 #include "pvr_vm.h" 17 18 #include <drm/drm_auth.h> 19 #include <drm/drm_managed.h> 20 21 #include <linux/bug.h> 22 #include <linux/errno.h> 23 #include <linux/kernel.h> 24 #include <linux/list.h> 25 #include <linux/sched.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/types.h> 30 #include <linux/xarray.h> 31 32 static int 33 remap_priority(struct pvr_file *pvr_file, s32 uapi_priority, 34 enum pvr_context_priority *priority_out) 35 { 36 switch (uapi_priority) { 37 case DRM_PVR_CTX_PRIORITY_LOW: 38 *priority_out = PVR_CTX_PRIORITY_LOW; 39 break; 40 case DRM_PVR_CTX_PRIORITY_NORMAL: 41 *priority_out = PVR_CTX_PRIORITY_MEDIUM; 42 break; 43 case DRM_PVR_CTX_PRIORITY_HIGH: 44 if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file))) 45 return -EACCES; 46 *priority_out = PVR_CTX_PRIORITY_HIGH; 47 break; 48 default: 49 return -EINVAL; 50 } 51 52 return 0; 53 } 54 55 static int get_fw_obj_size(enum drm_pvr_ctx_type type) 56 { 57 switch (type) { 58 case DRM_PVR_CTX_TYPE_RENDER: 59 return sizeof(struct rogue_fwif_fwrendercontext); 60 case DRM_PVR_CTX_TYPE_COMPUTE: 61 return sizeof(struct rogue_fwif_fwcomputecontext); 62 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 63 return sizeof(struct rogue_fwif_fwtransfercontext); 64 } 65 66 return -EINVAL; 67 } 68 69 static int 70 process_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs, 71 u64 stream_user_ptr, u32 stream_size, void *dest) 72 { 73 void *stream; 74 int err; 75 76 stream = kzalloc(stream_size, GFP_KERNEL); 77 if (!stream) 78 return -ENOMEM; 79 80 if (copy_from_user(stream, u64_to_user_ptr(stream_user_ptr), stream_size)) { 81 err = -EFAULT; 82 goto err_free; 83 } 84 85 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest); 86 if (err) 87 goto err_free; 88 89 kfree(stream); 90 91 return 0; 92 93 err_free: 94 kfree(stream); 95 96 return err; 97 } 98 99 static int init_render_fw_objs(struct pvr_context *ctx, 100 struct drm_pvr_ioctl_create_context_args *args, 101 void *fw_ctx_map) 102 { 103 struct rogue_fwif_static_rendercontext_state *static_rendercontext_state; 104 struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map; 105 106 if (!args->static_context_state_len) 107 return -EINVAL; 108 109 static_rendercontext_state = &fw_render_context->static_render_context_state; 110 111 /* Copy static render context state from userspace. */ 112 return process_static_context_state(ctx->pvr_dev, 113 &pvr_static_render_context_state_stream, 114 args->static_context_state, 115 args->static_context_state_len, 116 &static_rendercontext_state->ctxswitch_regs[0]); 117 } 118 119 static int init_compute_fw_objs(struct pvr_context *ctx, 120 struct drm_pvr_ioctl_create_context_args *args, 121 void *fw_ctx_map) 122 { 123 struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map; 124 struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs; 125 126 if (!args->static_context_state_len) 127 return -EINVAL; 128 129 ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs; 130 131 /* Copy static render context state from userspace. */ 132 return process_static_context_state(ctx->pvr_dev, 133 &pvr_static_compute_context_state_stream, 134 args->static_context_state, 135 args->static_context_state_len, 136 ctxswitch_regs); 137 } 138 139 static int init_transfer_fw_objs(struct pvr_context *ctx, 140 struct drm_pvr_ioctl_create_context_args *args, 141 void *fw_ctx_map) 142 { 143 if (args->static_context_state_len) 144 return -EINVAL; 145 146 return 0; 147 } 148 149 static int init_fw_objs(struct pvr_context *ctx, 150 struct drm_pvr_ioctl_create_context_args *args, 151 void *fw_ctx_map) 152 { 153 switch (ctx->type) { 154 case DRM_PVR_CTX_TYPE_RENDER: 155 return init_render_fw_objs(ctx, args, fw_ctx_map); 156 case DRM_PVR_CTX_TYPE_COMPUTE: 157 return init_compute_fw_objs(ctx, args, fw_ctx_map); 158 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 159 return init_transfer_fw_objs(ctx, args, fw_ctx_map); 160 } 161 162 return -EINVAL; 163 } 164 165 static void 166 ctx_fw_data_init(void *cpu_ptr, void *priv) 167 { 168 struct pvr_context *ctx = priv; 169 170 memcpy(cpu_ptr, ctx->data, ctx->data_size); 171 } 172 173 /** 174 * pvr_context_destroy_queues() - Destroy all queues attached to a context. 175 * @ctx: Context to destroy queues on. 176 * 177 * Should be called when the last reference to a context object is dropped. 178 * It releases all resources attached to the queues bound to this context. 179 */ 180 static void pvr_context_destroy_queues(struct pvr_context *ctx) 181 { 182 switch (ctx->type) { 183 case DRM_PVR_CTX_TYPE_RENDER: 184 pvr_queue_destroy(ctx->queues.fragment); 185 pvr_queue_destroy(ctx->queues.geometry); 186 break; 187 case DRM_PVR_CTX_TYPE_COMPUTE: 188 pvr_queue_destroy(ctx->queues.compute); 189 break; 190 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 191 pvr_queue_destroy(ctx->queues.transfer); 192 break; 193 } 194 } 195 196 /** 197 * pvr_context_create_queues() - Create all queues attached to a context. 198 * @ctx: Context to create queues on. 199 * @args: Context creation arguments passed by userspace. 200 * @fw_ctx_map: CPU mapping of the FW context object. 201 * 202 * Return: 203 * * 0 on success, or 204 * * A negative error code otherwise. 205 */ 206 static int pvr_context_create_queues(struct pvr_context *ctx, 207 struct drm_pvr_ioctl_create_context_args *args, 208 void *fw_ctx_map) 209 { 210 int err; 211 212 switch (ctx->type) { 213 case DRM_PVR_CTX_TYPE_RENDER: 214 ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY, 215 args, fw_ctx_map); 216 if (IS_ERR(ctx->queues.geometry)) { 217 err = PTR_ERR(ctx->queues.geometry); 218 ctx->queues.geometry = NULL; 219 goto err_destroy_queues; 220 } 221 222 ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT, 223 args, fw_ctx_map); 224 if (IS_ERR(ctx->queues.fragment)) { 225 err = PTR_ERR(ctx->queues.fragment); 226 ctx->queues.fragment = NULL; 227 goto err_destroy_queues; 228 } 229 return 0; 230 231 case DRM_PVR_CTX_TYPE_COMPUTE: 232 ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE, 233 args, fw_ctx_map); 234 if (IS_ERR(ctx->queues.compute)) { 235 err = PTR_ERR(ctx->queues.compute); 236 ctx->queues.compute = NULL; 237 goto err_destroy_queues; 238 } 239 return 0; 240 241 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 242 ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG, 243 args, fw_ctx_map); 244 if (IS_ERR(ctx->queues.transfer)) { 245 err = PTR_ERR(ctx->queues.transfer); 246 ctx->queues.transfer = NULL; 247 goto err_destroy_queues; 248 } 249 return 0; 250 } 251 252 return -EINVAL; 253 254 err_destroy_queues: 255 pvr_context_destroy_queues(ctx); 256 return err; 257 } 258 259 /** 260 * pvr_context_kill_queues() - Kill queues attached to context. 261 * @ctx: Context to kill queues on. 262 * 263 * Killing the queues implies making them unusable for future jobs, while still 264 * letting the currently submitted jobs a chance to finish. Queue resources will 265 * stay around until pvr_context_destroy_queues() is called. 266 */ 267 static void pvr_context_kill_queues(struct pvr_context *ctx) 268 { 269 switch (ctx->type) { 270 case DRM_PVR_CTX_TYPE_RENDER: 271 pvr_queue_kill(ctx->queues.fragment); 272 pvr_queue_kill(ctx->queues.geometry); 273 break; 274 case DRM_PVR_CTX_TYPE_COMPUTE: 275 pvr_queue_kill(ctx->queues.compute); 276 break; 277 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 278 pvr_queue_kill(ctx->queues.transfer); 279 break; 280 } 281 } 282 283 /** 284 * pvr_context_create() - Create a context. 285 * @pvr_file: File to attach the created context to. 286 * @args: Context creation arguments. 287 * 288 * Return: 289 * * 0 on success, or 290 * * A negative error code on failure. 291 */ 292 int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args) 293 { 294 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 295 struct pvr_context *ctx; 296 int ctx_size; 297 int err; 298 299 /* Context creation flags are currently unused and must be zero. */ 300 if (args->flags) 301 return -EINVAL; 302 303 ctx_size = get_fw_obj_size(args->type); 304 if (ctx_size < 0) 305 return ctx_size; 306 307 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 308 if (!ctx) 309 return -ENOMEM; 310 311 ctx->data_size = ctx_size; 312 ctx->type = args->type; 313 ctx->flags = args->flags; 314 ctx->pvr_dev = pvr_dev; 315 kref_init(&ctx->ref_count); 316 317 err = remap_priority(pvr_file, args->priority, &ctx->priority); 318 if (err) 319 goto err_free_ctx; 320 321 ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); 322 if (IS_ERR(ctx->vm_ctx)) { 323 err = PTR_ERR(ctx->vm_ctx); 324 goto err_free_ctx; 325 } 326 327 ctx->data = kzalloc(ctx_size, GFP_KERNEL); 328 if (!ctx->data) { 329 err = -ENOMEM; 330 goto err_put_vm; 331 } 332 333 err = pvr_context_create_queues(ctx, args, ctx->data); 334 if (err) 335 goto err_free_ctx_data; 336 337 err = init_fw_objs(ctx, args, ctx->data); 338 if (err) 339 goto err_destroy_queues; 340 341 err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 342 ctx_fw_data_init, ctx, &ctx->fw_obj); 343 if (err) 344 goto err_free_ctx_data; 345 346 err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); 347 if (err) 348 goto err_destroy_fw_obj; 349 350 err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); 351 if (err) { 352 /* 353 * It's possible that another thread could have taken a reference on the context at 354 * this point as it is in the ctx_ids xarray. Therefore instead of directly 355 * destroying the context, drop a reference instead. 356 */ 357 pvr_context_put(ctx); 358 return err; 359 } 360 361 spin_lock(&pvr_dev->ctx_list_lock); 362 list_add_tail(&ctx->file_link, &pvr_file->contexts); 363 spin_unlock(&pvr_dev->ctx_list_lock); 364 365 return 0; 366 367 err_destroy_fw_obj: 368 pvr_fw_object_destroy(ctx->fw_obj); 369 370 err_destroy_queues: 371 pvr_context_destroy_queues(ctx); 372 373 err_free_ctx_data: 374 kfree(ctx->data); 375 376 err_put_vm: 377 pvr_vm_context_put(ctx->vm_ctx); 378 379 err_free_ctx: 380 kfree(ctx); 381 return err; 382 } 383 384 static void 385 pvr_context_release(struct kref *ref_count) 386 { 387 struct pvr_context *ctx = 388 container_of(ref_count, struct pvr_context, ref_count); 389 struct pvr_device *pvr_dev = ctx->pvr_dev; 390 391 WARN_ON(in_interrupt()); 392 spin_lock(&pvr_dev->ctx_list_lock); 393 list_del(&ctx->file_link); 394 spin_unlock(&pvr_dev->ctx_list_lock); 395 396 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 397 pvr_context_destroy_queues(ctx); 398 pvr_fw_object_destroy(ctx->fw_obj); 399 kfree(ctx->data); 400 pvr_vm_context_put(ctx->vm_ctx); 401 kfree(ctx); 402 } 403 404 /** 405 * pvr_context_put() - Release reference on context 406 * @ctx: Target context. 407 */ 408 void 409 pvr_context_put(struct pvr_context *ctx) 410 { 411 if (ctx) 412 kref_put(&ctx->ref_count, pvr_context_release); 413 } 414 415 /** 416 * pvr_context_destroy() - Destroy context 417 * @pvr_file: Pointer to pvr_file structure. 418 * @handle: Userspace context handle. 419 * 420 * Removes context from context list and drops initial reference. Context will 421 * then be destroyed once all outstanding references are dropped. 422 * 423 * Return: 424 * * 0 on success, or 425 * * -%EINVAL if context not in context list. 426 */ 427 int 428 pvr_context_destroy(struct pvr_file *pvr_file, u32 handle) 429 { 430 struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle); 431 432 if (!ctx) 433 return -EINVAL; 434 435 /* Make sure nothing can be queued to the queues after that point. */ 436 pvr_context_kill_queues(ctx); 437 438 /* Release the reference held by the handle set. */ 439 pvr_context_put(ctx); 440 441 return 0; 442 } 443 444 /** 445 * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file 446 * @pvr_file: Pointer to pvr_file structure. 447 * 448 * Removes all contexts associated with @pvr_file from the device context list and drops initial 449 * references. Contexts will then be destroyed once all outstanding references are dropped. 450 */ 451 void pvr_destroy_contexts_for_file(struct pvr_file *pvr_file) 452 { 453 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 454 struct pvr_context *ctx; 455 unsigned long handle; 456 457 xa_for_each(&pvr_file->ctx_handles, handle, ctx) 458 pvr_context_destroy(pvr_file, handle); 459 460 spin_lock(&pvr_dev->ctx_list_lock); 461 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 462 463 while (!list_entry_is_head(ctx, &pvr_file->contexts, file_link)) { 464 list_del_init(&ctx->file_link); 465 466 if (pvr_context_get_if_referenced(ctx)) { 467 spin_unlock(&pvr_dev->ctx_list_lock); 468 469 pvr_vm_unmap_all(ctx->vm_ctx); 470 471 pvr_context_put(ctx); 472 spin_lock(&pvr_dev->ctx_list_lock); 473 } 474 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 475 } 476 spin_unlock(&pvr_dev->ctx_list_lock); 477 } 478 479 /** 480 * pvr_context_device_init() - Device level initialization for queue related resources. 481 * @pvr_dev: The device to initialize. 482 */ 483 void pvr_context_device_init(struct pvr_device *pvr_dev) 484 { 485 xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1); 486 spin_lock_init(&pvr_dev->ctx_list_lock); 487 } 488 489 /** 490 * pvr_context_device_fini() - Device level cleanup for queue related resources. 491 * @pvr_dev: The device to cleanup. 492 */ 493 void pvr_context_device_fini(struct pvr_device *pvr_dev) 494 { 495 WARN_ON(!xa_empty(&pvr_dev->ctx_ids)); 496 xa_destroy(&pvr_dev->ctx_ids); 497 } 498