1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4 #include "pvr_cccb.h" 5 #include "pvr_context.h" 6 #include "pvr_device.h" 7 #include "pvr_drv.h" 8 #include "pvr_gem.h" 9 #include "pvr_job.h" 10 #include "pvr_power.h" 11 #include "pvr_rogue_fwif.h" 12 #include "pvr_rogue_fwif_common.h" 13 #include "pvr_rogue_fwif_resetframework.h" 14 #include "pvr_stream.h" 15 #include "pvr_stream_defs.h" 16 #include "pvr_vm.h" 17 18 #include <drm/drm_auth.h> 19 #include <drm/drm_managed.h> 20 21 #include <linux/bug.h> 22 #include <linux/errno.h> 23 #include <linux/kernel.h> 24 #include <linux/list.h> 25 #include <linux/sched.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/types.h> 30 #include <linux/xarray.h> 31 32 static int 33 remap_priority(struct pvr_file *pvr_file, s32 uapi_priority, 34 enum pvr_context_priority *priority_out) 35 { 36 switch (uapi_priority) { 37 case DRM_PVR_CTX_PRIORITY_LOW: 38 *priority_out = PVR_CTX_PRIORITY_LOW; 39 break; 40 case DRM_PVR_CTX_PRIORITY_NORMAL: 41 *priority_out = PVR_CTX_PRIORITY_MEDIUM; 42 break; 43 case DRM_PVR_CTX_PRIORITY_HIGH: 44 if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file))) 45 return -EACCES; 46 *priority_out = PVR_CTX_PRIORITY_HIGH; 47 break; 48 default: 49 return -EINVAL; 50 } 51 52 return 0; 53 } 54 55 static int get_fw_obj_size(enum drm_pvr_ctx_type type) 56 { 57 switch (type) { 58 case DRM_PVR_CTX_TYPE_RENDER: 59 return sizeof(struct rogue_fwif_fwrendercontext); 60 case DRM_PVR_CTX_TYPE_COMPUTE: 61 return sizeof(struct rogue_fwif_fwcomputecontext); 62 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 63 return sizeof(struct rogue_fwif_fwtransfercontext); 64 } 65 66 return -EINVAL; 67 } 68 69 static int 70 process_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs, 71 u64 stream_user_ptr, u32 stream_size, void *dest) 72 { 73 void *stream; 74 int err; 75 76 stream = memdup_user(u64_to_user_ptr(stream_user_ptr), stream_size); 77 if (IS_ERR(stream)) 78 return PTR_ERR(stream); 79 80 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest); 81 82 kfree(stream); 83 84 return err; 85 } 86 87 static int init_render_fw_objs(struct pvr_context *ctx, 88 struct drm_pvr_ioctl_create_context_args *args, 89 void *fw_ctx_map) 90 { 91 struct rogue_fwif_static_rendercontext_state *static_rendercontext_state; 92 struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map; 93 94 if (!args->static_context_state_len) 95 return -EINVAL; 96 97 static_rendercontext_state = &fw_render_context->static_render_context_state; 98 99 /* Copy static render context state from userspace. */ 100 return process_static_context_state(ctx->pvr_dev, 101 &pvr_static_render_context_state_stream, 102 args->static_context_state, 103 args->static_context_state_len, 104 &static_rendercontext_state->ctxswitch_regs[0]); 105 } 106 107 static int init_compute_fw_objs(struct pvr_context *ctx, 108 struct drm_pvr_ioctl_create_context_args *args, 109 void *fw_ctx_map) 110 { 111 struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map; 112 struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs; 113 114 if (!args->static_context_state_len) 115 return -EINVAL; 116 117 ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs; 118 119 /* Copy static render context state from userspace. */ 120 return process_static_context_state(ctx->pvr_dev, 121 &pvr_static_compute_context_state_stream, 122 args->static_context_state, 123 args->static_context_state_len, 124 ctxswitch_regs); 125 } 126 127 static int init_transfer_fw_objs(struct pvr_context *ctx, 128 struct drm_pvr_ioctl_create_context_args *args, 129 void *fw_ctx_map) 130 { 131 if (args->static_context_state_len) 132 return -EINVAL; 133 134 return 0; 135 } 136 137 static int init_fw_objs(struct pvr_context *ctx, 138 struct drm_pvr_ioctl_create_context_args *args, 139 void *fw_ctx_map) 140 { 141 switch (ctx->type) { 142 case DRM_PVR_CTX_TYPE_RENDER: 143 return init_render_fw_objs(ctx, args, fw_ctx_map); 144 case DRM_PVR_CTX_TYPE_COMPUTE: 145 return init_compute_fw_objs(ctx, args, fw_ctx_map); 146 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 147 return init_transfer_fw_objs(ctx, args, fw_ctx_map); 148 } 149 150 return -EINVAL; 151 } 152 153 static void 154 ctx_fw_data_init(void *cpu_ptr, void *priv) 155 { 156 struct pvr_context *ctx = priv; 157 158 memcpy(cpu_ptr, ctx->data, ctx->data_size); 159 } 160 161 /** 162 * pvr_context_destroy_queues() - Destroy all queues attached to a context. 163 * @ctx: Context to destroy queues on. 164 * @cleanup_queue_entity: Whether to cleanup the queue entity e.g. context 165 * creation failure path. 166 * 167 * Should be called when the last reference to a context object is dropped. 168 * It releases all resources attached to the queues bound to this context. 169 */ 170 static void pvr_context_destroy_queues(struct pvr_context *ctx, bool cleanup_queue_entity) 171 { 172 switch (ctx->type) { 173 case DRM_PVR_CTX_TYPE_RENDER: 174 pvr_queue_destroy(ctx->queues.fragment, cleanup_queue_entity); 175 pvr_queue_destroy(ctx->queues.geometry, cleanup_queue_entity); 176 break; 177 case DRM_PVR_CTX_TYPE_COMPUTE: 178 pvr_queue_destroy(ctx->queues.compute, cleanup_queue_entity); 179 break; 180 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 181 pvr_queue_destroy(ctx->queues.transfer, cleanup_queue_entity); 182 break; 183 } 184 } 185 186 /** 187 * pvr_context_create_queues() - Create all queues attached to a context. 188 * @ctx: Context to create queues on. 189 * @args: Context creation arguments passed by userspace. 190 * @fw_ctx_map: CPU mapping of the FW context object. 191 * 192 * Return: 193 * * 0 on success, or 194 * * A negative error code otherwise. 195 */ 196 static int pvr_context_create_queues(struct pvr_context *ctx, 197 struct drm_pvr_ioctl_create_context_args *args, 198 void *fw_ctx_map) 199 { 200 int err; 201 202 switch (ctx->type) { 203 case DRM_PVR_CTX_TYPE_RENDER: 204 ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY, 205 args, fw_ctx_map); 206 if (IS_ERR(ctx->queues.geometry)) { 207 err = PTR_ERR(ctx->queues.geometry); 208 ctx->queues.geometry = NULL; 209 goto err_destroy_queues; 210 } 211 212 ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT, 213 args, fw_ctx_map); 214 if (IS_ERR(ctx->queues.fragment)) { 215 err = PTR_ERR(ctx->queues.fragment); 216 ctx->queues.fragment = NULL; 217 goto err_destroy_queues; 218 } 219 return 0; 220 221 case DRM_PVR_CTX_TYPE_COMPUTE: 222 ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE, 223 args, fw_ctx_map); 224 if (IS_ERR(ctx->queues.compute)) { 225 err = PTR_ERR(ctx->queues.compute); 226 ctx->queues.compute = NULL; 227 goto err_destroy_queues; 228 } 229 return 0; 230 231 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 232 ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG, 233 args, fw_ctx_map); 234 if (IS_ERR(ctx->queues.transfer)) { 235 err = PTR_ERR(ctx->queues.transfer); 236 ctx->queues.transfer = NULL; 237 goto err_destroy_queues; 238 } 239 return 0; 240 } 241 242 return -EINVAL; 243 244 err_destroy_queues: 245 pvr_context_destroy_queues(ctx, true); 246 return err; 247 } 248 249 /** 250 * pvr_context_kill_queues() - Kill queues attached to context. 251 * @ctx: Context to kill queues on. 252 * 253 * Killing the queues implies making them unusable for future jobs, while still 254 * letting the currently submitted jobs a chance to finish. Queue resources will 255 * stay around until pvr_context_destroy_queues() is called. 256 */ 257 static void pvr_context_kill_queues(struct pvr_context *ctx) 258 { 259 switch (ctx->type) { 260 case DRM_PVR_CTX_TYPE_RENDER: 261 pvr_queue_kill(ctx->queues.fragment); 262 pvr_queue_kill(ctx->queues.geometry); 263 break; 264 case DRM_PVR_CTX_TYPE_COMPUTE: 265 pvr_queue_kill(ctx->queues.compute); 266 break; 267 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 268 pvr_queue_kill(ctx->queues.transfer); 269 break; 270 } 271 } 272 273 /** 274 * pvr_context_create() - Create a context. 275 * @pvr_file: File to attach the created context to. 276 * @args: Context creation arguments. 277 * 278 * Return: 279 * * 0 on success, or 280 * * A negative error code on failure. 281 */ 282 int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args) 283 { 284 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 285 struct pvr_context *ctx; 286 int ctx_size; 287 int err; 288 289 /* Context creation flags are currently unused and must be zero. */ 290 if (args->flags) 291 return -EINVAL; 292 293 ctx_size = get_fw_obj_size(args->type); 294 if (ctx_size < 0) 295 return ctx_size; 296 297 ctx = kzalloc_obj(*ctx); 298 if (!ctx) 299 return -ENOMEM; 300 301 ctx->data_size = ctx_size; 302 ctx->type = args->type; 303 ctx->flags = args->flags; 304 ctx->pvr_dev = pvr_dev; 305 kref_init(&ctx->ref_count); 306 307 err = remap_priority(pvr_file, args->priority, &ctx->priority); 308 if (err) 309 goto err_free_ctx; 310 311 ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); 312 if (!ctx->vm_ctx) { 313 err = -EINVAL; 314 goto err_free_ctx; 315 } 316 317 ctx->data = kzalloc(ctx_size, GFP_KERNEL); 318 if (!ctx->data) { 319 err = -ENOMEM; 320 goto err_put_vm; 321 } 322 323 err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); 324 if (err) 325 goto err_free_ctx_data; 326 327 err = pvr_context_create_queues(ctx, args, ctx->data); 328 if (err) 329 goto err_free_ctx_id; 330 331 err = init_fw_objs(ctx, args, ctx->data); 332 if (err) 333 goto err_destroy_queues; 334 335 err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 336 ctx_fw_data_init, ctx, &ctx->fw_obj); 337 if (err) 338 goto err_destroy_queues; 339 340 err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); 341 if (err) 342 goto err_destroy_fw_obj; 343 344 spin_lock(&pvr_dev->ctx_list_lock); 345 list_add_tail(&ctx->file_link, &pvr_file->contexts); 346 spin_unlock(&pvr_dev->ctx_list_lock); 347 348 return 0; 349 350 err_destroy_fw_obj: 351 pvr_fw_object_destroy(ctx->fw_obj); 352 353 err_destroy_queues: 354 pvr_context_destroy_queues(ctx, true); 355 356 err_free_ctx_id: 357 /* 358 * Ctx_id is not exposed to userspace and not visible yet within 359 * the kernel/FW, plus a matching context handle (exposed to userspace) 360 * hasn't been allocated yet, so it is safe to remove ctx_id 361 * from the ctx_ids xarray. 362 */ 363 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 364 365 err_free_ctx_data: 366 kfree(ctx->data); 367 368 err_put_vm: 369 pvr_vm_context_put(ctx->vm_ctx); 370 371 err_free_ctx: 372 kfree(ctx); 373 return err; 374 } 375 376 static void 377 pvr_context_release(struct kref *ref_count) 378 { 379 struct pvr_context *ctx = 380 container_of(ref_count, struct pvr_context, ref_count); 381 struct pvr_device *pvr_dev = ctx->pvr_dev; 382 383 WARN_ON(in_interrupt()); 384 spin_lock(&pvr_dev->ctx_list_lock); 385 list_del(&ctx->file_link); 386 spin_unlock(&pvr_dev->ctx_list_lock); 387 388 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 389 pvr_context_destroy_queues(ctx, false); 390 pvr_fw_object_destroy(ctx->fw_obj); 391 kfree(ctx->data); 392 pvr_vm_context_put(ctx->vm_ctx); 393 kfree(ctx); 394 } 395 396 /** 397 * pvr_context_put() - Release reference on context 398 * @ctx: Target context. 399 */ 400 void 401 pvr_context_put(struct pvr_context *ctx) 402 { 403 if (ctx) 404 kref_put(&ctx->ref_count, pvr_context_release); 405 } 406 407 /** 408 * pvr_context_destroy() - Destroy context 409 * @pvr_file: Pointer to pvr_file structure. 410 * @handle: Userspace context handle. 411 * 412 * Removes context from context list and drops initial reference. Context will 413 * then be destroyed once all outstanding references are dropped. 414 * 415 * Return: 416 * * 0 on success, or 417 * * -%EINVAL if context not in context list. 418 */ 419 int 420 pvr_context_destroy(struct pvr_file *pvr_file, u32 handle) 421 { 422 struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle); 423 424 if (!ctx) 425 return -EINVAL; 426 427 /* Make sure nothing can be queued to the queues after that point. */ 428 pvr_context_kill_queues(ctx); 429 430 /* Release the reference held by the handle set. */ 431 pvr_context_put(ctx); 432 433 return 0; 434 } 435 436 /** 437 * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file 438 * @pvr_file: Pointer to pvr_file structure. 439 * 440 * Removes all contexts associated with @pvr_file from the device context list and drops initial 441 * references. Contexts will then be destroyed once all outstanding references are dropped. 442 */ 443 void pvr_destroy_contexts_for_file(struct pvr_file *pvr_file) 444 { 445 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 446 struct pvr_context *ctx; 447 unsigned long handle; 448 449 xa_for_each(&pvr_file->ctx_handles, handle, ctx) 450 pvr_context_destroy(pvr_file, handle); 451 452 spin_lock(&pvr_dev->ctx_list_lock); 453 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 454 455 while (!list_entry_is_head(ctx, &pvr_file->contexts, file_link)) { 456 list_del_init(&ctx->file_link); 457 458 if (pvr_context_get_if_referenced(ctx)) { 459 spin_unlock(&pvr_dev->ctx_list_lock); 460 461 pvr_vm_unmap_all(ctx->vm_ctx); 462 463 pvr_context_put(ctx); 464 spin_lock(&pvr_dev->ctx_list_lock); 465 } 466 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 467 } 468 spin_unlock(&pvr_dev->ctx_list_lock); 469 } 470 471 /** 472 * pvr_context_device_init() - Device level initialization for queue related resources. 473 * @pvr_dev: The device to initialize. 474 */ 475 void pvr_context_device_init(struct pvr_device *pvr_dev) 476 { 477 xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1); 478 spin_lock_init(&pvr_dev->ctx_list_lock); 479 } 480 481 /** 482 * pvr_context_device_fini() - Device level cleanup for queue related resources. 483 * @pvr_dev: The device to cleanup. 484 */ 485 void pvr_context_device_fini(struct pvr_device *pvr_dev) 486 { 487 WARN_ON(!xa_empty(&pvr_dev->ctx_ids)); 488 xa_destroy(&pvr_dev->ctx_ids); 489 } 490