1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4 #include "pvr_cccb.h" 5 #include "pvr_context.h" 6 #include "pvr_device.h" 7 #include "pvr_drv.h" 8 #include "pvr_gem.h" 9 #include "pvr_job.h" 10 #include "pvr_power.h" 11 #include "pvr_rogue_fwif.h" 12 #include "pvr_rogue_fwif_common.h" 13 #include "pvr_rogue_fwif_resetframework.h" 14 #include "pvr_stream.h" 15 #include "pvr_stream_defs.h" 16 #include "pvr_vm.h" 17 18 #include <drm/drm_auth.h> 19 #include <drm/drm_managed.h> 20 21 #include <linux/bug.h> 22 #include <linux/errno.h> 23 #include <linux/kernel.h> 24 #include <linux/list.h> 25 #include <linux/sched.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/types.h> 30 #include <linux/xarray.h> 31 32 static int 33 remap_priority(struct pvr_file *pvr_file, s32 uapi_priority, 34 enum pvr_context_priority *priority_out) 35 { 36 switch (uapi_priority) { 37 case DRM_PVR_CTX_PRIORITY_LOW: 38 *priority_out = PVR_CTX_PRIORITY_LOW; 39 break; 40 case DRM_PVR_CTX_PRIORITY_NORMAL: 41 *priority_out = PVR_CTX_PRIORITY_MEDIUM; 42 break; 43 case DRM_PVR_CTX_PRIORITY_HIGH: 44 if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file))) 45 return -EACCES; 46 *priority_out = PVR_CTX_PRIORITY_HIGH; 47 break; 48 default: 49 return -EINVAL; 50 } 51 52 return 0; 53 } 54 55 static int get_fw_obj_size(enum drm_pvr_ctx_type type) 56 { 57 switch (type) { 58 case DRM_PVR_CTX_TYPE_RENDER: 59 return sizeof(struct rogue_fwif_fwrendercontext); 60 case DRM_PVR_CTX_TYPE_COMPUTE: 61 return sizeof(struct rogue_fwif_fwcomputecontext); 62 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 63 return sizeof(struct rogue_fwif_fwtransfercontext); 64 } 65 66 return -EINVAL; 67 } 68 69 static int 70 process_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs, 71 u64 stream_user_ptr, u32 stream_size, void *dest) 72 { 73 void *stream; 74 int err; 75 76 stream = memdup_user(u64_to_user_ptr(stream_user_ptr), stream_size); 77 if (IS_ERR(stream)) 78 return PTR_ERR(stream); 79 80 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest); 81 82 kfree(stream); 83 84 return err; 85 } 86 87 static int init_render_fw_objs(struct pvr_context *ctx, 88 struct drm_pvr_ioctl_create_context_args *args, 89 void *fw_ctx_map) 90 { 91 struct rogue_fwif_static_rendercontext_state *static_rendercontext_state; 92 struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map; 93 94 if (!args->static_context_state_len) 95 return -EINVAL; 96 97 static_rendercontext_state = &fw_render_context->static_render_context_state; 98 99 /* Copy static render context state from userspace. */ 100 return process_static_context_state(ctx->pvr_dev, 101 &pvr_static_render_context_state_stream, 102 args->static_context_state, 103 args->static_context_state_len, 104 &static_rendercontext_state->ctxswitch_regs[0]); 105 } 106 107 static int init_compute_fw_objs(struct pvr_context *ctx, 108 struct drm_pvr_ioctl_create_context_args *args, 109 void *fw_ctx_map) 110 { 111 struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map; 112 struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs; 113 114 if (!args->static_context_state_len) 115 return -EINVAL; 116 117 ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs; 118 119 /* Copy static render context state from userspace. */ 120 return process_static_context_state(ctx->pvr_dev, 121 &pvr_static_compute_context_state_stream, 122 args->static_context_state, 123 args->static_context_state_len, 124 ctxswitch_regs); 125 } 126 127 static int init_transfer_fw_objs(struct pvr_context *ctx, 128 struct drm_pvr_ioctl_create_context_args *args, 129 void *fw_ctx_map) 130 { 131 if (args->static_context_state_len) 132 return -EINVAL; 133 134 return 0; 135 } 136 137 static int init_fw_objs(struct pvr_context *ctx, 138 struct drm_pvr_ioctl_create_context_args *args, 139 void *fw_ctx_map) 140 { 141 switch (ctx->type) { 142 case DRM_PVR_CTX_TYPE_RENDER: 143 return init_render_fw_objs(ctx, args, fw_ctx_map); 144 case DRM_PVR_CTX_TYPE_COMPUTE: 145 return init_compute_fw_objs(ctx, args, fw_ctx_map); 146 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 147 return init_transfer_fw_objs(ctx, args, fw_ctx_map); 148 } 149 150 return -EINVAL; 151 } 152 153 static void 154 ctx_fw_data_init(void *cpu_ptr, void *priv) 155 { 156 struct pvr_context *ctx = priv; 157 158 memcpy(cpu_ptr, ctx->data, ctx->data_size); 159 } 160 161 /** 162 * pvr_context_destroy_queues() - Destroy all queues attached to a context. 163 * @ctx: Context to destroy queues on. 164 * 165 * Should be called when the last reference to a context object is dropped. 166 * It releases all resources attached to the queues bound to this context. 167 */ 168 static void pvr_context_destroy_queues(struct pvr_context *ctx) 169 { 170 switch (ctx->type) { 171 case DRM_PVR_CTX_TYPE_RENDER: 172 pvr_queue_destroy(ctx->queues.fragment); 173 pvr_queue_destroy(ctx->queues.geometry); 174 break; 175 case DRM_PVR_CTX_TYPE_COMPUTE: 176 pvr_queue_destroy(ctx->queues.compute); 177 break; 178 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 179 pvr_queue_destroy(ctx->queues.transfer); 180 break; 181 } 182 } 183 184 /** 185 * pvr_context_create_queues() - Create all queues attached to a context. 186 * @ctx: Context to create queues on. 187 * @args: Context creation arguments passed by userspace. 188 * @fw_ctx_map: CPU mapping of the FW context object. 189 * 190 * Return: 191 * * 0 on success, or 192 * * A negative error code otherwise. 193 */ 194 static int pvr_context_create_queues(struct pvr_context *ctx, 195 struct drm_pvr_ioctl_create_context_args *args, 196 void *fw_ctx_map) 197 { 198 int err; 199 200 switch (ctx->type) { 201 case DRM_PVR_CTX_TYPE_RENDER: 202 ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY, 203 args, fw_ctx_map); 204 if (IS_ERR(ctx->queues.geometry)) { 205 err = PTR_ERR(ctx->queues.geometry); 206 ctx->queues.geometry = NULL; 207 goto err_destroy_queues; 208 } 209 210 ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT, 211 args, fw_ctx_map); 212 if (IS_ERR(ctx->queues.fragment)) { 213 err = PTR_ERR(ctx->queues.fragment); 214 ctx->queues.fragment = NULL; 215 goto err_destroy_queues; 216 } 217 return 0; 218 219 case DRM_PVR_CTX_TYPE_COMPUTE: 220 ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE, 221 args, fw_ctx_map); 222 if (IS_ERR(ctx->queues.compute)) { 223 err = PTR_ERR(ctx->queues.compute); 224 ctx->queues.compute = NULL; 225 goto err_destroy_queues; 226 } 227 return 0; 228 229 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 230 ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG, 231 args, fw_ctx_map); 232 if (IS_ERR(ctx->queues.transfer)) { 233 err = PTR_ERR(ctx->queues.transfer); 234 ctx->queues.transfer = NULL; 235 goto err_destroy_queues; 236 } 237 return 0; 238 } 239 240 return -EINVAL; 241 242 err_destroy_queues: 243 pvr_context_destroy_queues(ctx); 244 return err; 245 } 246 247 /** 248 * pvr_context_kill_queues() - Kill queues attached to context. 249 * @ctx: Context to kill queues on. 250 * 251 * Killing the queues implies making them unusable for future jobs, while still 252 * letting the currently submitted jobs a chance to finish. Queue resources will 253 * stay around until pvr_context_destroy_queues() is called. 254 */ 255 static void pvr_context_kill_queues(struct pvr_context *ctx) 256 { 257 switch (ctx->type) { 258 case DRM_PVR_CTX_TYPE_RENDER: 259 pvr_queue_kill(ctx->queues.fragment); 260 pvr_queue_kill(ctx->queues.geometry); 261 break; 262 case DRM_PVR_CTX_TYPE_COMPUTE: 263 pvr_queue_kill(ctx->queues.compute); 264 break; 265 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 266 pvr_queue_kill(ctx->queues.transfer); 267 break; 268 } 269 } 270 271 /** 272 * pvr_context_create() - Create a context. 273 * @pvr_file: File to attach the created context to. 274 * @args: Context creation arguments. 275 * 276 * Return: 277 * * 0 on success, or 278 * * A negative error code on failure. 279 */ 280 int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args) 281 { 282 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 283 struct pvr_context *ctx; 284 int ctx_size; 285 int err; 286 287 /* Context creation flags are currently unused and must be zero. */ 288 if (args->flags) 289 return -EINVAL; 290 291 ctx_size = get_fw_obj_size(args->type); 292 if (ctx_size < 0) 293 return ctx_size; 294 295 ctx = kzalloc_obj(*ctx); 296 if (!ctx) 297 return -ENOMEM; 298 299 ctx->data_size = ctx_size; 300 ctx->type = args->type; 301 ctx->flags = args->flags; 302 ctx->pvr_dev = pvr_dev; 303 kref_init(&ctx->ref_count); 304 305 err = remap_priority(pvr_file, args->priority, &ctx->priority); 306 if (err) 307 goto err_free_ctx; 308 309 ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); 310 if (IS_ERR(ctx->vm_ctx)) { 311 err = PTR_ERR(ctx->vm_ctx); 312 goto err_free_ctx; 313 } 314 315 ctx->data = kzalloc(ctx_size, GFP_KERNEL); 316 if (!ctx->data) { 317 err = -ENOMEM; 318 goto err_put_vm; 319 } 320 321 err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); 322 if (err) 323 goto err_free_ctx_data; 324 325 err = pvr_context_create_queues(ctx, args, ctx->data); 326 if (err) 327 goto err_free_ctx_id; 328 329 err = init_fw_objs(ctx, args, ctx->data); 330 if (err) 331 goto err_destroy_queues; 332 333 err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 334 ctx_fw_data_init, ctx, &ctx->fw_obj); 335 if (err) 336 goto err_destroy_queues; 337 338 err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); 339 if (err) 340 goto err_destroy_fw_obj; 341 342 spin_lock(&pvr_dev->ctx_list_lock); 343 list_add_tail(&ctx->file_link, &pvr_file->contexts); 344 spin_unlock(&pvr_dev->ctx_list_lock); 345 346 return 0; 347 348 err_destroy_fw_obj: 349 pvr_fw_object_destroy(ctx->fw_obj); 350 351 err_destroy_queues: 352 pvr_context_destroy_queues(ctx); 353 354 err_free_ctx_id: 355 /* 356 * Ctx_id is not exposed to userspace and not visible yet within 357 * the kernel/FW, plus a matching context handle (exposed to userspace) 358 * hasn't been allocated yet, so it is safe to remove ctx_id 359 * from the ctx_ids xarray. 360 */ 361 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 362 363 err_free_ctx_data: 364 kfree(ctx->data); 365 366 err_put_vm: 367 pvr_vm_context_put(ctx->vm_ctx); 368 369 err_free_ctx: 370 kfree(ctx); 371 return err; 372 } 373 374 static void 375 pvr_context_release(struct kref *ref_count) 376 { 377 struct pvr_context *ctx = 378 container_of(ref_count, struct pvr_context, ref_count); 379 struct pvr_device *pvr_dev = ctx->pvr_dev; 380 381 WARN_ON(in_interrupt()); 382 spin_lock(&pvr_dev->ctx_list_lock); 383 list_del(&ctx->file_link); 384 spin_unlock(&pvr_dev->ctx_list_lock); 385 386 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 387 pvr_context_destroy_queues(ctx); 388 pvr_fw_object_destroy(ctx->fw_obj); 389 kfree(ctx->data); 390 pvr_vm_context_put(ctx->vm_ctx); 391 kfree(ctx); 392 } 393 394 /** 395 * pvr_context_put() - Release reference on context 396 * @ctx: Target context. 397 */ 398 void 399 pvr_context_put(struct pvr_context *ctx) 400 { 401 if (ctx) 402 kref_put(&ctx->ref_count, pvr_context_release); 403 } 404 405 /** 406 * pvr_context_destroy() - Destroy context 407 * @pvr_file: Pointer to pvr_file structure. 408 * @handle: Userspace context handle. 409 * 410 * Removes context from context list and drops initial reference. Context will 411 * then be destroyed once all outstanding references are dropped. 412 * 413 * Return: 414 * * 0 on success, or 415 * * -%EINVAL if context not in context list. 416 */ 417 int 418 pvr_context_destroy(struct pvr_file *pvr_file, u32 handle) 419 { 420 struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle); 421 422 if (!ctx) 423 return -EINVAL; 424 425 /* Make sure nothing can be queued to the queues after that point. */ 426 pvr_context_kill_queues(ctx); 427 428 /* Release the reference held by the handle set. */ 429 pvr_context_put(ctx); 430 431 return 0; 432 } 433 434 /** 435 * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file 436 * @pvr_file: Pointer to pvr_file structure. 437 * 438 * Removes all contexts associated with @pvr_file from the device context list and drops initial 439 * references. Contexts will then be destroyed once all outstanding references are dropped. 440 */ 441 void pvr_destroy_contexts_for_file(struct pvr_file *pvr_file) 442 { 443 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 444 struct pvr_context *ctx; 445 unsigned long handle; 446 447 xa_for_each(&pvr_file->ctx_handles, handle, ctx) 448 pvr_context_destroy(pvr_file, handle); 449 450 spin_lock(&pvr_dev->ctx_list_lock); 451 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 452 453 while (!list_entry_is_head(ctx, &pvr_file->contexts, file_link)) { 454 list_del_init(&ctx->file_link); 455 456 if (pvr_context_get_if_referenced(ctx)) { 457 spin_unlock(&pvr_dev->ctx_list_lock); 458 459 pvr_vm_unmap_all(ctx->vm_ctx); 460 461 pvr_context_put(ctx); 462 spin_lock(&pvr_dev->ctx_list_lock); 463 } 464 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link); 465 } 466 spin_unlock(&pvr_dev->ctx_list_lock); 467 } 468 469 /** 470 * pvr_context_device_init() - Device level initialization for queue related resources. 471 * @pvr_dev: The device to initialize. 472 */ 473 void pvr_context_device_init(struct pvr_device *pvr_dev) 474 { 475 xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1); 476 spin_lock_init(&pvr_dev->ctx_list_lock); 477 } 478 479 /** 480 * pvr_context_device_fini() - Device level cleanup for queue related resources. 481 * @pvr_dev: The device to cleanup. 482 */ 483 void pvr_context_device_fini(struct pvr_device *pvr_dev) 484 { 485 WARN_ON(!xa_empty(&pvr_dev->ctx_ids)); 486 xa_destroy(&pvr_dev->ctx_ids); 487 } 488