1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4 #include "pvr_free_list.h" 5 #include "pvr_hwrt.h" 6 #include "pvr_gem.h" 7 #include "pvr_rogue_cr_defs_client.h" 8 #include "pvr_rogue_fwif.h" 9 10 #include <drm/drm_gem.h> 11 #include <linux/bitops.h> 12 #include <linux/math.h> 13 #include <linux/slab.h> 14 #include <linux/xarray.h> 15 #include <uapi/drm/pvr_drm.h> 16 17 static_assert(ROGUE_FWIF_NUM_RTDATAS == 2); 18 static_assert(ROGUE_FWIF_NUM_GEOMDATAS == 1); 19 static_assert(ROGUE_FWIF_NUM_RTDATA_FREELISTS == 2); 20 21 /* 22 * struct pvr_rt_mtile_info - Render target macrotile information 23 */ 24 struct pvr_rt_mtile_info { 25 u32 mtile_x[3]; 26 u32 mtile_y[3]; 27 u32 tile_max_x; 28 u32 tile_max_y; 29 u32 tile_size_x; 30 u32 tile_size_y; 31 u32 num_tiles_x; 32 u32 num_tiles_y; 33 }; 34 35 /* Size of Shadow Render Target Cache entry */ 36 #define SRTC_ENTRY_SIZE sizeof(u32) 37 /* Size of Renders Accumulation Array entry */ 38 #define RAA_ENTRY_SIZE sizeof(u32) 39 40 static int 41 hwrt_init_kernel_structure(struct pvr_file *pvr_file, 42 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 43 struct pvr_hwrt_dataset *hwrt) 44 { 45 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 46 int err; 47 48 hwrt->pvr_dev = pvr_dev; 49 hwrt->max_rts = args->layers; 50 51 /* Get pointers to the free lists */ 52 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 53 hwrt->free_lists[i] = pvr_free_list_lookup(pvr_file, args->free_list_handles[i]); 54 if (!hwrt->free_lists[i]) { 55 err = -EINVAL; 56 goto err_put_free_lists; 57 } 58 } 59 60 if (hwrt->free_lists[ROGUE_FW_LOCAL_FREELIST]->current_pages < 61 pvr_get_free_list_min_pages(pvr_dev)) { 62 err = -EINVAL; 63 goto err_put_free_lists; 64 } 65 66 return 0; 67 68 err_put_free_lists: 69 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 70 pvr_free_list_put(hwrt->free_lists[i]); 71 hwrt->free_lists[i] = NULL; 72 } 73 74 return err; 75 } 76 77 static void 78 hwrt_fini_kernel_structure(struct pvr_hwrt_dataset *hwrt) 79 { 80 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 81 pvr_free_list_put(hwrt->free_lists[i]); 82 hwrt->free_lists[i] = NULL; 83 } 84 } 85 86 static void 87 hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset *hwrt) 88 { 89 pvr_fw_object_destroy(hwrt->common_fw_obj); 90 } 91 92 static int 93 get_cr_isp_mtile_size_val(struct pvr_device *pvr_dev, u32 samples, 94 struct pvr_rt_mtile_info *info, u32 *value_out) 95 { 96 u32 x = info->mtile_x[0]; 97 u32 y = info->mtile_y[0]; 98 u32 samples_per_pixel; 99 int err; 100 101 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel); 102 if (err) 103 return err; 104 105 if (samples_per_pixel == 1) { 106 if (samples >= 4) 107 x <<= 1; 108 if (samples >= 2) 109 y <<= 1; 110 } else if (samples_per_pixel == 2) { 111 if (samples >= 8) 112 x <<= 1; 113 if (samples >= 4) 114 y <<= 1; 115 } else if (samples_per_pixel == 4) { 116 if (samples >= 8) 117 y <<= 1; 118 } else { 119 WARN(true, "Unsupported ISP samples per pixel value"); 120 return -EINVAL; 121 } 122 123 *value_out = ((x << ROGUE_CR_ISP_MTILE_SIZE_X_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_X_CLRMSK) | 124 ((y << ROGUE_CR_ISP_MTILE_SIZE_Y_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_Y_CLRMSK); 125 126 return 0; 127 } 128 129 static int 130 get_cr_multisamplectl_val(u32 samples, bool y_flip, u64 *value_out) 131 { 132 static const struct { 133 u8 x[8]; 134 u8 y[8]; 135 } sample_positions[4] = { 136 /* 1 sample */ 137 { 138 .x = { 8 }, 139 .y = { 8 }, 140 }, 141 /* 2 samples */ 142 { 143 .x = { 12, 4 }, 144 .y = { 12, 4 }, 145 }, 146 /* 4 samples */ 147 { 148 .x = { 6, 14, 2, 10 }, 149 .y = { 2, 6, 10, 14 }, 150 }, 151 /* 8 samples */ 152 { 153 .x = { 9, 7, 13, 5, 3, 1, 11, 15 }, 154 .y = { 5, 11, 9, 3, 13, 7, 15, 1 }, 155 }, 156 }; 157 const int idx = fls(samples) - 1; 158 u64 value = 0; 159 160 if (idx < 0 || idx > 3) 161 return -EINVAL; 162 163 for (u32 i = 0; i < 8; i++) { 164 value |= ((u64)sample_positions[idx].x[i]) << (i * 8); 165 if (y_flip) 166 value |= (((u64)(16 - sample_positions[idx].y[i]) & 0xf)) << (i * 8 + 4); 167 else 168 value |= ((u64)sample_positions[idx].y[i]) << (i * 8 + 4); 169 } 170 171 *value_out = value; 172 173 return 0; 174 } 175 176 static int 177 get_cr_te_aa_val(struct pvr_device *pvr_dev, u32 samples, u32 *value_out) 178 { 179 u32 samples_per_pixel; 180 u32 value = 0; 181 int err = 0; 182 183 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel); 184 if (err) 185 return err; 186 187 switch (samples_per_pixel) { 188 case 1: 189 if (samples >= 2) 190 value |= ROGUE_CR_TE_AA_Y_EN; 191 if (samples >= 4) 192 value |= ROGUE_CR_TE_AA_X_EN; 193 break; 194 case 2: 195 if (samples >= 2) 196 value |= ROGUE_CR_TE_AA_X2_EN; 197 if (samples >= 4) 198 value |= ROGUE_CR_TE_AA_Y_EN; 199 if (samples >= 8) 200 value |= ROGUE_CR_TE_AA_X_EN; 201 break; 202 case 4: 203 if (samples >= 2) 204 value |= ROGUE_CR_TE_AA_X2_EN; 205 if (samples >= 4) 206 value |= ROGUE_CR_TE_AA_Y2_EN; 207 if (samples >= 8) 208 value |= ROGUE_CR_TE_AA_Y_EN; 209 break; 210 default: 211 WARN(true, "Unsupported ISP samples per pixel value"); 212 return -EINVAL; 213 } 214 215 *value_out = value; 216 217 return 0; 218 } 219 220 static void 221 hwrtdata_common_init(void *cpu_ptr, void *priv) 222 { 223 struct pvr_hwrt_dataset *hwrt = priv; 224 225 memcpy(cpu_ptr, &hwrt->common, sizeof(hwrt->common)); 226 } 227 228 static int 229 hwrt_init_common_fw_structure(struct pvr_file *pvr_file, 230 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 231 struct pvr_hwrt_dataset *hwrt) 232 { 233 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args; 234 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 235 struct pvr_rt_mtile_info info; 236 int err; 237 238 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &info.tile_size_x); 239 if (WARN_ON(err)) 240 return err; 241 242 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &info.tile_size_y); 243 if (WARN_ON(err)) 244 return err; 245 246 info.num_tiles_x = DIV_ROUND_UP(args->width, info.tile_size_x); 247 info.num_tiles_y = DIV_ROUND_UP(args->height, info.tile_size_y); 248 249 if (PVR_HAS_FEATURE(pvr_dev, simple_parameter_format_version)) { 250 u32 parameter_format; 251 252 err = PVR_FEATURE_VALUE(pvr_dev, simple_parameter_format_version, 253 ¶meter_format); 254 if (WARN_ON(err)) 255 return err; 256 257 WARN_ON(parameter_format != 2); 258 259 /* 260 * Set up 16 macrotiles with a multiple of 2x2 tiles per macrotile, which is 261 * aligned to a tile group. 262 */ 263 info.mtile_x[0] = DIV_ROUND_UP(info.num_tiles_x, 8) * 2; 264 info.mtile_y[0] = DIV_ROUND_UP(info.num_tiles_y, 8) * 2; 265 info.mtile_x[1] = 0; 266 info.mtile_y[1] = 0; 267 info.mtile_x[2] = 0; 268 info.mtile_y[2] = 0; 269 info.tile_max_x = round_up(info.num_tiles_x, 2) - 1; 270 info.tile_max_y = round_up(info.num_tiles_y, 2) - 1; 271 } else { 272 /* Set up 16 macrotiles with a multiple of 4x4 tiles per macrotile. */ 273 info.mtile_x[0] = round_up(DIV_ROUND_UP(info.num_tiles_x, 4), 4); 274 info.mtile_y[0] = round_up(DIV_ROUND_UP(info.num_tiles_y, 4), 4); 275 info.mtile_x[1] = info.mtile_x[0] * 2; 276 info.mtile_y[1] = info.mtile_y[0] * 2; 277 info.mtile_x[2] = info.mtile_x[0] * 3; 278 info.mtile_y[2] = info.mtile_y[0] * 3; 279 info.tile_max_x = info.num_tiles_x - 1; 280 info.tile_max_y = info.num_tiles_y - 1; 281 } 282 283 hwrt->common.geom_caches_need_zeroing = false; 284 285 hwrt->common.isp_merge_lower_x = args->isp_merge_lower_x; 286 hwrt->common.isp_merge_lower_y = args->isp_merge_lower_y; 287 hwrt->common.isp_merge_upper_x = args->isp_merge_upper_x; 288 hwrt->common.isp_merge_upper_y = args->isp_merge_upper_y; 289 hwrt->common.isp_merge_scale_x = args->isp_merge_scale_x; 290 hwrt->common.isp_merge_scale_y = args->isp_merge_scale_y; 291 292 err = get_cr_multisamplectl_val(args->samples, false, 293 &hwrt->common.multi_sample_ctl); 294 if (err) 295 return err; 296 297 err = get_cr_multisamplectl_val(args->samples, true, 298 &hwrt->common.flipped_multi_sample_ctl); 299 if (err) 300 return err; 301 302 hwrt->common.mtile_stride = info.mtile_x[0] * info.mtile_y[0]; 303 304 err = get_cr_te_aa_val(pvr_dev, args->samples, &hwrt->common.teaa); 305 if (err) 306 return err; 307 308 hwrt->common.screen_pixel_max = 309 (((args->width - 1) << ROGUE_CR_PPP_SCREEN_PIXXMAX_SHIFT) & 310 ~ROGUE_CR_PPP_SCREEN_PIXXMAX_CLRMSK) | 311 (((args->height - 1) << ROGUE_CR_PPP_SCREEN_PIXYMAX_SHIFT) & 312 ~ROGUE_CR_PPP_SCREEN_PIXYMAX_CLRMSK); 313 314 hwrt->common.te_screen = 315 ((info.tile_max_x << ROGUE_CR_TE_SCREEN_XMAX_SHIFT) & 316 ~ROGUE_CR_TE_SCREEN_XMAX_CLRMSK) | 317 ((info.tile_max_y << ROGUE_CR_TE_SCREEN_YMAX_SHIFT) & 318 ~ROGUE_CR_TE_SCREEN_YMAX_CLRMSK); 319 hwrt->common.te_mtile1 = 320 ((info.mtile_x[0] << ROGUE_CR_TE_MTILE1_X1_SHIFT) & ~ROGUE_CR_TE_MTILE1_X1_CLRMSK) | 321 ((info.mtile_x[1] << ROGUE_CR_TE_MTILE1_X2_SHIFT) & ~ROGUE_CR_TE_MTILE1_X2_CLRMSK) | 322 ((info.mtile_x[2] << ROGUE_CR_TE_MTILE1_X3_SHIFT) & ~ROGUE_CR_TE_MTILE1_X3_CLRMSK); 323 hwrt->common.te_mtile2 = 324 ((info.mtile_y[0] << ROGUE_CR_TE_MTILE2_Y1_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y1_CLRMSK) | 325 ((info.mtile_y[1] << ROGUE_CR_TE_MTILE2_Y2_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y2_CLRMSK) | 326 ((info.mtile_y[2] << ROGUE_CR_TE_MTILE2_Y3_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y3_CLRMSK); 327 328 err = get_cr_isp_mtile_size_val(pvr_dev, args->samples, &info, 329 &hwrt->common.isp_mtile_size); 330 if (err) 331 return err; 332 333 hwrt->common.tpc_stride = geom_data_args->tpc_stride; 334 hwrt->common.tpc_size = geom_data_args->tpc_size; 335 336 hwrt->common.rgn_header_size = args->region_header_size; 337 338 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata_common), 339 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, hwrtdata_common_init, hwrt, 340 &hwrt->common_fw_obj); 341 342 return err; 343 } 344 345 static void 346 hwrt_fw_data_init(void *cpu_ptr, void *priv) 347 { 348 struct pvr_hwrt_data *hwrt_data = priv; 349 350 memcpy(cpu_ptr, &hwrt_data->data, sizeof(hwrt_data->data)); 351 } 352 353 static int 354 hwrt_data_init_fw_structure(struct pvr_file *pvr_file, 355 struct pvr_hwrt_dataset *hwrt, 356 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 357 struct drm_pvr_create_hwrt_rt_data_args *rt_data_args, 358 struct pvr_hwrt_data *hwrt_data) 359 { 360 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args; 361 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 362 struct rogue_fwif_rta_ctl *rta_ctl; 363 int err; 364 365 pvr_fw_object_get_fw_addr(hwrt->common_fw_obj, 366 &hwrt_data->data.hwrt_data_common_fw_addr); 367 368 for (int free_list_i = 0; free_list_i < ARRAY_SIZE(hwrt->free_lists); free_list_i++) { 369 pvr_fw_object_get_fw_addr(hwrt->free_lists[free_list_i]->fw_obj, 370 &hwrt_data->data.freelists_fw_addr[free_list_i]); 371 } 372 373 hwrt_data->data.tail_ptrs_dev_addr = geom_data_args->tpc_dev_addr; 374 hwrt_data->data.vheap_table_dev_addr = geom_data_args->vheap_table_dev_addr; 375 hwrt_data->data.rtc_dev_addr = geom_data_args->rtc_dev_addr; 376 377 hwrt_data->data.pm_mlist_dev_addr = rt_data_args->pm_mlist_dev_addr; 378 hwrt_data->data.macrotile_array_dev_addr = rt_data_args->macrotile_array_dev_addr; 379 hwrt_data->data.rgn_header_dev_addr = rt_data_args->region_header_dev_addr; 380 381 rta_ctl = &hwrt_data->data.rta_ctl; 382 383 rta_ctl->render_target_index = 0; 384 rta_ctl->active_render_targets = 0; 385 rta_ctl->valid_render_targets_fw_addr = 0; 386 rta_ctl->rta_num_partial_renders_fw_addr = 0; 387 rta_ctl->max_rts = args->layers; 388 389 if (args->layers > 1) { 390 err = pvr_fw_object_create(pvr_dev, args->layers * SRTC_ENTRY_SIZE, 391 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 392 NULL, NULL, &hwrt_data->srtc_obj); 393 if (err) 394 return err; 395 pvr_fw_object_get_fw_addr(hwrt_data->srtc_obj, 396 &rta_ctl->valid_render_targets_fw_addr); 397 398 err = pvr_fw_object_create(pvr_dev, args->layers * RAA_ENTRY_SIZE, 399 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 400 NULL, NULL, &hwrt_data->raa_obj); 401 if (err) 402 goto err_put_shadow_rt_cache; 403 pvr_fw_object_get_fw_addr(hwrt_data->raa_obj, 404 &rta_ctl->rta_num_partial_renders_fw_addr); 405 } 406 407 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata), 408 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 409 hwrt_fw_data_init, hwrt_data, &hwrt_data->fw_obj); 410 if (err) 411 goto err_put_raa_obj; 412 413 pvr_free_list_add_hwrt(hwrt->free_lists[0], hwrt_data); 414 415 return 0; 416 417 err_put_raa_obj: 418 if (args->layers > 1) 419 pvr_fw_object_destroy(hwrt_data->raa_obj); 420 421 err_put_shadow_rt_cache: 422 if (args->layers > 1) 423 pvr_fw_object_destroy(hwrt_data->srtc_obj); 424 425 return err; 426 } 427 428 static void 429 hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset *hwrt, int hwrt_nr) 430 { 431 struct pvr_hwrt_data *hwrt_data = &hwrt->data[hwrt_nr]; 432 433 pvr_free_list_remove_hwrt(hwrt->free_lists[0], hwrt_data); 434 435 if (hwrt->max_rts > 1) { 436 pvr_fw_object_destroy(hwrt_data->raa_obj); 437 pvr_fw_object_destroy(hwrt_data->srtc_obj); 438 } 439 440 pvr_fw_object_destroy(hwrt_data->fw_obj); 441 } 442 443 /** 444 * pvr_hwrt_dataset_create() - Create a new HWRT dataset 445 * @pvr_file: Pointer to pvr_file structure. 446 * @args: Creation arguments from userspace. 447 * 448 * Return: 449 * * Pointer to new HWRT, or 450 * * ERR_PTR(-%ENOMEM) on out of memory. 451 */ 452 struct pvr_hwrt_dataset * 453 pvr_hwrt_dataset_create(struct pvr_file *pvr_file, 454 struct drm_pvr_ioctl_create_hwrt_dataset_args *args) 455 { 456 struct pvr_hwrt_dataset *hwrt; 457 int err, i = 0; 458 459 /* Create and fill out the kernel structure */ 460 hwrt = kzalloc(sizeof(*hwrt), GFP_KERNEL); 461 462 if (!hwrt) 463 return ERR_PTR(-ENOMEM); 464 465 err = hwrt_init_kernel_structure(pvr_file, args, hwrt); 466 if (err < 0) 467 goto err_free; 468 469 err = hwrt_init_common_fw_structure(pvr_file, args, hwrt); 470 if (err < 0) 471 goto err_fini_kernel_structure; 472 473 for (; i < ARRAY_SIZE(hwrt->data); i++) { 474 err = hwrt_data_init_fw_structure(pvr_file, hwrt, args, 475 &args->rt_data_args[i], 476 &hwrt->data[i]); 477 if (err < 0) 478 goto err_fini_data_structures; 479 480 hwrt->data[i].hwrt_dataset = hwrt; 481 } 482 483 kref_init(&hwrt->ref_count); 484 return hwrt; 485 486 err_fini_data_structures: 487 while (--i >= 0) 488 hwrt_data_fini_fw_structure(hwrt, i); 489 490 err_fini_kernel_structure: 491 hwrt_fini_kernel_structure(hwrt); 492 493 err_free: 494 kfree(hwrt); 495 496 return ERR_PTR(err); 497 } 498 499 static void 500 pvr_hwrt_dataset_release(struct kref *ref_count) 501 { 502 struct pvr_hwrt_dataset *hwrt = 503 container_of(ref_count, struct pvr_hwrt_dataset, ref_count); 504 505 for (int i = ARRAY_SIZE(hwrt->data) - 1; i >= 0; i--) { 506 WARN_ON(pvr_fw_structure_cleanup(hwrt->pvr_dev, ROGUE_FWIF_CLEANUP_HWRTDATA, 507 hwrt->data[i].fw_obj, 0)); 508 hwrt_data_fini_fw_structure(hwrt, i); 509 } 510 511 hwrt_fini_common_fw_structure(hwrt); 512 hwrt_fini_kernel_structure(hwrt); 513 514 kfree(hwrt); 515 } 516 517 /** 518 * pvr_destroy_hwrt_datasets_for_file: Destroy any HWRT datasets associated 519 * with the given file. 520 * @pvr_file: Pointer to pvr_file structure. 521 * 522 * Removes all HWRT datasets associated with @pvr_file from the device 523 * hwrt_dataset list and drops initial references. HWRT datasets will then be 524 * destroyed once all outstanding references are dropped. 525 */ 526 void pvr_destroy_hwrt_datasets_for_file(struct pvr_file *pvr_file) 527 { 528 struct pvr_hwrt_dataset *hwrt; 529 unsigned long handle; 530 531 xa_for_each(&pvr_file->hwrt_handles, handle, hwrt) { 532 (void)hwrt; 533 pvr_hwrt_dataset_put(xa_erase(&pvr_file->hwrt_handles, handle)); 534 } 535 } 536 537 /** 538 * pvr_hwrt_dataset_put() - Release reference on HWRT dataset 539 * @hwrt: Pointer to HWRT dataset to release reference on 540 */ 541 void 542 pvr_hwrt_dataset_put(struct pvr_hwrt_dataset *hwrt) 543 { 544 if (hwrt) 545 kref_put(&hwrt->ref_count, pvr_hwrt_dataset_release); 546 } 547