1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4 #include "pvr_free_list.h"
5 #include "pvr_hwrt.h"
6 #include "pvr_gem.h"
7 #include "pvr_rogue_cr_defs_client.h"
8 #include "pvr_rogue_fwif.h"
9
10 #include <drm/drm_gem.h>
11 #include <linux/bitops.h>
12 #include <linux/math.h>
13 #include <linux/slab.h>
14 #include <linux/xarray.h>
15 #include <uapi/drm/pvr_drm.h>
16
17 static_assert(ROGUE_FWIF_NUM_RTDATAS == 2);
18 static_assert(ROGUE_FWIF_NUM_GEOMDATAS == 1);
19 static_assert(ROGUE_FWIF_NUM_RTDATA_FREELISTS == 2);
20
21 /*
22 * struct pvr_rt_mtile_info - Render target macrotile information
23 */
24 struct pvr_rt_mtile_info {
25 u32 mtile_x[3];
26 u32 mtile_y[3];
27 u32 tile_max_x;
28 u32 tile_max_y;
29 u32 tile_size_x;
30 u32 tile_size_y;
31 u32 num_tiles_x;
32 u32 num_tiles_y;
33 };
34
35 /* Size of Shadow Render Target Cache entry */
36 #define SRTC_ENTRY_SIZE sizeof(u32)
37 /* Size of Renders Accumulation Array entry */
38 #define RAA_ENTRY_SIZE sizeof(u32)
39
40 static int
hwrt_init_kernel_structure(struct pvr_file * pvr_file,struct drm_pvr_ioctl_create_hwrt_dataset_args * args,struct pvr_hwrt_dataset * hwrt)41 hwrt_init_kernel_structure(struct pvr_file *pvr_file,
42 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
43 struct pvr_hwrt_dataset *hwrt)
44 {
45 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
46 int err;
47 int i;
48
49 hwrt->pvr_dev = pvr_dev;
50 hwrt->max_rts = args->layers;
51
52 /* Get pointers to the free lists */
53 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
54 hwrt->free_lists[i] = pvr_free_list_lookup(pvr_file, args->free_list_handles[i]);
55 if (!hwrt->free_lists[i]) {
56 err = -EINVAL;
57 goto err_put_free_lists;
58 }
59 }
60
61 if (hwrt->free_lists[ROGUE_FW_LOCAL_FREELIST]->current_pages <
62 pvr_get_free_list_min_pages(pvr_dev)) {
63 err = -EINVAL;
64 goto err_put_free_lists;
65 }
66
67 return 0;
68
69 err_put_free_lists:
70 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
71 pvr_free_list_put(hwrt->free_lists[i]);
72 hwrt->free_lists[i] = NULL;
73 }
74
75 return err;
76 }
77
78 static void
hwrt_fini_kernel_structure(struct pvr_hwrt_dataset * hwrt)79 hwrt_fini_kernel_structure(struct pvr_hwrt_dataset *hwrt)
80 {
81 int i;
82
83 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
84 pvr_free_list_put(hwrt->free_lists[i]);
85 hwrt->free_lists[i] = NULL;
86 }
87 }
88
89 static void
hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset * hwrt)90 hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset *hwrt)
91 {
92 pvr_fw_object_destroy(hwrt->common_fw_obj);
93 }
94
95 static int
get_cr_isp_mtile_size_val(struct pvr_device * pvr_dev,u32 samples,struct pvr_rt_mtile_info * info,u32 * value_out)96 get_cr_isp_mtile_size_val(struct pvr_device *pvr_dev, u32 samples,
97 struct pvr_rt_mtile_info *info, u32 *value_out)
98 {
99 u32 x = info->mtile_x[0];
100 u32 y = info->mtile_y[0];
101 u32 samples_per_pixel;
102 int err;
103
104 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);
105 if (err)
106 return err;
107
108 if (samples_per_pixel == 1) {
109 if (samples >= 4)
110 x <<= 1;
111 if (samples >= 2)
112 y <<= 1;
113 } else if (samples_per_pixel == 2) {
114 if (samples >= 8)
115 x <<= 1;
116 if (samples >= 4)
117 y <<= 1;
118 } else if (samples_per_pixel == 4) {
119 if (samples >= 8)
120 y <<= 1;
121 } else {
122 WARN(true, "Unsupported ISP samples per pixel value");
123 return -EINVAL;
124 }
125
126 *value_out = ((x << ROGUE_CR_ISP_MTILE_SIZE_X_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_X_CLRMSK) |
127 ((y << ROGUE_CR_ISP_MTILE_SIZE_Y_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_Y_CLRMSK);
128
129 return 0;
130 }
131
132 static int
get_cr_multisamplectl_val(u32 samples,bool y_flip,u64 * value_out)133 get_cr_multisamplectl_val(u32 samples, bool y_flip, u64 *value_out)
134 {
135 static const struct {
136 u8 x[8];
137 u8 y[8];
138 } sample_positions[4] = {
139 /* 1 sample */
140 {
141 .x = { 8 },
142 .y = { 8 },
143 },
144 /* 2 samples */
145 {
146 .x = { 12, 4 },
147 .y = { 12, 4 },
148 },
149 /* 4 samples */
150 {
151 .x = { 6, 14, 2, 10 },
152 .y = { 2, 6, 10, 14 },
153 },
154 /* 8 samples */
155 {
156 .x = { 9, 7, 13, 5, 3, 1, 11, 15 },
157 .y = { 5, 11, 9, 3, 13, 7, 15, 1 },
158 },
159 };
160 const int idx = fls(samples) - 1;
161 u64 value = 0;
162
163 if (idx < 0 || idx > 3)
164 return -EINVAL;
165
166 for (u32 i = 0; i < 8; i++) {
167 value |= ((u64)sample_positions[idx].x[i]) << (i * 8);
168 if (y_flip)
169 value |= (((u64)(16 - sample_positions[idx].y[i]) & 0xf)) << (i * 8 + 4);
170 else
171 value |= ((u64)sample_positions[idx].y[i]) << (i * 8 + 4);
172 }
173
174 *value_out = value;
175
176 return 0;
177 }
178
179 static int
get_cr_te_aa_val(struct pvr_device * pvr_dev,u32 samples,u32 * value_out)180 get_cr_te_aa_val(struct pvr_device *pvr_dev, u32 samples, u32 *value_out)
181 {
182 u32 samples_per_pixel;
183 u32 value = 0;
184 int err = 0;
185
186 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);
187 if (err)
188 return err;
189
190 switch (samples_per_pixel) {
191 case 1:
192 if (samples >= 2)
193 value |= ROGUE_CR_TE_AA_Y_EN;
194 if (samples >= 4)
195 value |= ROGUE_CR_TE_AA_X_EN;
196 break;
197 case 2:
198 if (samples >= 2)
199 value |= ROGUE_CR_TE_AA_X2_EN;
200 if (samples >= 4)
201 value |= ROGUE_CR_TE_AA_Y_EN;
202 if (samples >= 8)
203 value |= ROGUE_CR_TE_AA_X_EN;
204 break;
205 case 4:
206 if (samples >= 2)
207 value |= ROGUE_CR_TE_AA_X2_EN;
208 if (samples >= 4)
209 value |= ROGUE_CR_TE_AA_Y2_EN;
210 if (samples >= 8)
211 value |= ROGUE_CR_TE_AA_Y_EN;
212 break;
213 default:
214 WARN(true, "Unsupported ISP samples per pixel value");
215 return -EINVAL;
216 }
217
218 *value_out = value;
219
220 return 0;
221 }
222
223 static void
hwrtdata_common_init(void * cpu_ptr,void * priv)224 hwrtdata_common_init(void *cpu_ptr, void *priv)
225 {
226 struct pvr_hwrt_dataset *hwrt = priv;
227
228 memcpy(cpu_ptr, &hwrt->common, sizeof(hwrt->common));
229 }
230
231 static int
hwrt_init_common_fw_structure(struct pvr_file * pvr_file,struct drm_pvr_ioctl_create_hwrt_dataset_args * args,struct pvr_hwrt_dataset * hwrt)232 hwrt_init_common_fw_structure(struct pvr_file *pvr_file,
233 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
234 struct pvr_hwrt_dataset *hwrt)
235 {
236 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;
237 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
238 struct pvr_rt_mtile_info info;
239 int err;
240
241 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &info.tile_size_x);
242 if (WARN_ON(err))
243 return err;
244
245 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &info.tile_size_y);
246 if (WARN_ON(err))
247 return err;
248
249 info.num_tiles_x = DIV_ROUND_UP(args->width, info.tile_size_x);
250 info.num_tiles_y = DIV_ROUND_UP(args->height, info.tile_size_y);
251
252 if (PVR_HAS_FEATURE(pvr_dev, simple_parameter_format_version)) {
253 u32 parameter_format;
254
255 err = PVR_FEATURE_VALUE(pvr_dev, simple_parameter_format_version,
256 ¶meter_format);
257 if (WARN_ON(err))
258 return err;
259
260 WARN_ON(parameter_format != 2);
261
262 /*
263 * Set up 16 macrotiles with a multiple of 2x2 tiles per macrotile, which is
264 * aligned to a tile group.
265 */
266 info.mtile_x[0] = DIV_ROUND_UP(info.num_tiles_x, 8) * 2;
267 info.mtile_y[0] = DIV_ROUND_UP(info.num_tiles_y, 8) * 2;
268 info.mtile_x[1] = 0;
269 info.mtile_y[1] = 0;
270 info.mtile_x[2] = 0;
271 info.mtile_y[2] = 0;
272 info.tile_max_x = round_up(info.num_tiles_x, 2) - 1;
273 info.tile_max_y = round_up(info.num_tiles_y, 2) - 1;
274 } else {
275 /* Set up 16 macrotiles with a multiple of 4x4 tiles per macrotile. */
276 info.mtile_x[0] = round_up(DIV_ROUND_UP(info.num_tiles_x, 4), 4);
277 info.mtile_y[0] = round_up(DIV_ROUND_UP(info.num_tiles_y, 4), 4);
278 info.mtile_x[1] = info.mtile_x[0] * 2;
279 info.mtile_y[1] = info.mtile_y[0] * 2;
280 info.mtile_x[2] = info.mtile_x[0] * 3;
281 info.mtile_y[2] = info.mtile_y[0] * 3;
282 info.tile_max_x = info.num_tiles_x - 1;
283 info.tile_max_y = info.num_tiles_y - 1;
284 }
285
286 hwrt->common.geom_caches_need_zeroing = false;
287
288 hwrt->common.isp_merge_lower_x = args->isp_merge_lower_x;
289 hwrt->common.isp_merge_lower_y = args->isp_merge_lower_y;
290 hwrt->common.isp_merge_upper_x = args->isp_merge_upper_x;
291 hwrt->common.isp_merge_upper_y = args->isp_merge_upper_y;
292 hwrt->common.isp_merge_scale_x = args->isp_merge_scale_x;
293 hwrt->common.isp_merge_scale_y = args->isp_merge_scale_y;
294
295 err = get_cr_multisamplectl_val(args->samples, false,
296 &hwrt->common.multi_sample_ctl);
297 if (err)
298 return err;
299
300 err = get_cr_multisamplectl_val(args->samples, true,
301 &hwrt->common.flipped_multi_sample_ctl);
302 if (err)
303 return err;
304
305 hwrt->common.mtile_stride = info.mtile_x[0] * info.mtile_y[0];
306
307 err = get_cr_te_aa_val(pvr_dev, args->samples, &hwrt->common.teaa);
308 if (err)
309 return err;
310
311 hwrt->common.screen_pixel_max =
312 (((args->width - 1) << ROGUE_CR_PPP_SCREEN_PIXXMAX_SHIFT) &
313 ~ROGUE_CR_PPP_SCREEN_PIXXMAX_CLRMSK) |
314 (((args->height - 1) << ROGUE_CR_PPP_SCREEN_PIXYMAX_SHIFT) &
315 ~ROGUE_CR_PPP_SCREEN_PIXYMAX_CLRMSK);
316
317 hwrt->common.te_screen =
318 ((info.tile_max_x << ROGUE_CR_TE_SCREEN_XMAX_SHIFT) &
319 ~ROGUE_CR_TE_SCREEN_XMAX_CLRMSK) |
320 ((info.tile_max_y << ROGUE_CR_TE_SCREEN_YMAX_SHIFT) &
321 ~ROGUE_CR_TE_SCREEN_YMAX_CLRMSK);
322 hwrt->common.te_mtile1 =
323 ((info.mtile_x[0] << ROGUE_CR_TE_MTILE1_X1_SHIFT) & ~ROGUE_CR_TE_MTILE1_X1_CLRMSK) |
324 ((info.mtile_x[1] << ROGUE_CR_TE_MTILE1_X2_SHIFT) & ~ROGUE_CR_TE_MTILE1_X2_CLRMSK) |
325 ((info.mtile_x[2] << ROGUE_CR_TE_MTILE1_X3_SHIFT) & ~ROGUE_CR_TE_MTILE1_X3_CLRMSK);
326 hwrt->common.te_mtile2 =
327 ((info.mtile_y[0] << ROGUE_CR_TE_MTILE2_Y1_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y1_CLRMSK) |
328 ((info.mtile_y[1] << ROGUE_CR_TE_MTILE2_Y2_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y2_CLRMSK) |
329 ((info.mtile_y[2] << ROGUE_CR_TE_MTILE2_Y3_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y3_CLRMSK);
330
331 err = get_cr_isp_mtile_size_val(pvr_dev, args->samples, &info,
332 &hwrt->common.isp_mtile_size);
333 if (err)
334 return err;
335
336 hwrt->common.tpc_stride = geom_data_args->tpc_stride;
337 hwrt->common.tpc_size = geom_data_args->tpc_size;
338
339 hwrt->common.rgn_header_size = args->region_header_size;
340
341 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata_common),
342 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, hwrtdata_common_init, hwrt,
343 &hwrt->common_fw_obj);
344
345 return err;
346 }
347
348 static void
hwrt_fw_data_init(void * cpu_ptr,void * priv)349 hwrt_fw_data_init(void *cpu_ptr, void *priv)
350 {
351 struct pvr_hwrt_data *hwrt_data = priv;
352
353 memcpy(cpu_ptr, &hwrt_data->data, sizeof(hwrt_data->data));
354 }
355
356 static int
hwrt_data_init_fw_structure(struct pvr_file * pvr_file,struct pvr_hwrt_dataset * hwrt,struct drm_pvr_ioctl_create_hwrt_dataset_args * args,struct drm_pvr_create_hwrt_rt_data_args * rt_data_args,struct pvr_hwrt_data * hwrt_data)357 hwrt_data_init_fw_structure(struct pvr_file *pvr_file,
358 struct pvr_hwrt_dataset *hwrt,
359 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
360 struct drm_pvr_create_hwrt_rt_data_args *rt_data_args,
361 struct pvr_hwrt_data *hwrt_data)
362 {
363 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;
364 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
365 struct rogue_fwif_rta_ctl *rta_ctl;
366 int free_list_i;
367 int err;
368
369 pvr_fw_object_get_fw_addr(hwrt->common_fw_obj,
370 &hwrt_data->data.hwrt_data_common_fw_addr);
371
372 for (free_list_i = 0; free_list_i < ARRAY_SIZE(hwrt->free_lists); free_list_i++) {
373 pvr_fw_object_get_fw_addr(hwrt->free_lists[free_list_i]->fw_obj,
374 &hwrt_data->data.freelists_fw_addr[free_list_i]);
375 }
376
377 hwrt_data->data.tail_ptrs_dev_addr = geom_data_args->tpc_dev_addr;
378 hwrt_data->data.vheap_table_dev_addr = geom_data_args->vheap_table_dev_addr;
379 hwrt_data->data.rtc_dev_addr = geom_data_args->rtc_dev_addr;
380
381 hwrt_data->data.pm_mlist_dev_addr = rt_data_args->pm_mlist_dev_addr;
382 hwrt_data->data.macrotile_array_dev_addr = rt_data_args->macrotile_array_dev_addr;
383 hwrt_data->data.rgn_header_dev_addr = rt_data_args->region_header_dev_addr;
384
385 rta_ctl = &hwrt_data->data.rta_ctl;
386
387 rta_ctl->render_target_index = 0;
388 rta_ctl->active_render_targets = 0;
389 rta_ctl->valid_render_targets_fw_addr = 0;
390 rta_ctl->rta_num_partial_renders_fw_addr = 0;
391 rta_ctl->max_rts = args->layers;
392
393 if (args->layers > 1) {
394 err = pvr_fw_object_create(pvr_dev, args->layers * SRTC_ENTRY_SIZE,
395 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
396 NULL, NULL, &hwrt_data->srtc_obj);
397 if (err)
398 return err;
399 pvr_fw_object_get_fw_addr(hwrt_data->srtc_obj,
400 &rta_ctl->valid_render_targets_fw_addr);
401
402 err = pvr_fw_object_create(pvr_dev, args->layers * RAA_ENTRY_SIZE,
403 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
404 NULL, NULL, &hwrt_data->raa_obj);
405 if (err)
406 goto err_put_shadow_rt_cache;
407 pvr_fw_object_get_fw_addr(hwrt_data->raa_obj,
408 &rta_ctl->rta_num_partial_renders_fw_addr);
409 }
410
411 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata),
412 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
413 hwrt_fw_data_init, hwrt_data, &hwrt_data->fw_obj);
414 if (err)
415 goto err_put_raa_obj;
416
417 pvr_free_list_add_hwrt(hwrt->free_lists[0], hwrt_data);
418
419 return 0;
420
421 err_put_raa_obj:
422 if (args->layers > 1)
423 pvr_fw_object_destroy(hwrt_data->raa_obj);
424
425 err_put_shadow_rt_cache:
426 if (args->layers > 1)
427 pvr_fw_object_destroy(hwrt_data->srtc_obj);
428
429 return err;
430 }
431
432 static void
hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset * hwrt,int hwrt_nr)433 hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset *hwrt, int hwrt_nr)
434 {
435 struct pvr_hwrt_data *hwrt_data = &hwrt->data[hwrt_nr];
436
437 pvr_free_list_remove_hwrt(hwrt->free_lists[0], hwrt_data);
438
439 if (hwrt->max_rts > 1) {
440 pvr_fw_object_destroy(hwrt_data->raa_obj);
441 pvr_fw_object_destroy(hwrt_data->srtc_obj);
442 }
443
444 pvr_fw_object_destroy(hwrt_data->fw_obj);
445 }
446
447 /**
448 * pvr_hwrt_dataset_create() - Create a new HWRT dataset
449 * @pvr_file: Pointer to pvr_file structure.
450 * @args: Creation arguments from userspace.
451 *
452 * Return:
453 * * Pointer to new HWRT, or
454 * * ERR_PTR(-%ENOMEM) on out of memory.
455 */
456 struct pvr_hwrt_dataset *
pvr_hwrt_dataset_create(struct pvr_file * pvr_file,struct drm_pvr_ioctl_create_hwrt_dataset_args * args)457 pvr_hwrt_dataset_create(struct pvr_file *pvr_file,
458 struct drm_pvr_ioctl_create_hwrt_dataset_args *args)
459 {
460 struct pvr_hwrt_dataset *hwrt;
461 int err, i = 0;
462
463 /* Create and fill out the kernel structure */
464 hwrt = kzalloc(sizeof(*hwrt), GFP_KERNEL);
465
466 if (!hwrt)
467 return ERR_PTR(-ENOMEM);
468
469 err = hwrt_init_kernel_structure(pvr_file, args, hwrt);
470 if (err < 0)
471 goto err_free;
472
473 err = hwrt_init_common_fw_structure(pvr_file, args, hwrt);
474 if (err < 0)
475 goto err_fini_kernel_structure;
476
477 for (; i < ARRAY_SIZE(hwrt->data); i++) {
478 err = hwrt_data_init_fw_structure(pvr_file, hwrt, args,
479 &args->rt_data_args[i],
480 &hwrt->data[i]);
481 if (err < 0)
482 goto err_fini_data_structures;
483
484 hwrt->data[i].hwrt_dataset = hwrt;
485 }
486
487 kref_init(&hwrt->ref_count);
488 return hwrt;
489
490 err_fini_data_structures:
491 while (--i >= 0)
492 hwrt_data_fini_fw_structure(hwrt, i);
493
494 err_fini_kernel_structure:
495 hwrt_fini_kernel_structure(hwrt);
496
497 err_free:
498 kfree(hwrt);
499
500 return ERR_PTR(err);
501 }
502
503 static void
pvr_hwrt_dataset_release(struct kref * ref_count)504 pvr_hwrt_dataset_release(struct kref *ref_count)
505 {
506 struct pvr_hwrt_dataset *hwrt =
507 container_of(ref_count, struct pvr_hwrt_dataset, ref_count);
508
509 for (int i = ARRAY_SIZE(hwrt->data) - 1; i >= 0; i--) {
510 WARN_ON(pvr_fw_structure_cleanup(hwrt->pvr_dev, ROGUE_FWIF_CLEANUP_HWRTDATA,
511 hwrt->data[i].fw_obj, 0));
512 hwrt_data_fini_fw_structure(hwrt, i);
513 }
514
515 hwrt_fini_common_fw_structure(hwrt);
516 hwrt_fini_kernel_structure(hwrt);
517
518 kfree(hwrt);
519 }
520
521 /**
522 * pvr_destroy_hwrt_datasets_for_file: Destroy any HWRT datasets associated
523 * with the given file.
524 * @pvr_file: Pointer to pvr_file structure.
525 *
526 * Removes all HWRT datasets associated with @pvr_file from the device
527 * hwrt_dataset list and drops initial references. HWRT datasets will then be
528 * destroyed once all outstanding references are dropped.
529 */
pvr_destroy_hwrt_datasets_for_file(struct pvr_file * pvr_file)530 void pvr_destroy_hwrt_datasets_for_file(struct pvr_file *pvr_file)
531 {
532 struct pvr_hwrt_dataset *hwrt;
533 unsigned long handle;
534
535 xa_for_each(&pvr_file->hwrt_handles, handle, hwrt) {
536 (void)hwrt;
537 pvr_hwrt_dataset_put(xa_erase(&pvr_file->hwrt_handles, handle));
538 }
539 }
540
541 /**
542 * pvr_hwrt_dataset_put() - Release reference on HWRT dataset
543 * @hwrt: Pointer to HWRT dataset to release reference on
544 */
545 void
pvr_hwrt_dataset_put(struct pvr_hwrt_dataset * hwrt)546 pvr_hwrt_dataset_put(struct pvr_hwrt_dataset *hwrt)
547 {
548 if (hwrt)
549 kref_put(&hwrt->ref_count, pvr_hwrt_dataset_release);
550 }
551