1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Syncpoints 4 * 5 * Copyright (c) 2010-2015, NVIDIA Corporation. 6 */ 7 8 #include <linux/module.h> 9 #include <linux/device.h> 10 #include <linux/dma-fence.h> 11 #include <linux/overflow.h> 12 #include <linux/slab.h> 13 14 #include <trace/events/host1x.h> 15 16 #include "syncpt.h" 17 #include "dev.h" 18 #include "intr.h" 19 #include "debug.h" 20 21 #define SYNCPT_CHECK_PERIOD (2 * HZ) 22 #define MAX_STUCK_CHECK_COUNT 15 23 24 static struct host1x_syncpt_base * 25 host1x_syncpt_base_request(struct host1x *host) 26 { 27 struct host1x_syncpt_base *bases = host->bases; 28 unsigned int i; 29 30 for (i = 0; i < host->info->nb_bases; i++) 31 if (!bases[i].requested) 32 break; 33 34 if (i >= host->info->nb_bases) 35 return NULL; 36 37 bases[i].requested = true; 38 return &bases[i]; 39 } 40 41 static void host1x_syncpt_base_free(struct host1x_syncpt_base *base) 42 { 43 if (base) 44 base->requested = false; 45 } 46 47 /** 48 * host1x_syncpt_alloc() - allocate a syncpoint 49 * @host: host1x device data 50 * @flags: bitfield of HOST1X_SYNCPT_* flags 51 * @name: name for the syncpoint for use in debug prints 52 * 53 * Allocates a hardware syncpoint for the caller's use. The caller then has 54 * the sole authority to mutate the syncpoint's value until it is freed again. 55 * 56 * If no free syncpoints are available, or a NULL name was specified, returns 57 * NULL. 58 */ 59 struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host, 60 unsigned long flags, 61 const char *name) 62 { 63 struct host1x_syncpt *sp = host->syncpt; 64 char *full_name; 65 unsigned int i; 66 67 if (!name) 68 return NULL; 69 70 mutex_lock(&host->syncpt_mutex); 71 72 for (i = 0; i < host->info->nb_pts && kref_read(&sp->ref); i++, sp++) 73 ; 74 75 if (i >= host->info->nb_pts) 76 goto unlock; 77 78 if (flags & HOST1X_SYNCPT_HAS_BASE) { 79 sp->base = host1x_syncpt_base_request(host); 80 if (!sp->base) 81 goto unlock; 82 } 83 84 full_name = kasprintf(GFP_KERNEL, "%u-%s", sp->id, name); 85 if (!full_name) 86 goto free_base; 87 88 sp->name = full_name; 89 90 if (flags & HOST1X_SYNCPT_CLIENT_MANAGED) 91 sp->client_managed = true; 92 else 93 sp->client_managed = false; 94 95 kref_init(&sp->ref); 96 97 mutex_unlock(&host->syncpt_mutex); 98 return sp; 99 100 free_base: 101 host1x_syncpt_base_free(sp->base); 102 sp->base = NULL; 103 unlock: 104 mutex_unlock(&host->syncpt_mutex); 105 return NULL; 106 } 107 EXPORT_SYMBOL(host1x_syncpt_alloc); 108 109 /** 110 * host1x_syncpt_id() - retrieve syncpoint ID 111 * @sp: host1x syncpoint 112 * 113 * Given a pointer to a struct host1x_syncpt, retrieves its ID. This ID is 114 * often used as a value to program into registers that control how hardware 115 * blocks interact with syncpoints. 116 */ 117 u32 host1x_syncpt_id(struct host1x_syncpt *sp) 118 { 119 return sp->id; 120 } 121 EXPORT_SYMBOL(host1x_syncpt_id); 122 123 /** 124 * host1x_syncpt_incr_max() - update the value sent to hardware 125 * @sp: host1x syncpoint 126 * @incrs: number of increments 127 */ 128 u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs) 129 { 130 /* 131 * Syncpoint values are intended to be modulo 2^32, so overflow 132 * here is intended. 133 */ 134 return (u32)atomic_add_return(incrs, &sp->max_val); 135 } 136 EXPORT_SYMBOL(host1x_syncpt_incr_max); 137 138 /* 139 * Write cached syncpoint and waitbase values to hardware. 140 */ 141 void host1x_syncpt_restore(struct host1x *host) 142 { 143 struct host1x_syncpt *sp_base = host->syncpt; 144 unsigned int i; 145 146 for (i = 0; i < host1x_syncpt_nb_pts(host); i++) { 147 /* 148 * Unassign syncpt from channels for purposes of Tegra186 149 * syncpoint protection. This prevents any channel from 150 * accessing it until it is reassigned. 151 */ 152 host1x_hw_syncpt_assign_to_channel(host, sp_base + i, NULL); 153 host1x_hw_syncpt_restore(host, sp_base + i); 154 } 155 156 for (i = 0; i < host1x_syncpt_nb_bases(host); i++) 157 host1x_hw_syncpt_restore_wait_base(host, sp_base + i); 158 159 host1x_hw_syncpt_enable_protection(host); 160 161 wmb(); 162 } 163 164 /* 165 * Update the cached syncpoint and waitbase values by reading them 166 * from the registers. 167 */ 168 void host1x_syncpt_save(struct host1x *host) 169 { 170 struct host1x_syncpt *sp_base = host->syncpt; 171 unsigned int i; 172 173 for (i = 0; i < host1x_syncpt_nb_pts(host); i++) { 174 if (host1x_syncpt_client_managed(sp_base + i)) 175 host1x_hw_syncpt_load(host, sp_base + i); 176 else 177 WARN_ON(!host1x_syncpt_idle(sp_base + i)); 178 } 179 180 for (i = 0; i < host1x_syncpt_nb_bases(host); i++) 181 host1x_hw_syncpt_load_wait_base(host, sp_base + i); 182 } 183 184 /* 185 * Updates the cached syncpoint value by reading a new value from the hardware 186 * register 187 */ 188 u32 host1x_syncpt_load(struct host1x_syncpt *sp) 189 { 190 u32 val; 191 192 val = host1x_hw_syncpt_load(sp->host, sp); 193 trace_host1x_syncpt_load_min(sp->id, val); 194 195 return val; 196 } 197 198 /* 199 * Get the current syncpoint base 200 */ 201 u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp) 202 { 203 host1x_hw_syncpt_load_wait_base(sp->host, sp); 204 205 return sp->base_val; 206 } 207 208 /** 209 * host1x_syncpt_incr() - increment syncpoint value from CPU, updating cache 210 * @sp: host1x syncpoint 211 */ 212 int host1x_syncpt_incr(struct host1x_syncpt *sp) 213 { 214 return host1x_hw_syncpt_cpu_incr(sp->host, sp); 215 } 216 EXPORT_SYMBOL(host1x_syncpt_incr); 217 218 /** 219 * host1x_syncpt_wait() - wait for a syncpoint to reach a given value 220 * @sp: host1x syncpoint 221 * @thresh: threshold 222 * @timeout: maximum time to wait for the syncpoint to reach the given value 223 * @value: return location for the syncpoint value 224 */ 225 int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout, 226 u32 *value) 227 { 228 struct dma_fence *fence; 229 long wait_err; 230 u32 curr; 231 232 curr = host1x_syncpt_load(sp); 233 234 if (value) 235 *value = curr; 236 237 if (host1x_syncpt_is_expired(sp, thresh)) 238 return 0; 239 240 if (timeout < 0) 241 timeout = LONG_MAX; 242 else if (timeout == 0) 243 return -EAGAIN; 244 245 fence = host1x_fence_create(sp, thresh, false); 246 if (IS_ERR(fence)) 247 return PTR_ERR(fence); 248 249 wait_err = dma_fence_wait_timeout(fence, true, timeout); 250 if (wait_err == 0) 251 host1x_fence_cancel(fence); 252 dma_fence_put(fence); 253 254 /* 255 * Don't rely on dma_fence_wait_timeout return value, 256 * since it returns zero both on timeout and if the 257 * wait completed with 0 jiffies left. 258 */ 259 if (wait_err == 0 && !host1x_syncpt_is_expired(sp, thresh)) { 260 if (value) 261 *value = host1x_syncpt_load(sp); 262 263 return -EAGAIN; 264 } else if (wait_err < 0) { 265 return wait_err; 266 } else { 267 /* Success, read the value cached by ISR */ 268 if (value) 269 *value = host1x_syncpt_read_min(sp); 270 271 return 0; 272 } 273 } 274 EXPORT_SYMBOL(host1x_syncpt_wait); 275 276 /* 277 * Returns true if syncpoint is expired, false if we may need to wait 278 */ 279 bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh) 280 { 281 u32 current_val; 282 283 smp_rmb(); 284 285 current_val = (u32)atomic_read(&sp->min_val); 286 287 return (wrapping_sub(u32, current_val, thresh) & 0x80000000U) == 0U; 288 } 289 290 int host1x_syncpt_init(struct host1x *host) 291 { 292 struct host1x_syncpt_base *bases; 293 struct host1x_syncpt *syncpt; 294 unsigned int i; 295 296 syncpt = devm_kcalloc(host->dev, host->info->nb_pts, sizeof(*syncpt), 297 GFP_KERNEL); 298 if (!syncpt) 299 return -ENOMEM; 300 301 bases = devm_kcalloc(host->dev, host->info->nb_bases, sizeof(*bases), 302 GFP_KERNEL); 303 if (!bases) 304 return -ENOMEM; 305 306 for (i = 0; i < host->info->nb_pts; i++) { 307 syncpt[i].id = i; 308 syncpt[i].host = host; 309 } 310 311 for (i = 0; i < host->info->nb_bases; i++) 312 bases[i].id = i; 313 314 mutex_init(&host->syncpt_mutex); 315 host->syncpt = syncpt; 316 host->bases = bases; 317 318 /* Allocate sync point to use for clearing waits for expired fences */ 319 host->nop_sp = host1x_syncpt_alloc(host, 0, "reserved-nop"); 320 if (!host->nop_sp) 321 return -ENOMEM; 322 323 if (host->info->reserve_vblank_syncpts) { 324 kref_init(&host->syncpt[26].ref); 325 kref_init(&host->syncpt[27].ref); 326 } 327 328 return 0; 329 } 330 331 /** 332 * host1x_syncpt_request() - request a syncpoint 333 * @client: client requesting the syncpoint 334 * @flags: flags 335 * 336 * host1x client drivers can use this function to allocate a syncpoint for 337 * subsequent use. A syncpoint returned by this function will be reserved for 338 * use by the client exclusively. When no longer using a syncpoint, a host1x 339 * client driver needs to release it using host1x_syncpt_put(). 340 */ 341 struct host1x_syncpt *host1x_syncpt_request(struct host1x_client *client, 342 unsigned long flags) 343 { 344 struct host1x *host = dev_get_drvdata(client->host->parent); 345 346 return host1x_syncpt_alloc(host, flags, dev_name(client->dev)); 347 } 348 EXPORT_SYMBOL(host1x_syncpt_request); 349 350 static void syncpt_release(struct kref *ref) 351 { 352 struct host1x_syncpt *sp = container_of(ref, struct host1x_syncpt, ref); 353 354 atomic_set(&sp->max_val, host1x_syncpt_read(sp)); 355 356 sp->locked = false; 357 358 host1x_syncpt_base_free(sp->base); 359 kfree(sp->name); 360 sp->base = NULL; 361 sp->name = NULL; 362 sp->client_managed = false; 363 364 mutex_unlock(&sp->host->syncpt_mutex); 365 } 366 367 /** 368 * host1x_syncpt_put() - free a requested syncpoint 369 * @sp: host1x syncpoint 370 * 371 * Release a syncpoint previously allocated using host1x_syncpt_request(). A 372 * host1x client driver should call this when the syncpoint is no longer in 373 * use. 374 */ 375 void host1x_syncpt_put(struct host1x_syncpt *sp) 376 { 377 if (!sp) 378 return; 379 380 kref_put_mutex(&sp->ref, syncpt_release, &sp->host->syncpt_mutex); 381 } 382 EXPORT_SYMBOL(host1x_syncpt_put); 383 384 void host1x_syncpt_deinit(struct host1x *host) 385 { 386 struct host1x_syncpt *sp = host->syncpt; 387 unsigned int i; 388 389 for (i = 0; i < host->info->nb_pts; i++, sp++) 390 kfree(sp->name); 391 } 392 393 /** 394 * host1x_syncpt_read_max() - read maximum syncpoint value 395 * @sp: host1x syncpoint 396 * 397 * The maximum syncpoint value indicates how many operations there are in 398 * queue, either in channel or in a software thread. 399 */ 400 u32 host1x_syncpt_read_max(struct host1x_syncpt *sp) 401 { 402 smp_rmb(); 403 404 return (u32)atomic_read(&sp->max_val); 405 } 406 EXPORT_SYMBOL(host1x_syncpt_read_max); 407 408 /** 409 * host1x_syncpt_read_min() - read minimum syncpoint value 410 * @sp: host1x syncpoint 411 * 412 * The minimum syncpoint value is a shadow of the current sync point value in 413 * hardware. 414 */ 415 u32 host1x_syncpt_read_min(struct host1x_syncpt *sp) 416 { 417 smp_rmb(); 418 419 return (u32)atomic_read(&sp->min_val); 420 } 421 EXPORT_SYMBOL(host1x_syncpt_read_min); 422 423 /** 424 * host1x_syncpt_read() - read the current syncpoint value 425 * @sp: host1x syncpoint 426 */ 427 u32 host1x_syncpt_read(struct host1x_syncpt *sp) 428 { 429 return host1x_syncpt_load(sp); 430 } 431 EXPORT_SYMBOL(host1x_syncpt_read); 432 433 unsigned int host1x_syncpt_nb_pts(struct host1x *host) 434 { 435 return host->info->nb_pts; 436 } 437 438 unsigned int host1x_syncpt_nb_bases(struct host1x *host) 439 { 440 return host->info->nb_bases; 441 } 442 443 unsigned int host1x_syncpt_nb_mlocks(struct host1x *host) 444 { 445 return host->info->nb_mlocks; 446 } 447 448 /** 449 * host1x_syncpt_get_by_id() - obtain a syncpoint by ID 450 * @host: host1x controller 451 * @id: syncpoint ID 452 */ 453 struct host1x_syncpt *host1x_syncpt_get_by_id(struct host1x *host, 454 unsigned int id) 455 { 456 if (id >= host->info->nb_pts) 457 return NULL; 458 459 if (kref_get_unless_zero(&host->syncpt[id].ref)) 460 return &host->syncpt[id]; 461 else 462 return NULL; 463 } 464 EXPORT_SYMBOL(host1x_syncpt_get_by_id); 465 466 /** 467 * host1x_syncpt_get_by_id_noref() - obtain a syncpoint by ID but don't 468 * increase the refcount. 469 * @host: host1x controller 470 * @id: syncpoint ID 471 */ 472 struct host1x_syncpt *host1x_syncpt_get_by_id_noref(struct host1x *host, 473 unsigned int id) 474 { 475 if (id >= host->info->nb_pts) 476 return NULL; 477 478 return &host->syncpt[id]; 479 } 480 EXPORT_SYMBOL(host1x_syncpt_get_by_id_noref); 481 482 /** 483 * host1x_syncpt_get() - increment syncpoint refcount 484 * @sp: syncpoint 485 */ 486 struct host1x_syncpt *host1x_syncpt_get(struct host1x_syncpt *sp) 487 { 488 kref_get(&sp->ref); 489 490 return sp; 491 } 492 EXPORT_SYMBOL(host1x_syncpt_get); 493 494 /** 495 * host1x_syncpt_get_base() - obtain the wait base associated with a syncpoint 496 * @sp: host1x syncpoint 497 */ 498 struct host1x_syncpt_base *host1x_syncpt_get_base(struct host1x_syncpt *sp) 499 { 500 return sp ? sp->base : NULL; 501 } 502 EXPORT_SYMBOL(host1x_syncpt_get_base); 503 504 /** 505 * host1x_syncpt_base_id() - retrieve the ID of a syncpoint wait base 506 * @base: host1x syncpoint wait base 507 */ 508 u32 host1x_syncpt_base_id(struct host1x_syncpt_base *base) 509 { 510 return base->id; 511 } 512 EXPORT_SYMBOL(host1x_syncpt_base_id); 513 514 static void do_nothing(struct kref *ref) 515 { 516 } 517 518 /** 519 * host1x_syncpt_release_vblank_reservation() - Make VBLANK syncpoint 520 * available for allocation 521 * 522 * @client: host1x bus client 523 * @syncpt_id: syncpoint ID to make available 524 * 525 * Makes VBLANK<i> syncpoint available for allocatation if it was 526 * reserved at initialization time. This should be called by the display 527 * driver after it has ensured that any VBLANK increment programming configured 528 * by the boot chain has been disabled. 529 */ 530 void host1x_syncpt_release_vblank_reservation(struct host1x_client *client, 531 u32 syncpt_id) 532 { 533 struct host1x *host = dev_get_drvdata(client->host->parent); 534 535 if (!host->info->reserve_vblank_syncpts) 536 return; 537 538 kref_put(&host->syncpt[syncpt_id].ref, do_nothing); 539 } 540 EXPORT_SYMBOL(host1x_syncpt_release_vblank_reservation); 541