1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Syncpoints 4 * 5 * Copyright (c) 2010-2015, NVIDIA Corporation. 6 */ 7 8 #include <linux/module.h> 9 #include <linux/device.h> 10 #include <linux/dma-fence.h> 11 #include <linux/overflow.h> 12 #include <linux/slab.h> 13 14 #include <trace/events/host1x.h> 15 16 #include "syncpt.h" 17 #include "dev.h" 18 #include "intr.h" 19 #include "debug.h" 20 21 #define SYNCPT_CHECK_PERIOD (2 * HZ) 22 #define MAX_STUCK_CHECK_COUNT 15 23 24 static struct host1x_syncpt_base * 25 host1x_syncpt_base_request(struct host1x *host) 26 { 27 struct host1x_syncpt_base *bases = host->bases; 28 unsigned int i; 29 30 for (i = 0; i < host->info->nb_bases; i++) 31 if (!bases[i].requested) 32 break; 33 34 if (i >= host->info->nb_bases) 35 return NULL; 36 37 bases[i].requested = true; 38 return &bases[i]; 39 } 40 41 static void host1x_syncpt_base_free(struct host1x_syncpt_base *base) 42 { 43 if (base) 44 base->requested = false; 45 } 46 47 /** 48 * host1x_syncpt_alloc() - allocate a syncpoint 49 * @host: host1x device data 50 * @flags: bitfield of HOST1X_SYNCPT_* flags 51 * @name: name for the syncpoint for use in debug prints 52 * 53 * Allocates a hardware syncpoint for the caller's use. The caller then has 54 * the sole authority to mutate the syncpoint's value until it is freed again. 55 * 56 * If no free syncpoints are available, or a NULL name was specified, returns 57 * NULL. 58 */ 59 struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host, 60 unsigned long flags, 61 const char *name) 62 { 63 struct host1x_syncpt *sp = host->syncpt + host->syncpt_base; 64 char *full_name; 65 unsigned int i; 66 67 if (!name) 68 return NULL; 69 70 mutex_lock(&host->syncpt_mutex); 71 72 for (i = host->syncpt_base; i < host->syncpt_end && kref_read(&sp->ref); i++, sp++) 73 ; 74 75 if (i >= host->syncpt_end) 76 goto unlock; 77 78 if (flags & HOST1X_SYNCPT_HAS_BASE) { 79 sp->base = host1x_syncpt_base_request(host); 80 if (!sp->base) 81 goto unlock; 82 } 83 84 full_name = kasprintf(GFP_KERNEL, "%u-%s", sp->id, name); 85 if (!full_name) 86 goto free_base; 87 88 sp->name = full_name; 89 90 if (flags & HOST1X_SYNCPT_CLIENT_MANAGED) 91 sp->client_managed = true; 92 else 93 sp->client_managed = false; 94 95 kref_init(&sp->ref); 96 97 mutex_unlock(&host->syncpt_mutex); 98 return sp; 99 100 free_base: 101 host1x_syncpt_base_free(sp->base); 102 sp->base = NULL; 103 unlock: 104 mutex_unlock(&host->syncpt_mutex); 105 return NULL; 106 } 107 EXPORT_SYMBOL(host1x_syncpt_alloc); 108 109 /** 110 * host1x_syncpt_id() - retrieve syncpoint ID 111 * @sp: host1x syncpoint 112 * 113 * Given a pointer to a struct host1x_syncpt, retrieves its ID. This ID is 114 * often used as a value to program into registers that control how hardware 115 * blocks interact with syncpoints. 116 */ 117 u32 host1x_syncpt_id(struct host1x_syncpt *sp) 118 { 119 return sp->id; 120 } 121 EXPORT_SYMBOL(host1x_syncpt_id); 122 123 /** 124 * host1x_syncpt_incr_max() - update the value sent to hardware 125 * @sp: host1x syncpoint 126 * @incrs: number of increments 127 */ 128 u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs) 129 { 130 /* 131 * Syncpoint values are intended to be modulo 2^32, so overflow 132 * here is intended. 133 */ 134 return (u32)atomic_add_return(incrs, &sp->max_val); 135 } 136 EXPORT_SYMBOL(host1x_syncpt_incr_max); 137 138 /* 139 * Write cached syncpoint and waitbase values to hardware. 140 */ 141 void host1x_syncpt_restore(struct host1x *host) 142 { 143 struct host1x_syncpt *sp_base = host->syncpt; 144 unsigned int i; 145 146 for (i = host->syncpt_base; i < host->syncpt_end; i++) { 147 /* 148 * Unassign syncpt from channels for purposes of Tegra186 149 * syncpoint protection. This prevents any channel from 150 * accessing it until it is reassigned. 151 */ 152 host1x_hw_syncpt_assign_to_channel(host, sp_base + i, NULL); 153 host1x_hw_syncpt_restore(host, sp_base + i); 154 } 155 156 for (i = 0; i < host1x_syncpt_nb_bases(host); i++) 157 host1x_hw_syncpt_restore_wait_base(host, sp_base + i); 158 159 host1x_hw_syncpt_enable_protection(host); 160 161 wmb(); 162 } 163 164 /* 165 * Update the cached syncpoint and waitbase values by reading them 166 * from the registers. 167 */ 168 void host1x_syncpt_save(struct host1x *host) 169 { 170 struct host1x_syncpt *sp_base = host->syncpt; 171 unsigned int i; 172 173 for (i = 0; i < host1x_syncpt_nb_pts(host); i++) { 174 if (host1x_syncpt_client_managed(sp_base + i)) 175 host1x_hw_syncpt_load(host, sp_base + i); 176 else 177 WARN_ON(!host1x_syncpt_idle(sp_base + i)); 178 } 179 180 for (i = 0; i < host1x_syncpt_nb_bases(host); i++) 181 host1x_hw_syncpt_load_wait_base(host, sp_base + i); 182 } 183 184 /* 185 * Updates the cached syncpoint value by reading a new value from the hardware 186 * register 187 */ 188 u32 host1x_syncpt_load(struct host1x_syncpt *sp) 189 { 190 u32 val; 191 192 val = host1x_hw_syncpt_load(sp->host, sp); 193 trace_host1x_syncpt_load_min(sp->id, val); 194 195 return val; 196 } 197 198 /* 199 * Get the current syncpoint base 200 */ 201 u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp) 202 { 203 host1x_hw_syncpt_load_wait_base(sp->host, sp); 204 205 return sp->base_val; 206 } 207 208 /** 209 * host1x_syncpt_incr() - increment syncpoint value from CPU, updating cache 210 * @sp: host1x syncpoint 211 */ 212 int host1x_syncpt_incr(struct host1x_syncpt *sp) 213 { 214 return host1x_hw_syncpt_cpu_incr(sp->host, sp); 215 } 216 EXPORT_SYMBOL(host1x_syncpt_incr); 217 218 /** 219 * host1x_syncpt_wait() - wait for a syncpoint to reach a given value 220 * @sp: host1x syncpoint 221 * @thresh: threshold 222 * @timeout: maximum time to wait for the syncpoint to reach the given value 223 * @value: return location for the syncpoint value 224 */ 225 int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout, 226 u32 *value) 227 { 228 struct dma_fence *fence; 229 long wait_err; 230 u32 curr; 231 232 curr = host1x_syncpt_load(sp); 233 234 if (value) 235 *value = curr; 236 237 if (host1x_syncpt_is_expired(sp, thresh)) 238 return 0; 239 240 if (timeout < 0) 241 timeout = LONG_MAX; 242 else if (timeout == 0) 243 return -EAGAIN; 244 245 fence = host1x_fence_create(sp, thresh, false); 246 if (IS_ERR(fence)) 247 return PTR_ERR(fence); 248 249 wait_err = dma_fence_wait_timeout(fence, true, timeout); 250 if (wait_err == 0) 251 host1x_fence_cancel(fence); 252 dma_fence_put(fence); 253 254 /* 255 * Don't rely on dma_fence_wait_timeout return value, 256 * since it returns zero both on timeout and if the 257 * wait completed with 0 jiffies left. 258 */ 259 if (wait_err == 0 && !host1x_syncpt_is_expired(sp, thresh)) { 260 if (value) 261 *value = host1x_syncpt_load(sp); 262 263 return -EAGAIN; 264 } else if (wait_err < 0) { 265 return wait_err; 266 } else { 267 /* Success, read the value cached by ISR */ 268 if (value) 269 *value = host1x_syncpt_read_min(sp); 270 271 return 0; 272 } 273 } 274 EXPORT_SYMBOL(host1x_syncpt_wait); 275 276 /* 277 * Returns true if syncpoint is expired, false if we may need to wait 278 */ 279 bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh) 280 { 281 u32 current_val; 282 283 smp_rmb(); 284 285 current_val = (u32)atomic_read(&sp->min_val); 286 287 return (wrapping_sub(u32, current_val, thresh) & 0x80000000U) == 0U; 288 } 289 290 int host1x_syncpt_init(struct host1x *host) 291 { 292 struct host1x_syncpt_base *bases; 293 struct host1x_syncpt *syncpt; 294 unsigned int i; 295 296 syncpt = devm_kcalloc(host->dev, host->info->nb_pts, sizeof(*syncpt), 297 GFP_KERNEL); 298 if (!syncpt) 299 return -ENOMEM; 300 301 bases = devm_kcalloc(host->dev, host->info->nb_bases, sizeof(*bases), 302 GFP_KERNEL); 303 if (!bases) 304 return -ENOMEM; 305 306 for (i = 0; i < host->info->nb_pts; i++) { 307 syncpt[i].id = i; 308 syncpt[i].host = host; 309 310 /* Default to client managed for syncpoints not owned by us */ 311 syncpt[i].client_managed = true; 312 } 313 314 for (i = 0; i < host->info->nb_bases; i++) 315 bases[i].id = i; 316 317 mutex_init(&host->syncpt_mutex); 318 host->syncpt = syncpt; 319 host->bases = bases; 320 321 /* Prevent syncpoint 0 from being allocated by users */ 322 if (host->syncpt_base == 0) { 323 host->nop_sp = host1x_syncpt_alloc(host, 0, "reserved-nop"); 324 if (!host->nop_sp) 325 return -ENOMEM; 326 } 327 328 if (host->info->reserve_vblank_syncpts) { 329 kref_init(&host->syncpt[26].ref); 330 kref_init(&host->syncpt[27].ref); 331 } 332 333 return 0; 334 } 335 336 /** 337 * host1x_syncpt_request() - request a syncpoint 338 * @client: client requesting the syncpoint 339 * @flags: flags 340 * 341 * host1x client drivers can use this function to allocate a syncpoint for 342 * subsequent use. A syncpoint returned by this function will be reserved for 343 * use by the client exclusively. When no longer using a syncpoint, a host1x 344 * client driver needs to release it using host1x_syncpt_put(). 345 */ 346 struct host1x_syncpt *host1x_syncpt_request(struct host1x_client *client, 347 unsigned long flags) 348 { 349 struct host1x *host = dev_get_drvdata(client->host->parent); 350 351 return host1x_syncpt_alloc(host, flags, dev_name(client->dev)); 352 } 353 EXPORT_SYMBOL(host1x_syncpt_request); 354 355 static void syncpt_release(struct kref *ref) 356 { 357 struct host1x_syncpt *sp = container_of(ref, struct host1x_syncpt, ref); 358 359 atomic_set(&sp->max_val, host1x_syncpt_read(sp)); 360 361 sp->locked = false; 362 363 host1x_syncpt_base_free(sp->base); 364 kfree(sp->name); 365 sp->base = NULL; 366 sp->name = NULL; 367 sp->client_managed = false; 368 369 mutex_unlock(&sp->host->syncpt_mutex); 370 } 371 372 /** 373 * host1x_syncpt_put() - free a requested syncpoint 374 * @sp: host1x syncpoint 375 * 376 * Release a syncpoint previously allocated using host1x_syncpt_request(). A 377 * host1x client driver should call this when the syncpoint is no longer in 378 * use. 379 */ 380 void host1x_syncpt_put(struct host1x_syncpt *sp) 381 { 382 if (!sp) 383 return; 384 385 kref_put_mutex(&sp->ref, syncpt_release, &sp->host->syncpt_mutex); 386 } 387 EXPORT_SYMBOL(host1x_syncpt_put); 388 389 void host1x_syncpt_deinit(struct host1x *host) 390 { 391 struct host1x_syncpt *sp = host->syncpt; 392 unsigned int i; 393 394 for (i = 0; i < host->info->nb_pts; i++, sp++) 395 kfree(sp->name); 396 } 397 398 /** 399 * host1x_syncpt_read_max() - read maximum syncpoint value 400 * @sp: host1x syncpoint 401 * 402 * The maximum syncpoint value indicates how many operations there are in 403 * queue, either in channel or in a software thread. 404 */ 405 u32 host1x_syncpt_read_max(struct host1x_syncpt *sp) 406 { 407 smp_rmb(); 408 409 return (u32)atomic_read(&sp->max_val); 410 } 411 EXPORT_SYMBOL(host1x_syncpt_read_max); 412 413 /** 414 * host1x_syncpt_read_min() - read minimum syncpoint value 415 * @sp: host1x syncpoint 416 * 417 * The minimum syncpoint value is a shadow of the current sync point value in 418 * hardware. 419 */ 420 u32 host1x_syncpt_read_min(struct host1x_syncpt *sp) 421 { 422 smp_rmb(); 423 424 return (u32)atomic_read(&sp->min_val); 425 } 426 EXPORT_SYMBOL(host1x_syncpt_read_min); 427 428 /** 429 * host1x_syncpt_read() - read the current syncpoint value 430 * @sp: host1x syncpoint 431 */ 432 u32 host1x_syncpt_read(struct host1x_syncpt *sp) 433 { 434 return host1x_syncpt_load(sp); 435 } 436 EXPORT_SYMBOL(host1x_syncpt_read); 437 438 unsigned int host1x_syncpt_nb_pts(struct host1x *host) 439 { 440 return host->info->nb_pts; 441 } 442 443 unsigned int host1x_syncpt_nb_bases(struct host1x *host) 444 { 445 return host->info->nb_bases; 446 } 447 448 unsigned int host1x_syncpt_nb_mlocks(struct host1x *host) 449 { 450 return host->info->nb_mlocks; 451 } 452 453 /** 454 * host1x_syncpt_get_by_id() - obtain a syncpoint by ID 455 * @host: host1x controller 456 * @id: syncpoint ID 457 */ 458 struct host1x_syncpt *host1x_syncpt_get_by_id(struct host1x *host, 459 unsigned int id) 460 { 461 if (id >= host->info->nb_pts) 462 return NULL; 463 464 if (kref_get_unless_zero(&host->syncpt[id].ref)) 465 return &host->syncpt[id]; 466 else 467 return NULL; 468 } 469 EXPORT_SYMBOL(host1x_syncpt_get_by_id); 470 471 /** 472 * host1x_syncpt_get_by_id_noref() - obtain a syncpoint by ID but don't 473 * increase the refcount. 474 * @host: host1x controller 475 * @id: syncpoint ID 476 */ 477 struct host1x_syncpt *host1x_syncpt_get_by_id_noref(struct host1x *host, 478 unsigned int id) 479 { 480 if (id >= host->info->nb_pts) 481 return NULL; 482 483 return &host->syncpt[id]; 484 } 485 EXPORT_SYMBOL(host1x_syncpt_get_by_id_noref); 486 487 /** 488 * host1x_syncpt_get() - increment syncpoint refcount 489 * @sp: syncpoint 490 */ 491 struct host1x_syncpt *host1x_syncpt_get(struct host1x_syncpt *sp) 492 { 493 kref_get(&sp->ref); 494 495 return sp; 496 } 497 EXPORT_SYMBOL(host1x_syncpt_get); 498 499 /** 500 * host1x_syncpt_get_base() - obtain the wait base associated with a syncpoint 501 * @sp: host1x syncpoint 502 */ 503 struct host1x_syncpt_base *host1x_syncpt_get_base(struct host1x_syncpt *sp) 504 { 505 return sp ? sp->base : NULL; 506 } 507 EXPORT_SYMBOL(host1x_syncpt_get_base); 508 509 /** 510 * host1x_syncpt_base_id() - retrieve the ID of a syncpoint wait base 511 * @base: host1x syncpoint wait base 512 */ 513 u32 host1x_syncpt_base_id(struct host1x_syncpt_base *base) 514 { 515 return base->id; 516 } 517 EXPORT_SYMBOL(host1x_syncpt_base_id); 518 519 static void do_nothing(struct kref *ref) 520 { 521 } 522 523 /** 524 * host1x_syncpt_release_vblank_reservation() - Make VBLANK syncpoint 525 * available for allocation 526 * 527 * @client: host1x bus client 528 * @syncpt_id: syncpoint ID to make available 529 * 530 * Makes VBLANK<i> syncpoint available for allocatation if it was 531 * reserved at initialization time. This should be called by the display 532 * driver after it has ensured that any VBLANK increment programming configured 533 * by the boot chain has been disabled. 534 */ 535 void host1x_syncpt_release_vblank_reservation(struct host1x_client *client, 536 u32 syncpt_id) 537 { 538 struct host1x *host = dev_get_drvdata(client->host->parent); 539 540 if (!host->info->reserve_vblank_syncpts) 541 return; 542 543 kref_put(&host->syncpt[syncpt_id].ref, do_nothing); 544 } 545 EXPORT_SYMBOL(host1x_syncpt_release_vblank_reservation); 546