Lines Matching refs:ctx

78 static void kcpc_restore(kcpc_ctx_t *ctx);
79 static void kcpc_save(kcpc_ctx_t *ctx);
80 static void kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx);
93 #define KCPC_CTX_FLAG_SET(ctx, flag) atomic_or_uint(&(ctx)->kc_flags, (flag)) argument
94 #define KCPC_CTX_FLAG_CLR(ctx, flag) atomic_and_uint(&(ctx)->kc_flags, ~(flag)) argument
173 kcpc_ctx_t *ctx; in kcpc_bind_cpu() local
177 ctx = kcpc_ctx_alloc(KM_SLEEP); in kcpc_bind_cpu()
179 if (kcpc_assign_reqs(set, ctx) != 0) { in kcpc_bind_cpu()
180 kcpc_ctx_free(ctx); in kcpc_bind_cpu()
185 ctx->kc_cpuid = cpuid; in kcpc_bind_cpu()
186 ctx->kc_thread = curthread; in kcpc_bind_cpu()
190 if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) { in kcpc_bind_cpu()
192 kcpc_ctx_free(ctx); in kcpc_bind_cpu()
196 set->ks_ctx = ctx; in kcpc_bind_cpu()
197 ctx->kc_set = set; in kcpc_bind_cpu()
239 kcpc_program(ctx, B_FALSE, B_TRUE); in kcpc_bind_cpu()
258 kcpc_ctx_free(ctx); in kcpc_bind_cpu()
265 kcpc_ctx_t *ctx; in kcpc_bind_thread() local
276 ctx = kcpc_ctx_alloc(KM_SLEEP); in kcpc_bind_thread()
283 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE); in kcpc_bind_thread()
284 ctx->kc_hrtime = gethrtime(); in kcpc_bind_thread()
286 if (kcpc_assign_reqs(set, ctx) != 0) { in kcpc_bind_thread()
287 kcpc_ctx_free(ctx); in kcpc_bind_thread()
292 ctx->kc_cpuid = -1; in kcpc_bind_thread()
294 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_LWPINHERIT); in kcpc_bind_thread()
295 ctx->kc_thread = t; in kcpc_bind_thread()
296 t->t_cpc_ctx = ctx; in kcpc_bind_thread()
300 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_NONPRIV); in kcpc_bind_thread()
307 if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) { in kcpc_bind_thread()
309 kcpc_ctx_free(ctx); in kcpc_bind_thread()
314 set->ks_ctx = ctx; in kcpc_bind_thread()
315 ctx->kc_set = set; in kcpc_bind_thread()
320 installctx(t, ctx, kcpc_save, kcpc_restore, NULL, in kcpc_bind_thread()
331 kcpc_program(ctx, B_TRUE, B_TRUE); in kcpc_bind_thread()
343 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE); in kcpc_bind_thread()
359 kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode) in kcpc_configure_reqs() argument
373 ASSERT(ctx->kc_pics[n].kp_req == NULL); in kcpc_configure_reqs()
386 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_SIGOVF); in kcpc_configure_reqs()
392 &(rp->kr_config), (void *)ctx)) != 0) { in kcpc_configure_reqs()
404 ctx->kc_pics[n].kp_req = rp; in kcpc_configure_reqs()
405 rp->kr_picp = &ctx->kc_pics[n]; in kcpc_configure_reqs()
430 kcpc_ctx_t *ctx = set->ks_ctx; in kcpc_sample() local
448 if (ctx->kc_flags & KCPC_CTX_INVALID) { in kcpc_sample()
454 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) { in kcpc_sample()
455 if (ctx->kc_cpuid != -1) { in kcpc_sample()
456 if (curthread->t_bind_cpu != ctx->kc_cpuid) { in kcpc_sample()
463 if (ctx->kc_thread == curthread) { in kcpc_sample()
466 ctx->kc_hrtime = gethrtime_waitfree(); in kcpc_sample()
467 pcbe_ops->pcbe_sample(ctx); in kcpc_sample()
468 ctx->kc_vtick += curtick - ctx->kc_rawtick; in kcpc_sample()
469 ctx->kc_rawtick = curtick; in kcpc_sample()
476 if (ctx->kc_flags & KCPC_CTX_INVALID) { in kcpc_sample()
490 if (copyout(&ctx->kc_hrtime, hrtime, sizeof (uint64_t)) == -1) in kcpc_sample()
492 if (copyout(&ctx->kc_vtick, tick, sizeof (uint64_t)) == -1) in kcpc_sample()
502 kcpc_stop_hw(kcpc_ctx_t *ctx) in kcpc_stop_hw() argument
508 if (ctx->kc_cpuid == CPU->cpu_id) { in kcpc_stop_hw()
511 cp = cpu_get(ctx->kc_cpuid); in kcpc_stop_hw()
514 ASSERT(cp != NULL && cp->cpu_cpc_ctx == ctx); in kcpc_stop_hw()
523 kcpc_ctx_t *ctx; in kcpc_unbind() local
536 ctx = set->ks_ctx; in kcpc_unbind()
541 mutex_enter(&ctx->kc_lock); in kcpc_unbind()
542 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID); in kcpc_unbind()
543 mutex_exit(&ctx->kc_lock); in kcpc_unbind()
545 if (ctx->kc_cpuid == -1) { in kcpc_unbind()
546 t = ctx->kc_thread; in kcpc_unbind()
557 if (!(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) in kcpc_unbind()
558 kcpc_unprogram(ctx, B_TRUE); in kcpc_unbind()
563 if (removectx(t, ctx, kcpc_save, kcpc_restore, NULL, in kcpc_unbind()
566 (void *)ctx, (void *)t); in kcpc_unbind()
568 (void) removectx(t, ctx, kcpc_save, kcpc_restore, NULL, in kcpc_unbind()
587 cp = cpu_get(ctx->kc_cpuid); in kcpc_unbind()
593 if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) in kcpc_unbind()
594 kcpc_stop_hw(ctx); in kcpc_unbind()
595 ASSERT(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED); in kcpc_unbind()
599 if (ctx->kc_thread == curthread) { in kcpc_unbind()
600 kcpc_free(ctx, 0); in kcpc_unbind()
633 kcpc_ctx_t *ctx = set->ks_ctx; in kcpc_restart() local
638 ASSERT(ctx->kc_thread == curthread); in kcpc_restart()
639 ASSERT(ctx->kc_cpuid == -1); in kcpc_restart()
654 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) in kcpc_restart()
660 ctx->kc_rawtick = KCPC_GET_TICK(); in kcpc_restart()
661 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE); in kcpc_restart()
662 pcbe_ops->pcbe_program(ctx); in kcpc_restart()
675 kcpc_ctx_t *ctx = t->t_cpc_ctx; in kcpc_enable() local
684 if (ctx == NULL) { in kcpc_enable()
692 } else if (ctx->kc_flags & KCPC_CTX_INVALID) in kcpc_enable()
696 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) in kcpc_enable()
699 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE); in kcpc_enable()
700 kcpc_restore(ctx); in kcpc_enable()
703 if (ctx->kc_flags & KCPC_CTX_FREEZE) in kcpc_enable()
706 kcpc_save(ctx); in kcpc_enable()
707 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE); in kcpc_enable()
719 KCPC_CTX_FLAG_SET(ctx, in kcpc_enable()
760 kcpc_ctx_t *ctx = (kcpc_ctx_t *)token; in kcpc_next_config() local
769 if (ctx->kc_pics[i].kp_req != NULL) in kcpc_next_config()
781 pic = &ctx->kc_pics[i]; in kcpc_next_config()
793 pic = &ctx->kc_pics[i]; in kcpc_next_config()
803 *data = ctx->kc_pics[i].kp_req->kr_data; in kcpc_next_config()
806 return (ctx->kc_pics[i].kp_req->kr_config); in kcpc_next_config()
813 kcpc_ctx_t *ctx; in kcpc_ctx_alloc() local
816 ctx = (kcpc_ctx_t *)kmem_zalloc(sizeof (kcpc_ctx_t), kmem_flags); in kcpc_ctx_alloc()
817 if (ctx == NULL) in kcpc_ctx_alloc()
820 hash = CPC_HASH_CTX(ctx); in kcpc_ctx_alloc()
822 ctx->kc_next = kcpc_ctx_list[hash]; in kcpc_ctx_alloc()
823 kcpc_ctx_list[hash] = ctx; in kcpc_ctx_alloc()
826 ctx->kc_pics = (kcpc_pic_t *)kmem_zalloc(sizeof (kcpc_pic_t) * in kcpc_ctx_alloc()
829 ctx->kc_cpuid = -1; in kcpc_ctx_alloc()
831 return (ctx); in kcpc_ctx_alloc()
839 kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx) in kcpc_ctx_clone() argument
841 kcpc_set_t *ks = ctx->kc_set, *cks; in kcpc_ctx_clone()
893 kcpc_ctx_free(kcpc_ctx_t *ctx) in kcpc_ctx_free() argument
896 long hash = CPC_HASH_CTX(ctx); in kcpc_ctx_free()
901 while (*loc != ctx) in kcpc_ctx_free()
903 *loc = ctx->kc_next; in kcpc_ctx_free()
906 kmem_free(ctx->kc_pics, cpc_ncounters * sizeof (kcpc_pic_t)); in kcpc_ctx_free()
907 cv_destroy(&ctx->kc_condv); in kcpc_ctx_free()
908 mutex_destroy(&ctx->kc_lock); in kcpc_ctx_free()
909 kmem_free(ctx, sizeof (*ctx)); in kcpc_ctx_free()
922 kcpc_ctx_t *ctx; in kcpc_overflow_intr() local
948 ctx = NULL; in kcpc_overflow_intr()
951 ctx = t->t_cpc_ctx; in kcpc_overflow_intr()
954 ctx = t->t_cpc_ctx; in kcpc_overflow_intr()
956 if (ctx == NULL) { in kcpc_overflow_intr()
966 ctx = curthread->t_cpu->cpu_cpc_ctx; in kcpc_overflow_intr()
967 if (ctx != NULL) { in kcpc_overflow_intr()
973 return (ctx); in kcpc_overflow_intr()
1012 } else if ((ctx->kc_flags & KCPC_CTX_INVALID) == 0) { in kcpc_overflow_intr()
1025 if (ctx->kc_pics[i].kp_req != NULL && in kcpc_overflow_intr()
1027 ctx->kc_pics[i].kp_req->kr_flags & in kcpc_overflow_intr()
1034 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE); in kcpc_overflow_intr()
1035 atomic_or_uint(&ctx->kc_pics[i].kp_flags, in kcpc_overflow_intr()
1040 } else if (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) { in kcpc_overflow_intr()
1059 kcpc_ctx_t *ctx; in kcpc_hw_overflow_intr() local
1091 ctx = curthread->t_cpu->cpu_cpc_ctx; in kcpc_hw_overflow_intr()
1092 if (ctx == NULL) { in kcpc_hw_overflow_intr()
1101 for (i = 0; i < ctx->kc_set->ks_nreqs; i++) { in kcpc_hw_overflow_intr()
1102 req = ctx->kc_set->ks_req[i]; in kcpc_hw_overflow_intr()
1109 (void *)ctx); in kcpc_hw_overflow_intr()
1112 pcbe_ops->pcbe_program(ctx); in kcpc_hw_overflow_intr()
1145 if ((ctx = kcpc_overflow_intr(arg1, bitmap)) != NULL) { in kcpc_hw_overflow_intr()
1148 ctx->kc_hrtime = gethrtime_waitfree(); in kcpc_hw_overflow_intr()
1149 ctx->kc_vtick += curtick - ctx->kc_rawtick; in kcpc_hw_overflow_intr()
1150 ctx->kc_rawtick = curtick; in kcpc_hw_overflow_intr()
1151 pcbe_ops->pcbe_sample(ctx); in kcpc_hw_overflow_intr()
1152 pcbe_ops->pcbe_program(ctx); in kcpc_hw_overflow_intr()
1166 kcpc_ctx_t *ctx = curthread->t_cpc_ctx; in kcpc_overflow_ast() local
1171 ASSERT(ctx != NULL); /* Beware of interrupt skid. */ in kcpc_overflow_ast()
1179 ctx->kc_hrtime = gethrtime_waitfree(); in kcpc_overflow_ast()
1180 pcbe_ops->pcbe_sample(ctx); in kcpc_overflow_ast()
1183 ctx->kc_vtick += curtick - ctx->kc_rawtick; in kcpc_overflow_ast()
1194 if (ctx->kc_pics[i].kp_flags & KCPC_PIC_OVERFLOWED) { in kcpc_overflow_ast()
1195 atomic_and_uint(&ctx->kc_pics[i].kp_flags, in kcpc_overflow_ast()
1207 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE); in kcpc_overflow_ast()
1208 pcbe_ops->pcbe_program(ctx); in kcpc_overflow_ast()
1217 kcpc_save(kcpc_ctx_t *ctx) in kcpc_save() argument
1225 if (ctx->kc_flags & KCPC_CTX_INVALID) { in kcpc_save()
1226 if (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) { in kcpc_save()
1235 kcpc_unprogram(ctx, B_TRUE); in kcpc_save()
1242 if (ctx->kc_flags & KCPC_CTX_FREEZE) { in kcpc_save()
1251 ctx->kc_hrtime = gethrtime_waitfree(); in kcpc_save()
1252 ctx->kc_vtick += KCPC_GET_TICK() - ctx->kc_rawtick; in kcpc_save()
1253 pcbe_ops->pcbe_sample(ctx); in kcpc_save()
1259 ASSERT(ctx->kc_cpuid == -1); in kcpc_save()
1266 kcpc_restore(kcpc_ctx_t *ctx) in kcpc_restore() argument
1270 mutex_enter(&ctx->kc_lock); in kcpc_restore()
1272 if ((ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED)) == in kcpc_restore()
1279 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID_STOPPED); in kcpc_restore()
1282 if (ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_FREEZE)) { in kcpc_restore()
1283 mutex_exit(&ctx->kc_lock); in kcpc_restore()
1296 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_RESTORE); in kcpc_restore()
1297 mutex_exit(&ctx->kc_lock); in kcpc_restore()
1306 kcpc_program(ctx, B_TRUE, B_TRUE); in kcpc_restore()
1313 mutex_enter(&ctx->kc_lock); in kcpc_restore()
1314 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_RESTORE); in kcpc_restore()
1315 cv_signal(&ctx->kc_condv); in kcpc_restore()
1316 mutex_exit(&ctx->kc_lock); in kcpc_restore()
1378 kcpc_ctx_t *ctx = t->t_cpc_ctx, *cctx; in kcpc_lwp_create() local
1381 if (ctx == NULL || (ctx->kc_flags & KCPC_CTX_LWPINHERIT) == 0) in kcpc_lwp_create()
1385 if (ctx->kc_flags & KCPC_CTX_INVALID) { in kcpc_lwp_create()
1390 kcpc_ctx_clone(ctx, cctx); in kcpc_lwp_create()
1397 KCPC_CTX_FLAG_SET(cctx, ctx->kc_flags); in kcpc_lwp_create()
1412 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE); in kcpc_lwp_create()
1464 kcpc_free(kcpc_ctx_t *ctx, int isexec) in kcpc_free() argument
1467 kcpc_set_t *set = ctx->kc_set; in kcpc_free()
1474 mutex_enter(&ctx->kc_lock); in kcpc_free()
1475 while (ctx->kc_flags & KCPC_CTX_RESTORE) in kcpc_free()
1476 cv_wait(&ctx->kc_condv, &ctx->kc_lock); in kcpc_free()
1477 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID); in kcpc_free()
1478 mutex_exit(&ctx->kc_lock); in kcpc_free()
1487 if (ctx->kc_cpuid != -1) { in kcpc_free()
1495 cp = cpu_get(ctx->kc_cpuid); in kcpc_free()
1503 kcpc_stop_hw(ctx); in kcpc_free()
1516 kcpc_unprogram(ctx, B_TRUE); in kcpc_free()
1528 ASSERT(ctx->kc_thread == curthread); in kcpc_free()
1542 kcpc_ctx_free(ctx); in kcpc_free()
1578 kcpc_ctx_t *ctx; in kcpc_invalidate_all() local
1583 for (ctx = kcpc_ctx_list[hash]; ctx; ctx = ctx->kc_next) in kcpc_invalidate_all()
1584 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID); in kcpc_invalidate_all()
1596 kcpc_ctx_t *ctx = token; in kcpc_invalidate_config() local
1598 ASSERT(ctx != NULL); in kcpc_invalidate_config()
1600 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID); in kcpc_invalidate_config()
1609 kcpc_ctx_t *ctx = curthread->t_cpc_ctx; in kcpc_passivate() local
1616 if (ctx == NULL) { in kcpc_passivate()
1644 if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) { in kcpc_passivate()
1645 kcpc_unprogram(ctx, B_TRUE); in kcpc_passivate()
1646 KCPC_CTX_FLAG_SET(ctx, in kcpc_passivate()
1667 kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx) in kcpc_assign_reqs() argument
1815 kcpc_ctx_t *ctx = t->t_cpc_ctx; in kcpc_invalidate() local
1817 if (ctx != NULL) in kcpc_invalidate()
1818 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID); in kcpc_invalidate()
1897 kcpc_ctx_t *ctx; in kcpc_cpu_ctx_create() local
1904 ctx = kcpc_ctx_alloc(kmem_flags); in kcpc_cpu_ctx_create()
1907 kcpc_ctx_free(ctx); in kcpc_cpu_ctx_create()
1915 if (kcpc_assign_reqs(set, ctx) != 0) { in kcpc_cpu_ctx_create()
1927 kcpc_ctx_free(ctx); in kcpc_cpu_ctx_create()
1930 if (kcpc_assign_reqs(set, ctx) != 0) { in kcpc_cpu_ctx_create()
1937 kcpc_ctx_free(ctx); in kcpc_cpu_ctx_create()
1951 kcpc_ctx_free(ctx); in kcpc_cpu_ctx_create()
1958 if (kcpc_configure_reqs(ctx, set, &subcode) != 0) { in kcpc_cpu_ctx_create()
1969 kcpc_ctx_free(ctx); in kcpc_cpu_ctx_create()
1977 set->ks_ctx = ctx; in kcpc_cpu_ctx_create()
1978 ctx->kc_set = set; in kcpc_cpu_ctx_create()
1979 ctx->kc_cpuid = cp->cpu_id; in kcpc_cpu_ctx_create()
1980 ctx->kc_thread = curthread; in kcpc_cpu_ctx_create()
1982 ctx_ptrs[nctx] = ctx; in kcpc_cpu_ctx_create()
2068 kcpc_program(kcpc_ctx_t *ctx, boolean_t for_thread, boolean_t cu_interpose) in kcpc_program() argument
2079 ASSERT(ctx != NULL && (ctx->kc_cpuid == CPU->cpu_id || in kcpc_program()
2080 ctx->kc_cpuid == -1) && curthread->t_preempt > 0); in kcpc_program()
2081 if (ctx == NULL || (ctx->kc_cpuid != CPU->cpu_id && in kcpc_program()
2082 ctx->kc_cpuid != -1) || curthread->t_preempt < 1) in kcpc_program()
2092 kcpc_set_t *set = ctx->kc_set; in kcpc_program()
2121 ctx->kc_rawtick = KCPC_GET_TICK(); in kcpc_program()
2122 pcbe_ops->pcbe_program(ctx); in kcpc_program()
2129 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE); in kcpc_program()
2131 CPU->cpu_cpc_ctx = ctx; in kcpc_program()
2145 kcpc_unprogram(kcpc_ctx_t *ctx, boolean_t cu_interpose) in kcpc_unprogram() argument
2156 ASSERT(ctx != NULL && (ctx->kc_cpuid == CPU->cpu_id || in kcpc_unprogram()
2157 ctx->kc_cpuid == -1) && curthread->t_preempt > 0); in kcpc_unprogram()
2159 if (ctx == NULL || (ctx->kc_cpuid != CPU->cpu_id && in kcpc_unprogram()
2160 ctx->kc_cpuid != -1) || curthread->t_preempt < 1 || in kcpc_unprogram()
2161 (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) != 0) { in kcpc_unprogram()
2169 ASSERT(CPU->cpu_cpc_ctx == ctx || curthread->t_cpc_ctx == ctx); in kcpc_unprogram()
2175 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID_STOPPED); in kcpc_unprogram()
2193 kcpc_ctx_t *ctx; in kcpc_read() local
2206 ctx = CPU->cpu_cpc_ctx; in kcpc_read()
2207 if (ctx == NULL) { in kcpc_read()
2215 pcbe_ops->pcbe_sample(ctx); in kcpc_read()
2217 set = ctx->kc_set; in kcpc_read()
2464 kcpc_remoteprogram_func(kcpc_ctx_t *ctx, uintptr_t arg) in kcpc_remoteprogram_func() argument
2468 ASSERT(ctx != NULL); in kcpc_remoteprogram_func()
2471 kcpc_program(ctx, for_thread, B_TRUE); in kcpc_remoteprogram_func()
2479 kcpc_cpu_program(cpu_t *cp, kcpc_ctx_t *ctx) in kcpc_cpu_program() argument
2481 cpu_call(cp, (cpu_call_func_t)kcpc_remoteprogram_func, (uintptr_t)ctx, in kcpc_cpu_program()