Lines Matching refs:cpu
688 #define CYC_TRACE(cpu, level, why, arg0, arg1) \
689 CYC_TRACE_IMPL(&cpu->cyp_trace[level], level, why, arg0, arg1)
712 #define CYC_TRACE(cpu, level, why, arg0, arg1)
717 #define CYC_TRACE0(cpu, level, why) CYC_TRACE(cpu, level, why, 0, 0)
718 #define CYC_TRACE1(cpu, level, why, arg0) CYC_TRACE(cpu, level, why, arg0, 0)
734 cyclic_upheap(cyc_cpu_t *cpu, cyc_index_t ndx)
744 heap = cpu->cyp_heap;
745 cyclics = cpu->cyp_cyclics;
777 cyclic_downheap(cyc_cpu_t *cpu, cyc_index_t ndx)
779 cyclic_t *cyclics = cpu->cyp_cyclics;
780 cyc_index_t *heap = cpu->cyp_heap;
784 cyc_index_t nelems = cpu->cyp_nelems;
852 cyclic_expire(cyc_cpu_t *cpu, cyc_index_t ndx, cyclic_t *cyclic)
854 cyc_backend_t *be = cpu->cyp_backend;
865 CYC_TRACE(cpu, CY_HIGH_LEVEL, "handler-in", handler, arg);
871 CYC_TRACE(cpu, CY_HIGH_LEVEL, "handler-out", handler, arg);
882 cyc_softbuf_t *softbuf = &cpu->cyp_softbuf[level];
888 CYC_TRACE(cpu, CY_HIGH_LEVEL, "expire-enq", cyclic,
901 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "expire-wrap", cyclic);
905 CYC_TRACE(cpu, CY_HIGH_LEVEL, "expire-bump", cyclic, 0);
943 cyc_cpu_t *cpu = c->cpu_cyclic;
944 cyc_backend_t *be = cpu->cyp_backend;
945 cyc_index_t *heap = cpu->cyp_heap;
946 cyclic_t *cyclic, *cyclics = cpu->cyp_cyclics;
951 CYC_TRACE(cpu, CY_HIGH_LEVEL, "fire", now, 0);
953 if (cpu->cyp_nelems == 0) {
958 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "fire-spurious");
969 CYC_TRACE(cpu, CY_HIGH_LEVEL, "fire-check", cyclic,
975 cyclic_expire(cpu, ndx, cyclic);
1021 CYC_TRACE(cpu, CY_HIGH_LEVEL, exp == interval ?
1028 cyclic_downheap(cpu, 0);
1039 cyclic_remove_pend(cyc_cpu_t *cpu, cyc_level_t level, cyclic_t *cyclic)
1043 uint32_t i, rpend = cpu->cyp_rpend - 1;
1047 ASSERT(cpu->cyp_state == CYS_REMOVING);
1048 ASSERT(cpu->cyp_rpend > 0);
1050 CYC_TRACE(cpu, level, "remove-rpend", cyclic, cpu->cyp_rpend);
1057 CYC_TRACE(cpu, level, "rpend-in", handler, arg);
1063 CYC_TRACE(cpu, level, "rpend-out", handler, arg);
1069 sema_v(&cpu->cyp_modify_wait);
1073 * cyclic_softint(cpu_t *cpu, cyc_level_t level)
1123 cyc_cpu_t *cpu = c->cpu_cyclic;
1127 cyclic_t *cyclics = cpu->cyp_cyclics;
1130 CYC_TRACE(cpu, level, "softint", cyclics, 0);
1134 softbuf = &cpu->cyp_softbuf[level];
1144 CYC_TRACE(cpu, level, "softint-top", cyclics, pc);
1153 ASSERT(buf[consmasked] < cpu->cyp_size);
1154 CYC_TRACE(cpu, level, "consuming", consndx, cyclic);
1178 CYC_TRACE(cpu, level, "handler-in", handler, arg);
1184 CYC_TRACE(cpu, level, "handler-out", handler, arg);
1190 if (cpu->cyp_state == CYS_REMOVING) {
1202 cyclic_remove_pend(cpu, level, cyclic);
1209 CYC_TRACE(cpu, level, "resize-int", cyclics, 0);
1210 ASSERT(cpu->cyp_state == CYS_EXPANDING);
1211 ASSERT(cyclics != cpu->cyp_cyclics);
1215 cyclics = cpu->cyp_cyclics;
1249 CYC_TRACE(cpu, level, "cas-fail", opend, pend);
1251 ((cyclics != cpu->cyp_cyclics &&
1252 cpu->cyp_state == CYS_EXPANDING) ||
1253 (cpu->cyp_state == CYS_REMOVING &&
1278 CYC_TRACE(cpu, level, "buffer-grow", 0, 0);
1279 ASSERT(cpu->cyp_state == CYS_EXPANDING);
1290 if (cpu->cyp_cyclics != cyclics) {
1291 CYC_TRACE1(cpu, level, "resize-int-int", consndx);
1292 cyclics = cpu->cyp_cyclics;
1311 ASSERT(cpu->cyp_state == CYS_EXPANDING);
1314 lev = cpu->cyp_modify_levels;
1316 } while (atomic_cas_32(&cpu->cyp_modify_levels, lev, nlev) !=
1325 CYC_TRACE0(cpu, level, "resize-kick");
1326 sema_v(&cpu->cyp_modify_wait);
1330 cyc_backend_t *be = cpu->cyp_backend;
1332 CYC_TRACE0(cpu, level, "resize-post");
1342 cyc_cpu_t *cpu = arg->cyx_cpu;
1343 cyc_backend_t *be = cpu->cyp_backend;
1346 cyc_index_t new_size = arg->cyx_size, size = cpu->cyp_size, i;
1348 cyclic_t *cyclics = cpu->cyp_cyclics, *new_cyclics = arg->cyx_cyclics;
1350 ASSERT(cpu->cyp_state == CYS_EXPANDING);
1361 CYC_TRACE(cpu, CY_HIGH_LEVEL, "expand", new_size, 0);
1368 ASSERT(cpu->cyp_heap != NULL && cpu->cyp_cyclics != NULL);
1370 bcopy(cpu->cyp_heap, new_heap, sizeof (cyc_index_t) * size);
1394 cpu->cyp_heap = new_heap;
1395 cpu->cyp_cyclics = new_cyclics;
1396 cpu->cyp_size = new_size;
1403 cyc_softbuf_t *softbuf = &cpu->cyp_softbuf[i];
1438 cyclic_expand(cyc_cpu_t *cpu)
1444 cyc_backend_t *be = cpu->cyp_backend;
1449 ASSERT(cpu->cyp_state == CYS_ONLINE);
1451 cpu->cyp_state = CYS_EXPANDING;
1453 old_heap = cpu->cyp_heap;
1454 old_cyclics = cpu->cyp_cyclics;
1456 if ((new_size = ((old_size = cpu->cyp_size) << 1)) == 0) {
1473 old_hard = cpu->cyp_softbuf[0].cys_hard;
1476 cyc_softbuf_t *softbuf = &cpu->cyp_softbuf[i];
1490 arg.cyx_cpu = cpu;
1495 cpu->cyp_modify_levels = 0;
1497 be->cyb_xcall(be->cyb_arg, cpu->cyp_cpu,
1503 sema_p(&cpu->cyp_modify_wait);
1504 ASSERT(cpu->cyp_modify_levels == CY_SOFT_LEVELS);
1510 cyc_softbuf_t *softbuf = &cpu->cyp_softbuf[i];
1534 ASSERT(cpu->cyp_state == CYS_EXPANDING);
1535 cpu->cyp_state = CYS_ONLINE;
1556 CYC_PTRACE("pick-cpu", part, bound);
1567 CYC_PTRACE("pick-cpu-bound", bound, avoid);
1640 CYC_PTRACE("pick-cpu-found", c, avoid);
1650 cyc_cpu_t *cpu = arg->cyx_cpu;
1653 cyc_backend_t *be = cpu->cyp_backend;
1659 ASSERT(cpu->cyp_nelems < cpu->cyp_size);
1663 CYC_TRACE(cpu, CY_HIGH_LEVEL,
1666 nelems = cpu->cyp_nelems++;
1673 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "enabled");
1677 ndx = cpu->cyp_heap[nelems];
1678 cyclic = &cpu->cyp_cyclics[ndx];
1699 if (cyclic_upheap(cpu, nelems)) {
1702 CYC_TRACE(cpu, CY_HIGH_LEVEL, "add-reprog", cyclic, exp);
1716 cyclic_add_here(cyc_cpu_t *cpu, cyc_handler_t *hdlr,
1719 cyc_backend_t *be = cpu->cyp_backend;
1723 CYC_PTRACE("add-cpu", cpu, hdlr->cyh_func);
1725 ASSERT(cpu->cyp_state == CYS_ONLINE);
1726 ASSERT(!(cpu->cyp_cpu->cpu_flags & CPU_OFFLINE));
1729 if (cpu->cyp_nelems == cpu->cyp_size) {
1734 cyclic_expand(cpu);
1735 ASSERT(cpu->cyp_nelems < cpu->cyp_size);
1743 arg.cyx_cpu = cpu;
1748 be->cyb_xcall(bar, cpu->cyp_cpu, (cyc_func_t)cyclic_add_xcall, &arg);
1750 CYC_PTRACE("add-cpu-done", cpu, arg.cyx_ndx);
1758 cyc_cpu_t *cpu = arg->cyx_cpu;
1759 cyc_backend_t *be = cpu->cyp_backend;
1769 ASSERT(cpu->cyp_state == CYS_REMOVING);
1773 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "remove-xcall", ndx);
1775 heap = cpu->cyp_heap;
1776 nelems = cpu->cyp_nelems;
1778 cyclic = &cpu->cyp_cyclics[ndx];
1807 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "remove-pend", cyclic->cy_pend);
1808 cpu->cyp_rpend = cyclic->cy_pend;
1828 cpu->cyp_nelems = --nelems;
1835 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "disabled");
1844 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "remove-bottom");
1861 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "remove-root");
1862 cyclic_downheap(cpu, 0);
1864 if (cyclic_upheap(cpu, i) == 0) {
1869 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "remove-no-root");
1871 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "remove-no-up");
1872 cyclic_downheap(cpu, i);
1883 cyclic = &cpu->cyp_cyclics[heap[0]];
1885 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "remove-reprog");
1894 cyclic_remove_here(cyc_cpu_t *cpu, cyc_index_t ndx, cyc_time_t *when, int wait)
1896 cyc_backend_t *be = cpu->cyp_backend;
1898 cyclic_t *cyclic = &cpu->cyp_cyclics[ndx];
1902 ASSERT(cpu->cyp_rpend == 0);
1906 arg.cyx_cpu = cpu;
1910 ASSERT(cpu->cyp_state == CYS_ONLINE);
1911 cpu->cyp_state = CYS_REMOVING;
1913 be->cyb_xcall(be->cyb_arg, cpu->cyp_cpu,
1921 ASSERT(!(level == CY_HIGH_LEVEL && cpu->cyp_rpend != 0));
1922 ASSERT(!(wait == CY_NOWAIT && cpu->cyp_rpend != 0));
1923 ASSERT(!(arg.cyx_wait == CY_NOWAIT && cpu->cyp_rpend != 0));
1932 ASSERT(cpu->cyp_state == CYS_REMOVING);
1933 cpu->cyp_state = CYS_ONLINE;
1938 if (cpu->cyp_rpend != 0)
1939 sema_p(&cpu->cyp_modify_wait);
1941 ASSERT(cpu->cyp_state == CYS_REMOVING);
1943 cpu->cyp_rpend = 0;
1944 cpu->cyp_state = CYS_ONLINE;
1955 cyclic_reprogram_cyclic(cyc_cpu_t *cpu, cyc_index_t ndx, hrtime_t expire)
1957 cyc_backend_t *be = cpu->cyp_backend;
1968 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "reprog-xcall", ndx);
1970 nelems = cpu->cyp_nelems;
1972 heap = cpu->cyp_heap;
1987 cyclic = &cpu->cyp_cyclics[ndx];
1993 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "reprog-down", i);
1994 cyclic_downheap(cpu, i);
1996 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "reprog-up", i);
1997 reprog = cyclic_upheap(cpu, i);
2000 if (reprog && (cpu->cyp_state != CYS_SUSPENDED)) {
2004 CYC_TRACE0(cpu, CY_HIGH_LEVEL, "reprog-root");
2005 cyclic = &cpu->cyp_cyclics[heap[0]];
2020 cyclic_reprogram_here(cyc_cpu_t *cpu, cyc_index_t ndx, hrtime_t expiration)
2022 cyc_backend_t *be = cpu->cyp_backend;
2029 arg.cyx_cpu = cpu;
2033 be->cyb_xcall(be->cyb_arg, cpu->cyp_cpu,
2146 cyc_cpu_t *cpu = idp->cyi_cpu, *dest;
2147 cyclic_t *cyclic = &cpu->cyp_cyclics[ndx];
2148 cpu_t *c = cpu->cyp_cpu;
2151 CYC_PTRACE("juggle-one", idp, cpu);
2154 ASSERT(cpu->cyp_state == CYS_ONLINE);
2161 CYC_PTRACE("juggle-fail", idp, cpu)
2174 cyc_cpu_t *cpu = idp->cyi_cpu;
2175 cpu_t *c = cpu->cyp_cpu;
2176 cyclic_t *cyclic = &cpu->cyp_cyclics[idp->cyi_ndx];
2178 CYC_PTRACE("unbind-cpu", id, cpu);
2180 ASSERT(cpu->cyp_state == CYS_ONLINE);
2195 ASSERT((res && idp->cyi_cpu != cpu) ||
2204 cyc_cpu_t *dest = d->cpu_cyclic, *cpu = idp->cyi_cpu;
2205 cpu_t *c = cpu->cyp_cpu;
2206 cyclic_t *cyclic = &cpu->cyp_cyclics[idp->cyi_ndx];
2209 CYC_PTRACE("bind-cpu", id, dest);
2213 ASSERT(cpu->cyp_state == CYS_ONLINE);
2221 if (dest != cpu) {
2233 cyc_cpu_t *cpu = idp->cyi_cpu;
2234 cpu_t *c = cpu->cyp_cpu;
2235 cyclic_t *cyc = &cpu->cyp_cyclics[idp->cyi_ndx];
2239 ASSERT(cpu->cyp_state == CYS_ONLINE);
2252 ASSERT(res && idp->cyi_cpu != cpu);
2260 cyc_cpu_t *cpu = idp->cyi_cpu, *dest;
2261 cpu_t *c = cpu->cyp_cpu;
2262 cyclic_t *cyc = &cpu->cyp_cyclics[idp->cyi_ndx];
2267 ASSERT(cpu->cyp_state == CYS_ONLINE);
2274 if (dest != cpu) {
2285 cyc_cpu_t *cpu = kmem_zalloc(sizeof (cyc_cpu_t), KM_SLEEP);
2289 CYC_PTRACE1("configure", cpu);
2296 cpu->cyp_cpu = c;
2298 sema_init(&cpu->cyp_modify_wait, 0, NULL, SEMA_DEFAULT, NULL);
2300 cpu->cyp_size = 1;
2301 cpu->cyp_heap = kmem_zalloc(sizeof (cyc_index_t), KM_SLEEP);
2302 cpu->cyp_cyclics = kmem_zalloc(sizeof (cyclic_t), KM_SLEEP);
2303 cpu->cyp_cyclics->cy_flags = CYF_FREE;
2310 cpu->cyp_softbuf[i].cys_buf[0].cypc_buf =
2314 cpu->cyp_state = CYS_OFFLINE;
2321 cpu->cyp_backend = nbe;
2330 c->cpu_cyclic = cpu;
2336 cyc_cpu_t *cpu = c->cpu_cyclic;
2337 cyc_backend_t *be = cpu->cyp_backend;
2341 CYC_PTRACE1("unconfigure", cpu);
2343 ASSERT(cpu->cyp_state == CYS_OFFLINE);
2344 ASSERT(cpu->cyp_nelems == 0);
2352 cpu->cyp_backend = NULL;
2358 cyc_softbuf_t *softbuf = &cpu->cyp_softbuf[i];
2379 kmem_free(cpu->cyp_cyclics, cpu->cyp_size * sizeof (cyclic_t));
2380 kmem_free(cpu->cyp_heap, cpu->cyp_size * sizeof (cyc_index_t));
2381 kmem_free(cpu, sizeof (cyc_cpu_t));
2391 * cpu array for this CPU.
2393 cpu_t *c = cpu[id];
2419 cyc_cpu_t *cpu = arg->cyx_cpu;
2420 cyc_backend_t *be = cpu->cyp_backend;
2426 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "suspend-xcall", cpu->cyp_nelems);
2427 ASSERT(cpu->cyp_state == CYS_ONLINE || cpu->cyp_state == CYS_OFFLINE);
2434 if (cpu->cyp_nelems > 0) {
2435 ASSERT(cpu->cyp_state == CYS_ONLINE);
2439 if (cpu->cyp_state == CYS_ONLINE)
2440 cpu->cyp_state = CYS_SUSPENDED;
2449 cyc_cpu_t *cpu = arg->cyx_cpu;
2450 cyc_backend_t *be = cpu->cyp_backend;
2453 cyc_state_t state = cpu->cyp_state;
2457 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "resume-xcall", cpu->cyp_nelems);
2466 if (cpu->cyp_nelems > 0) {
2467 cyclic_t *cyclic = &cpu->cyp_cyclics[cpu->cyp_heap[0]];
2470 CYC_TRACE(cpu, CY_HIGH_LEVEL, "resume-reprog", cyclic, exp);
2477 cpu->cyp_state = CYS_ONLINE;
2479 CYC_TRACE1(cpu, CY_HIGH_LEVEL, "resume-done", cpu->cyp_nelems);
2484 cyclic_omni_start(cyc_id_t *idp, cyc_cpu_t *cpu)
2491 CYC_PTRACE("omni-start", cpu, idp);
2493 ASSERT(cpu->cyp_state == CYS_ONLINE);
2503 omni->cyo_online(omni->cyo_arg, cpu->cyp_cpu, &hdlr, &when);
2509 ocpu->cyo_cpu = cpu;
2511 ocpu->cyo_ndx = cyclic_add_here(cpu, &hdlr, &when, 0);
2517 cyclic_omni_stop(cyc_id_t *idp, cyc_cpu_t *cpu)
2524 CYC_PTRACE("omni-stop", cpu, idp);
2526 ASSERT(cpu->cyp_state == CYS_ONLINE);
2537 while (ocpu != NULL && ocpu->cyo_cpu != cpu) {
2606 omni->cyo_offline(omni->cyo_arg, cpu->cyp_cpu, ocpu->cyo_arg);
2833 cyc_cpu_t *cpu;
2843 if ((cpu = c->cpu_cyclic) == NULL)
2846 if (cpu->cyp_state != CYS_ONLINE) {
2847 ASSERT(cpu->cyp_state == CYS_OFFLINE);
2851 cyclic_omni_start(idp, cpu);
2899 cyc_cpu_t *cpu = idp->cyi_cpu;
2904 if (cpu != NULL) {
2905 (void) cyclic_remove_here(cpu, idp->cyi_ndx, NULL, CY_WAIT);
2999 cyc_cpu_t *cpu = idp->cyi_cpu;
3007 if (cpu == NULL) {
3012 c = cpu->cyp_cpu;
3013 flags = cpu->cyp_cyclics[idp->cyi_ndx].cy_flags;
3019 * Reload our cpu (we may have migrated). We don't have to reload
3023 cpu = idp->cyi_cpu;
3024 c = cpu->cyp_cpu;
3034 cpu = idp->cyi_cpu;
3035 c = cpu->cyp_cpu;
3036 flags = cpu->cyp_cyclics[idp->cyi_ndx].cy_flags;
3051 cyc_cpu_t *cpu;
3068 cpu = CPU->cpu_cyclic;
3076 if (ocpu->cyo_cpu == cpu)
3095 cpu = idp->cyi_cpu;
3099 if (cpu->cyp_cpu == CPU)
3100 cyclic_reprogram_cyclic(cpu, ndx, expiration);
3102 cyclic_reprogram_here(cpu, ndx, expiration);
3205 cyc_cpu_t *cpu = c->cpu_cyclic;
3217 if (idp->cyi_cpu != cpu)
3225 ASSERT(idp->cyi_cpu != cpu);
3266 cyc_cpu_t *cpu = c->cpu_cyclic;
3269 CYC_PTRACE1("offline", cpu);
3288 cyclic_omni_stop(idp, cpu);
3291 ASSERT(cpu->cyp_state == CYS_ONLINE);
3292 cpu->cyp_state = CYS_OFFLINE;
3323 cyc_cpu_t *cpu = c->cpu_cyclic;
3326 CYC_PTRACE1("online", cpu);
3329 ASSERT(cpu->cyp_state == CYS_OFFLINE);
3331 cpu->cyp_state = CYS_ONLINE;
3341 cyclic_omni_start(idp, cpu);
3395 cyc_cpu_t *cpu = idp->cyi_cpu;
3401 if (cpu == NULL)
3404 c = cpu->cyp_cpu;
3409 cyclic = &cpu->cyp_cyclics[idp->cyi_ndx];
3470 cyc_cpu_t *cpu = c->cpu_cyclic, *dest;
3471 cyclic_t *cyclic, *cyclics = cpu->cyp_cyclics;
3474 CYC_PTRACE1("move-out", cpu);
3483 if (idp->cyi_cpu != cpu)
3499 CYC_PTRACE("move-out-fail", cpu, idp);
3505 CYC_PTRACE1("move-out-done", cpu);
3561 cyc_cpu_t *cpu;
3570 cpu = c->cpu_cyclic;
3571 be = cpu->cyp_backend;
3572 arg.cyx_cpu = cpu;
3613 cyc_cpu_t *cpu;
3623 cpu = c->cpu_cyclic;
3624 be = cpu->cyp_backend;
3625 arg.cyx_cpu = cpu;