Lines Matching refs:ct
156 #define CALLOUT_APPEND(ct, cp) \
157 CALLOUT_HASH_APPEND(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \
161 #define CALLOUT_DELETE(ct, cp) \
162 CALLOUT_HASH_DELETE(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \
193 #define CALLOUT_EXEC_COMPUTE(ct, nextexp, exec) \
197 cl = ct->ct_expired.ch_head; \
242 #define CALLOUT_LIST_FREE(ct, cl) \
244 cl->cl_next = ct->ct_lfree; \
245 ct->ct_lfree = cl; \
252 #define CALLOUT_FREE(ct, cl) \
254 cp->c_idnext = ct->ct_free; \
255 ct->ct_free = cp; \
266 callout_alloc(callout_table_t *ct)
271 ASSERT(MUTEX_HELD(&ct->ct_mutex));
272 mutex_exit(&ct->ct_mutex);
274 cp = kmem_cache_alloc(ct->ct_cache, KM_NOSLEEP);
284 mutex_enter(&ct->ct_mutex);
285 ct->ct_allocations++;
296 callout_list_alloc(callout_table_t *ct)
301 ASSERT(MUTEX_HELD(&ct->ct_mutex));
302 mutex_exit(&ct->ct_mutex);
304 cl = kmem_cache_alloc(ct->ct_lcache, KM_NOSLEEP);
311 mutex_enter(&ct->ct_mutex);
312 CALLOUT_LIST_FREE(ct, cl);
319 callout_list_get(callout_table_t *ct, hrtime_t expiration, int flags, int hash)
324 ASSERT(MUTEX_HELD(&ct->ct_mutex));
335 for (cl = ct->ct_clhash[hash].ch_head; (cl != NULL); cl = cl->cl_next) {
357 callout_queue_add(callout_table_t *ct, callout_list_t *cl)
363 nextcl = ct->ct_queue.ch_head;
365 CALLOUT_LIST_INSERT(ct->ct_queue, cl);
376 CALLOUT_LIST_APPEND(ct->ct_queue, cl);
386 callout_queue_insert(callout_table_t *ct, callout_list_t *cl)
400 if (callout_queue_add(ct, cl) && (ct->ct_suspend == 0))
401 (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration);
408 callout_queue_delete(callout_table_t *ct)
413 ASSERT(MUTEX_HELD(&ct->ct_mutex));
416 while ((cl = ct->ct_queue.ch_head) != NULL) {
420 CALLOUT_LIST_DELETE(ct->ct_queue, cl);
421 CALLOUT_LIST_APPEND(ct->ct_expired, cl);
428 if ((cl == NULL) || (ct->ct_suspend > 0))
431 (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration);
437 callout_queue_process(callout_table_t *ct, hrtime_t delta, int timechange)
444 ASSERT(MUTEX_HELD(&ct->ct_mutex));
446 firstcl = ct->ct_queue.ch_head;
456 temp = ct->ct_queue;
457 ct->ct_queue.ch_head = NULL;
458 ct->ct_queue.ch_tail = NULL;
475 CALLOUT_LIST_APPEND(ct->ct_expired, cl);
491 (void) callout_queue_add(ct, cl);
500 if (ct->ct_expired.ch_head != NULL)
503 cl = ct->ct_queue.ch_head;
515 callout_heap_init(callout_table_t *ct)
519 ASSERT(MUTEX_HELD(&ct->ct_mutex));
520 ASSERT(ct->ct_heap == NULL);
522 ct->ct_heap_num = 0;
523 ct->ct_heap_max = callout_chunk;
525 ct->ct_heap = kmem_alloc(size, KM_SLEEP);
533 callout_heap_expand(callout_table_t *ct)
538 ASSERT(MUTEX_HELD(&ct->ct_mutex));
539 ASSERT(ct->ct_heap_num <= ct->ct_heap_max);
541 while (ct->ct_heap_num == ct->ct_heap_max) {
542 max = ct->ct_heap_max;
543 mutex_exit(&ct->ct_mutex);
549 mutex_enter(&ct->ct_mutex);
555 if (ct->ct_nreap > 0)
556 (void) callout_heap_process(ct, 0, 0);
561 if (ct->ct_heap_num == ct->ct_heap_max)
565 if (max < ct->ct_heap_max) {
574 bcopy(ct->ct_heap, heap, osize);
575 kmem_free(ct->ct_heap, osize);
576 ct->ct_heap = heap;
577 ct->ct_heap_max = size / sizeof (callout_heap_t);
589 callout_upheap(callout_table_t *ct)
594 ASSERT(MUTEX_HELD(&ct->ct_mutex));
595 ASSERT(ct->ct_heap_num >= 1);
597 if (ct->ct_heap_num == 1) {
601 heap = ct->ct_heap;
602 current = ct->ct_heap_num - 1;
637 callout_heap_insert(callout_table_t *ct, callout_list_t *cl)
639 ASSERT(MUTEX_HELD(&ct->ct_mutex));
640 ASSERT(ct->ct_heap_num < ct->ct_heap_max);
647 ct->ct_heap[ct->ct_heap_num].ch_expiration = cl->cl_expiration;
648 ct->ct_heap[ct->ct_heap_num].ch_list = cl;
649 ct->ct_heap_num++;
661 if (callout_upheap(ct) && (ct->ct_suspend == 0))
662 (void) cyclic_reprogram(ct->ct_cyclic, cl->cl_expiration);
670 callout_downheap(callout_table_t *ct)
675 ASSERT(MUTEX_HELD(&ct->ct_mutex));
676 ASSERT(ct->ct_heap_num >= 1);
678 heap = ct->ct_heap;
680 nelems = ct->ct_heap_num;
749 callout_heap_delete(callout_table_t *ct)
756 ASSERT(MUTEX_HELD(&ct->ct_mutex));
758 if (CALLOUT_CLEANUP(ct)) {
763 (void) callout_heap_process(ct, 0, 0);
767 heap = ct->ct_heap;
769 while (ct->ct_heap_num > 0) {
780 CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
781 CALLOUT_LIST_FREE(ct, cl);
782 ct->ct_nreap--;
797 CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
798 CALLOUT_LIST_APPEND(ct->ct_expired, cl);
805 ct->ct_heap_num--;
806 if (ct->ct_heap_num > 0) {
807 heap[0] = heap[ct->ct_heap_num];
808 callout_downheap(ct);
817 if ((ct->ct_heap_num == 0) || (ct->ct_suspend > 0))
826 if (ct->ct_heap_num > 2) {
833 (void) cyclic_reprogram(ct->ct_cyclic, expiration);
859 callout_heap_process(callout_table_t *ct, hrtime_t delta, int timechange)
867 ASSERT(MUTEX_HELD(&ct->ct_mutex));
869 if (ct->ct_heap_num == 0)
872 if (ct->ct_nreap > 0)
873 ct->ct_cleanups++;
875 heap = ct->ct_heap;
893 num = ct->ct_heap_num;
894 ct->ct_heap_num = 0;
905 CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
906 CALLOUT_LIST_FREE(ct, cl);
921 CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
922 CALLOUT_LIST_APPEND(ct->ct_expired, cl);
933 CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
941 CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl);
943 CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl);
947 heap[ct->ct_heap_num] = heap[i];
948 ct->ct_heap_num++;
949 (void) callout_upheap(ct);
952 ct->ct_nreap = 0;
960 if (ct->ct_expired.ch_head != NULL)
963 if (ct->ct_heap_num == 0)
983 callout_table_t *ct;
1011 ct = &callout_table[CALLOUT_TABLE(type, CPU->cpu_seqid)];
1012 mutex_enter(&ct->ct_mutex);
1014 if (ct->ct_cyclic == CYCLIC_NONE) {
1015 mutex_exit(&ct->ct_mutex);
1021 ct = &callout_boot_ct[type];
1022 mutex_enter(&ct->ct_mutex);
1025 if (CALLOUT_CLEANUP(ct)) {
1032 (void) callout_heap_process(ct, 0, 0);
1035 if ((cp = ct->ct_free) == NULL)
1036 cp = callout_alloc(ct);
1038 ct->ct_free = cp->c_idnext;
1075 id = (ct->ct_long_id - callout_counter_low);
1077 ct->ct_long_id = id;
1079 id = (ct->ct_short_id - callout_counter_low);
1081 ct->ct_short_id = id;
1084 id = (ct->ct_gen_id - callout_counter_low);
1089 ct->ct_gen_id = id;
1107 cl = callout_list_get(ct, expiration, clflags, hash);
1113 if ((cl = ct->ct_lfree) == NULL) {
1114 callout_list_alloc(ct);
1126 ct->ct_lfree = cl->cl_next;
1134 if (ct->ct_heap_num == ct->ct_heap_max) {
1135 if (callout_heap_expand(ct) == 0) {
1139 callout_queue_insert(ct, cl);
1155 CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl);
1157 CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl);
1165 callout_heap_insert(ct, cl);
1173 ct->ct_nreap--;
1177 CALLOUT_APPEND(ct, cp);
1179 ct->ct_timeouts++;
1180 ct->ct_timeouts_pending++;
1182 mutex_exit(&ct->ct_mutex);
1280 callout_table_t *ct;
1287 ct = &callout_table[CALLOUT_ID_TO_TABLE(id)];
1290 mutex_enter(&ct->ct_mutex);
1295 for (cp = ct->ct_idhash[hash].ch_head; cp; cp = cp->c_idnext) {
1319 CALLOUT_DELETE(ct, cp);
1320 CALLOUT_FREE(ct, cp);
1321 ct->ct_untimeouts_unexpired++;
1322 ct->ct_timeouts_pending--;
1335 ct->ct_nreap++;
1337 CALLOUT_LIST_DELETE(ct->ct_queue, cl);
1338 CALLOUT_LIST_FREE(ct, cl);
1340 CALLOUT_LIST_DELETE(ct->ct_expired, cl);
1341 CALLOUT_LIST_FREE(ct, cl);
1344 mutex_exit(&ct->ct_mutex);
1353 ct->ct_untimeouts_executing++;
1368 mutex_exit(&ct->ct_mutex);
1382 cv_wait(&cp->c_done, &ct->ct_mutex);
1385 mutex_exit(&ct->ct_mutex);
1390 ct->ct_untimeouts_expired++;
1392 mutex_exit(&ct->ct_mutex);
1453 callout_list_expire(callout_table_t *ct, callout_list_t *cl)
1457 ASSERT(MUTEX_HELD(&ct->ct_mutex));
1477 mutex_exit(&ct->ct_mutex);
1483 mutex_enter(&ct->ct_mutex);
1485 ct->ct_expirations++;
1486 ct->ct_timeouts_pending--;
1499 CALLOUT_DELETE(ct, cp);
1500 CALLOUT_FREE(ct, cp);
1513 callout_expire(callout_table_t *ct)
1517 ASSERT(MUTEX_HELD(&ct->ct_mutex));
1519 for (cl = ct->ct_expired.ch_head; (cl != NULL); cl = clnext) {
1523 callout_list_expire(ct, cl);
1530 CALLOUT_LIST_DELETE(ct->ct_expired, cl);
1531 CALLOUT_LIST_FREE(ct, cl);
1558 callout_realtime(callout_table_t *ct)
1560 mutex_enter(&ct->ct_mutex);
1561 (void) callout_heap_delete(ct);
1562 callout_expire(ct);
1563 mutex_exit(&ct->ct_mutex);
1567 callout_queue_realtime(callout_table_t *ct)
1569 mutex_enter(&ct->ct_mutex);
1570 (void) callout_queue_delete(ct);
1571 callout_expire(ct);
1572 mutex_exit(&ct->ct_mutex);
1576 callout_execute(callout_table_t *ct)
1578 mutex_enter(&ct->ct_mutex);
1579 callout_expire(ct);
1580 mutex_exit(&ct->ct_mutex);
1587 callout_normal(callout_table_t *ct)
1592 mutex_enter(&ct->ct_mutex);
1593 exp = callout_heap_delete(ct);
1594 CALLOUT_EXEC_COMPUTE(ct, exp, exec);
1595 mutex_exit(&ct->ct_mutex);
1598 ASSERT(ct->ct_taskq != NULL);
1599 (void) taskq_dispatch(ct->ct_taskq,
1600 (task_func_t *)callout_execute, ct, TQ_NOSLEEP);
1605 callout_queue_normal(callout_table_t *ct)
1610 mutex_enter(&ct->ct_mutex);
1611 exp = callout_queue_delete(ct);
1612 CALLOUT_EXEC_COMPUTE(ct, exp, exec);
1613 mutex_exit(&ct->ct_mutex);
1616 ASSERT(ct->ct_taskq != NULL);
1617 (void) taskq_dispatch(ct->ct_taskq,
1618 (task_func_t *)callout_execute, ct, TQ_NOSLEEP);
1629 callout_table_t *ct;
1641 ct = &callout_table[CALLOUT_TABLE(t, f)];
1643 mutex_enter(&ct->ct_mutex);
1644 ct->ct_suspend++;
1645 if (ct->ct_cyclic == CYCLIC_NONE) {
1646 mutex_exit(&ct->ct_mutex);
1649 if (ct->ct_suspend == 1) {
1650 (void) cyclic_reprogram(ct->ct_cyclic,
1652 (void) cyclic_reprogram(ct->ct_qcyclic,
1655 mutex_exit(&ct->ct_mutex);
1668 callout_table_t *ct;
1677 ct = &callout_table[CALLOUT_TABLE(t, f)];
1679 mutex_enter(&ct->ct_mutex);
1680 if (ct->ct_cyclic == CYCLIC_NONE) {
1681 ct->ct_suspend--;
1682 mutex_exit(&ct->ct_mutex);
1693 hexp = callout_heap_process(ct, delta, timechange);
1694 qexp = callout_queue_process(ct, delta, timechange);
1696 ct->ct_suspend--;
1697 if (ct->ct_suspend == 0) {
1698 (void) cyclic_reprogram(ct->ct_cyclic, hexp);
1699 (void) cyclic_reprogram(ct->ct_qcyclic, qexp);
1702 mutex_exit(&ct->ct_mutex);
1758 callout_hrestime_one(callout_table_t *ct)
1762 mutex_enter(&ct->ct_mutex);
1763 if (ct->ct_cyclic == CYCLIC_NONE) {
1764 mutex_exit(&ct->ct_mutex);
1771 hexp = callout_heap_process(ct, 0, 1);
1772 qexp = callout_queue_process(ct, 0, 1);
1774 if (ct->ct_suspend == 0) {
1775 (void) cyclic_reprogram(ct->ct_cyclic, hexp);
1776 (void) cyclic_reprogram(ct->ct_qcyclic, qexp);
1779 mutex_exit(&ct->ct_mutex);
1791 callout_table_t *ct;
1803 ct = &callout_table[CALLOUT_TABLE(t, f)];
1804 callout_hrestime_one(ct);
1813 callout_hash_init(callout_table_t *ct)
1817 ASSERT(MUTEX_HELD(&ct->ct_mutex));
1818 ASSERT((ct->ct_idhash == NULL) && (ct->ct_clhash == NULL));
1821 ct->ct_idhash = kmem_zalloc(size, KM_SLEEP);
1822 ct->ct_clhash = kmem_zalloc(size, KM_SLEEP);
1829 callout_kstat_init(callout_table_t *ct)
1835 ASSERT(MUTEX_HELD(&ct->ct_mutex));
1836 ASSERT(ct->ct_kstats == NULL);
1838 ndx = ct - callout_table;
1844 (void *)ct);
1846 ct_kstats->ks_data = ct->ct_kstat_data;
1848 kstat_named_init(&ct->ct_kstat_data[stat],
1850 ct->ct_kstats = ct_kstats;
1856 callout_cyclic_init(callout_table_t *ct)
1864 ASSERT(MUTEX_HELD(&ct->ct_mutex));
1866 t = ct->ct_type;
1867 seqid = CALLOUT_TABLE_SEQID(ct);
1875 ASSERT(ct->ct_taskq == NULL);
1885 * be unnecessary (because "callout_excute(ct)"
1888 ct->ct_taskq =
1899 ASSERT(ct->ct_heap_num == 0);
1909 mutex_exit(&ct->ct_mutex);
1921 ASSERT(ct->ct_cyclic == CYCLIC_NONE);
1930 hdlr.cyh_arg = ct;
1943 mutex_enter(&ct->ct_mutex);
1944 ct->ct_cyclic = cyclic;
1945 ct->ct_qcyclic = qcyclic;
1954 callout_table_t *ct;
1991 ct = &callout_table[CALLOUT_TABLE(t, seqid)];
1993 mutex_enter(&ct->ct_mutex);
2000 ct->ct_cache = cache->cc_cache;
2001 ct->ct_lcache = cache->cc_lcache;
2007 if (ct->ct_heap == NULL) {
2008 callout_heap_init(ct);
2009 callout_hash_init(ct);
2010 callout_kstat_init(ct);
2011 callout_cyclic_init(ct);
2014 mutex_exit(&ct->ct_mutex);
2019 cyclic_bind(ct->ct_cyclic, cp, NULL);
2020 cyclic_bind(ct->ct_qcyclic, cp, NULL);
2027 callout_table_t *ct;
2036 ct = &callout_table[CALLOUT_TABLE(t, seqid)];
2042 cyclic_bind(ct->ct_cyclic, NULL, NULL);
2043 cyclic_bind(ct->ct_qcyclic, NULL, NULL);
2091 callout_table_t *ct;
2146 ct = &callout_table[table_id];
2147 ct->ct_type = t;
2148 mutex_init(&ct->ct_mutex, NULL, MUTEX_DEFAULT, NULL);
2154 ct->ct_short_id = CALLOUT_SHORT_ID(table_id);
2155 ct->ct_long_id = CALLOUT_LONG_ID(table_id);
2162 ct->ct_gen_id = CALLOUT_SHORT_ID(table_id);
2169 ct->ct_cyclic = CYCLIC_NONE;
2170 ct->ct_qcyclic = CYCLIC_NONE;
2171 ct->ct_kstat_data = kmem_zalloc(size, KM_SLEEP);