31  * buckets. The core counter contains a lock for the entire counter, as well
32  * as the current upper and lower bounds on the value of the counter. The
83 aggsum_init(aggsum_t *as, uint64_t value)  in aggsum_init()  argument
85 	bzero(as, sizeof (*as));  in aggsum_init()
86 	as->as_lower_bound = as->as_upper_bound = value;  in aggsum_init()
87 	mutex_init(&as->as_lock, NULL, MUTEX_DEFAULT, NULL);  in aggsum_init()
88 	as->as_numbuckets = boot_ncpus;  in aggsum_init()
89 	as->as_buckets = kmem_zalloc(boot_ncpus * sizeof (aggsum_bucket_t),  in aggsum_init()
91 	for (int i = 0; i < as->as_numbuckets; i++) {  in aggsum_init()
92 		mutex_init(&as->as_buckets[i].asc_lock,  in aggsum_init()
98 aggsum_fini(aggsum_t *as)  in aggsum_fini()  argument
100 	for (int i = 0; i < as->as_numbuckets; i++)  in aggsum_fini()
101 		mutex_destroy(&as->as_buckets[i].asc_lock);  in aggsum_fini()
102 	kmem_free(as->as_buckets, as->as_numbuckets * sizeof (aggsum_bucket_t));  in aggsum_fini()
103 	mutex_destroy(&as->as_lock);  in aggsum_fini()
107 aggsum_lower_bound(aggsum_t *as)  in aggsum_lower_bound()  argument
109 	return (as->as_lower_bound);  in aggsum_lower_bound()
113 aggsum_upper_bound(aggsum_t *as)  in aggsum_upper_bound()  argument
115 	return (as->as_upper_bound);  in aggsum_upper_bound()
119 aggsum_flush_bucket(aggsum_t *as, struct aggsum_bucket *asb)  in aggsum_flush_bucket()  argument
121 	ASSERT(MUTEX_HELD(&as->as_lock));  in aggsum_flush_bucket()
128 	atomic_add_64((volatile uint64_t *)&as->as_lower_bound, asb->asc_delta);  in aggsum_flush_bucket()
129 	atomic_add_64((volatile uint64_t *)&as->as_upper_bound, asb->asc_delta);  in aggsum_flush_bucket()
131 	atomic_add_64((volatile uint64_t *)&as->as_upper_bound,  in aggsum_flush_bucket()
133 	atomic_add_64((volatile uint64_t *)&as->as_lower_bound,  in aggsum_flush_bucket()
139 aggsum_value(aggsum_t *as)  in aggsum_value()  argument
143 	mutex_enter(&as->as_lock);  in aggsum_value()
144 	if (as->as_lower_bound == as->as_upper_bound) {  in aggsum_value()
145 		rv = as->as_lower_bound;  in aggsum_value()
146 		for (int i = 0; i < as->as_numbuckets; i++) {  in aggsum_value()
147 			ASSERT0(as->as_buckets[i].asc_delta);  in aggsum_value()
148 			ASSERT0(as->as_buckets[i].asc_borrowed);  in aggsum_value()
150 		mutex_exit(&as->as_lock);  in aggsum_value()
153 	for (int i = 0; i < as->as_numbuckets; i++) {  in aggsum_value()
154 		struct aggsum_bucket *asb = &as->as_buckets[i];  in aggsum_value()
156 		aggsum_flush_bucket(as, asb);  in aggsum_value()
159 	VERIFY3U(as->as_lower_bound, ==, as->as_upper_bound);  in aggsum_value()
160 	rv = as->as_lower_bound;  in aggsum_value()
161 	mutex_exit(&as->as_lock);  in aggsum_value()
167 aggsum_borrow(aggsum_t *as, int64_t delta, struct aggsum_bucket *asb)  in aggsum_borrow()  argument
170 	mutex_enter(&as->as_lock);  in aggsum_borrow()
173 	aggsum_flush_bucket(as, asb);  in aggsum_borrow()
175 	atomic_add_64((volatile uint64_t *)&as->as_upper_bound, abs_delta);  in aggsum_borrow()
176 	atomic_add_64((volatile uint64_t *)&as->as_lower_bound, -abs_delta);  in aggsum_borrow()
180 	mutex_exit(&as->as_lock);  in aggsum_borrow()
184 aggsum_add(aggsum_t *as, int64_t delta)  in aggsum_add()  argument
187 	    &as->as_buckets[CPU_SEQID % as->as_numbuckets];  in aggsum_add()
198 		aggsum_borrow(as, delta * aggsum_borrow_multiplier, asb);  in aggsum_add()
208 aggsum_compare(aggsum_t *as, uint64_t target)  in aggsum_compare()  argument
210 	if (as->as_upper_bound < target)  in aggsum_compare()
212 	if (as->as_lower_bound > target)  in aggsum_compare()
214 	mutex_enter(&as->as_lock);  in aggsum_compare()
215 	for (int i = 0; i < as->as_numbuckets; i++) {  in aggsum_compare()
216 		struct aggsum_bucket *asb = &as->as_buckets[i];  in aggsum_compare()
218 		aggsum_flush_bucket(as, asb);  in aggsum_compare()
220 		if (as->as_upper_bound < target) {  in aggsum_compare()
221 			mutex_exit(&as->as_lock);  in aggsum_compare()
224 		if (as->as_lower_bound > target) {  in aggsum_compare()
225 			mutex_exit(&as->as_lock);  in aggsum_compare()
229 	VERIFY3U(as->as_lower_bound, ==, as->as_upper_bound);  in aggsum_compare()
230 	ASSERT3U(as->as_lower_bound, ==, target);  in aggsum_compare()
231 	mutex_exit(&as->as_lock);  in aggsum_compare()