67 prealloc_setup(struct alloc_cache* alloc)  in prealloc_setup()  argument
78 		alloc_set_special_next(p, alloc->quar);  in prealloc_setup()
79 		alloc->quar = p;  in prealloc_setup()
80 		alloc->num_quar++;  in prealloc_setup()
86 prealloc_blocks(struct alloc_cache* alloc, size_t num)  in prealloc_blocks()  argument
96 		r->next = (char*)alloc->reg_list;  in prealloc_blocks()
97 		alloc->reg_list = r;  in prealloc_blocks()
98 		alloc->num_reg_blocks ++;  in prealloc_blocks()
103 alloc_init(struct alloc_cache* alloc, struct alloc_cache* super,  in alloc_init()  argument
106 	memset(alloc, 0, sizeof(*alloc));  in alloc_init()
107 	alloc->super = super;  in alloc_init()
108 	alloc->thread_num = thread_num;  in alloc_init()
109 	alloc->next_id = (uint64_t)thread_num; 	/* in steps, so that type */  in alloc_init()
110 	alloc->next_id <<= THRNUM_SHIFT; 	/* of *_id is used. */  in alloc_init()
111 	alloc->last_id = 1; 			/* so no 64bit constants, */  in alloc_init()
112 	alloc->last_id <<= THRNUM_SHIFT; 	/* or implicit 'int' ops. */  in alloc_init()
113 	alloc->last_id -= 1; 			/* for compiler portability. */  in alloc_init()
114 	alloc->last_id |= alloc->next_id;  in alloc_init()
115 	alloc->next_id += 1;			/* because id=0 is special. */  in alloc_init()
116 	alloc->max_reg_blocks = 100;  in alloc_init()
117 	alloc->num_reg_blocks = 0;  in alloc_init()
118 	alloc->reg_list = NULL;  in alloc_init()
119 	alloc->cleanup = NULL;  in alloc_init()
120 	alloc->cleanup_arg = NULL;  in alloc_init()
121 	if(alloc->super)  in alloc_init()
122 		prealloc_blocks(alloc, alloc->max_reg_blocks);  in alloc_init()
123 	if(!alloc->super) {  in alloc_init()
124 		lock_quick_init(&alloc->lock);  in alloc_init()
125 		lock_protect(&alloc->lock, alloc, sizeof(*alloc));  in alloc_init()
131 alloc_clear_special_list(struct alloc_cache* alloc)  in alloc_clear_special_list()  argument
135 	p = alloc->quar;  in alloc_clear_special_list()
146 alloc_clear_special(struct alloc_cache* alloc)  in alloc_clear_special()  argument
148 	if(!alloc->super) {  in alloc_clear_special()
149 		lock_quick_lock(&alloc->lock);  in alloc_clear_special()
151 	alloc_clear_special_list(alloc);  in alloc_clear_special()
152 	alloc->quar = 0;  in alloc_clear_special()
153 	alloc->num_quar = 0;  in alloc_clear_special()
154 	if(!alloc->super) {  in alloc_clear_special()
155 		lock_quick_unlock(&alloc->lock);  in alloc_clear_special()
160 alloc_clear(struct alloc_cache* alloc)  in alloc_clear()  argument
164 	if(!alloc)  in alloc_clear()
166 	if(!alloc->super) {  in alloc_clear()
167 		lock_quick_destroy(&alloc->lock);  in alloc_clear()
169 	if(alloc->super && alloc->quar) {  in alloc_clear()
171 		p = alloc->quar;  in alloc_clear()
174 		lock_quick_lock(&alloc->super->lock);  in alloc_clear()
175 		alloc_set_special_next(p, alloc->super->quar);  in alloc_clear()
176 		alloc->super->quar = alloc->quar;  in alloc_clear()
177 		alloc->super->num_quar += alloc->num_quar;  in alloc_clear()
178 		lock_quick_unlock(&alloc->super->lock);  in alloc_clear()
180 		alloc_clear_special_list(alloc);  in alloc_clear()
182 	alloc->quar = 0;  in alloc_clear()
183 	alloc->num_quar = 0;  in alloc_clear()
184 	r = alloc->reg_list;  in alloc_clear()
190 	alloc->reg_list = NULL;  in alloc_clear()
191 	alloc->num_reg_blocks = 0;  in alloc_clear()
195 alloc_get_id(struct alloc_cache* alloc)  in alloc_get_id()  argument
197 	uint64_t id = alloc->next_id++;  in alloc_get_id()
198 	if(id == alloc->last_id) {  in alloc_get_id()
200 		fptr_ok(fptr_whitelist_alloc_cleanup(alloc->cleanup));  in alloc_get_id()
201 		(*alloc->cleanup)(alloc->cleanup_arg);  in alloc_get_id()
204 		alloc->next_id = (uint64_t)alloc->thread_num; 	  in alloc_get_id()
205 		alloc->next_id <<= THRNUM_SHIFT; 	/* in steps for comp. */  in alloc_get_id()
206 		alloc->next_id += 1;			/* portability. */  in alloc_get_id()
208 		id = alloc->next_id++;  in alloc_get_id()
214 alloc_special_obtain(struct alloc_cache* alloc)  in alloc_special_obtain()  argument
217 	log_assert(alloc);  in alloc_special_obtain()
219 	if(alloc->quar) {  in alloc_special_obtain()
220 		p = alloc->quar;  in alloc_special_obtain()
221 		alloc->quar = alloc_special_next(p);  in alloc_special_obtain()
222 		alloc->num_quar--;  in alloc_special_obtain()
223 		p->id = alloc_get_id(alloc);  in alloc_special_obtain()
227 	if(alloc->super) {  in alloc_special_obtain()
230 		lock_quick_lock(&alloc->super->lock);  in alloc_special_obtain()
231 		if((p = alloc->super->quar)) {  in alloc_special_obtain()
232 			alloc->super->quar = alloc_special_next(p);  in alloc_special_obtain()
233 			alloc->super->num_quar--;  in alloc_special_obtain()
235 		lock_quick_unlock(&alloc->super->lock);  in alloc_special_obtain()
237 			p->id = alloc_get_id(alloc);  in alloc_special_obtain()
242 	prealloc_setup(alloc);  in alloc_special_obtain()
248 	p->id = alloc_get_id(alloc);  in alloc_special_obtain()
254 pushintosuper(struct alloc_cache* alloc, alloc_special_type* mem)  in pushintosuper()  argument
257 	alloc_special_type *p = alloc->quar;  in pushintosuper()
259 	log_assert(alloc && alloc->super &&   in pushintosuper()
260 		alloc->num_quar >= ALLOC_SPECIAL_MAX);  in pushintosuper()
262 	alloc_set_special_next(mem, alloc->quar);  in pushintosuper()
266 	alloc->quar = alloc_special_next(p);  in pushintosuper()
267 	alloc->num_quar -= ALLOC_SPECIAL_MAX/2;  in pushintosuper()
270 	lock_quick_lock(&alloc->super->lock);  in pushintosuper()
271 	alloc_set_special_next(p, alloc->super->quar);  in pushintosuper()
272 	alloc->super->quar = mem;  in pushintosuper()
273 	alloc->super->num_quar += ALLOC_SPECIAL_MAX/2 + 1;  in pushintosuper()
274 	lock_quick_unlock(&alloc->super->lock);  in pushintosuper()
279 alloc_special_release(struct alloc_cache* alloc, alloc_special_type* mem)  in alloc_special_release()  argument
281 	log_assert(alloc);  in alloc_special_release()
284 	if(!alloc->super) {   in alloc_special_release()
285 		lock_quick_lock(&alloc->lock); /* superalloc needs locking */  in alloc_special_release()
289 	if(alloc->super && alloc->num_quar >= ALLOC_SPECIAL_MAX) {  in alloc_special_release()
291 		pushintosuper(alloc, mem);  in alloc_special_release()
295 	alloc_set_special_next(mem, alloc->quar);  in alloc_special_release()
296 	alloc->quar = mem;  in alloc_special_release()
297 	alloc->num_quar++;  in alloc_special_release()
298 	if(!alloc->super) {  in alloc_special_release()
299 		lock_quick_unlock(&alloc->lock);  in alloc_special_release()
304 alloc_stats(struct alloc_cache* alloc)  in alloc_stats()  argument
306 	log_info("%salloc: %d in cache, %d blocks.", alloc->super?"":"sup",  in alloc_stats()
307 		(int)alloc->num_quar, (int)alloc->num_reg_blocks);  in alloc_stats()
310 size_t alloc_get_mem(struct alloc_cache* alloc)  in alloc_get_mem()  argument
313 	size_t s = sizeof(*alloc);  in alloc_get_mem()
314 	if(!alloc->super) {   in alloc_get_mem()
315 		lock_quick_lock(&alloc->lock); /* superalloc needs locking */  in alloc_get_mem()
317 	s += sizeof(alloc_special_type) * alloc->num_quar;  in alloc_get_mem()
318 	for(p = alloc->quar; p; p = alloc_special_next(p)) {  in alloc_get_mem()
321 	s += alloc->num_reg_blocks * ALLOC_REG_SIZE;  in alloc_get_mem()
322 	if(!alloc->super) {  in alloc_get_mem()
323 		lock_quick_unlock(&alloc->lock);  in alloc_get_mem()
329 alloc_reg_obtain(struct alloc_cache* alloc)  in alloc_reg_obtain()  argument
331 	if(alloc->num_reg_blocks > 0) {  in alloc_reg_obtain()
332 		struct regional* r = alloc->reg_list;  in alloc_reg_obtain()
333 		alloc->reg_list = (struct regional*)r->next;  in alloc_reg_obtain()
335 		alloc->num_reg_blocks--;  in alloc_reg_obtain()
342 alloc_reg_release(struct alloc_cache* alloc, struct regional* r)  in alloc_reg_release()  argument
344 	if(alloc->num_reg_blocks >= alloc->max_reg_blocks) {  in alloc_reg_release()
351 	r->next = (char*)alloc->reg_list;  in alloc_reg_release()
352 	alloc->reg_list = r;  in alloc_reg_release()
353 	alloc->num_reg_blocks++;  in alloc_reg_release()
357 alloc_set_id_cleanup(struct alloc_cache* alloc, void (*cleanup)(void*),  in alloc_set_id_cleanup()  argument
360 	alloc->cleanup = cleanup;  in alloc_set_id_cleanup()
361 	alloc->cleanup_arg = arg;  in alloc_set_id_cleanup()