12 static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)  in get_lower_chunk()  argument
16 	lockdep_assert_held(&pid_list->lock);  in get_lower_chunk()
18 	if (!pid_list->lower_list)  in get_lower_chunk()
21 	chunk = pid_list->lower_list;  in get_lower_chunk()
22 	pid_list->lower_list = chunk->next;  in get_lower_chunk()
23 	pid_list->free_lower_chunks--;  in get_lower_chunk()
24 	WARN_ON_ONCE(pid_list->free_lower_chunks < 0);  in get_lower_chunk()
30 	if (pid_list->free_lower_chunks <= CHUNK_REALLOC)  in get_lower_chunk()
31 		irq_work_queue(&pid_list->refill_irqwork);  in get_lower_chunk()
36 static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)  in get_upper_chunk()  argument
40 	lockdep_assert_held(&pid_list->lock);  in get_upper_chunk()
42 	if (!pid_list->upper_list)  in get_upper_chunk()
45 	chunk = pid_list->upper_list;  in get_upper_chunk()
46 	pid_list->upper_list = chunk->next;  in get_upper_chunk()
47 	pid_list->free_upper_chunks--;  in get_upper_chunk()
48 	WARN_ON_ONCE(pid_list->free_upper_chunks < 0);  in get_upper_chunk()
54 	if (pid_list->free_upper_chunks <= CHUNK_REALLOC)  in get_upper_chunk()
55 		irq_work_queue(&pid_list->refill_irqwork);  in get_upper_chunk()
60 static inline void put_lower_chunk(struct trace_pid_list *pid_list,  in put_lower_chunk()  argument
63 	lockdep_assert_held(&pid_list->lock);  in put_lower_chunk()
65 	chunk->next = pid_list->lower_list;  in put_lower_chunk()
66 	pid_list->lower_list = chunk;  in put_lower_chunk()
67 	pid_list->free_lower_chunks++;  in put_lower_chunk()
70 static inline void put_upper_chunk(struct trace_pid_list *pid_list,  in put_upper_chunk()  argument
73 	lockdep_assert_held(&pid_list->lock);  in put_upper_chunk()
75 	chunk->next = pid_list->upper_list;  in put_upper_chunk()
76 	pid_list->upper_list = chunk;  in put_upper_chunk()
77 	pid_list->free_upper_chunks++;  in put_upper_chunk()
125 bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_is_set()  argument
135 	if (!pid_list)  in trace_pid_list_is_set()
141 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_is_set()
142 	upper_chunk = pid_list->upper[upper1];  in trace_pid_list_is_set()
148 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_is_set()
164 int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_set()  argument
174 	if (!pid_list)  in trace_pid_list_set()
180 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_set()
181 	upper_chunk = pid_list->upper[upper1];  in trace_pid_list_set()
183 		upper_chunk = get_upper_chunk(pid_list);  in trace_pid_list_set()
188 		pid_list->upper[upper1] = upper_chunk;  in trace_pid_list_set()
192 		lower_chunk = get_lower_chunk(pid_list);  in trace_pid_list_set()
202 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_set()
217 int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_clear()  argument
226 	if (!pid_list)  in trace_pid_list_clear()
232 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_clear()
233 	upper_chunk = pid_list->upper[upper1];  in trace_pid_list_clear()
245 		put_lower_chunk(pid_list, lower_chunk);  in trace_pid_list_clear()
248 			put_upper_chunk(pid_list, upper_chunk);  in trace_pid_list_clear()
249 			pid_list->upper[upper1] = NULL;  in trace_pid_list_clear()
253 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_clear()
269 int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,  in trace_pid_list_next()  argument
279 	if (!pid_list)  in trace_pid_list_next()
285 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_next()
287 		upper_chunk = pid_list->upper[upper1];  in trace_pid_list_next()
305 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_next()
323 int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)  in trace_pid_list_first()  argument
325 	return trace_pid_list_next(pid_list, 0, pid);  in trace_pid_list_first()
330 	struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,  in pid_list_refill_irq()  local
342 	raw_spin_lock(&pid_list->lock);  in pid_list_refill_irq()
343 	upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;  in pid_list_refill_irq()
344 	lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;  in pid_list_refill_irq()
345 	raw_spin_unlock(&pid_list->lock);  in pid_list_refill_irq()
372 	raw_spin_lock(&pid_list->lock);  in pid_list_refill_irq()
374 		*upper_next = pid_list->upper_list;  in pid_list_refill_irq()
375 		pid_list->upper_list = upper;  in pid_list_refill_irq()
376 		pid_list->free_upper_chunks += ucnt;  in pid_list_refill_irq()
379 		*lower_next = pid_list->lower_list;  in pid_list_refill_irq()
380 		pid_list->lower_list = lower;  in pid_list_refill_irq()
381 		pid_list->free_lower_chunks += lcnt;  in pid_list_refill_irq()
383 	raw_spin_unlock(&pid_list->lock);  in pid_list_refill_irq()
409 	struct trace_pid_list *pid_list;  in trace_pid_list_alloc()  local
415 	pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);  in trace_pid_list_alloc()
416 	if (!pid_list)  in trace_pid_list_alloc()
419 	init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);  in trace_pid_list_alloc()
421 	raw_spin_lock_init(&pid_list->lock);  in trace_pid_list_alloc()
429 		chunk->next = pid_list->upper_list;  in trace_pid_list_alloc()
430 		pid_list->upper_list = chunk;  in trace_pid_list_alloc()
431 		pid_list->free_upper_chunks++;  in trace_pid_list_alloc()
440 		chunk->next = pid_list->lower_list;  in trace_pid_list_alloc()
441 		pid_list->lower_list = chunk;  in trace_pid_list_alloc()
442 		pid_list->free_lower_chunks++;  in trace_pid_list_alloc()
445 	return pid_list;  in trace_pid_list_alloc()
454 void trace_pid_list_free(struct trace_pid_list *pid_list)  in trace_pid_list_free()  argument
460 	if (!pid_list)  in trace_pid_list_free()
463 	irq_work_sync(&pid_list->refill_irqwork);  in trace_pid_list_free()
465 	while (pid_list->lower_list) {  in trace_pid_list_free()
468 		chunk = pid_list->lower_list;  in trace_pid_list_free()
469 		pid_list->lower_list = pid_list->lower_list->next;  in trace_pid_list_free()
473 	while (pid_list->upper_list) {  in trace_pid_list_free()
476 		chunk = pid_list->upper_list;  in trace_pid_list_free()
477 		pid_list->upper_list = pid_list->upper_list->next;  in trace_pid_list_free()
482 		upper = pid_list->upper[i];  in trace_pid_list_free()
491 	kfree(pid_list);  in trace_pid_list_free()