13 static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)  in get_lower_chunk()  argument
17 	lockdep_assert_held(&pid_list->lock);  in get_lower_chunk()
19 	if (!pid_list->lower_list)  in get_lower_chunk()
22 	chunk = pid_list->lower_list;  in get_lower_chunk()
23 	pid_list->lower_list = chunk->next;  in get_lower_chunk()
24 	pid_list->free_lower_chunks--;  in get_lower_chunk()
25 	WARN_ON_ONCE(pid_list->free_lower_chunks < 0);  in get_lower_chunk()
31 	if (pid_list->free_lower_chunks <= CHUNK_REALLOC)  in get_lower_chunk()
32 		irq_work_queue(&pid_list->refill_irqwork);  in get_lower_chunk()
37 static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)  in get_upper_chunk()  argument
41 	lockdep_assert_held(&pid_list->lock);  in get_upper_chunk()
43 	if (!pid_list->upper_list)  in get_upper_chunk()
46 	chunk = pid_list->upper_list;  in get_upper_chunk()
47 	pid_list->upper_list = chunk->next;  in get_upper_chunk()
48 	pid_list->free_upper_chunks--;  in get_upper_chunk()
49 	WARN_ON_ONCE(pid_list->free_upper_chunks < 0);  in get_upper_chunk()
55 	if (pid_list->free_upper_chunks <= CHUNK_REALLOC)  in get_upper_chunk()
56 		irq_work_queue(&pid_list->refill_irqwork);  in get_upper_chunk()
61 static inline void put_lower_chunk(struct trace_pid_list *pid_list,  in put_lower_chunk()  argument
64 	lockdep_assert_held(&pid_list->lock);  in put_lower_chunk()
66 	chunk->next = pid_list->lower_list;  in put_lower_chunk()
67 	pid_list->lower_list = chunk;  in put_lower_chunk()
68 	pid_list->free_lower_chunks++;  in put_lower_chunk()
71 static inline void put_upper_chunk(struct trace_pid_list *pid_list,  in put_upper_chunk()  argument
74 	lockdep_assert_held(&pid_list->lock);  in put_upper_chunk()
76 	chunk->next = pid_list->upper_list;  in put_upper_chunk()
77 	pid_list->upper_list = chunk;  in put_upper_chunk()
78 	pid_list->free_upper_chunks++;  in put_upper_chunk()
126 bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_is_set()  argument
136 	if (!pid_list)  in trace_pid_list_is_set()
143 		seq = read_seqcount_begin(&pid_list->seqcount);  in trace_pid_list_is_set()
145 		upper_chunk = pid_list->upper[upper1];  in trace_pid_list_is_set()
151 	} while (read_seqcount_retry(&pid_list->seqcount, seq));  in trace_pid_list_is_set()
167 int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_set()  argument
177 	if (!pid_list)  in trace_pid_list_set()
183 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_set()
184 	write_seqcount_begin(&pid_list->seqcount);  in trace_pid_list_set()
185 	upper_chunk = pid_list->upper[upper1];  in trace_pid_list_set()
187 		upper_chunk = get_upper_chunk(pid_list);  in trace_pid_list_set()
192 		pid_list->upper[upper1] = upper_chunk;  in trace_pid_list_set()
196 		lower_chunk = get_lower_chunk(pid_list);  in trace_pid_list_set()
206 	write_seqcount_end(&pid_list->seqcount);  in trace_pid_list_set()
207 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_set()
222 int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)  in trace_pid_list_clear()  argument
231 	if (!pid_list)  in trace_pid_list_clear()
237 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_clear()
238 	write_seqcount_begin(&pid_list->seqcount);  in trace_pid_list_clear()
239 	upper_chunk = pid_list->upper[upper1];  in trace_pid_list_clear()
251 		put_lower_chunk(pid_list, lower_chunk);  in trace_pid_list_clear()
254 			put_upper_chunk(pid_list, upper_chunk);  in trace_pid_list_clear()
255 			pid_list->upper[upper1] = NULL;  in trace_pid_list_clear()
259 	write_seqcount_end(&pid_list->seqcount);  in trace_pid_list_clear()
260 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_clear()
276 int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,  in trace_pid_list_next()  argument
286 	if (!pid_list)  in trace_pid_list_next()
292 	raw_spin_lock_irqsave(&pid_list->lock, flags);  in trace_pid_list_next()
294 		upper_chunk = pid_list->upper[upper1];  in trace_pid_list_next()
312 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);  in trace_pid_list_next()
330 int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)  in trace_pid_list_first()  argument
332 	return trace_pid_list_next(pid_list, 0, pid);  in trace_pid_list_first()
337 	struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,  in pid_list_refill_irq()  local
349 	raw_spin_lock(&pid_list->lock);  in pid_list_refill_irq()
350 	write_seqcount_begin(&pid_list->seqcount);  in pid_list_refill_irq()
351 	upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;  in pid_list_refill_irq()
352 	lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;  in pid_list_refill_irq()
353 	write_seqcount_end(&pid_list->seqcount);  in pid_list_refill_irq()
354 	raw_spin_unlock(&pid_list->lock);  in pid_list_refill_irq()
381 	raw_spin_lock(&pid_list->lock);  in pid_list_refill_irq()
382 	write_seqcount_begin(&pid_list->seqcount);  in pid_list_refill_irq()
384 		*upper_next = pid_list->upper_list;  in pid_list_refill_irq()
385 		pid_list->upper_list = upper;  in pid_list_refill_irq()
386 		pid_list->free_upper_chunks += ucnt;  in pid_list_refill_irq()
389 		*lower_next = pid_list->lower_list;  in pid_list_refill_irq()
390 		pid_list->lower_list = lower;  in pid_list_refill_irq()
391 		pid_list->free_lower_chunks += lcnt;  in pid_list_refill_irq()
393 	write_seqcount_end(&pid_list->seqcount);  in pid_list_refill_irq()
394 	raw_spin_unlock(&pid_list->lock);  in pid_list_refill_irq()
420 	struct trace_pid_list *pid_list;  in trace_pid_list_alloc()  local
426 	pid_list = kzalloc_obj(*pid_list);  in trace_pid_list_alloc()
427 	if (!pid_list)  in trace_pid_list_alloc()
430 	init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);  in trace_pid_list_alloc()
432 	raw_spin_lock_init(&pid_list->lock);  in trace_pid_list_alloc()
433 	seqcount_raw_spinlock_init(&pid_list->seqcount, &pid_list->lock);  in trace_pid_list_alloc()
441 		chunk->next = pid_list->upper_list;  in trace_pid_list_alloc()
442 		pid_list->upper_list = chunk;  in trace_pid_list_alloc()
443 		pid_list->free_upper_chunks++;  in trace_pid_list_alloc()
452 		chunk->next = pid_list->lower_list;  in trace_pid_list_alloc()
453 		pid_list->lower_list = chunk;  in trace_pid_list_alloc()
454 		pid_list->free_lower_chunks++;  in trace_pid_list_alloc()
457 	return pid_list;  in trace_pid_list_alloc()
466 void trace_pid_list_free(struct trace_pid_list *pid_list)  in trace_pid_list_free()  argument
472 	if (!pid_list)  in trace_pid_list_free()
475 	irq_work_sync(&pid_list->refill_irqwork);  in trace_pid_list_free()
477 	while (pid_list->lower_list) {  in trace_pid_list_free()
480 		chunk = pid_list->lower_list;  in trace_pid_list_free()
481 		pid_list->lower_list = pid_list->lower_list->next;  in trace_pid_list_free()
485 	while (pid_list->upper_list) {  in trace_pid_list_free()
488 		chunk = pid_list->upper_list;  in trace_pid_list_free()
489 		pid_list->upper_list = pid_list->upper_list->next;  in trace_pid_list_free()
494 		upper = pid_list->upper[i];  in trace_pid_list_free()
503 	kfree(pid_list);  in trace_pid_list_free()