15 		struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu);  in pcpu_freelist_init()  local
17 		raw_spin_lock_init(&head->lock);  in pcpu_freelist_init()
18 		head->first = NULL;  in pcpu_freelist_init()
30 static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,  in pcpu_freelist_push_node()  argument
33 	node->next = head->first;  in pcpu_freelist_push_node()
34 	WRITE_ONCE(head->first, node);  in pcpu_freelist_push_node()
37 static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,  in ___pcpu_freelist_push()  argument
40 	raw_spin_lock(&head->lock);  in ___pcpu_freelist_push()
41 	pcpu_freelist_push_node(head, node);  in ___pcpu_freelist_push()
42 	raw_spin_unlock(&head->lock);  in ___pcpu_freelist_push()
64 			struct pcpu_freelist_head *head;  in ___pcpu_freelist_push_nmi()  local
66 			head = per_cpu_ptr(s->freelist, cpu);  in ___pcpu_freelist_push_nmi()
67 			if (raw_spin_trylock(&head->lock)) {  in ___pcpu_freelist_push_nmi()
68 				pcpu_freelist_push_node(head, node);  in ___pcpu_freelist_push_nmi()
69 				raw_spin_unlock(&head->lock);  in ___pcpu_freelist_push_nmi()
102 	struct pcpu_freelist_head *head;  in pcpu_freelist_populate()  local
110 		head = per_cpu_ptr(s->freelist, cpu);  in pcpu_freelist_populate()
114 			pcpu_freelist_push_node(head, buf);  in pcpu_freelist_populate()
123 	struct pcpu_freelist_head *head;  in ___pcpu_freelist_pop()  local
128 		head = per_cpu_ptr(s->freelist, cpu);  in ___pcpu_freelist_pop()
129 		if (!READ_ONCE(head->first))  in ___pcpu_freelist_pop()
131 		raw_spin_lock(&head->lock);  in ___pcpu_freelist_pop()
132 		node = head->first;  in ___pcpu_freelist_pop()
134 			WRITE_ONCE(head->first, node->next);  in ___pcpu_freelist_pop()
135 			raw_spin_unlock(&head->lock);  in ___pcpu_freelist_pop()
138 		raw_spin_unlock(&head->lock);  in ___pcpu_freelist_pop()
155 	struct pcpu_freelist_head *head;  in ___pcpu_freelist_pop_nmi()  local
160 		head = per_cpu_ptr(s->freelist, cpu);  in ___pcpu_freelist_pop_nmi()
161 		if (!READ_ONCE(head->first))  in ___pcpu_freelist_pop_nmi()
163 		if (raw_spin_trylock(&head->lock)) {  in ___pcpu_freelist_pop_nmi()
164 			node = head->first;  in ___pcpu_freelist_pop_nmi()
166 				WRITE_ONCE(head->first, node->next);  in ___pcpu_freelist_pop_nmi()
167 				raw_spin_unlock(&head->lock);  in ___pcpu_freelist_pop_nmi()
170 			raw_spin_unlock(&head->lock);  in ___pcpu_freelist_pop_nmi()