103 linuxkpi_napi_schedule_prep(struct napi_struct *napi)  in linuxkpi_napi_schedule_prep()  argument
107 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
111 		old = READ_ONCE(napi->state);  in linuxkpi_napi_schedule_prep()
115 			NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
125 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in linuxkpi_napi_schedule_prep()
127 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
132 lkpi___napi_schedule_dd(struct napi_struct *napi)  in lkpi___napi_schedule_dd()  argument
139 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
140 	if (napi->poll != NULL)  in lkpi___napi_schedule_dd()
141 		rc = napi->poll(napi, napi->budget);  in lkpi___napi_schedule_dd()
142 	napi->rx_count += rc;  in lkpi___napi_schedule_dd()
146 	if (rc >= napi->budget)  in lkpi___napi_schedule_dd()
150 	if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state))  in lkpi___napi_schedule_dd()
154 		new = old = READ_ONCE(napi->state);  in lkpi___napi_schedule_dd()
157 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in lkpi___napi_schedule_dd()
159 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
163 linuxkpi___napi_schedule(struct napi_struct *napi)  in linuxkpi___napi_schedule()  argument
167 	NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
168 	if (test_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state)) {  in linuxkpi___napi_schedule()
169 		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state);  in linuxkpi___napi_schedule()
170 		clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);  in linuxkpi___napi_schedule()
171 		NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
176 		lkpi___napi_schedule_dd(napi);  in linuxkpi___napi_schedule()
178 		rc = taskqueue_enqueue(napi->dev->napi_tq, &napi->napi_task);  in linuxkpi___napi_schedule()
179 		NAPI_TRACE2D(napi, rc);  in linuxkpi___napi_schedule()
188 linuxkpi_napi_schedule(struct napi_struct *napi)  in linuxkpi_napi_schedule()  argument
191 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule()
197 	if (napi_schedule_prep(napi)) {  in linuxkpi_napi_schedule()
198 		__napi_schedule(napi);  in linuxkpi_napi_schedule()
206 linuxkpi_napi_reschedule(struct napi_struct *napi)  in linuxkpi_napi_reschedule()  argument
209 	NAPI_TRACE(napi);  in linuxkpi_napi_reschedule()
212 	if (napi_schedule_prep(napi))  in linuxkpi_napi_reschedule()
213 		__napi_schedule(napi);  in linuxkpi_napi_reschedule()
217 linuxkpi_napi_complete_done(struct napi_struct *napi, int ret)  in linuxkpi_napi_complete_done()  argument
221 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
226 		new = old = READ_ONCE(napi->state);  in linuxkpi_napi_complete_done()
235 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in linuxkpi_napi_complete_done()
237 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
241 		__napi_schedule(napi);  in linuxkpi_napi_complete_done()
249 linuxkpi_napi_complete(struct napi_struct *napi)  in linuxkpi_napi_complete()  argument
252 	NAPI_TRACE(napi);  in linuxkpi_napi_complete()
253 	return (napi_complete_done(napi, 0));  in linuxkpi_napi_complete()
257 linuxkpi_napi_disable(struct napi_struct *napi)  in linuxkpi_napi_disable()  argument
259 	NAPI_TRACE(napi);  in linuxkpi_napi_disable()
260 	set_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);  in linuxkpi_napi_disable()
261 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))  in linuxkpi_napi_disable()
263 	clear_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);  in linuxkpi_napi_disable()
267 linuxkpi_napi_enable(struct napi_struct *napi)  in linuxkpi_napi_enable()  argument
270 	NAPI_TRACE(napi);  in linuxkpi_napi_enable()
271 	KASSERT(!test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state),  in linuxkpi_napi_enable()
272 	    ("%s: enabling napi %p already scheduled\n", __func__, napi));  in linuxkpi_napi_enable()
275 	clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);  in linuxkpi_napi_enable()
279 linuxkpi_napi_synchronize(struct napi_struct *napi)  in linuxkpi_napi_synchronize()  argument
281 	NAPI_TRACE(napi);  in linuxkpi_napi_synchronize()
284 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))  in linuxkpi_napi_synchronize()
296 	struct napi_struct *napi;  in lkpi_napi_task()  local
301 	napi = ctx;  in lkpi_napi_task()
302 	KASSERT(napi->poll != NULL, ("%s: napi %p poll is NULL\n",  in lkpi_napi_task()
303 	    __func__, napi));  in lkpi_napi_task()
305 	NAPI_TRACE_TASK(napi, pending, napi->budget);  in lkpi_napi_task()
306 	count = napi->poll(napi, napi->budget);  in lkpi_napi_task()
307 	napi->rx_count += count;  in lkpi_napi_task()
308 	NAPI_TRACE_TASK(napi, pending, count);  in lkpi_napi_task()
315 	if (count >= napi->budget) {  in lkpi_napi_task()
322 		__napi_schedule(napi);  in lkpi_napi_task()
329 linuxkpi_netif_napi_add(struct net_device *ndev, struct napi_struct *napi,  in linuxkpi_netif_napi_add()  argument
333 	napi->dev = ndev;  in linuxkpi_netif_napi_add()
334 	napi->poll = napi_poll;  in linuxkpi_netif_napi_add()
335 	napi->budget = NAPI_POLL_WEIGHT;  in linuxkpi_netif_napi_add()
337 	INIT_LIST_HEAD(&napi->rx_list);  in linuxkpi_netif_napi_add()
338 	napi->rx_count = 0;  in linuxkpi_netif_napi_add()
340 	TASK_INIT(&napi->napi_task, 0, lkpi_napi_task, napi);  in linuxkpi_netif_napi_add()
343 	TAILQ_INSERT_TAIL(&ndev->napi_head, napi, entry);  in linuxkpi_netif_napi_add()
347 	clear_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);  in linuxkpi_netif_napi_add()
351 lkpi_netif_napi_del_locked(struct napi_struct *napi)  in lkpi_netif_napi_del_locked()  argument
355 	ndev = napi->dev;  in lkpi_netif_napi_del_locked()
358 	set_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);  in lkpi_netif_napi_del_locked()
359 	TAILQ_REMOVE(&ndev->napi_head, napi, entry);  in lkpi_netif_napi_del_locked()
360 	while (taskqueue_cancel(ndev->napi_tq, &napi->napi_task, NULL) != 0)  in lkpi_netif_napi_del_locked()
361 		taskqueue_drain(ndev->napi_tq, &napi->napi_task);  in lkpi_netif_napi_del_locked()
365 linuxkpi_netif_napi_del(struct napi_struct *napi)  in linuxkpi_netif_napi_del()  argument
369 	ndev = napi->dev;  in linuxkpi_netif_napi_del()
371 	lkpi_netif_napi_del_locked(napi);  in linuxkpi_netif_napi_del()
421 	struct napi_struct *napi, *temp;  in linuxkpi_free_netdev()  local
424 	TAILQ_FOREACH_SAFE(napi, &ndev->napi_head, entry, temp) {  in linuxkpi_free_netdev()
425 		lkpi_netif_napi_del_locked(napi);  in linuxkpi_free_netdev()