76  * received by the AIO job so far.
82 static void t4_aio_cancel_active(struct kaiocb *job);
83 static void t4_aio_cancel_queued(struct kaiocb *job);
164 ddp_complete_one(struct kaiocb *job, int error)  in ddp_complete_one()  argument
169 	 * If this job had copied data out of the socket buffer before  in ddp_complete_one()
173 	copied = job->aio_received;  in ddp_complete_one()
175 		aio_complete(job, copied, 0);  in ddp_complete_one()
177 		aio_complete(job, -1, error);  in ddp_complete_one()
275 	if (db->job) {  in free_ddp_buffer()
282 		if (!aio_clear_cancel_function(db->job))  in free_ddp_buffer()
283 			ddp_complete_one(db->job, 0);  in free_ddp_buffer()
285 		db->job = NULL;  in free_ddp_buffer()
356 			MPASS(toep->ddp.db[i].job == NULL);  in ddp_assert_empty()
381 				KASSERT(toep->ddp.db[db_idx ^ 1].job == NULL,  in complete_ddp_buffer()
401 		db->job = NULL;  in complete_ddp_buffer()
467 	struct kaiocb *job;  in insert_ddp_data()  local
505 		job = db->job;  in insert_ddp_data()
506 		copied = job->aio_received;  in insert_ddp_data()
508 		if (placed > job->uaiocb.aio_nbytes - copied)  in insert_ddp_data()
509 			placed = job->uaiocb.aio_nbytes - copied;  in insert_ddp_data()
511 			job->msgrcv = 1;  in insert_ddp_data()
515 		if (!aio_clear_cancel_function(job)) {  in insert_ddp_data()
521 			job->aio_received += placed;  in insert_ddp_data()
525 			    __func__, job, copied, placed);  in insert_ddp_data()
527 			aio_complete(job, copied + placed, 0);  in insert_ddp_data()
528 		} else if (aio_set_cancel_function(job, t4_aio_cancel_queued)) {  in insert_ddp_data()
529 			TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);  in insert_ddp_data()
532 			aio_cancel(job);  in insert_ddp_data()
648 	struct kaiocb *job;  in handle_ddp_data_aio()  local
665 	job = db->job;  in handle_ddp_data_aio()
674 		if (aio_clear_cancel_function(job))  in handle_ddp_data_aio()
675 			ddp_complete_one(job, ECONNRESET);  in handle_ddp_data_aio()
724 	job->msgrcv = 1;  in handle_ddp_data_aio()
729 		 * Update the job's length but defer completion to the  in handle_ddp_data_aio()
732 		job->aio_received += len;  in handle_ddp_data_aio()
734 	} else if (!aio_clear_cancel_function(job)) {  in handle_ddp_data_aio()
739 		job->aio_received += len;  in handle_ddp_data_aio()
741 		copied = job->aio_received;  in handle_ddp_data_aio()
745 		    __func__, toep->tid, job, copied, len);  in handle_ddp_data_aio()
747 		aio_complete(job, copied + len, 0);  in handle_ddp_data_aio()
1023 	struct kaiocb *job;  in do_ddp_tcb_rpl()  local
1046 		 * handle_ddp_data() should leave the job around until  in do_ddp_tcb_rpl()
1050 		MPASS(db->job != NULL);  in do_ddp_tcb_rpl()
1063 		job = db->job;  in do_ddp_tcb_rpl()
1064 		copied = job->aio_received;  in do_ddp_tcb_rpl()
1066 			CTR2(KTR_CXGBE, "%s: cancelling %p", __func__, job);  in do_ddp_tcb_rpl()
1067 			aio_cancel(job);  in do_ddp_tcb_rpl()
1070 			    __func__, job, copied);  in do_ddp_tcb_rpl()
1071 			aio_complete(job, copied, 0);  in do_ddp_tcb_rpl()
1095 	struct kaiocb *job;  in handle_ddp_close()  local
1137 		job = db->job;  in handle_ddp_close()
1138 		copied = job->aio_received;  in handle_ddp_close()
1140 		if (placed > job->uaiocb.aio_nbytes - copied)  in handle_ddp_close()
1141 			placed = job->uaiocb.aio_nbytes - copied;  in handle_ddp_close()
1143 			job->msgrcv = 1;  in handle_ddp_close()
1147 		if (!aio_clear_cancel_function(job)) {  in handle_ddp_close()
1153 			job->aio_received += placed;  in handle_ddp_close()
1157 			aio_complete(job, copied + placed, 0);  in handle_ddp_close()
2165 hold_aio(struct toepcb *toep, struct kaiocb *job, struct pageset **pps)  in hold_aio()  argument
2180 	vm = job->userproc->p_vmspace;  in hold_aio()
2182 	start = (uintptr_t)job->uaiocb.aio_buf;  in hold_aio()
2184 	end = round_page(start + job->uaiocb.aio_nbytes);  in hold_aio()
2199 		    __func__, toep->tid, (unsigned long)job->uaiocb.aio_nbytes,  in hold_aio()
2201 		job->uaiocb.aio_nbytes = end - (start + pgoff);  in hold_aio()
2213 		    job->uaiocb.aio_nbytes) == 0) {  in hold_aio()
2256 	ps->len = job->uaiocb.aio_nbytes;  in hold_aio()
2261 	CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d",  in hold_aio()
2262 	    __func__, toep->tid, ps, job, ps->npages);  in hold_aio()
2270 	struct kaiocb *job;  in ddp_complete_all()  local
2275 		job = TAILQ_FIRST(&toep->ddp.aiojobq);  in ddp_complete_all()
2276 		TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);  in ddp_complete_all()
2278 		if (aio_clear_cancel_function(job))  in ddp_complete_all()
2279 			ddp_complete_one(job, error);  in ddp_complete_all()
2284 aio_ddp_cancel_one(struct kaiocb *job)  in aio_ddp_cancel_one()  argument
2289 	 * If this job had copied data out of the socket buffer before  in aio_ddp_cancel_one()
2293 	copied = job->aio_received;  in aio_ddp_cancel_one()
2295 		aio_complete(job, copied, 0);  in aio_ddp_cancel_one()
2297 		aio_cancel(job);  in aio_ddp_cancel_one()
2301  * Called when the main loop wants to requeue a job to retry it later.
2302  * Deals with the race of the job being cancelled while it was being
2306 aio_ddp_requeue_one(struct toepcb *toep, struct kaiocb *job)  in aio_ddp_requeue_one()  argument
2311 	    aio_set_cancel_function(job, t4_aio_cancel_queued)) {  in aio_ddp_requeue_one()
2312 		TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);  in aio_ddp_requeue_one()
2315 		aio_ddp_cancel_one(job);  in aio_ddp_requeue_one()
2325 	struct kaiocb *job;  in aio_ddp_requeue()  local
2348 	job = TAILQ_FIRST(&toep->ddp.aiojobq);  in aio_ddp_requeue()
2349 	so = job->fd_file->f_data;  in aio_ddp_requeue()
2367 		TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);  in aio_ddp_requeue()
2368 		if (!aio_clear_cancel_function(job)) {  in aio_ddp_requeue()
2374 		 * If this job has previously copied some data, report  in aio_ddp_requeue()
2378 		copied = job->aio_received;  in aio_ddp_requeue()
2381 			aio_complete(job, copied, 0);  in aio_ddp_requeue()
2387 		aio_complete(job, -1, error);  in aio_ddp_requeue()
2435 	/* Take the next job to prep it for DDP. */  in aio_ddp_requeue()
2437 	TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);  in aio_ddp_requeue()
2438 	if (!aio_clear_cancel_function(job))  in aio_ddp_requeue()
2440 	toep->ddp.queueing = job;  in aio_ddp_requeue()
2443 	error = hold_aio(toep, job, &ps);  in aio_ddp_requeue()
2445 		ddp_complete_one(job, error);  in aio_ddp_requeue()
2452 		copied = job->aio_received;  in aio_ddp_requeue()
2456 			aio_complete(job, copied, 0);  in aio_ddp_requeue()
2465 		aio_complete(job, -1, error);  in aio_ddp_requeue()
2479 			aio_ddp_requeue_one(toep, job);  in aio_ddp_requeue()
2483 		ddp_complete_one(job, 0);  in aio_ddp_requeue()
2502 	offset = ps->offset + job->aio_received;  in aio_ddp_requeue()
2503 	MPASS(job->aio_received <= job->uaiocb.aio_nbytes);  in aio_ddp_requeue()
2504 	resid = job->uaiocb.aio_nbytes - job->aio_received;  in aio_ddp_requeue()
2538 		job->aio_received += copied;  in aio_ddp_requeue()
2539 		job->msgrcv = 1;  in aio_ddp_requeue()
2540 		copied = job->aio_received;  in aio_ddp_requeue()
2545 			 * the AIO job should keep 'sb' and 'inp' stable.  in aio_ddp_requeue()
2572 			aio_complete(job, copied, 0);  in aio_ddp_requeue()
2585 			aio_ddp_requeue_one(toep, job);  in aio_ddp_requeue()
2602 		aio_ddp_requeue_one(toep, job);  in aio_ddp_requeue()
2614 	if (toep->ddp.db[0].job == NULL) {  in aio_ddp_requeue()
2617 		MPASS(toep->ddp.db[1].job == NULL);  in aio_ddp_requeue()
2652 	 * end, the AIO job holds a reference on this end of the socket  in aio_ddp_requeue()
2654 	 * after the job is completed.  in aio_ddp_requeue()
2657 	    job->aio_received, ps->len, ddp_flags, ddp_flags_mask);  in aio_ddp_requeue()
2660 		aio_ddp_requeue_one(toep, job);  in aio_ddp_requeue()
2675 	if (!aio_set_cancel_function(job, t4_aio_cancel_active)) {  in aio_ddp_requeue()
2678 		aio_ddp_cancel_one(job);  in aio_ddp_requeue()
2686 	    toep->tid, job, db_idx, ddp_flags, ddp_flags_mask);  in aio_ddp_requeue()
2692 	db->job = job;  in aio_ddp_requeue()
2732 t4_aio_cancel_active(struct kaiocb *job)  in t4_aio_cancel_active()  argument
2734 	struct socket *so = job->fd_file->f_data;  in t4_aio_cancel_active()
2742 	if (aio_cancel_cleared(job)) {  in t4_aio_cancel_active()
2744 		aio_ddp_cancel_one(job);  in t4_aio_cancel_active()
2749 		if (toep->ddp.db[i].job == job) {  in t4_aio_cancel_active()
2750 			/* Should only ever get one cancel request for a job. */  in t4_aio_cancel_active()
2765 			    __func__, job);  in t4_aio_cancel_active()
2773 t4_aio_cancel_queued(struct kaiocb *job)  in t4_aio_cancel_queued()  argument
2775 	struct socket *so = job->fd_file->f_data;  in t4_aio_cancel_queued()
2780 	if (!aio_cancel_cleared(job)) {  in t4_aio_cancel_queued()
2781 		TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);  in t4_aio_cancel_queued()
2786 	CTR2(KTR_CXGBE, "%s: request %p cancelled", __func__, job);  in t4_aio_cancel_queued()
2789 	aio_ddp_cancel_one(job);  in t4_aio_cancel_queued()
2793 t4_aio_queue_ddp(struct socket *so, struct kaiocb *job)  in t4_aio_queue_ddp()  argument
2800 	if (job->uaiocb.aio_lio_opcode != LIO_READ)  in t4_aio_queue_ddp()
2838 	CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);  in t4_aio_queue_ddp()
2840 	if (!aio_set_cancel_function(job, t4_aio_cancel_queued))  in t4_aio_queue_ddp()
2841 		panic("new job was cancelled");  in t4_aio_queue_ddp()
2842 	TAILQ_INSERT_TAIL(&toep->ddp.aiojobq, job, list);  in t4_aio_queue_ddp()