77 static void	t4_aiotx_cancel(struct kaiocb *job);
2027 jobtotid(struct kaiocb *job)  in jobtotid()  argument
2033 	so = job->fd_file->f_data;  in jobtotid()
2041 aiotx_free_job(struct kaiocb *job)  in aiotx_free_job()  argument
2046 	if (refcount_release(&job->aio_refs) == 0)  in aiotx_free_job()
2049 	error = (intptr_t)job->aio_error;  in aiotx_free_job()
2050 	status = job->aio_sent;  in aiotx_free_job()
2053 	    jobtotid(job), job, status, error);  in aiotx_free_job()
2058 		aio_cancel(job);  in aiotx_free_job()
2060 		aio_complete(job, -1, error);  in aiotx_free_job()
2062 		job->msgsnd = 1;  in aiotx_free_job()
2063 		aio_complete(job, status, 0);  in aiotx_free_job()
2070 	struct kaiocb *job;  in aiotx_free_pgs()  local
2074 	job = m->m_ext.ext_arg1;  in aiotx_free_pgs()
2077 	    m->m_len, jobtotid(job));  in aiotx_free_pgs()
2085 	aiotx_free_job(job);  in aiotx_free_pgs()
2093 alloc_aiotx_mbuf(struct kaiocb *job, int len)  in alloc_aiotx_mbuf()  argument
2102 	KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes,  in alloc_aiotx_mbuf()
2104 	    job, len));  in alloc_aiotx_mbuf()
2111 	vm = job->userproc->p_vmspace;  in alloc_aiotx_mbuf()
2113 	start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent;  in alloc_aiotx_mbuf()
2146 		m->m_ext.ext_arg1 = job;  in alloc_aiotx_mbuf()
2147 		refcount_acquire(&job->aio_refs);  in alloc_aiotx_mbuf()
2151 		    __func__, jobtotid(job), m, job, npages);  in alloc_aiotx_mbuf()
2169 t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job)  in t4_aiotx_process_job()  argument
2184 	error = mac_socket_check_send(job->fd_file->f_cred, so);  in t4_aiotx_process_job()
2200 			PROC_LOCK(job->userproc);  in t4_aiotx_process_job()
2201 			kern_psignal(job->userproc, SIGPIPE);  in t4_aiotx_process_job()
2202 			PROC_UNLOCK(job->userproc);  in t4_aiotx_process_job()
2221 		MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO));  in t4_aiotx_process_job()
2227 		if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {  in t4_aiotx_process_job()
2233 		TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);  in t4_aiotx_process_job()
2244 	if (len > job->uaiocb.aio_nbytes - job->aio_sent) {  in t4_aiotx_process_job()
2245 		len = job->uaiocb.aio_nbytes - job->aio_sent;  in t4_aiotx_process_job()
2260 	m = alloc_aiotx_mbuf(job, len);  in t4_aiotx_process_job()
2279 	job->aio_sent += sent;  in t4_aiotx_process_job()
2313 	if (job->aio_sent < job->uaiocb.aio_nbytes &&  in t4_aiotx_process_job()
2316 		if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {  in t4_aiotx_process_job()
2321 		TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);  in t4_aiotx_process_job()
2332 	aiotx_free_job(job);  in t4_aiotx_process_job()
2337 		job->aio_error = (void *)(intptr_t)error;  in t4_aiotx_process_job()
2338 		aiotx_free_job(job);  in t4_aiotx_process_job()
2349 	struct kaiocb *job;  in t4_aiotx_task()  local
2357 		job = TAILQ_FIRST(&toep->aiotx_jobq);  in t4_aiotx_task()
2358 		TAILQ_REMOVE(&toep->aiotx_jobq, job, list);  in t4_aiotx_task()
2359 		if (!aio_clear_cancel_function(job))  in t4_aiotx_task()
2362 		t4_aiotx_process_job(toep, so, job);  in t4_aiotx_task()
2391 t4_aiotx_cancel(struct kaiocb *job)  in t4_aiotx_cancel()  argument
2398 	so = job->fd_file->f_data;  in t4_aiotx_cancel()
2401 	MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE);  in t4_aiotx_cancel()
2405 	if (!aio_cancel_cleared(job))  in t4_aiotx_cancel()
2406 		TAILQ_REMOVE(&toep->aiotx_jobq, job, list);  in t4_aiotx_cancel()
2409 	job->aio_error = (void *)(intptr_t)ECANCELED;  in t4_aiotx_cancel()
2410 	aiotx_free_job(job);  in t4_aiotx_cancel()
2414 t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job)  in t4_aio_queue_aiotx()  argument
2421 	if (job->uaiocb.aio_lio_opcode != LIO_WRITE)  in t4_aio_queue_aiotx()
2432 	CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);  in t4_aio_queue_aiotx()
2434 	if (!aio_set_cancel_function(job, t4_aiotx_cancel))  in t4_aio_queue_aiotx()
2436 	refcount_init(&job->aio_refs, 1);  in t4_aio_queue_aiotx()
2437 	TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);  in t4_aio_queue_aiotx()