170 	union nvme_tcp_pdu	pdu;  member
295 static inline void nvmet_tcp_hdgst(void *pdu, size_t len)  in nvmet_tcp_hdgst()  argument
297 	put_unaligned_le32(~crc32c(~0, pdu, len), pdu + len);  in nvmet_tcp_hdgst()
301 	void *pdu, size_t len)  in nvmet_tcp_verify_hdgst()  argument
303 	struct nvme_tcp_hdr *hdr = pdu;  in nvmet_tcp_verify_hdgst()
313 	recv_digest = *(__le32 *)(pdu + hdr->hlen);  in nvmet_tcp_verify_hdgst()
314 	nvmet_tcp_hdgst(pdu, len);  in nvmet_tcp_verify_hdgst()
315 	exp_digest = *(__le32 *)(pdu + hdr->hlen);  in nvmet_tcp_verify_hdgst()
326 static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)  in nvmet_tcp_check_ddgst()  argument
328 	struct nvme_tcp_hdr *hdr = pdu;  in nvmet_tcp_check_ddgst()
459 	struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;  in nvmet_setup_c2h_data_pdu()  local
467 	pdu->hdr.type = nvme_tcp_c2h_data;  in nvmet_setup_c2h_data_pdu()
468 	pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ?  in nvmet_setup_c2h_data_pdu()
470 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_c2h_data_pdu()
471 	pdu->hdr.pdo = pdu->hdr.hlen + hdgst;  in nvmet_setup_c2h_data_pdu()
472 	pdu->hdr.plen =  in nvmet_setup_c2h_data_pdu()
473 		cpu_to_le32(pdu->hdr.hlen + hdgst +  in nvmet_setup_c2h_data_pdu()
475 	pdu->command_id = cmd->req.cqe->command_id;  in nvmet_setup_c2h_data_pdu()
476 	pdu->data_length = cpu_to_le32(cmd->req.transfer_len);  in nvmet_setup_c2h_data_pdu()
477 	pdu->data_offset = cpu_to_le32(cmd->wbytes_done);  in nvmet_setup_c2h_data_pdu()
480 		pdu->hdr.flags |= NVME_TCP_F_DDGST;  in nvmet_setup_c2h_data_pdu()
485 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_c2h_data_pdu()
486 		nvmet_tcp_hdgst(pdu, sizeof(*pdu));  in nvmet_setup_c2h_data_pdu()
492 	struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;  in nvmet_setup_r2t_pdu()  local
498 	pdu->hdr.type = nvme_tcp_r2t;  in nvmet_setup_r2t_pdu()
499 	pdu->hdr.flags = 0;  in nvmet_setup_r2t_pdu()
500 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_r2t_pdu()
501 	pdu->hdr.pdo = 0;  in nvmet_setup_r2t_pdu()
502 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvmet_setup_r2t_pdu()
504 	pdu->command_id = cmd->req.cmd->common.command_id;  in nvmet_setup_r2t_pdu()
505 	pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd);  in nvmet_setup_r2t_pdu()
506 	pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);  in nvmet_setup_r2t_pdu()
507 	pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);  in nvmet_setup_r2t_pdu()
509 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_r2t_pdu()
510 		nvmet_tcp_hdgst(pdu, sizeof(*pdu));  in nvmet_setup_r2t_pdu()
516 	struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;  in nvmet_setup_response_pdu()  local
522 	pdu->hdr.type = nvme_tcp_rsp;  in nvmet_setup_response_pdu()
523 	pdu->hdr.flags = 0;  in nvmet_setup_response_pdu()
524 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_response_pdu()
525 	pdu->hdr.pdo = 0;  in nvmet_setup_response_pdu()
526 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvmet_setup_response_pdu()
528 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_response_pdu()
529 		nvmet_tcp_hdgst(pdu, sizeof(*pdu));  in nvmet_setup_response_pdu()
865 	struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq;  in nvmet_tcp_handle_icreq()
866 	struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp;  in nvmet_tcp_handle_icreq()
872 		pr_err("bad nvme-tcp pdu length (%d)\n",  in nvmet_tcp_handle_icreq()
927 	 * we don't, we can simply prepare for the next pdu and bail out,  in nvmet_tcp_handle_req_failure()
951 	struct nvme_tcp_data_pdu *data = &queue->pdu.data;  in nvmet_tcp_handle_h2c_data_pdu()
982 		pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len);  in nvmet_tcp_handle_h2c_data_pdu()
1000 	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;  in nvmet_tcp_done_recv_pdu()
1001 	struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;  in nvmet_tcp_done_recv_pdu()
1007 			pr_err("unexpected pdu type (%d) before icreq\n",  in nvmet_tcp_done_recv_pdu()
1016 		pr_err("queue %d: received icreq pdu in state %d\n",  in nvmet_tcp_done_recv_pdu()
1146 	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;  in nvmet_tcp_try_recv_pdu()
1153 	iov.iov_base = (void *)&queue->pdu + queue->offset;  in nvmet_tcp_try_recv_pdu()
1178 			pr_err("unexpected pdu type %d\n", hdr->type);  in nvmet_tcp_try_recv_pdu()
1184 			pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);  in nvmet_tcp_try_recv_pdu()
1193 	    nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) {  in nvmet_tcp_try_recv_pdu()
1199 	    nvmet_tcp_check_ddgst(queue, &queue->pdu)) {  in nvmet_tcp_try_recv_pdu()
1281 		pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",  in nvmet_tcp_try_recv_ddgst()
1283 			queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),  in nvmet_tcp_try_recv_ddgst()
1715 	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;  in nvmet_tcp_try_peek_pdu()
1718 		.iov_base = (u8 *)&queue->pdu + queue->offset,  in nvmet_tcp_try_peek_pdu()