Lines Matching +full:long +full:- +full:term
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
30 * per-RDMAP message basis. Please keep order of initializer. All MPA len
36 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
44 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
51 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
59 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
66 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
73 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
80 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
87 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
100 read_lock(&sk->sk_callback_lock);
102 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
107 if (likely(!qp->rx_stream.rx_suspend &&
108 down_read_trylock(&qp->state_lock))) {
111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
120 up_read(&qp->state_lock);
123 qp->rx_stream.rx_suspend);
126 read_unlock(&sk->sk_callback_lock);
132 siw_qp_state_to_string[qp->attrs.state]);
134 down_write(&qp->state_lock);
136 qp->rx_stream.rx_suspend = 1;
137 qp->tx_ctx.tx_suspend = 1;
138 qp->attrs.sk = NULL;
140 switch (qp->attrs.state) {
145 qp->attrs.state = SIW_QP_STATE_ERROR;
154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
155 qp->attrs.state = SIW_QP_STATE_ERROR;
157 qp->attrs.state = SIW_QP_STATE_IDLE;
162 siw_qp_state_to_string[qp->attrs.state]);
171 if (qp->cep) {
172 siw_cep_put(qp->cep);
173 qp->cep = NULL;
176 up_write(&qp->state_lock);
179 siw_qp_state_to_string[qp->attrs.state]);
190 read_lock(&sk->sk_callback_lock);
194 cep->sk_write_space(sk);
196 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
197 (void)siw_sq_start(cep->qp);
200 read_unlock(&sk->sk_callback_lock);
207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe));
208 if (!qp->irq) {
209 qp->attrs.irq_size = 0;
210 return -ENOMEM;
215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe));
216 if (!qp->orq) {
217 qp->attrs.orq_size = 0;
218 qp->attrs.irq_size = 0;
219 vfree(qp->irq);
220 return -ENOMEM;
223 qp->attrs.irq_size = irq_size;
224 qp->attrs.orq_size = orq_size;
239 unsigned long flags;
242 spin_lock_irqsave(&qp->sq_lock, flags);
244 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
245 spin_unlock_irqrestore(&qp->sq_lock, flags);
246 return -EIO;
248 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
250 wqe->wr_status = SIW_WR_QUEUED;
251 wqe->sqe.flags = 0;
252 wqe->sqe.num_sge = 1;
253 wqe->sqe.sge[0].length = 0;
254 wqe->sqe.sge[0].laddr = 0;
255 wqe->sqe.sge[0].lkey = 0;
260 wqe->sqe.rkey = 1;
261 wqe->sqe.raddr = 0;
262 wqe->processed = 0;
265 wqe->sqe.opcode = SIW_OP_WRITE;
269 wqe->sqe.opcode = SIW_OP_READ;
271 spin_lock(&qp->orq_lock);
273 if (qp->attrs.orq_size)
276 siw_read_to_orq(rreq, &wqe->sqe);
277 qp->orq_put++;
279 rv = -EIO;
281 spin_unlock(&qp->orq_lock);
283 rv = -EINVAL;
286 wqe->wr_status = SIW_WR_IDLE;
288 spin_unlock_irqrestore(&qp->sq_lock, flags);
343 if (!qp->term_info.valid) {
344 memset(&qp->term_info, 0, sizeof(qp->term_info));
345 qp->term_info.layer = layer;
346 qp->term_info.etype = etype;
347 qp->term_info.ecode = ecode;
348 qp->term_info.in_tx = in_tx;
349 qp->term_info.valid = 1;
351 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
357 * Sending TERMINATE messages is best effort - such messages
359 * not have another outbound message in-progress, i.e. the
367 struct iwarp_terminate *term = NULL;
369 struct socket *s = qp->attrs.sk;
370 struct siw_rx_stream *srx = &qp->rx_stream;
371 union iwarp_hdr *rx_hdr = &srx->hdr;
375 if (!qp->term_info.valid)
378 qp->term_info.valid = 0;
380 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
385 if (!s && qp->cep)
387 s = qp->cep->sock;
394 term = kzalloc(sizeof(*term), GFP_KERNEL);
395 if (!term)
398 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
399 term->ddp_mo = 0;
400 term->ddp_msn = cpu_to_be32(1);
402 iov[0].iov_base = term;
403 iov[0].iov_len = sizeof(*term);
405 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
406 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
407 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
410 kfree(term);
414 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
417 __rdmap_term_set_layer(term, qp->term_info.layer);
418 __rdmap_term_set_etype(term, qp->term_info.etype);
419 __rdmap_term_set_ecode(term, qp->term_info.ecode);
421 switch (qp->term_info.layer) {
423 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
427 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
432 term->flag_m = 1;
433 term->flag_d = 1;
434 term->flag_r = 1;
436 if (qp->term_info.in_tx) {
447 memcpy(&rreq->ctrl,
451 rreq->rsvd = 0;
452 rreq->ddp_qn =
456 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
458 rreq->ddp_mo = htonl(wqe->processed);
459 rreq->sink_stag = htonl(wqe->sqe.rkey);
460 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
461 rreq->read_size = htonl(wqe->sqe.sge[0].length);
462 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
463 rreq->source_to =
464 cpu_to_be64(wqe->sqe.sge[0].laddr);
476 if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
488 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
489 (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
495 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
501 term->flag_m = 1;
502 term->flag_d = 1;
504 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
516 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
517 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
518 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
519 (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
524 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
529 term->flag_m = 1;
530 term->flag_d = 1;
536 if (term->flag_m || term->flag_d || term->flag_r) {
539 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
544 len_terminate = sizeof(*term) + MPA_CRC_SIZE;
549 if (term->flag_m) {
550 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
551 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
553 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
554 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
557 term->ctrl.mpa_len =
558 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
559 if (qp->tx_ctx.mpa_crc_enabled) {
560 siw_crc_init(&qp->tx_ctx.mpa_crc);
561 siw_crc_update(&qp->tx_ctx.mpa_crc,
564 siw_crc_update(&qp->tx_ctx.mpa_crc,
567 siw_crc_final(&qp->tx_ctx.mpa_crc, (u8 *)&crc);
571 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
573 __rdmap_term_layer(term), __rdmap_term_etype(term),
574 __rdmap_term_ecode(term), rv);
575 kfree(term);
587 if (attrs->flags & SIW_RDMA_BIND_ENABLED)
588 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
590 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
592 if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
593 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
595 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
597 if (attrs->flags & SIW_RDMA_READ_ENABLED)
598 qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
600 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
610 switch (attrs->state) {
612 if (attrs->flags & SIW_MPA_CRC) {
613 siw_crc_init(&qp->tx_ctx.mpa_crc);
614 qp->tx_ctx.mpa_crc_enabled = true;
615 siw_crc_init(&qp->rx_stream.mpa_crc);
616 qp->rx_stream.mpa_crc_enabled = true;
620 rv = -EINVAL;
625 rv = -EINVAL;
631 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
632 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
633 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
638 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
639 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
640 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
646 rv = siw_qp_readq_init(qp, attrs->irq_size,
647 attrs->orq_size);
651 qp->attrs.sk = attrs->sk;
652 qp->attrs.state = SIW_QP_STATE_RTS;
655 attrs->flags & SIW_MPA_CRC ? "y" : "n",
656 qp->attrs.orq_size, qp->attrs.irq_size);
661 qp->attrs.state = SIW_QP_STATE_ERROR;
662 if (qp->cep) {
663 siw_cep_put(qp->cep);
664 qp->cep = NULL;
679 switch (attrs->state) {
689 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
690 qp->attrs.state = SIW_QP_STATE_CLOSING;
692 qp->attrs.state = SIW_QP_STATE_ERROR;
701 qp->attrs.state = SIW_QP_STATE_TERMINATE;
719 * Esp., how to handle the non-empty IRQ case?
725 qp->attrs.state = SIW_QP_STATE_ERROR;
738 switch (attrs->state) {
741 qp->attrs.state = SIW_QP_STATE_ERROR;
743 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
757 switch (attrs->state) {
759 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
760 qp->attrs.state = SIW_QP_STATE_IDLE;
775 qp->attrs.state = SIW_QP_STATE_ERROR;
777 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
785 siw_qp_state_to_string[qp->attrs.state],
786 siw_qp_state_to_string[attrs->state]);
788 rv = -ECONNABORTED;
794 * Caller must hold qp->state_lock
805 siw_qp_state_to_string[qp->attrs.state],
806 siw_qp_state_to_string[attrs->state]);
814 switch (qp->attrs.state) {
842 rreq->id = sqe->id;
843 rreq->opcode = sqe->opcode;
844 rreq->sge[0].laddr = sqe->sge[0].laddr;
845 rreq->sge[0].length = sqe->sge[0].length;
846 rreq->sge[0].lkey = sqe->sge[0].lkey;
847 rreq->sge[1].lkey = sqe->sge[1].lkey;
848 rreq->flags = sqe->flags | SIW_WQE_VALID;
849 rreq->num_sge = 1;
862 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
863 wqe->wr_status = SIW_WR_QUEUED;
866 memcpy(&wqe->sqe, sqe, sizeof(*sqe));
868 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
869 rv = -EINVAL;
872 if (wqe->sqe.flags & SIW_WQE_INLINE) {
873 if (wqe->sqe.opcode != SIW_OP_SEND &&
874 wqe->sqe.opcode != SIW_OP_WRITE) {
875 rv = -EINVAL;
878 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
879 rv = -EINVAL;
882 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];
883 wqe->sqe.sge[0].lkey = 0;
884 wqe->sqe.num_sge = 1;
886 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
888 if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
889 wqe->sqe.opcode ==
892 rv = -EINVAL;
895 spin_lock(&qp->orq_lock);
897 if (qp->attrs.orq_size && !siw_orq_empty(qp)) {
898 qp->tx_ctx.orq_fence = 1;
901 spin_unlock(&qp->orq_lock);
903 } else if (wqe->sqe.opcode == SIW_OP_READ ||
904 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
907 if (unlikely(!qp->attrs.orq_size)) {
909 rv = -EINVAL;
912 wqe->sqe.num_sge = 1;
914 spin_lock(&qp->orq_lock);
922 siw_read_to_orq(rreq, &wqe->sqe);
923 qp->orq_put++;
925 qp->tx_ctx.orq_fence = 1;
928 spin_unlock(&qp->orq_lock);
931 /* Clear SQE, can be re-used by application */
932 smp_store_mb(sqe->flags, 0);
933 qp->sq_get++;
937 wqe->wr_status = SIW_WR_IDLE;
945 * the active IRQ will not be served after qp->irq_burst, if the
953 if (!qp->attrs.irq_size)
956 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
958 if (!(irqe->flags & SIW_WQE_VALID))
965 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
966 qp->irq_burst = 0;
969 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
970 wqe->wr_status = SIW_WR_QUEUED;
973 wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
974 wqe->sqe.flags = 0;
975 if (irqe->num_sge) {
976 wqe->sqe.num_sge = 1;
977 wqe->sqe.sge[0].length = irqe->sge[0].length;
978 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
979 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
981 wqe->sqe.num_sge = 0;
987 wqe->sqe.sge[1].length = irqe->sge[1].length;
989 wqe->sqe.rkey = irqe->rkey;
990 wqe->sqe.raddr = irqe->raddr;
992 wqe->processed = 0;
993 qp->irq_get++;
996 smp_store_mb(irqe->flags, 0);
1009 if (!cq->base_cq.comp_handler)
1013 cq_notify = READ_ONCE(cq->notify->flags);
1019 * CQ notification is one-shot: Since the
1021 * the CQ gets dis-aremd and must be re-aremd
1024 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1034 struct siw_cq *cq = qp->scq;
1038 u32 sqe_flags = sqe->flags;
1041 unsigned long flags;
1043 spin_lock_irqsave(&cq->lock, flags);
1045 idx = cq->cq_put % cq->num_cqe;
1046 cqe = &cq->queue[idx];
1048 if (!READ_ONCE(cqe->flags)) {
1051 cqe->id = sqe->id;
1052 cqe->opcode = sqe->opcode;
1053 cqe->status = status;
1054 cqe->imm_data = 0;
1055 cqe->bytes = bytes;
1057 if (rdma_is_kernel_res(&cq->base_cq.res))
1058 cqe->base_qp = &qp->base_qp;
1060 cqe->qp_id = qp_id(qp);
1063 WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1065 smp_store_mb(sqe->flags, 0);
1067 cq->cq_put++;
1070 spin_unlock_irqrestore(&cq->lock, flags);
1074 cq->base_cq.comp_handler(&cq->base_cq,
1075 cq->base_cq.cq_context);
1078 spin_unlock_irqrestore(&cq->lock, flags);
1079 rv = -ENOMEM;
1084 smp_store_mb(sqe->flags, 0);
1092 struct siw_cq *cq = qp->rcq;
1098 unsigned long flags;
1100 spin_lock_irqsave(&cq->lock, flags);
1102 idx = cq->cq_put % cq->num_cqe;
1103 cqe = &cq->queue[idx];
1105 if (!READ_ONCE(cqe->flags)) {
1109 cqe->id = rqe->id;
1110 cqe->opcode = SIW_OP_RECEIVE;
1111 cqe->status = status;
1112 cqe->imm_data = 0;
1113 cqe->bytes = bytes;
1115 if (rdma_is_kernel_res(&cq->base_cq.res)) {
1116 cqe->base_qp = &qp->base_qp;
1119 cqe->inval_stag = inval_stag;
1122 cqe->qp_id = qp_id(qp);
1125 WRITE_ONCE(cqe->flags, cqe_flags);
1127 smp_store_mb(rqe->flags, 0);
1129 cq->cq_put++;
1132 spin_unlock_irqrestore(&cq->lock, flags);
1136 cq->base_cq.comp_handler(&cq->base_cq,
1137 cq->base_cq.cq_context);
1140 spin_unlock_irqrestore(&cq->lock, flags);
1141 rv = -ENOMEM;
1146 smp_store_mb(rqe->flags, 0);
1168 while (qp->attrs.orq_size) {
1169 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1170 if (!READ_ONCE(sqe->flags))
1176 WRITE_ONCE(sqe->flags, 0);
1177 qp->orq_get++;
1180 * Flush an in-progress WQE if present
1182 if (wqe->wr_status != SIW_WR_IDLE) {
1184 tx_type(wqe), wqe->wr_status);
1191 wqe->wr_status == SIW_WR_QUEUED))
1193 * An in-progress Read Request is already in
1196 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1199 wqe->wr_status = SIW_WR_IDLE;
1204 while (qp->attrs.sq_size) {
1205 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1206 if (!READ_ONCE(sqe->flags))
1217 WRITE_ONCE(sqe->flags, 0);
1218 qp->sq_get++;
1237 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1240 * Flush an in-progress untagged operation if present
1242 if (wqe->wr_status != SIW_WR_IDLE) {
1244 rx_type(wqe), wqe->wr_status);
1249 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1254 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1256 wqe->wr_status = SIW_WR_IDLE;
1258 wqe = &qp->rx_tagged.wqe_active;
1260 if (wqe->wr_status != SIW_WR_IDLE) {
1262 wqe->wr_status = SIW_WR_IDLE;
1267 while (qp->attrs.rq_size) {
1269 &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1271 if (!READ_ONCE(rqe->flags))
1277 WRITE_ONCE(rqe->flags, 0);
1278 qp->rq_get++;
1284 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,
1288 kref_init(&qp->ref);
1289 qp->sdev = sdev;
1298 struct siw_device *sdev = qp->sdev;
1299 unsigned long flags;
1301 if (qp->cep)
1302 siw_cep_put(qp->cep);
1304 found = xa_erase(&sdev->qp_xa, qp_id(qp));
1306 spin_lock_irqsave(&sdev->lock, flags);
1307 list_del(&qp->devq);
1308 spin_unlock_irqrestore(&sdev->lock, flags);
1310 vfree(qp->sendq);
1311 vfree(qp->recvq);
1312 vfree(qp->irq);
1313 vfree(qp->orq);
1315 siw_put_tx_cpu(qp->tx_cpu);
1316 complete(&qp->qp_free);
1317 atomic_dec(&sdev->num_qp);