Lines Matching +full:hdr +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright(c) 2015 - 2018 Intel Corporation.
16 return (gid->global.interface_id == id && in gid_ok()
17 (gid->global.subnet_prefix == gid_prefix || in gid_ok()
18 gid->global.subnet_prefix == IB_DEFAULT_GID_PREFIX)); in gid_ok()
31 struct rvt_qp *qp = packet->qp; in hfi1_ruc_check_hdr()
32 u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; in hfi1_ruc_check_hdr()
33 u32 dlid = packet->dlid; in hfi1_ruc_check_hdr()
34 u32 slid = packet->slid; in hfi1_ruc_check_hdr()
35 u32 sl = packet->sl; in hfi1_ruc_check_hdr()
36 bool migrated = packet->migrated; in hfi1_ruc_check_hdr()
37 u16 pkey = packet->pkey; in hfi1_ruc_check_hdr()
39 if (qp->s_mig_state == IB_MIG_ARMED && migrated) { in hfi1_ruc_check_hdr()
40 if (!packet->grh) { in hfi1_ruc_check_hdr()
41 if ((rdma_ah_get_ah_flags(&qp->alt_ah_attr) & in hfi1_ruc_check_hdr()
43 (packet->etype != RHF_RCV_TYPE_BYPASS)) in hfi1_ruc_check_hdr()
48 if (!(rdma_ah_get_ah_flags(&qp->alt_ah_attr) & in hfi1_ruc_check_hdr()
51 grh = rdma_ah_read_grh(&qp->alt_ah_attr); in hfi1_ruc_check_hdr()
52 guid = get_sguid(ibp, grh->sgid_index); in hfi1_ruc_check_hdr()
53 if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix, in hfi1_ruc_check_hdr()
57 &packet->grh->sgid, in hfi1_ruc_check_hdr()
58 grh->dgid.global.subnet_prefix, in hfi1_ruc_check_hdr()
59 grh->dgid.global.interface_id)) in hfi1_ruc_check_hdr()
64 hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, in hfi1_ruc_check_hdr()
69 if (slid != rdma_ah_get_dlid(&qp->alt_ah_attr) || in hfi1_ruc_check_hdr()
70 ppd_from_ibp(ibp)->port != in hfi1_ruc_check_hdr()
71 rdma_ah_get_port_num(&qp->alt_ah_attr)) in hfi1_ruc_check_hdr()
73 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_ruc_check_hdr()
75 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_ruc_check_hdr()
77 if (!packet->grh) { in hfi1_ruc_check_hdr()
78 if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & in hfi1_ruc_check_hdr()
80 (packet->etype != RHF_RCV_TYPE_BYPASS)) in hfi1_ruc_check_hdr()
85 if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & in hfi1_ruc_check_hdr()
88 grh = rdma_ah_read_grh(&qp->remote_ah_attr); in hfi1_ruc_check_hdr()
89 guid = get_sguid(ibp, grh->sgid_index); in hfi1_ruc_check_hdr()
90 if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix, in hfi1_ruc_check_hdr()
94 &packet->grh->sgid, in hfi1_ruc_check_hdr()
95 grh->dgid.global.subnet_prefix, in hfi1_ruc_check_hdr()
96 grh->dgid.global.interface_id)) in hfi1_ruc_check_hdr()
101 hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, in hfi1_ruc_check_hdr()
106 if ((slid != rdma_ah_get_dlid(&qp->remote_ah_attr)) || in hfi1_ruc_check_hdr()
107 ppd_from_ibp(ibp)->port != qp->port_num) in hfi1_ruc_check_hdr()
109 if (qp->s_mig_state == IB_MIG_REARM && !migrated) in hfi1_ruc_check_hdr()
110 qp->s_mig_state = IB_MIG_ARMED; in hfi1_ruc_check_hdr()
117 * hfi1_make_grh - construct a GRH header
119 * @hdr: a pointer to the GRH header being constructed
126 u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, in hfi1_make_grh() argument
129 hdr->version_tclass_flow = in hfi1_make_grh()
131 (grh->traffic_class << IB_GRH_TCLASS_SHIFT) | in hfi1_make_grh()
132 (grh->flow_label << IB_GRH_FLOW_SHIFT)); in hfi1_make_grh()
133 hdr->paylen = cpu_to_be16((hwords + nwords) << 2); in hfi1_make_grh()
134 /* next_hdr is defined by C8-7 in ch. 8.4.1 */ in hfi1_make_grh()
135 hdr->next_hdr = IB_GRH_NEXT_HDR; in hfi1_make_grh()
136 hdr->hop_limit = grh->hop_limit; in hfi1_make_grh()
137 /* The SGID is 32-bit aligned. */ in hfi1_make_grh()
138 hdr->sgid.global.subnet_prefix = ibp->rvp.gid_prefix; in hfi1_make_grh()
139 hdr->sgid.global.interface_id = in hfi1_make_grh()
140 grh->sgid_index < HFI1_GUIDS_PER_PORT ? in hfi1_make_grh()
141 get_sguid(ibp, grh->sgid_index) : in hfi1_make_grh()
143 hdr->dgid = grh->dgid; in hfi1_make_grh()
145 /* GRH header size in 32-bit words. */ in hfi1_make_grh()
150 hdr.ibh.u.oth.bth[2]) / 4)
153 * build_ahg - create ahg in s_ahg
165 struct hfi1_qp_priv *priv = qp->priv; in build_ahg()
167 if (unlikely(qp->s_flags & HFI1_S_AHG_CLEAR)) in build_ahg()
169 if (!(qp->s_flags & HFI1_S_AHG_VALID)) { in build_ahg()
171 if (qp->s_ahgidx < 0) in build_ahg()
172 qp->s_ahgidx = sdma_ahg_alloc(priv->s_sde); in build_ahg()
173 if (qp->s_ahgidx >= 0) { in build_ahg()
174 qp->s_ahgpsn = npsn; in build_ahg()
175 priv->s_ahg->tx_flags |= SDMA_TXREQ_F_AHG_COPY; in build_ahg()
177 priv->s_ahg->ahgidx = qp->s_ahgidx; in build_ahg()
178 qp->s_flags |= HFI1_S_AHG_VALID; in build_ahg()
182 if (qp->s_ahgidx >= 0) { in build_ahg()
183 priv->s_ahg->tx_flags |= SDMA_TXREQ_F_USE_AHG; in build_ahg()
184 priv->s_ahg->ahgidx = qp->s_ahgidx; in build_ahg()
185 priv->s_ahg->ahgcount++; in build_ahg()
186 priv->s_ahg->ahgdesc[0] = in build_ahg()
193 (qp->s_ahgpsn & 0xffff0000)) { in build_ahg()
194 priv->s_ahg->ahgcount++; in build_ahg()
195 priv->s_ahg->ahgdesc[1] = in build_ahg()
211 ohdr->bth[0] = cpu_to_be32(bth0); in hfi1_make_ruc_bth()
212 ohdr->bth[1] = cpu_to_be32(bth1); in hfi1_make_ruc_bth()
213 ohdr->bth[2] = cpu_to_be32(bth2); in hfi1_make_ruc_bth()
217 * hfi1_make_ruc_header_16B - build a 16B header
227 * - packet needs a GRH
228 * - BECN needed
229 * - migration state not IB_MIG_MIGRATED
237 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_ruc_header_16B()
238 struct hfi1_ibport *ibp = ps->ibp; in hfi1_make_ruc_header_16B()
241 u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_ruc_header_16B()
244 (ps->s_txreq->hdr_dwords << 2), in hfi1_make_ruc_header_16B()
245 ps->s_txreq->s_cur_size); in hfi1_make_ruc_header_16B()
246 u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + in hfi1_make_ruc_header_16B()
250 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && in hfi1_make_ruc_header_16B()
251 hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { in hfi1_make_ruc_header_16B()
254 rdma_ah_retrieve_grh(&qp->remote_ah_attr); in hfi1_make_ruc_header_16B()
259 if (grd->sgid_index == OPA_GID_INDEX) in hfi1_make_ruc_header_16B()
260 grd->sgid_index = 0; in hfi1_make_ruc_header_16B()
261 grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; in hfi1_make_ruc_header_16B()
263 ps->s_txreq->hdr_dwords += in hfi1_make_ruc_header_16B()
265 ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, in hfi1_make_ruc_header_16B()
270 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_ruc_header_16B()
275 if (qp->s_flags & RVT_S_ECN) { in hfi1_make_ruc_header_16B()
276 qp->s_flags &= ~RVT_S_ECN; in hfi1_make_ruc_header_16B()
284 qp->s_flags &= ~HFI1_S_AHG_VALID; in hfi1_make_ruc_header_16B()
290 if (!ppd->lid) in hfi1_make_ruc_header_16B()
293 slid = ppd->lid | in hfi1_make_ruc_header_16B()
294 (rdma_ah_get_path_bits(&qp->remote_ah_attr) & in hfi1_make_ruc_header_16B()
295 ((1 << ppd->lmc) - 1)); in hfi1_make_ruc_header_16B()
297 hfi1_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah, in hfi1_make_ruc_header_16B()
299 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), in hfi1_make_ruc_header_16B()
301 (ps->s_txreq->hdr_dwords + nwords) >> 1, in hfi1_make_ruc_header_16B()
302 pkey, becn, 0, l4, priv->s_sc); in hfi1_make_ruc_header_16B()
306 * hfi1_make_ruc_header_9B - build a 9B header
316 * - packet needs a GRH
317 * - BECN needed
318 * - migration state not IB_MIG_MIGRATED
326 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_ruc_header_9B()
327 struct hfi1_ibport *ibp = ps->ibp; in hfi1_make_ruc_header_9B()
328 u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_ruc_header_9B()
330 u8 extra_bytes = -ps->s_txreq->s_cur_size & 3; in hfi1_make_ruc_header_9B()
331 u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + in hfi1_make_ruc_header_9B()
334 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { in hfi1_make_ruc_header_9B()
335 struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; in hfi1_make_ruc_header_9B()
338 ps->s_txreq->hdr_dwords += in hfi1_make_ruc_header_9B()
340 rdma_ah_read_grh(&qp->remote_ah_attr), in hfi1_make_ruc_header_9B()
341 ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, in hfi1_make_ruc_header_9B()
345 lrh0 |= (priv->s_sc & 0xf) << 12 | in hfi1_make_ruc_header_9B()
346 (rdma_ah_get_sl(&qp->remote_ah_attr) & 0xf) << 4; in hfi1_make_ruc_header_9B()
348 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_ruc_header_9B()
353 if (qp->s_flags & RVT_S_ECN) { in hfi1_make_ruc_header_9B()
354 qp->s_flags &= ~RVT_S_ECN; in hfi1_make_ruc_header_9B()
362 qp->s_flags &= ~HFI1_S_AHG_VALID; in hfi1_make_ruc_header_9B()
367 hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh, in hfi1_make_ruc_header_9B()
369 ps->s_txreq->hdr_dwords + nwords, in hfi1_make_ruc_header_9B()
370 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), in hfi1_make_ruc_header_9B()
371 ppd_from_ibp(ibp)->lid | in hfi1_make_ruc_header_9B()
372 rdma_ah_get_path_bits(&qp->remote_ah_attr)); in hfi1_make_ruc_header_9B()
380 /* We support only two types - 9B and 16B for now */
390 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_ruc_header()
396 * are at a non-AHG default to protect in hfi1_make_ruc_header()
403 priv->s_ahg->tx_flags = 0; in hfi1_make_ruc_header()
404 priv->s_ahg->ahgcount = 0; in hfi1_make_ruc_header()
405 priv->s_ahg->ahgidx = 0; in hfi1_make_ruc_header()
408 hfi1_ruc_header_tbl[priv->hdr_type](qp, ohdr, bth0, bth1, bth2, middle, in hfi1_make_ruc_header()
416 * hfi1_schedule_send_yield - test for a yield required for QP
417 * send engine
420 * the send engine progress
432 ps->pkts_sent = true; in hfi1_schedule_send_yield()
434 if (unlikely(time_after(jiffies, ps->timeout))) { in hfi1_schedule_send_yield()
435 if (!ps->in_thread || in hfi1_schedule_send_yield()
436 workqueue_congested(ps->cpu, ps->ppd->hfi1_wq)) { in hfi1_schedule_send_yield()
437 spin_lock_irqsave(&qp->s_lock, ps->flags); in hfi1_schedule_send_yield()
439 qp->s_flags &= ~RVT_S_BUSY; in hfi1_schedule_send_yield()
442 struct hfi1_qp_priv *priv = qp->priv; in hfi1_schedule_send_yield()
444 if (priv->s_flags & in hfi1_schedule_send_yield()
446 qp->s_flags &= ~RVT_S_BUSY; in hfi1_schedule_send_yield()
447 priv->s_flags &= in hfi1_schedule_send_yield()
451 priv->s_flags &= ~RVT_S_BUSY; in hfi1_schedule_send_yield()
456 spin_unlock_irqrestore(&qp->s_lock, ps->flags); in hfi1_schedule_send_yield()
457 this_cpu_inc(*ps->ppd->dd->send_schedule); in hfi1_schedule_send_yield()
463 this_cpu_inc(*ps->ppd->dd->send_schedule); in hfi1_schedule_send_yield()
464 ps->timeout = jiffies + ps->timeout_int; in hfi1_schedule_send_yield()
479 struct rvt_qp *qp = iowait_to_qp(w->iow); in _hfi1_do_send()
485 * hfi1_do_send - perform a send on a QP
496 struct hfi1_qp_priv *priv = qp->priv; in hfi1_do_send()
499 ps.dev = to_idev(qp->ibqp.device); in hfi1_do_send()
500 ps.ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_do_send()
503 ps.wait = iowait_get_ib_work(&priv->s_iowait); in hfi1_do_send()
507 switch (qp->ibqp.qp_type) { in hfi1_do_send()
509 if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & in hfi1_do_send()
510 ~((1 << ps.ppd->lmc) - 1)) == in hfi1_do_send()
511 ps.ppd->lid)) { in hfi1_do_send()
516 ps.timeout_int = qp->timeout_jiffies; in hfi1_do_send()
519 if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & in hfi1_do_send()
520 ~((1 << ps.ppd->lmc) - 1)) == in hfi1_do_send()
521 ps.ppd->lid)) { in hfi1_do_send()
533 spin_lock_irqsave(&qp->s_lock, ps.flags); in hfi1_do_send()
537 if (qp->s_flags & HFI1_S_ANY_WAIT_IO) in hfi1_do_send()
538 iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_IB); in hfi1_do_send()
539 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_send()
543 qp->s_flags |= RVT_S_BUSY; in hfi1_do_send()
547 ps.cpu = priv->s_sde ? priv->s_sde->cpu : in hfi1_do_send()
548 cpumask_first(cpumask_of_node(ps.ppd->dd->node)); in hfi1_do_send()
551 /* insure a pre-built packet is handled */ in hfi1_do_send()
556 if (priv->s_flags & HFI1_S_TID_BUSY_SET) in hfi1_do_send()
557 qp->s_flags |= RVT_S_BUSY; in hfi1_do_send()
558 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_send()
561 * the send engine will be woken up later. in hfi1_do_send()
570 spin_lock_irqsave(&qp->s_lock, ps.flags); in hfi1_do_send()
573 iowait_starve_clear(ps.pkts_sent, &priv->s_iowait); in hfi1_do_send()
574 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_send()