Lines Matching +full:hp +full:- +full:cfg
1 // SPDX-License-Identifier: GPL-2.0
2 /* ldc.c: Logical Domain Channel link-layer protocol driver.
19 #include <asm/iommu-common.h>
40 * When in RAW mode, packets are simply straight 64-byte payloads
70 u8 u_data[LDC_PACKET_SIZE - 8];
74 u8 r_data[LDC_PACKET_SIZE - 8 - 8];
141 struct ldc_channel_config cfg; member
177 do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
178 printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
212 return __advance(off, lp->rx_num_entries); in rx_advance()
217 return __advance(off, lp->tx_num_entries); in tx_advance()
226 t = tx_advance(lp, lp->tx_tail); in handshake_get_tx_packet()
227 if (t == lp->tx_head) in handshake_get_tx_packet()
232 p = lp->tx_base; in handshake_get_tx_packet()
233 return p + (lp->tx_tail / LDC_PACKET_SIZE); in handshake_get_tx_packet()
241 * lp->tx_acked.
245 if (lp->cfg.mode == LDC_MODE_STREAM) in head_for_data()
246 return lp->tx_acked; in head_for_data()
247 return lp->tx_head; in head_for_data()
256 tail = lp->tx_tail; in tx_has_space_for()
262 diff = limit - new_tail; in tx_has_space_for()
265 ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); in tx_has_space_for()
267 mss = lp->mss; in tx_has_space_for()
282 t = tx_advance(lp, lp->tx_tail); in data_get_tx_packet()
288 p = lp->tx_base; in data_get_tx_packet()
289 return p + (lp->tx_tail / LDC_PACKET_SIZE); in data_get_tx_packet()
294 unsigned long orig_tail = lp->tx_tail; in set_tx_tail()
297 lp->tx_tail = tail; in set_tx_tail()
298 while (limit-- > 0) { in set_tx_tail()
301 err = sun4v_ldc_tx_set_qtail(lp->id, tail); in set_tx_tail()
306 lp->tx_tail = orig_tail; in set_tx_tail()
307 return -EINVAL; in set_tx_tail()
312 lp->tx_tail = orig_tail; in set_tx_tail()
313 return -EBUSY; in set_tx_tail()
324 while (limit-- > 0) { in __set_rx_head()
327 err = sun4v_ldc_rx_set_qhead(lp->id, head); in __set_rx_head()
332 return -EINVAL; in __set_rx_head()
337 return -EBUSY; in __set_rx_head()
344 BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); in send_tx_packet()
358 p->type = LDC_CTRL; in handshake_compose_ctrl()
359 p->stype = stype; in handshake_compose_ctrl()
360 p->ctrl = ctrl; in handshake_compose_ctrl()
362 memcpy(p->u.u_data, data, dlen); in handshake_compose_ctrl()
376 ver->major, ver->minor); in start_handshake()
383 lp->flags &= ~LDC_FLAG_RESET; in start_handshake()
386 return -EBUSY; in start_handshake()
407 return -EBUSY; in send_version_nack()
420 vp->major, vp->minor); in send_version_ack()
424 return -EBUSY; in send_version_ack()
435 p->env = lp->cfg.mode; in send_rts()
436 p->seqid = 0; in send_rts()
437 lp->rcv_nxt = 0; in send_rts()
440 p->env, p->seqid); in send_rts()
444 return -EBUSY; in send_rts()
455 p->env = lp->cfg.mode; in send_rtr()
456 p->seqid = 0; in send_rtr()
459 p->env, p->seqid); in send_rtr()
463 return -EBUSY; in send_rtr()
474 p->env = 0; in send_rdx()
475 p->seqid = ++lp->snd_nxt; in send_rdx()
476 p->u.r.ackid = lp->rcv_nxt; in send_rdx()
479 p->env, p->seqid, p->u.r.ackid); in send_rdx()
483 return -EBUSY; in send_rdx()
494 return -EBUSY; in send_data_nack()
496 p->type = data_pkt->type; in send_data_nack()
497 p->stype = LDC_NACK; in send_data_nack()
498 p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; in send_data_nack()
499 p->seqid = lp->snd_nxt + 1; in send_data_nack()
500 p->u.r.ackid = lp->rcv_nxt; in send_data_nack()
503 p->type, p->ctrl, p->seqid, p->u.r.ackid); in send_data_nack()
507 lp->snd_nxt++; in send_data_nack()
522 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); in ldc_abort()
526 lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); in ldc_abort()
528 hv_err = sun4v_ldc_tx_get_state(lp->id, in ldc_abort()
529 &lp->tx_head, in ldc_abort()
530 &lp->tx_tail, in ldc_abort()
531 &lp->chan_state); in ldc_abort()
535 lp->id, hv_err); in ldc_abort()
537 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); in ldc_abort()
541 lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); in ldc_abort()
546 hv_err = sun4v_ldc_rx_get_state(lp->id, in ldc_abort()
547 &lp->rx_head, in ldc_abort()
548 &lp->rx_tail, in ldc_abort()
549 &lp->chan_state); in ldc_abort()
553 lp->id, hv_err); in ldc_abort()
555 return -ECONNRESET; in ldc_abort()
565 if (v->major <= major) { in find_by_major()
579 vp->major, vp->minor); in process_ver_info()
581 if (lp->hs_state == LDC_HS_GOTVERS) { in process_ver_info()
582 lp->hs_state = LDC_HS_OPEN; in process_ver_info()
583 memset(&lp->ver, 0, sizeof(lp->ver)); in process_ver_info()
586 vap = find_by_major(vp->major); in process_ver_info()
589 } else if (vap->major != vp->major) { in process_ver_info()
590 err = send_version_nack(lp, vap->major, vap->minor); in process_ver_info()
593 if (ver.minor > vap->minor) in process_ver_info()
594 ver.minor = vap->minor; in process_ver_info()
597 lp->ver = ver; in process_ver_info()
598 lp->hs_state = LDC_HS_GOTVERS; in process_ver_info()
610 vp->major, vp->minor); in process_ver_ack()
612 if (lp->hs_state == LDC_HS_GOTVERS) { in process_ver_ack()
613 if (lp->ver.major != vp->major || in process_ver_ack()
614 lp->ver.minor != vp->minor) in process_ver_ack()
617 lp->ver = *vp; in process_ver_ack()
618 lp->hs_state = LDC_HS_GOTVERS; in process_ver_ack()
631 if (vp->major == 0 && vp->minor == 0) in process_ver_nack()
634 vap = find_by_major(vp->major); in process_ver_nack()
652 vp = (struct ldc_version *) p->u.u_data; in process_version()
654 switch (p->stype) { in process_version()
673 p->stype, p->seqid, p->env); in process_rts()
675 if (p->stype != LDC_INFO || in process_rts()
676 lp->hs_state != LDC_HS_GOTVERS || in process_rts()
677 p->env != lp->cfg.mode) in process_rts()
680 lp->snd_nxt = p->seqid; in process_rts()
681 lp->rcv_nxt = p->seqid; in process_rts()
682 lp->hs_state = LDC_HS_SENTRTR; in process_rts()
693 p->stype, p->seqid, p->env); in process_rtr()
695 if (p->stype != LDC_INFO || in process_rtr()
696 p->env != lp->cfg.mode) in process_rtr()
699 lp->snd_nxt = p->seqid; in process_rtr()
700 lp->hs_state = LDC_HS_COMPLETE; in process_rtr()
709 return lp->rcv_nxt + 1 == seqid; in rx_seq_ok()
716 p->stype, p->seqid, p->env, p->u.r.ackid); in process_rdx()
718 if (p->stype != LDC_INFO || in process_rdx()
719 !(rx_seq_ok(lp, p->seqid))) in process_rdx()
722 lp->rcv_nxt = p->seqid; in process_rdx()
724 lp->hs_state = LDC_HS_COMPLETE; in process_rdx()
733 switch (p->ctrl) { in process_control_frame()
760 unsigned long head = lp->tx_acked; in process_data_ack()
761 u32 ackid = ack->u.r.ackid; in process_data_ack()
764 struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); in process_data_ack()
768 if (p->seqid == ackid) { in process_data_ack()
769 lp->tx_acked = head; in process_data_ack()
772 if (head == lp->tx_tail) in process_data_ack()
782 lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); in send_events()
784 lp->cfg.event(lp->event_arg, LDC_EVENT_UP); in send_events()
786 lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); in send_events()
795 spin_lock_irqsave(&lp->lock, flags); in ldc_rx()
797 orig_state = lp->chan_state; in ldc_rx()
802 sun4v_ldc_rx_get_state(lp->id, in ldc_rx()
803 &lp->rx_head, in ldc_rx()
804 &lp->rx_tail, in ldc_rx()
805 &lp->chan_state); in ldc_rx()
808 orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); in ldc_rx()
812 if (lp->cfg.mode == LDC_MODE_RAW && in ldc_rx()
813 lp->chan_state == LDC_CHANNEL_UP) { in ldc_rx()
814 lp->hs_state = LDC_HS_COMPLETE; in ldc_rx()
823 orig_state = lp->chan_state; in ldc_rx()
830 if (lp->flags & LDC_FLAG_RESET) { in ldc_rx()
839 if (lp->hs_state == LDC_HS_COMPLETE) { in ldc_rx()
841 if (lp->chan_state != orig_state) { in ldc_rx()
844 if (lp->chan_state == LDC_CHANNEL_UP) in ldc_rx()
849 if (lp->rx_head != lp->rx_tail) in ldc_rx()
855 if (lp->chan_state != orig_state) in ldc_rx()
858 while (lp->rx_head != lp->rx_tail) { in ldc_rx()
863 p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); in ldc_rx()
865 switch (p->type) { in ldc_rx()
889 new = lp->rx_head; in ldc_rx()
891 if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) in ldc_rx()
893 lp->rx_head = new; in ldc_rx()
900 if (lp->hs_state == LDC_HS_COMPLETE) in ldc_rx()
905 spin_unlock_irqrestore(&lp->lock, flags); in ldc_rx()
918 spin_lock_irqsave(&lp->lock, flags); in ldc_tx()
920 orig_state = lp->chan_state; in ldc_tx()
925 sun4v_ldc_tx_get_state(lp->id, in ldc_tx()
926 &lp->tx_head, in ldc_tx()
927 &lp->tx_tail, in ldc_tx()
928 &lp->chan_state); in ldc_tx()
931 orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); in ldc_tx()
933 if (lp->cfg.mode == LDC_MODE_RAW && in ldc_tx()
934 lp->chan_state == LDC_CHANNEL_UP) { in ldc_tx()
935 lp->hs_state = LDC_HS_COMPLETE; in ldc_tx()
944 orig_state = lp->chan_state; in ldc_tx()
948 spin_unlock_irqrestore(&lp->lock, flags); in ldc_tx()
969 if (lp->id == id) in __ldc_channel_exists()
988 return -ENOMEM; in alloc_queue()
1029 base = iommu->page_table + entry; in ldc_demap()
1031 if (base->cookie) in ldc_demap()
1033 base->cookie); in ldc_demap()
1034 base->mte = 0; in ldc_demap()
1044 struct ldc_iommu *ldc_iommu = &lp->iommu; in ldc_iommu_init()
1045 struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; in ldc_iommu_init()
1052 spin_lock_init(&ldc_iommu->lock); in ldc_iommu_init()
1056 iommu->map = kzalloc(sz, GFP_KERNEL); in ldc_iommu_init()
1057 if (!iommu->map) { in ldc_iommu_init()
1059 return -ENOMEM; in ldc_iommu_init()
1070 err = -ENOMEM; in ldc_iommu_init()
1079 ldc_iommu->page_table = table; in ldc_iommu_init()
1081 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), in ldc_iommu_init()
1083 err = -EINVAL; in ldc_iommu_init()
1091 ldc_iommu->page_table = NULL; in ldc_iommu_init()
1094 kfree(iommu->map); in ldc_iommu_init()
1095 iommu->map = NULL; in ldc_iommu_init()
1102 struct ldc_iommu *ldc_iommu = &lp->iommu; in ldc_iommu_release()
1103 struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; in ldc_iommu_release()
1106 (void) sun4v_ldc_set_map_table(lp->id, 0, 0); in ldc_iommu_release()
1108 num_tsb_entries = iommu->poolsize * iommu->nr_pools; in ldc_iommu_release()
1112 free_pages((unsigned long) ldc_iommu->page_table, order); in ldc_iommu_release()
1113 ldc_iommu->page_table = NULL; in ldc_iommu_release()
1115 kfree(iommu->map); in ldc_iommu_release()
1116 iommu->map = NULL; in ldc_iommu_release()
1130 err = -ENODEV; in ldc_alloc()
1134 err = -EINVAL; in ldc_alloc()
1140 switch (cfgp->mode) { in ldc_alloc()
1148 mss = LDC_PACKET_SIZE - 8; in ldc_alloc()
1153 mss = LDC_PACKET_SIZE - 8 - 8; in ldc_alloc()
1160 if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) in ldc_alloc()
1164 err = -ENODEV; in ldc_alloc()
1168 err = -EEXIST; in ldc_alloc()
1175 err = -ENOMEM; in ldc_alloc()
1179 spin_lock_init(&lp->lock); in ldc_alloc()
1181 lp->id = id; in ldc_alloc()
1187 lp->mops = mops; in ldc_alloc()
1188 lp->mss = mss; in ldc_alloc()
1190 lp->cfg = *cfgp; in ldc_alloc()
1191 if (!lp->cfg.mtu) in ldc_alloc()
1192 lp->cfg.mtu = LDC_DEFAULT_MTU; in ldc_alloc()
1194 if (lp->cfg.mode == LDC_MODE_STREAM) { in ldc_alloc()
1195 mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); in ldc_alloc()
1197 err = -ENOMEM; in ldc_alloc()
1200 lp->mssbuf = mssbuf; in ldc_alloc()
1203 lp->event_arg = event_arg; in ldc_alloc()
1208 lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; in ldc_alloc()
1209 lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; in ldc_alloc()
1211 err = alloc_queue("TX", lp->tx_num_entries, in ldc_alloc()
1212 &lp->tx_base, &lp->tx_ra); in ldc_alloc()
1216 err = alloc_queue("RX", lp->rx_num_entries, in ldc_alloc()
1217 &lp->rx_base, &lp->rx_ra); in ldc_alloc()
1221 lp->flags |= LDC_FLAG_ALLOCED_QUEUES; in ldc_alloc()
1223 lp->hs_state = LDC_HS_CLOSED; in ldc_alloc()
1226 INIT_HLIST_NODE(&lp->list); in ldc_alloc()
1227 hlist_add_head(&lp->list, &ldc_channel_list); in ldc_alloc()
1229 INIT_HLIST_HEAD(&lp->mh_list); in ldc_alloc()
1231 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); in ldc_alloc()
1232 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); in ldc_alloc()
1234 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, in ldc_alloc()
1235 lp->rx_irq_name, lp); in ldc_alloc()
1239 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, in ldc_alloc()
1240 lp->tx_irq_name, lp); in ldc_alloc()
1242 free_irq(lp->cfg.rx_irq, lp); in ldc_alloc()
1249 free_queue(lp->tx_num_entries, lp->tx_base); in ldc_alloc()
1267 if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { in ldc_unbind()
1268 free_irq(lp->cfg.rx_irq, lp); in ldc_unbind()
1269 free_irq(lp->cfg.tx_irq, lp); in ldc_unbind()
1270 lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; in ldc_unbind()
1273 if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { in ldc_unbind()
1274 sun4v_ldc_tx_qconf(lp->id, 0, 0); in ldc_unbind()
1275 sun4v_ldc_rx_qconf(lp->id, 0, 0); in ldc_unbind()
1276 lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; in ldc_unbind()
1278 if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { in ldc_unbind()
1279 free_queue(lp->tx_num_entries, lp->tx_base); in ldc_unbind()
1280 free_queue(lp->rx_num_entries, lp->rx_base); in ldc_unbind()
1281 lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; in ldc_unbind()
1291 hlist_del(&lp->list); in ldc_free()
1292 kfree(lp->mssbuf); in ldc_free()
1300 * the hypervisor and puts the channel into a pseudo-listening
1307 int err = -EINVAL; in ldc_bind()
1309 if (lp->state != LDC_STATE_INIT) in ldc_bind()
1310 return -EINVAL; in ldc_bind()
1312 spin_lock_irqsave(&lp->lock, flags); in ldc_bind()
1314 enable_irq(lp->cfg.rx_irq); in ldc_bind()
1315 enable_irq(lp->cfg.tx_irq); in ldc_bind()
1317 lp->flags |= LDC_FLAG_REGISTERED_IRQS; in ldc_bind()
1319 err = -ENODEV; in ldc_bind()
1320 hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); in ldc_bind()
1324 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); in ldc_bind()
1328 hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); in ldc_bind()
1332 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); in ldc_bind()
1336 lp->flags |= LDC_FLAG_REGISTERED_QUEUES; in ldc_bind()
1338 hv_err = sun4v_ldc_tx_get_state(lp->id, in ldc_bind()
1339 &lp->tx_head, in ldc_bind()
1340 &lp->tx_tail, in ldc_bind()
1341 &lp->chan_state); in ldc_bind()
1342 err = -EBUSY; in ldc_bind()
1346 lp->tx_acked = lp->tx_head; in ldc_bind()
1348 lp->hs_state = LDC_HS_OPEN; in ldc_bind()
1351 if (lp->cfg.mode == LDC_MODE_RAW) { in ldc_bind()
1356 lp->hs_state = LDC_HS_COMPLETE; in ldc_bind()
1359 spin_unlock_irqrestore(&lp->lock, flags); in ldc_bind()
1364 lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; in ldc_bind()
1365 sun4v_ldc_rx_qconf(lp->id, 0, 0); in ldc_bind()
1368 sun4v_ldc_tx_qconf(lp->id, 0, 0); in ldc_bind()
1371 lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; in ldc_bind()
1372 free_irq(lp->cfg.tx_irq, lp); in ldc_bind()
1373 free_irq(lp->cfg.rx_irq, lp); in ldc_bind()
1375 spin_unlock_irqrestore(&lp->lock, flags); in ldc_bind()
1386 if (lp->cfg.mode == LDC_MODE_RAW) in ldc_connect()
1387 return -EINVAL; in ldc_connect()
1389 spin_lock_irqsave(&lp->lock, flags); in ldc_connect()
1391 if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || in ldc_connect()
1392 !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || in ldc_connect()
1393 lp->hs_state != LDC_HS_OPEN) in ldc_connect()
1394 err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); in ldc_connect()
1398 spin_unlock_irqrestore(&lp->lock, flags); in ldc_connect()
1409 if (lp->cfg.mode == LDC_MODE_RAW) in ldc_disconnect()
1410 return -EINVAL; in ldc_disconnect()
1412 if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || in ldc_disconnect()
1413 !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) in ldc_disconnect()
1414 return -EINVAL; in ldc_disconnect()
1416 spin_lock_irqsave(&lp->lock, flags); in ldc_disconnect()
1418 err = -ENODEV; in ldc_disconnect()
1419 hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); in ldc_disconnect()
1423 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); in ldc_disconnect()
1427 hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); in ldc_disconnect()
1431 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); in ldc_disconnect()
1436 lp->hs_state = LDC_HS_OPEN; in ldc_disconnect()
1437 lp->flags |= LDC_FLAG_RESET; in ldc_disconnect()
1439 spin_unlock_irqrestore(&lp->lock, flags); in ldc_disconnect()
1444 sun4v_ldc_tx_qconf(lp->id, 0, 0); in ldc_disconnect()
1445 sun4v_ldc_rx_qconf(lp->id, 0, 0); in ldc_disconnect()
1446 free_irq(lp->cfg.tx_irq, lp); in ldc_disconnect()
1447 free_irq(lp->cfg.rx_irq, lp); in ldc_disconnect()
1448 lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | in ldc_disconnect()
1452 spin_unlock_irqrestore(&lp->lock, flags); in ldc_disconnect()
1460 return lp->state; in ldc_state()
1466 ldcdbg(STATE, "STATE (%s) --> (%s)\n", in ldc_set_state()
1467 state_to_str(lp->state), in ldc_set_state()
1470 lp->state = state; in ldc_set_state()
1476 return lp->cfg.mode; in ldc_mode()
1482 return __set_rx_head(lp, lp->rx_tail); in ldc_rx_reset()
1492 caller, lp->id, lp->flags, state_to_str(lp->state), in __ldc_print()
1493 lp->chan_state, lp->hs_state, in __ldc_print()
1494 lp->rx_head, lp->rx_tail, lp->rx_num_entries, in __ldc_print()
1495 lp->tx_head, lp->tx_tail, lp->tx_num_entries, in __ldc_print()
1496 lp->rcv_nxt, lp->snd_nxt); in __ldc_print()
1506 hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, in write_raw()
1507 &lp->chan_state); in write_raw()
1509 return -EBUSY; in write_raw()
1511 if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) in write_raw()
1515 return -EMSGSIZE; in write_raw()
1519 return -EAGAIN; in write_raw()
1537 return -EINVAL; in read_raw()
1539 hv_err = sun4v_ldc_rx_get_state(lp->id, in read_raw()
1540 &lp->rx_head, in read_raw()
1541 &lp->rx_tail, in read_raw()
1542 &lp->chan_state); in read_raw()
1546 if (lp->chan_state == LDC_CHANNEL_DOWN || in read_raw()
1547 lp->chan_state == LDC_CHANNEL_RESETTING) in read_raw()
1548 return -ECONNRESET; in read_raw()
1550 if (lp->rx_head == lp->rx_tail) in read_raw()
1553 p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); in read_raw()
1556 new = rx_advance(lp, lp->rx_head); in read_raw()
1557 lp->rx_head = new; in read_raw()
1561 err = -ECONNRESET; in read_raw()
1581 hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, in write_nonraw()
1582 &lp->chan_state); in write_nonraw()
1584 return -EBUSY; in write_nonraw()
1586 if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) in write_nonraw()
1590 return -EAGAIN; in write_nonraw()
1592 seq = lp->snd_nxt; in write_nonraw()
1594 tail = lp->tx_tail; in write_nonraw()
1596 struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); in write_nonraw()
1597 u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? in write_nonraw()
1598 p->u.u_data : in write_nonraw()
1599 p->u.r.r_data); in write_nonraw()
1602 p->type = LDC_DATA; in write_nonraw()
1603 p->stype = LDC_INFO; in write_nonraw()
1604 p->ctrl = 0; in write_nonraw()
1606 data_len = size - copied; in write_nonraw()
1607 if (data_len > lp->mss) in write_nonraw()
1608 data_len = lp->mss; in write_nonraw()
1612 p->env = (data_len | in write_nonraw()
1614 (data_len == size - copied ? LDC_STOP : 0)); in write_nonraw()
1616 p->seqid = ++seq; in write_nonraw()
1619 p->type, in write_nonraw()
1620 p->stype, in write_nonraw()
1621 p->ctrl, in write_nonraw()
1622 p->env, in write_nonraw()
1623 p->seqid); in write_nonraw()
1634 lp->snd_nxt = seq; in write_nonraw()
1647 lp->rcv_nxt = first_frag->seqid - 1; in rx_bad_seq()
1662 if (p->stype & LDC_ACK) { in data_ack_nack()
1667 if (p->stype & LDC_NACK) in data_ack_nack()
1679 cur_head, lp->rx_head, lp->rx_tail); in rx_data_wait()
1680 while (limit-- > 0) { in rx_data_wait()
1683 hv_err = sun4v_ldc_rx_get_state(lp->id, in rx_data_wait()
1685 &lp->rx_tail, in rx_data_wait()
1686 &lp->chan_state); in rx_data_wait()
1690 if (lp->chan_state == LDC_CHANNEL_DOWN || in rx_data_wait()
1691 lp->chan_state == LDC_CHANNEL_RESETTING) in rx_data_wait()
1692 return -ECONNRESET; in rx_data_wait()
1694 if (cur_head != lp->rx_tail) { in rx_data_wait()
1697 dummy, lp->rx_tail, lp->chan_state); in rx_data_wait()
1703 return -EAGAIN; in rx_data_wait()
1713 lp->rx_head = head; in rx_set_head()
1727 p->type = LDC_DATA; in send_data_ack()
1728 p->stype = LDC_ACK; in send_data_ack()
1729 p->ctrl = 0; in send_data_ack()
1730 p->seqid = lp->snd_nxt + 1; in send_data_ack()
1731 p->u.r.ackid = lp->rcv_nxt; in send_data_ack()
1735 lp->snd_nxt++; in send_data_ack()
1745 hv_err = sun4v_ldc_rx_get_state(lp->id, in read_nonraw()
1746 &lp->rx_head, in read_nonraw()
1747 &lp->rx_tail, in read_nonraw()
1748 &lp->chan_state); in read_nonraw()
1752 if (lp->chan_state == LDC_CHANNEL_DOWN || in read_nonraw()
1753 lp->chan_state == LDC_CHANNEL_RESETTING) in read_nonraw()
1754 return -ECONNRESET; in read_nonraw()
1756 if (lp->rx_head == lp->rx_tail) in read_nonraw()
1761 new = lp->rx_head; in read_nonraw()
1766 BUG_ON(new == lp->rx_tail); in read_nonraw()
1767 p = lp->rx_base + (new / LDC_PACKET_SIZE); in read_nonraw()
1771 p->type, in read_nonraw()
1772 p->stype, in read_nonraw()
1773 p->ctrl, in read_nonraw()
1774 p->env, in read_nonraw()
1775 p->seqid, in read_nonraw()
1776 p->u.r.ackid, in read_nonraw()
1777 lp->rcv_nxt); in read_nonraw()
1779 if (unlikely(!rx_seq_ok(lp, p->seqid))) { in read_nonraw()
1785 if (p->type & LDC_CTRL) { in read_nonraw()
1792 lp->rcv_nxt = p->seqid; in read_nonraw()
1795 * If this is a control-only packet, there is nothing in read_nonraw()
1799 if (!(p->type & LDC_DATA)) { in read_nonraw()
1803 if (p->stype & (LDC_ACK | LDC_NACK)) { in read_nonraw()
1808 if (!(p->stype & LDC_INFO)) { in read_nonraw()
1816 pkt_len = p->env & LDC_LEN; in read_nonraw()
1831 if ((first_frag == NULL && !(p->env & LDC_START)) || in read_nonraw()
1832 (first_frag != NULL && (p->env & LDC_START))) { in read_nonraw()
1846 if (pkt_len > size - copied) { in read_nonraw()
1856 * RX buffer for ldc_read() calls and use -EMSGSIZE in read_nonraw()
1859 err = -EMSGSIZE; in read_nonraw()
1867 (lp->cfg.mode == LDC_MODE_UNRELIABLE ? in read_nonraw()
1868 p->u.u_data : p->u.r.r_data), pkt_len); in read_nonraw()
1872 if (p->env & LDC_STOP) in read_nonraw()
1876 if (new == lp->rx_tail) { in read_nonraw()
1887 lp->rcv_nxt = first_frag->seqid - 1; in read_nonraw()
1891 if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) in read_nonraw()
1906 if (size > lp->cfg.mtu) in write_stream()
1907 size = lp->cfg.mtu; in write_stream()
1913 if (!lp->mssbuf_len) { in read_stream()
1914 int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); in read_stream()
1918 lp->mssbuf_len = err; in read_stream()
1919 lp->mssbuf_off = 0; in read_stream()
1922 if (size > lp->mssbuf_len) in read_stream()
1923 size = lp->mssbuf_len; in read_stream()
1924 memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); in read_stream()
1926 lp->mssbuf_off += size; in read_stream()
1927 lp->mssbuf_len -= size; in read_stream()
1943 return -EINVAL; in ldc_write()
1948 spin_lock_irqsave(&lp->lock, flags); in ldc_write()
1950 if (lp->hs_state != LDC_HS_COMPLETE) in ldc_write()
1951 err = -ENOTCONN; in ldc_write()
1953 err = lp->mops->write(lp, buf, size); in ldc_write()
1955 spin_unlock_irqrestore(&lp->lock, flags); in ldc_write()
1969 return -EINVAL; in ldc_read()
1974 spin_lock_irqsave(&lp->lock, flags); in ldc_read()
1976 if (lp->hs_state != LDC_HS_COMPLETE) in ldc_read()
1977 err = -ENOTCONN; in ldc_read()
1979 err = lp->mops->read(lp, buf, size); in ldc_read()
1981 spin_unlock_irqrestore(&lp->lock, flags); in ldc_read()
1984 lp->cfg.mode, lp->rx_head, lp->rx_tail, err); in ldc_read()
2022 entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, in alloc_npages()
2023 npages, NULL, (unsigned long)-1, 0); in alloc_npages()
2027 return iommu->page_table + entry; in alloc_npages()
2067 len -= (new - base); in pages_in_region()
2091 sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; in fill_cookies()
2095 tlen = PAGE_SIZE - off; in fill_cookies()
2099 this_cookie = make_cookie(sp->pte_idx, in fill_cookies()
2104 if (this_cookie == sp->prev_cookie) { in fill_cookies()
2105 sp->cookies[sp->nc - 1].cookie_size += tlen; in fill_cookies()
2107 sp->cookies[sp->nc].cookie_addr = this_cookie; in fill_cookies()
2108 sp->cookies[sp->nc].cookie_size = tlen; in fill_cookies()
2109 sp->nc++; in fill_cookies()
2111 sp->prev_cookie = this_cookie + tlen; in fill_cookies()
2113 sp->pte_idx++; in fill_cookies()
2115 len -= tlen; in fill_cookies()
2123 long len = sg->length; in sg_count_one()
2125 if ((sg->offset | len) & (8UL - 1)) in sg_count_one()
2126 return -EFAULT; in sg_count_one()
2128 return pages_in_region(base + sg->offset, len); in sg_count_one()
2160 return -EINVAL; in ldc_map_sg()
2168 return -EMSGSIZE; in ldc_map_sg()
2170 iommu = &lp->iommu; in ldc_map_sg()
2175 return -ENOMEM; in ldc_map_sg()
2177 state.page_table = iommu->page_table; in ldc_map_sg()
2181 state.pte_idx = (base - iommu->page_table); in ldc_map_sg()
2186 s->offset, s->length); in ldc_map_sg()
2204 return -EINVAL; in ldc_map_single()
2207 if ((pa | len) & (8UL - 1)) in ldc_map_single()
2208 return -EFAULT; in ldc_map_single()
2212 iommu = &lp->iommu; in ldc_map_single()
2217 return -ENOMEM; in ldc_map_single()
2219 state.page_table = iommu->page_table; in ldc_map_single()
2223 state.pte_idx = (base - iommu->page_table); in ldc_map_single()
2242 iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); in free_npages()
2248 struct ldc_iommu *iommu = &lp->iommu; in ldc_unmap()
2252 spin_lock_irqsave(&iommu->lock, flags); in ldc_unmap()
2257 free_npages(lp->id, iommu, addr, size); in ldc_unmap()
2259 spin_unlock_irqrestore(&iommu->lock, flags); in ldc_unmap()
2273 lp->id, copy_dir); in ldc_copy()
2274 return -EINVAL; in ldc_copy()
2278 if ((ra | len | offset) & (8UL - 1)) { in ldc_copy()
2281 lp->id, ra, len, offset); in ldc_copy()
2282 return -EFAULT; in ldc_copy()
2285 if (lp->hs_state != LDC_HS_COMPLETE || in ldc_copy()
2286 (lp->flags & LDC_FLAG_RESET)) { in ldc_copy()
2288 "flags[%x]\n", lp->id, lp->hs_state, lp->flags); in ldc_copy()
2289 return -ECONNRESET; in ldc_copy()
2304 offset -= this_off; in ldc_copy()
2305 this_len -= this_off; in ldc_copy()
2317 hv_err = sun4v_ldc_copy(lp->id, copy_dir, in ldc_copy()
2323 lp->id, hv_err); in ldc_copy()
2324 if (lp->hs_state != LDC_HS_COMPLETE || in ldc_copy()
2325 (lp->flags & LDC_FLAG_RESET)) in ldc_copy()
2326 return -ECONNRESET; in ldc_copy()
2328 return -EFAULT; in ldc_copy()
2333 len -= actual_len; in ldc_copy()
2337 this_len -= actual_len; in ldc_copy()
2349 return orig_len - len; in ldc_copy()
2360 if (len & (8UL - 1)) in ldc_alloc_exp_dring()
2361 return ERR_PTR(-EINVAL); in ldc_alloc_exp_dring()
2365 return ERR_PTR(-ENOMEM); in ldc_alloc_exp_dring()
2389 struct mdesc_handle *hp; in ldc_init() local
2394 hp = mdesc_grab(); in ldc_init()
2395 if (!hp) in ldc_init()
2396 return -ENODEV; in ldc_init()
2398 mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); in ldc_init()
2399 err = -ENODEV; in ldc_init()
2403 v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); in ldc_init()
2424 mdesc_release(hp); in ldc_init()