Lines Matching +full:smc +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
11 * - support for alternate links postponed
19 #define KMSG_COMPONENT "smc"
38 #include <net/smc.h>
45 #include "smc.h"
81 if (cb_ctx->pos[0])
84 hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
88 return -ENOMEM;
91 sock_net(skb->sk)->smc.limit_smc_hs))
95 cb_ctx->pos[0] = 1;
97 return skb->len;
100 return -EMSGSIZE;
105 sock_net(skb->sk)->smc.limit_smc_hs = true;
111 sock_net(skb->sk)->smc.limit_smc_hs = false;
117 struct smc_sock *smc = smc_sk(sk);
119 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
129 struct smc_sock *smc;
132 smc = smc_clcsock_user_data(sk);
134 if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) >
135 sk->sk_max_ack_backlog)
138 if (sk_acceptq_is_full(&smc->sk)) {
144 child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash,
146 /* child must not inherit smc or its ops */
150 /* v4-mapped sockets don't inherit parent ops. Don't restore. */
151 if (inet_csk(child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops)
152 inet_csk(child)->icsk_af_ops = smc->ori_af_ops;
164 const struct smc_sock *smc;
166 smc = smc_clcsock_user_data(sk);
168 if (!smc)
187 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
190 head = &h->ht;
192 write_lock_bh(&h->lock);
194 write_unlock_bh(&h->lock);
195 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
202 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
204 write_lock_bh(&h->lock);
206 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
207 write_unlock_bh(&h->lock);
216 struct smc_sock *smc = smc_sk(sk);
218 if (smc->conn.tx_in_release_sock) {
219 smc_tx_pending(&smc->conn);
220 smc->conn.tx_in_release_sock = false;
225 .name = "SMC",
250 static void smc_fback_restore_callbacks(struct smc_sock *smc)
252 struct sock *clcsk = smc->clcsock->sk;
254 write_lock_bh(&clcsk->sk_callback_lock);
255 clcsk->sk_user_data = NULL;
257 smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change);
258 smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready);
259 smc_clcsock_restore_cb(&clcsk->sk_write_space, &smc->clcsk_write_space);
260 smc_clcsock_restore_cb(&clcsk->sk_error_report, &smc->clcsk_error_report);
262 write_unlock_bh(&clcsk->sk_callback_lock);
265 static void smc_restore_fallback_changes(struct smc_sock *smc)
267 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
268 smc->clcsock->file->private_data = smc->sk.sk_socket;
269 smc->clcsock->file = NULL;
270 smc_fback_restore_callbacks(smc);
274 static int __smc_release(struct smc_sock *smc)
276 struct sock *sk = &smc->sk;
279 if (!smc->use_fallback) {
280 rc = smc_close_active(smc);
282 sk->sk_shutdown |= SHUTDOWN_MASK;
284 if (sk->sk_state != SMC_CLOSED) {
285 if (sk->sk_state != SMC_LISTEN &&
286 sk->sk_state != SMC_INIT)
288 if (sk->sk_state == SMC_LISTEN) {
290 rc = kernel_sock_shutdown(smc->clcsock,
293 sk->sk_state = SMC_CLOSED;
294 sk->sk_state_change(sk);
296 smc_restore_fallback_changes(smc);
299 sk->sk_prot->unhash(sk);
301 if (sk->sk_state == SMC_CLOSED) {
302 if (smc->clcsock) {
304 smc_clcsock_release(smc);
307 if (!smc->use_fallback)
308 smc_conn_free(&smc->conn);
316 struct sock *sk = sock->sk;
317 struct smc_sock *smc;
324 smc = smc_sk(sk);
326 old_state = sk->sk_state;
328 /* cleanup for a dangling non-blocking connect */
329 if (smc->connect_nonblock && old_state == SMC_INIT)
330 tcp_abort(smc->clcsock->sk, ECONNABORTED);
332 if (cancel_work_sync(&smc->connect_work))
333 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
335 if (sk->sk_state == SMC_LISTEN)
343 if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
344 !smc->use_fallback)
345 smc_close_active_abort(smc);
347 rc = __smc_release(smc);
351 sock->sk = NULL;
362 if (sk->sk_state != SMC_CLOSED)
366 switch (sk->sk_family) {
383 struct smc_sock *smc = smc_sk(sk);
385 sk->sk_state = SMC_INIT;
386 sk->sk_destruct = smc_destruct;
387 sk->sk_protocol = protocol;
388 WRITE_ONCE(sk->sk_sndbuf, 2 * READ_ONCE(net->smc.sysctl_wmem));
389 WRITE_ONCE(sk->sk_rcvbuf, 2 * READ_ONCE(net->smc.sysctl_rmem));
390 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
391 INIT_WORK(&smc->connect_work, smc_connect_work);
392 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
393 INIT_LIST_HEAD(&smc->accept_q);
394 sock_lock_init_class_and_name(sk, "slock-AF_SMC", &smc_slock_key,
395 "sk_lock-AF_SMC", &smc_key);
396 spin_lock_init(&smc->accept_q_lock);
397 spin_lock_init(&smc->conn.send_lock);
398 sk->sk_prot->hash(sk);
399 mutex_init(&smc->clcsock_release_lock);
400 smc_init_saved_callbacks(smc);
401 smc->limit_smc_hs = net->smc.limit_smc_hs;
402 smc->use_fallback = false; /* assume rdma capability first */
403 smc->fallback_rsn = 0;
404 smc_close_init(smc);
428 struct sock *sk = sock->sk;
429 struct smc_sock *smc;
432 smc = smc_sk(sk);
435 rc = -EINVAL;
439 rc = -EAFNOSUPPORT;
440 if (addr->sin_family != AF_INET &&
441 addr->sin_family != AF_INET6 &&
442 addr->sin_family != AF_UNSPEC)
445 if (addr->sin_family == AF_UNSPEC &&
446 addr->sin_addr.s_addr != htonl(INADDR_ANY))
452 rc = -EINVAL;
453 if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
456 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
457 smc->clcsock->sk->sk_reuseport = sk->sk_reuseport;
458 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
466 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
467 * clc socket (since smc is not called for these options from net/core)
486 /* if set, use value set by setsockopt() - else use IPv4 or SMC sysctl value */
490 nsk->sk_userlocks = osk->sk_userlocks;
491 if (osk->sk_userlocks & SOCK_SNDBUF_LOCK)
492 nsk->sk_sndbuf = osk->sk_sndbuf;
493 if (osk->sk_userlocks & SOCK_RCVBUF_LOCK)
494 nsk->sk_rcvbuf = osk->sk_rcvbuf;
501 nsk->sk_type = osk->sk_type;
502 nsk->sk_sndtimeo = READ_ONCE(osk->sk_sndtimeo);
503 nsk->sk_rcvtimeo = READ_ONCE(osk->sk_rcvtimeo);
504 nsk->sk_mark = READ_ONCE(osk->sk_mark);
505 nsk->sk_priority = READ_ONCE(osk->sk_priority);
506 nsk->sk_rcvlowat = osk->sk_rcvlowat;
507 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
508 nsk->sk_err = osk->sk_err;
510 nsk->sk_flags &= ~mask;
511 nsk->sk_flags |= osk->sk_flags & mask;
516 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
518 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
525 /* copy only settings and flags relevant for smc from clc to smc socket */
526 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
528 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
535 struct smc_link_group *lgr = link->lgr;
538 if (!snd_desc->is_vm)
539 return -EINVAL;
542 down_write(&lgr->llc_conf_mutex);
544 if (!smc_link_active(&lgr->lnk[i]))
546 rc = smcr_link_reg_buf(&lgr->lnk[i], snd_desc);
550 up_write(&lgr->llc_conf_mutex);
558 struct smc_link_group *lgr = link->lgr;
566 down_read(&lgr->llc_conf_mutex);
568 if (!smc_link_active(&lgr->lnk[i]))
570 if (!rmb_desc->is_reg_mr[link->link_idx]) {
571 up_read(&lgr->llc_conf_mutex);
582 down_write(&lgr->llc_conf_mutex);
584 if (!smc_link_active(&lgr->lnk[i]))
586 rc = smcr_link_reg_buf(&lgr->lnk[i], rmb_desc);
594 rc = -EFAULT;
597 rmb_desc->is_conf_rkey = true;
599 do_slow ? up_write(&lgr->llc_conf_mutex) : up_read(&lgr->llc_conf_mutex);
600 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
604 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
606 struct smc_link *link = smc->conn.lnk;
615 qentry = smc_llc_wait(link->lgr, NULL, 2 * SMC_LLC_WAIT_TIME,
620 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
622 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
626 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
637 if (smc->conn.sndbuf_desc->is_vm) {
638 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc))
643 if (smcr_link_reg_buf(link, smc->conn.rmb_desc))
647 smc->conn.rmb_desc->is_conf_rkey = true;
655 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
657 if (link->lgr->max_links > 1) {
659 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
664 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
666 if (rc == -EAGAIN)
670 smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
686 static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
692 if (clc->hdr.version == SMC_V1 ||
693 !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
696 if (smc->conn.lgr->is_smcd) {
697 memcpy(smc->conn.lgr->negotiated_eid, clc->d1.eid,
701 memcpy(smc->conn.lgr->negotiated_eid, clc->r1.eid,
706 smc->conn.lgr->peer_os = fce->os_type;
707 smc->conn.lgr->peer_smc_release = fce->release;
708 if (smc_isascii(fce->hostname))
709 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
713 static void smcr_conn_save_peer_info(struct smc_sock *smc,
716 int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
718 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
719 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
720 smc->conn.peer_rmbe_size = bufsize;
721 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
722 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
725 static void smcd_conn_save_peer_info(struct smc_sock *smc,
728 int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
730 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
731 smc->conn.peer_token = ntohll(clc->d0.token);
733 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
734 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
735 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
738 static void smc_conn_save_peer_info(struct smc_sock *smc,
741 if (smc->conn.lgr->is_smcd)
742 smcd_conn_save_peer_info(smc, clc);
744 smcr_conn_save_peer_info(smc, clc);
745 smc_conn_save_peer_info_fce(smc, clc);
752 link->peer_qpn = ntoh24(clc->r0.qpn);
753 memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
754 memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
755 link->peer_psn = ntoh24(clc->r0.psn);
756 link->peer_mtu = clc->r0.qp_mtu;
759 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
765 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
770 fback_arr[cnt].fback_code = smc->fallback_rsn;
777 static void smc_stat_fallback(struct smc_sock *smc)
779 struct net *net = sock_net(&smc->sk);
781 mutex_lock(&net->smc.mutex_fback_rsn);
782 if (smc->listen_smc) {
783 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
784 net->smc.fback_rsn->srv_fback_cnt++;
786 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
787 net->smc.fback_rsn->clnt_fback_cnt++;
789 mutex_unlock(&net->smc.mutex_fback_rsn);
793 static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
798 wq = rcu_dereference(smc->sk.sk_wq);
802 /* wake up smc sk->sk_wq */
805 wake_up_interruptible_all(&wq->wait);
810 wake_up_interruptible_sync_poll(&wq->wait, flags);
813 wake_up_interruptible_poll(&wq->wait, flags);
823 mark->woken = true;
824 mark->key = key;
828 static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
837 wq = rcu_dereference(clcsk->sk_wq);
845 smc_fback_wakeup_waitqueue(smc, mark.key);
852 struct smc_sock *smc;
854 read_lock_bh(&clcsk->sk_callback_lock);
855 smc = smc_clcsock_user_data(clcsk);
856 if (smc)
857 smc_fback_forward_wakeup(smc, clcsk,
858 smc->clcsk_state_change);
859 read_unlock_bh(&clcsk->sk_callback_lock);
864 struct smc_sock *smc;
866 read_lock_bh(&clcsk->sk_callback_lock);
867 smc = smc_clcsock_user_data(clcsk);
868 if (smc)
869 smc_fback_forward_wakeup(smc, clcsk,
870 smc->clcsk_data_ready);
871 read_unlock_bh(&clcsk->sk_callback_lock);
876 struct smc_sock *smc;
878 read_lock_bh(&clcsk->sk_callback_lock);
879 smc = smc_clcsock_user_data(clcsk);
880 if (smc)
881 smc_fback_forward_wakeup(smc, clcsk,
882 smc->clcsk_write_space);
883 read_unlock_bh(&clcsk->sk_callback_lock);
888 struct smc_sock *smc;
890 read_lock_bh(&clcsk->sk_callback_lock);
891 smc = smc_clcsock_user_data(clcsk);
892 if (smc)
893 smc_fback_forward_wakeup(smc, clcsk,
894 smc->clcsk_error_report);
895 read_unlock_bh(&clcsk->sk_callback_lock);
898 static void smc_fback_replace_callbacks(struct smc_sock *smc)
900 struct sock *clcsk = smc->clcsock->sk;
902 write_lock_bh(&clcsk->sk_callback_lock);
903 clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
905 smc_clcsock_replace_cb(&clcsk->sk_state_change, smc_fback_state_change,
906 &smc->clcsk_state_change);
907 smc_clcsock_replace_cb(&clcsk->sk_data_ready, smc_fback_data_ready,
908 &smc->clcsk_data_ready);
909 smc_clcsock_replace_cb(&clcsk->sk_write_space, smc_fback_write_space,
910 &smc->clcsk_write_space);
911 smc_clcsock_replace_cb(&clcsk->sk_error_report, smc_fback_error_report,
912 &smc->clcsk_error_report);
914 write_unlock_bh(&clcsk->sk_callback_lock);
917 static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
921 mutex_lock(&smc->clcsock_release_lock);
922 if (!smc->clcsock) {
923 rc = -EBADF;
927 smc->use_fallback = true;
928 smc->fallback_rsn = reason_code;
929 smc_stat_fallback(smc);
930 trace_smc_switch_to_fallback(smc, reason_code);
931 if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
932 smc->clcsock->file = smc->sk.sk_socket->file;
933 smc->clcsock->file->private_data = smc->clcsock;
934 smc->clcsock->wq.fasync_list =
935 smc->sk.sk_socket->wq.fasync_list;
936 smc->sk.sk_socket->wq.fasync_list = NULL;
939 * in smc sk->sk_wq and they should be woken up
942 smc_fback_replace_callbacks(smc);
945 mutex_unlock(&smc->clcsock_release_lock);
950 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
952 struct net *net = sock_net(&smc->sk);
955 rc = smc_switch_to_fallback(smc, reason_code);
957 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
958 if (smc->sk.sk_state == SMC_INIT)
959 sock_put(&smc->sk); /* passive closing */
962 smc_copy_sock_settings_to_clc(smc);
963 smc->connect_nonblock = 0;
964 if (smc->sk.sk_state == SMC_INIT)
965 smc->sk.sk_state = SMC_ACTIVE;
970 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
973 struct net *net = sock_net(&smc->sk);
977 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
978 if (smc->sk.sk_state == SMC_INIT)
979 sock_put(&smc->sk); /* passive closing */
983 rc = smc_clc_send_decline(smc, reason_code, version);
985 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
986 if (smc->sk.sk_state == SMC_INIT)
987 sock_put(&smc->sk); /* passive closing */
991 return smc_connect_fallback(smc, reason_code);
994 static void smc_conn_abort(struct smc_sock *smc, int local_first)
996 struct smc_connection *conn = &smc->conn;
997 struct smc_link_group *lgr = conn->lgr;
1010 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
1016 smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
1017 if (!ini->check_smcrv2 && !ini->ib_dev)
1019 if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
1026 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
1029 smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
1030 if (!ini->ism_dev[0])
1033 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
1041 int i = (!ini->ism_dev[0]) ? 1 : 0;
1044 if (ini->ism_chid[i] == chid)
1052 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
1061 if (smcd_indicated(ini->smc_type_v1))
1065 if (smcd->going_away || smcd == ini->ism_dev[0])
1071 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
1072 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
1074 /* It's the last GID-CHID entry left in CLC
1075 * Proposal SMC-Dv2 extension, but an Emulated-
1080 ini->ism_dev[i] = smcd;
1081 ini->ism_chid[i] = chid;
1082 ini->is_smcd = true;
1091 ini->ism_offered_cnt = i - 1;
1092 if (!ini->ism_dev[0] && !ini->ism_dev[1])
1093 ini->smcd_version = 0;
1098 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
1101 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
1106 static int smc_find_proposal_devices(struct smc_sock *smc,
1112 if (!(ini->smcd_version & SMC_V1) ||
1113 smc_find_ism_device(smc, ini) ||
1115 ini->smcd_version &= ~SMC_V1;
1119 if (!(ini->smcr_version & SMC_V1) ||
1120 smc_find_rdma_device(smc, ini))
1121 ini->smcr_version &= ~SMC_V1;
1124 ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
1125 ini->smcr_version & SMC_V1);
1128 if (!(ini->smcd_version & SMC_V2) ||
1130 smc_find_ism_v2_device_clnt(smc, ini))
1131 ini->smcd_version &= ~SMC_V2;
1134 ini->check_smcrv2 = true;
1135 ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
1136 if (!(ini->smcr_version & SMC_V2) ||
1138 (smc->clcsock->sk->sk_family == AF_INET6 &&
1139 !ipv6_addr_v4mapped(&smc->clcsock->sk->sk_v6_rcv_saddr)) ||
1142 smc_find_rdma_device(smc, ini))
1143 ini->smcr_version &= ~SMC_V2;
1144 ini->check_smcrv2 = false;
1146 ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
1147 ini->smcr_version & SMC_V2);
1150 if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
1156 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
1157 * used, the VLAN ID will be registered again during the connection setup.
1161 if (!smcd_indicated(ini->smc_type_v1))
1163 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
1174 static int smc_connect_clc(struct smc_sock *smc,
1181 rc = smc_clc_send_proposal(smc, ini);
1184 /* receive SMC Accept CLC message */
1185 return smc_clc_wait_msg(smc, aclc, SMC_CLC_MAX_ACCEPT_LEN,
1196 memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
1202 alt_ini->vlan_id = lgr->vlan_id;
1203 alt_ini->check_smcrv2 = true;
1204 alt_ini->smcrv2.saddr = lgr->saddr;
1207 if (!alt_ini->smcrv2.ib_dev_v2)
1210 memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
1217 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
1223 struct net *net = sock_net(&smc->sk);
1226 if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
1229 if (fce->v2_direct) {
1230 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
1231 ini->smcrv2.uses_gateway = false;
1233 if (smc_ib_find_route(net, smc->clcsock->sk->sk_rcv_saddr,
1234 smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
1235 ini->smcrv2.nexthop_mac,
1236 &ini->smcrv2.uses_gateway))
1238 if (!ini->smcrv2.uses_gateway) {
1244 ini->release_nr = fce->release;
1253 static int smc_connect_rdma(struct smc_sock *smc,
1261 ini->is_smcd = false;
1262 ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
1263 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1264 memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1265 memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1266 memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1267 ini->max_conns = SMC_CONN_PER_LGR_MAX;
1268 ini->max_links = SMC_LINKS_ADD_LNK_MAX;
1270 reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1275 reason_code = smc_conn_create(smc, ini);
1281 smc_conn_save_peer_info(smc, aclc);
1283 if (ini->first_contact_local) {
1284 link = smc->conn.lnk;
1289 struct smc_link *l = &smc->conn.lgr->lnk[i];
1291 if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
1292 !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
1294 (aclc->hdr.version > SMC_V1 ||
1295 !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
1296 sizeof(l->peer_mac)))) {
1305 smc_switch_link_and_count(&smc->conn, link);
1309 if (smc_buf_create(smc, false)) {
1314 if (ini->first_contact_local)
1317 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
1322 smc_rx_init(smc);
1324 if (ini->first_contact_local) {
1331 if (smc->conn.sndbuf_desc->is_vm) {
1332 if (smcr_lgr_reg_sndbufs(link, smc->conn.sndbuf_desc)) {
1337 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
1343 if (aclc->hdr.version > SMC_V1) {
1344 eid = aclc->r1.eid;
1345 if (ini->first_contact_local)
1346 smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
1347 link->smcibdev, link->gid);
1350 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
1351 aclc->hdr.version, eid, ini);
1355 smc_tx_init(smc);
1357 if (ini->first_contact_local) {
1359 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1360 reason_code = smcr_clnt_conf_first_link(smc);
1361 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1367 smc_copy_sock_settings_to_clc(smc);
1368 smc->connect_nonblock = 0;
1369 if (smc->sk.sk_state == SMC_INIT)
1370 smc->sk.sk_state = SMC_ACTIVE;
1374 smc_conn_abort(smc, ini->first_contact_local);
1376 smc->connect_nonblock = 0;
1390 for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1391 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1392 ini->ism_selected = i;
1397 return -EPROTO;
1401 static int smc_connect_ism(struct smc_sock *smc,
1408 ini->is_smcd = true;
1409 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1411 if (aclc->hdr.version == SMC_V2) {
1412 if (ini->first_contact_peer) {
1416 ini->release_nr = fce->release;
1426 if (__smc_ism_is_emulated(ini->ism_chid[ini->ism_selected]))
1427 ini->ism_peer_gid[ini->ism_selected].gid_ext =
1428 ntohll(aclc->d1.gid_ext);
1429 /* for non-Emulated-ISM devices, peer gid_ext remains 0. */
1431 ini->ism_peer_gid[ini->ism_selected].gid = ntohll(aclc->d0.gid);
1433 /* there is only one lgr role for SMC-D; use server lock */
1435 rc = smc_conn_create(smc, ini);
1442 rc = smc_buf_create(smc, true);
1444 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1448 smc_conn_save_peer_info(smc, aclc);
1450 if (smc_ism_support_dmb_nocopy(smc->conn.lgr->smcd)) {
1451 rc = smcd_buf_attach(smc);
1457 smc_rx_init(smc);
1458 smc_tx_init(smc);
1460 if (aclc->hdr.version > SMC_V1)
1461 eid = aclc->d1.eid;
1463 rc = smc_clc_send_confirm(smc, ini->first_contact_local,
1464 aclc->hdr.version, eid, ini);
1469 smc_copy_sock_settings_to_clc(smc);
1470 smc->connect_nonblock = 0;
1471 if (smc->sk.sk_state == SMC_INIT)
1472 smc->sk.sk_state = SMC_ACTIVE;
1476 smc_conn_abort(smc, ini->first_contact_local);
1478 smc->connect_nonblock = 0;
1487 if (aclc->hdr.version >= SMC_V2) {
1488 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1489 !smcr_indicated(ini->smc_type_v2)) ||
1490 (aclc->hdr.typev1 == SMC_TYPE_D &&
1491 !smcd_indicated(ini->smc_type_v2)))
1494 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1495 !smcr_indicated(ini->smc_type_v1)) ||
1496 (aclc->hdr.typev1 == SMC_TYPE_D &&
1497 !smcd_indicated(ini->smc_type_v1)))
1505 static int __smc_connect(struct smc_sock *smc)
1513 if (smc->use_fallback)
1514 return smc_connect_fallback(smc, smc->fallback_rsn);
1516 /* if peer has not signalled SMC-capability, fall back */
1517 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1518 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1520 /* IPSec connections opt out of SMC optimizations */
1521 if (using_ipsec(smc))
1522 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1527 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1530 ini->smcd_version = SMC_V1 | SMC_V2;
1531 ini->smcr_version = SMC_V1 | SMC_V2;
1532 ini->smc_type_v1 = SMC_TYPE_B;
1533 ini->smc_type_v2 = SMC_TYPE_B;
1535 /* get vlan id from IP device */
1536 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
1537 ini->smcd_version &= ~SMC_V1;
1538 ini->smcr_version = 0;
1539 ini->smc_type_v1 = SMC_TYPE_N;
1542 rc = smc_find_proposal_devices(smc, ini);
1554 rc = smc_connect_clc(smc, aclc, ini);
1556 /* -EAGAIN on timeout, see tcp_recvmsg() */
1557 if (rc == -EAGAIN) {
1558 rc = -ETIMEDOUT;
1559 smc->sk.sk_err = ETIMEDOUT;
1564 /* check if smc modes and versions of CLC proposal and accept match */
1566 version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1571 if (aclc->hdr.typev1 == SMC_TYPE_R) {
1572 ini->smcr_version = version;
1573 rc = smc_connect_rdma(smc, aclc, ini);
1574 } else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1575 ini->smcd_version = version;
1576 rc = smc_connect_ism(smc, aclc, ini);
1581 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1592 return smc_connect_decline_fallback(smc, rc, version);
1597 struct smc_sock *smc = container_of(work, struct smc_sock,
1599 long timeo = READ_ONCE(smc->sk.sk_sndtimeo);
1604 lock_sock(smc->clcsock->sk);
1605 if (smc->clcsock->sk->sk_err) {
1606 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1607 } else if ((1 << smc->clcsock->sk->sk_state) &
1609 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1610 if ((rc == -EPIPE) &&
1611 ((1 << smc->clcsock->sk->sk_state) &
1615 release_sock(smc->clcsock->sk);
1616 lock_sock(&smc->sk);
1617 if (rc != 0 || smc->sk.sk_err) {
1618 smc->sk.sk_state = SMC_CLOSED;
1619 if (rc == -EPIPE || rc == -EAGAIN)
1620 smc->sk.sk_err = EPIPE;
1621 else if (rc == -ECONNREFUSED)
1622 smc->sk.sk_err = ECONNREFUSED;
1624 smc->sk.sk_err = -sock_intr_errno(timeo);
1625 sock_put(&smc->sk); /* passive closing */
1629 rc = __smc_connect(smc);
1631 smc->sk.sk_err = -rc;
1634 if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1635 if (smc->sk.sk_err) {
1636 smc->sk.sk_state_change(&smc->sk);
1638 smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1639 smc->sk.sk_write_space(&smc->sk);
1642 release_sock(&smc->sk);
1648 struct sock *sk = sock->sk;
1649 struct smc_sock *smc;
1650 int rc = -EINVAL;
1652 smc = smc_sk(sk);
1654 /* separate smc parameter checking to be safe */
1655 if (alen < sizeof(addr->sa_family))
1657 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1661 switch (sock->state) {
1663 rc = -EINVAL;
1666 rc = sk->sk_state == SMC_ACTIVE ? -EISCONN : -EINVAL;
1669 if (sk->sk_state == SMC_ACTIVE)
1673 sock->state = SS_CONNECTING;
1677 switch (sk->sk_state) {
1681 rc = sock_error(sk) ? : -ECONNABORTED;
1682 sock->state = SS_UNCONNECTED;
1685 rc = -EISCONN;
1691 smc_copy_sock_settings_to_clc(smc);
1692 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1693 if (smc->connect_nonblock) {
1694 rc = -EALREADY;
1697 rc = kernel_connect(smc->clcsock, addr, alen, flags);
1698 if (rc && rc != -EINPROGRESS)
1701 if (smc->use_fallback) {
1702 sock->state = rc ? SS_CONNECTING : SS_CONNECTED;
1705 sock_hold(&smc->sk); /* sock put in passive closing */
1707 if (queue_work(smc_hs_wq, &smc->connect_work))
1708 smc->connect_nonblock = 1;
1709 rc = -EINPROGRESS;
1712 rc = __smc_connect(smc);
1719 sock->state = SS_CONNECTED;
1729 struct sock *lsk = &lsmc->sk;
1731 int rc = -EINVAL;
1734 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1736 rc = -ENOMEM;
1737 lsk->sk_err = ENOMEM;
1744 mutex_lock(&lsmc->clcsock_release_lock);
1745 if (lsmc->clcsock)
1746 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1747 mutex_unlock(&lsmc->clcsock_release_lock);
1749 if (rc < 0 && rc != -EAGAIN)
1750 lsk->sk_err = -rc;
1751 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1752 new_sk->sk_prot->unhash(new_sk);
1755 new_sk->sk_state = SMC_CLOSED;
1762 /* new clcsock has inherited the smc listen-specific sk_data_ready
1765 new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1767 /* if new clcsock has also inherited the fallback-specific callback
1770 if (lsmc->use_fallback) {
1771 if (lsmc->clcsk_state_change)
1772 new_clcsock->sk->sk_state_change = lsmc->clcsk_state_change;
1773 if (lsmc->clcsk_write_space)
1774 new_clcsock->sk->sk_write_space = lsmc->clcsk_write_space;
1775 if (lsmc->clcsk_error_report)
1776 new_clcsock->sk->sk_error_report = lsmc->clcsk_error_report;
1779 (*new_smc)->clcsock = new_clcsock;
1792 spin_lock(&par->accept_q_lock);
1793 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1794 spin_unlock(&par->accept_q_lock);
1801 struct smc_sock *par = smc_sk(sk)->listen_smc;
1803 spin_lock(&par->accept_q_lock);
1804 list_del_init(&smc_sk(sk)->accept_q);
1805 spin_unlock(&par->accept_q_lock);
1806 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1819 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1823 if (new_sk->sk_state == SMC_CLOSED) {
1824 new_sk->sk_prot->unhash(new_sk);
1825 if (isk->clcsock) {
1826 sock_release(isk->clcsock);
1827 isk->clcsock = NULL;
1834 new_sock->state = SS_CONNECTED;
1835 if (isk->use_fallback) {
1836 smc_sk(new_sk)->clcsock->file = new_sock->file;
1837 isk->clcsock->file->private_data = isk->clcsock;
1848 struct smc_sock *smc = smc_sk(sk);
1852 if (!sk->sk_lingertime)
1854 WRITE_ONCE(sk->sk_lingertime, SMC_MAX_STREAM_WAIT_TIMEOUT);
1855 __smc_release(smc);
1861 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1863 struct smc_link *link = smc->conn.lnk;
1868 if (smc->conn.sndbuf_desc->is_vm) {
1869 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc))
1874 if (smcr_link_reg_buf(link, smc->conn.rmb_desc))
1883 qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1888 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1890 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1894 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1899 smc->conn.rmb_desc->is_conf_rkey = true;
1902 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1904 if (link->lgr->max_links > 1) {
1905 down_write(&link->lgr->llc_conf_mutex);
1906 /* initial contact - try to establish second link */
1908 up_write(&link->lgr->llc_conf_mutex);
1916 struct smc_sock *lsmc = new_smc->listen_smc;
1917 struct sock *newsmcsk = &new_smc->sk;
1919 if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
1920 atomic_dec(&lsmc->queued_smc_hs);
1923 if (lsmc->sk.sk_state == SMC_LISTEN) {
1924 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1925 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1926 release_sock(&lsmc->sk);
1932 lsmc->sk.sk_data_ready(&lsmc->sk);
1933 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1939 struct sock *newsmcsk = &new_smc->sk;
1941 if (newsmcsk->sk_state == SMC_INIT)
1942 newsmcsk->sk_state = SMC_ACTIVE;
1950 struct sock *newsmcsk = &new_smc->sk;
1953 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1954 if (newsmcsk->sk_state == SMC_INIT)
1955 sock_put(&new_smc->sk); /* passive closing */
1956 newsmcsk->sk_state = SMC_CLOSED;
1991 ini->smc_type_v1 = pclc->hdr.typev1;
1992 ini->smc_type_v2 = pclc->hdr.typev2;
1993 ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1994 ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1995 if (pclc->hdr.version > SMC_V1) {
1996 if (smcd_indicated(ini->smc_type_v2))
1997 ini->smcd_version |= SMC_V2;
1998 if (smcr_indicated(ini->smc_type_v2))
1999 ini->smcr_version |= SMC_V2;
2001 if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
2007 ini->smcd_version &= ~SMC_V2;
2008 ini->smcr_version &= ~SMC_V2;
2013 if (ini->smcd_version & SMC_V2) {
2015 ini->smcd_version &= ~SMC_V2;
2018 ini->smcd_version &= ~SMC_V2;
2020 } else if (!pclc_v2_ext->hdr.eid_cnt &&
2021 !pclc_v2_ext->hdr.flag.seid) {
2022 ini->smcd_version &= ~SMC_V2;
2026 if (ini->smcr_version & SMC_V2) {
2027 if (!pclc_v2_ext->hdr.eid_cnt) {
2028 ini->smcr_version &= ~SMC_V2;
2033 ini->release_nr = pclc_v2_ext->hdr.flag.release;
2034 if (pclc_v2_ext->hdr.flag.release > SMC_RELEASE)
2035 ini->release_nr = SMC_RELEASE;
2038 if (!ini->smcd_version && !ini->smcr_version)
2049 struct socket *newclcsock = new_smc->clcsock;
2051 if (pclc->hdr.typev1 == SMC_TYPE_N)
2055 return -EPROTO;
2075 smc_conn_abort(new_smc, ini->first_contact_local);
2082 /* listen worker: initialize connection and buffers for SMC-D */
2095 smc_conn_abort(new_smc, ini->first_contact_local);
2096 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
2110 if (smcd == ini->ism_dev[i])
2125 if (smcd->going_away)
2131 ini->ism_peer_gid[*matches].gid = proposed_gid->gid;
2133 ini->ism_peer_gid[*matches].gid_ext =
2134 proposed_gid->gid_ext;
2135 /* non-Emulated-ISM's peer gid_ext remains 0. */
2136 ini->ism_dev[*matches] = smcd;
2145 if (!ini->rc)
2146 ini->rc = rc;
2163 if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
2173 if (pclc_smcd->ism.chid) {
2175 smcd_gid.gid = ntohll(pclc_smcd->ism.gid);
2177 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
2180 for (i = 0; i < smc_v2_ext->hdr.ism_gid_cnt; i++) {
2181 /* check for ISM devices matching proposed non-native ISM
2184 smcd_gid.gid = ntohll(smcd_v2_ext->gidchid[i].gid);
2186 chid = ntohs(smcd_v2_ext->gidchid[i].chid);
2188 if ((i + 1) == smc_v2_ext->hdr.ism_gid_cnt ||
2189 chid != ntohs(smcd_v2_ext->gidchid[i + 1].chid))
2190 /* each Emulated-ISM device takes two GID-CHID
2194 * So check if the next GID-CHID entry exists
2199 ntohll(smcd_v2_ext->gidchid[++i].gid);
2205 if (!ini->ism_dev[0]) {
2211 if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
2212 smcd_v2_ext->system_eid, eid))
2215 /* separate - outside the smcd_dev_list.lock */
2216 smcd_version = ini->smcd_version;
2218 ini->smcd_version = SMC_V2;
2219 ini->is_smcd = true;
2220 ini->ism_selected = i;
2230 ini->smcd_version = smcd_version; /* restore original value */
2231 ini->negotiated_eid[0] = 0;
2234 ini->smcd_version &= ~SMC_V2;
2235 ini->ism_dev[0] = NULL;
2236 ini->is_smcd = false;
2247 if (!(ini->smcd_version & SMC_V1) ||
2248 !smcd_indicated(ini->smc_type_v1) ||
2251 ini->is_smcd = true; /* prepare ISM check */
2252 ini->ism_peer_gid[0].gid = ntohll(pclc_smcd->ism.gid);
2253 ini->ism_peer_gid[0].gid_ext = 0;
2257 ini->ism_selected = 0;
2264 ini->smcd_version &= ~SMC_V1;
2265 ini->ism_dev[0] = NULL;
2266 ini->is_smcd = false;
2272 struct smc_connection *conn = &new_smc->conn;
2276 if (conn->sndbuf_desc->is_vm) {
2277 if (smcr_lgr_reg_sndbufs(conn->lnk,
2278 conn->sndbuf_desc))
2281 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
2296 if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
2301 !smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
2305 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2306 memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
2307 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2308 ini->check_smcrv2 = true;
2309 ini->smcrv2.clc_sk = new_smc->clcsock->sk;
2310 ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
2311 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
2317 if (!ini->smcrv2.uses_gateway)
2318 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
2320 smcr_version = ini->smcr_version;
2321 ini->smcr_version = SMC_V2;
2324 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2326 smc_conn_abort(new_smc, ini->first_contact_local);
2330 ini->smcr_version = smcr_version;
2334 ini->smcr_version &= ~SMC_V2;
2335 ini->smcrv2.ib_dev_v2 = NULL;
2336 ini->check_smcrv2 = false;
2345 if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
2349 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2350 memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
2351 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2360 return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2372 if (ini->ism_dev[0])
2380 /* get vlan id from IP device */
2381 if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
2382 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
2387 if (ini->ism_dev[0])
2390 if (!smcr_indicated(pclc->hdr.typev1) &&
2391 !smcr_indicated(pclc->hdr.typev2))
2393 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
2397 if (ini->smcrv2.ib_dev_v2)
2406 return (!rc) ? 0 : ini->rc;
2417 struct smc_link *link = new_smc->conn.lnk;
2423 if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
2430 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
2432 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
2442 struct socket *newclcsock = new_smc->clcsock;
2451 lock_sock(&new_smc->sk); /* release in smc_listen_out() */
2452 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2455 if (new_smc->use_fallback) {
2460 /* check if peer is smc capable */
2461 if (!tcp_sk(newclcsock->sk)->syn_smc) {
2470 /* do inband token exchange -
2471 * wait for and receive SMC Proposal CLC message
2484 if (pclc->hdr.version > SMC_V1)
2487 /* IPSec connections opt out of SMC optimizations */
2517 /* send SMC Accept CLC message */
2518 accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2519 rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
2520 accept_version, ini->negotiated_eid, ini);
2524 /* SMC-D does not need this lock any more */
2525 if (ini->is_smcd)
2528 /* receive SMC Confirm CLC message */
2534 if (!ini->is_smcd)
2541 if (!ini->is_smcd)
2546 /* fce smc release version is needed in smc_listen_rdma_finish,
2552 if (!ini->is_smcd) {
2554 ini->first_contact_local, ini);
2561 if (ini->is_smcd &&
2562 smc_ism_support_dmb_nocopy(new_smc->conn.lgr->smcd)) {
2568 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2576 smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
2587 struct sock *lsk = &lsmc->sk;
2592 while (lsk->sk_state == SMC_LISTEN) {
2599 if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
2600 atomic_inc(&lsmc->queued_smc_hs);
2602 new_smc->listen_smc = lsmc;
2603 new_smc->use_fallback = lsmc->use_fallback;
2604 new_smc->fallback_rsn = lsmc->fallback_rsn;
2606 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2608 sock_hold(&new_smc->sk); /* sock_put in passive closing */
2609 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
2610 sock_put(&new_smc->sk);
2615 sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2622 read_lock_bh(&listen_clcsock->sk_callback_lock);
2626 lsmc->clcsk_data_ready(listen_clcsock);
2627 if (lsmc->sk.sk_state == SMC_LISTEN) {
2628 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2629 if (!queue_work(smc_tcp_ls_wq, &lsmc->tcp_listen_work))
2630 sock_put(&lsmc->sk);
2633 read_unlock_bh(&listen_clcsock->sk_callback_lock);
2638 struct sock *sk = sock->sk;
2639 struct smc_sock *smc;
2642 smc = smc_sk(sk);
2645 rc = -EINVAL;
2646 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2647 smc->connect_nonblock || sock->state != SS_UNCONNECTED)
2651 if (sk->sk_state == SMC_LISTEN) {
2652 sk->sk_max_ack_backlog = backlog;
2656 * them to the clc socket -- copy smc socket options to clc socket
2658 smc_copy_sock_settings_to_clc(smc);
2659 if (!smc->use_fallback)
2660 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2663 * smc-specific sk_data_ready function
2665 write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2666 smc->clcsock->sk->sk_user_data =
2667 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2668 smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready,
2669 smc_clcsock_data_ready, &smc->clcsk_data_ready);
2670 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2673 smc->ori_af_ops = inet_csk(smc->clcsock->sk)->icsk_af_ops;
2675 smc->af_ops = *smc->ori_af_ops;
2676 smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock;
2678 inet_csk(smc->clcsock->sk)->icsk_af_ops = &smc->af_ops;
2680 if (smc->limit_smc_hs)
2681 tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested;
2683 rc = kernel_listen(smc->clcsock, backlog);
2685 write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2686 smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready,
2687 &smc->clcsk_data_ready);
2688 smc->clcsock->sk->sk_user_data = NULL;
2689 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2692 sk->sk_max_ack_backlog = backlog;
2693 sk->sk_ack_backlog = 0;
2694 sk->sk_state = SMC_LISTEN;
2704 struct sock *sk = sock->sk, *nsk;
2714 if (lsmc->sk.sk_state != SMC_LISTEN) {
2715 rc = -EINVAL;
2721 timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK);
2726 rc = -EAGAIN;
2748 if (lsmc->sockopt_defer_accept && !(arg->flags & O_NONBLOCK)) {
2750 timeo = secs_to_jiffies(lsmc->sockopt_defer_accept);
2751 if (smc_sk(nsk)->use_fallback) {
2752 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2755 if (skb_queue_empty(&clcsk->sk_receive_queue))
2758 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2773 struct smc_sock *smc;
2775 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2776 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2777 return -ENOTCONN;
2779 smc = smc_sk(sock->sk);
2781 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2786 struct sock *sk = sock->sk;
2787 struct smc_sock *smc;
2790 smc = smc_sk(sk);
2793 /* SMC does not support connect with fastopen */
2794 if (msg->msg_flags & MSG_FASTOPEN) {
2796 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2797 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2801 rc = -EINVAL;
2804 } else if ((sk->sk_state != SMC_ACTIVE) &&
2805 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2806 (sk->sk_state != SMC_INIT)) {
2807 rc = -EPIPE;
2811 if (smc->use_fallback) {
2812 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2814 rc = smc_tx_sendmsg(smc, msg, len);
2815 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2825 struct sock *sk = sock->sk;
2826 struct smc_sock *smc;
2827 int rc = -ENOTCONN;
2829 smc = smc_sk(sk);
2831 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2836 if ((sk->sk_state == SMC_INIT) ||
2837 (sk->sk_state == SMC_LISTEN) ||
2838 (sk->sk_state == SMC_CLOSED))
2841 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2846 if (smc->use_fallback) {
2847 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2849 msg->msg_namelen = 0;
2850 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2851 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2864 spin_lock(&isk->accept_q_lock);
2865 if (!list_empty(&isk->accept_q))
2867 spin_unlock(&isk->accept_q_lock);
2875 struct sock *sk = sock->sk;
2876 struct smc_sock *smc;
2882 smc = smc_sk(sock->sk);
2883 if (smc->use_fallback) {
2885 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2886 sk->sk_err = smc->clcsock->sk->sk_err;
2888 if (sk->sk_state != SMC_CLOSED)
2890 if (sk->sk_err)
2892 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2893 (sk->sk_state == SMC_CLOSED))
2895 if (sk->sk_state == SMC_LISTEN) {
2898 } else if (smc->use_fallback) { /* as result of connect_work()*/
2899 mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2901 sk->sk_err = smc->clcsock->sk->sk_err;
2903 if ((sk->sk_state != SMC_INIT &&
2904 atomic_read(&smc->conn.sndbuf_space)) ||
2905 sk->sk_shutdown & SEND_SHUTDOWN) {
2909 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2911 if (sk->sk_state != SMC_INIT) {
2914 if (atomic_read(&smc->conn.sndbuf_space))
2918 if (atomic_read(&smc->conn.bytes_to_rcv))
2920 if (sk->sk_shutdown & RCV_SHUTDOWN)
2922 if (sk->sk_state == SMC_APPCLOSEWAIT1)
2924 if (smc->conn.urg_state == SMC_URG_VALID)
2934 struct sock *sk = sock->sk;
2936 struct smc_sock *smc;
2937 int rc = -EINVAL;
2941 smc = smc_sk(sk);
2948 if (sock->state == SS_CONNECTING) {
2949 if (sk->sk_state == SMC_ACTIVE)
2950 sock->state = SS_CONNECTED;
2951 else if (sk->sk_state == SMC_PEERCLOSEWAIT1 ||
2952 sk->sk_state == SMC_PEERCLOSEWAIT2 ||
2953 sk->sk_state == SMC_APPCLOSEWAIT1 ||
2954 sk->sk_state == SMC_APPCLOSEWAIT2 ||
2955 sk->sk_state == SMC_APPFINCLOSEWAIT)
2956 sock->state = SS_DISCONNECTING;
2959 rc = -ENOTCONN;
2960 if ((sk->sk_state != SMC_ACTIVE) &&
2961 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2962 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2963 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2964 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2965 (sk->sk_state != SMC_APPFINCLOSEWAIT))
2967 if (smc->use_fallback) {
2968 rc = kernel_sock_shutdown(smc->clcsock, how);
2969 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2970 if (sk->sk_shutdown == SHUTDOWN_MASK) {
2971 sk->sk_state = SMC_CLOSED;
2972 sk->sk_socket->state = SS_UNCONNECTED;
2979 old_state = sk->sk_state;
2980 rc = smc_close_active(smc);
2982 sk->sk_state == SMC_PEERCLOSEWAIT1)
2986 rc = smc_close_shutdown_write(smc);
2993 if (do_shutdown && smc->clcsock)
2994 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2996 sk->sk_shutdown |= how + 1;
2998 if (sk->sk_state == SMC_CLOSED)
2999 sock->state = SS_UNCONNECTED;
3001 sock->state = SS_DISCONNECTING;
3010 struct smc_sock *smc;
3013 smc = smc_sk(sock->sk);
3016 return -EFAULT;
3021 return -EINVAL;
3025 val = smc->limit_smc_hs;
3028 return -EOPNOTSUPP;
3032 return -EFAULT;
3034 return -EFAULT;
3042 struct sock *sk = sock->sk;
3043 struct smc_sock *smc;
3046 smc = smc_sk(sk);
3052 rc = -EINVAL;
3056 rc = -EFAULT;
3060 smc->limit_smc_hs = !!val;
3064 rc = -EOPNOTSUPP;
3075 struct sock *sk = sock->sk;
3076 struct smc_sock *smc;
3080 return -EOPNOTSUPP;
3084 smc = smc_sk(sk);
3089 mutex_lock(&smc->clcsock_release_lock);
3090 if (!smc->clcsock) {
3091 mutex_unlock(&smc->clcsock_release_lock);
3092 return -EBADF;
3094 if (unlikely(!smc->clcsock->ops->setsockopt))
3095 rc = -EOPNOTSUPP;
3097 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
3099 if (smc->clcsock->sk->sk_err) {
3100 sk->sk_err = smc->clcsock->sk->sk_err;
3103 mutex_unlock(&smc->clcsock_release_lock);
3106 return -EINVAL;
3108 return -EFAULT;
3111 if (rc || smc->use_fallback)
3118 /* option not supported by SMC */
3119 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
3120 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
3122 rc = -EINVAL;
3126 if (sk->sk_state != SMC_INIT &&
3127 sk->sk_state != SMC_LISTEN &&
3128 sk->sk_state != SMC_CLOSED) {
3130 SMC_STAT_INC(smc, ndly_cnt);
3131 smc_tx_pending(&smc->conn);
3132 cancel_delayed_work(&smc->conn.tx_work);
3137 if (sk->sk_state != SMC_INIT &&
3138 sk->sk_state != SMC_LISTEN &&
3139 sk->sk_state != SMC_CLOSED) {
3141 SMC_STAT_INC(smc, cork_cnt);
3142 smc_tx_pending(&smc->conn);
3143 cancel_delayed_work(&smc->conn.tx_work);
3148 smc->sockopt_defer_accept = val;
3162 struct smc_sock *smc;
3168 smc = smc_sk(sock->sk);
3169 mutex_lock(&smc->clcsock_release_lock);
3170 if (!smc->clcsock) {
3171 mutex_unlock(&smc->clcsock_release_lock);
3172 return -EBADF;
3175 if (unlikely(!smc->clcsock->ops->getsockopt)) {
3176 mutex_unlock(&smc->clcsock_release_lock);
3177 return -EOPNOTSUPP;
3179 rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
3181 mutex_unlock(&smc->clcsock_release_lock);
3190 struct smc_sock *smc;
3193 smc = smc_sk(sock->sk);
3194 conn = &smc->conn;
3195 lock_sock(&smc->sk);
3196 if (smc->use_fallback) {
3197 if (!smc->clcsock) {
3198 release_sock(&smc->sk);
3199 return -EBADF;
3201 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
3202 release_sock(&smc->sk);
3207 if (smc->sk.sk_state == SMC_LISTEN) {
3208 release_sock(&smc->sk);
3209 return -EINVAL;
3211 if (smc->sk.sk_state == SMC_INIT ||
3212 smc->sk.sk_state == SMC_CLOSED)
3215 answ = atomic_read(&smc->conn.bytes_to_rcv);
3219 if (smc->sk.sk_state == SMC_LISTEN) {
3220 release_sock(&smc->sk);
3221 return -EINVAL;
3223 if (smc->sk.sk_state == SMC_INIT ||
3224 smc->sk.sk_state == SMC_CLOSED)
3227 answ = smc->conn.sndbuf_desc->len -
3228 atomic_read(&smc->conn.sndbuf_space);
3232 if (smc->sk.sk_state == SMC_LISTEN) {
3233 release_sock(&smc->sk);
3234 return -EINVAL;
3236 if (smc->sk.sk_state == SMC_INIT ||
3237 smc->sk.sk_state == SMC_CLOSED)
3240 answ = smc_tx_prepared_sends(&smc->conn);
3243 if (smc->sk.sk_state == SMC_LISTEN) {
3244 release_sock(&smc->sk);
3245 return -EINVAL;
3247 if (smc->sk.sk_state == SMC_INIT ||
3248 smc->sk.sk_state == SMC_CLOSED) {
3251 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
3252 smc_curs_copy(&urg, &conn->urg_curs, conn);
3253 answ = smc_curs_diff(conn->rmb_desc->len,
3258 release_sock(&smc->sk);
3259 return -ENOIOCTLCMD;
3261 release_sock(&smc->sk);
3267 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
3276 struct sock *sk = sock->sk;
3277 struct smc_sock *smc;
3278 int rc = -ENOTCONN;
3280 smc = smc_sk(sk);
3282 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
3287 if (sk->sk_state == SMC_INIT ||
3288 sk->sk_state == SMC_LISTEN ||
3289 sk->sk_state == SMC_CLOSED)
3292 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
3297 if (smc->use_fallback) {
3298 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
3302 rc = -ESPIPE;
3309 SMC_STAT_INC(smc, splice_cnt);
3310 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
3342 struct smc_sock *smc = smc_sk(sk);
3346 &smc->clcsock);
3350 /* smc_clcsock_release() does not wait smc->clcsock->sk's
3352 * smc->sk is close()d, and TCP timers can be fired later,
3355 sk = smc->clcsock->sk;
3364 struct smc_sock *smc;
3368 rc = -ESOCKTNOSUPPORT;
3369 if (sock->type != SOCK_STREAM)
3372 rc = -EPROTONOSUPPORT;
3376 rc = -ENOBUFS;
3377 sock->ops = &smc_sock_ops;
3378 sock->state = SS_UNCONNECTED;
3384 smc = smc_sk(sk);
3388 smc->clcsock = clcsock;
3394 sock->sk = NULL;
3414 struct socket *tcp = sk->sk_socket;
3420 if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP ||
3421 (sk->sk_family != AF_INET && sk->sk_family != AF_INET6))
3422 return -ESOCKTNOSUPPORT;
3424 if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list)
3425 return -ENOTCONN;
3427 if (sk->sk_family == AF_INET)
3434 return -ENFILE;
3436 smcsock->type = SOCK_STREAM;
3444 /* replace tcp socket to smc */
3445 smcsock->file = tcp->file;
3446 smcsock->file->private_data = smcsock;
3447 smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */
3448 smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */
3449 tcp->file = NULL;
3460 icsk->icsk_ulp_ops = NULL;
3464 .name = "smc",
3501 .id = &smc_net_id,
3535 rc = -ENOMEM;
3664 MODULE_DESCRIPTION("smc socket address family");
3667 MODULE_ALIAS_TCP_ULP("smc");