1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications Direct over loopback-ism device. 4 * 5 * Functions for loopback-ism device. 6 * 7 * Copyright (c) 2024, Alibaba Inc. 8 * 9 * Author: Wen Gu <guwen@linux.alibaba.com> 10 * Tony Lu <tonylu@linux.alibaba.com> 11 * 12 */ 13 14 #include <linux/device.h> 15 #include <linux/types.h> 16 #include <net/smc.h> 17 18 #include "smc_cdc.h" 19 #include "smc_ism.h" 20 #include "smc_loopback.h" 21 22 #define SMC_LO_V2_CAPABLE 0x1 /* loopback-ism acts as ISMv2 */ 23 #define SMC_LO_SUPPORT_NOCOPY 0x1 24 #define SMC_DMA_ADDR_INVALID (~(dma_addr_t)0) 25 26 static const char smc_lo_dev_name[] = "loopback-ism"; 27 static struct smc_lo_dev *lo_dev; 28 29 static void smc_lo_generate_ids(struct smc_lo_dev *ldev) 30 { 31 struct smcd_gid *lgid = &ldev->local_gid; 32 uuid_t uuid; 33 34 uuid_gen(&uuid); 35 memcpy(&lgid->gid, &uuid, sizeof(lgid->gid)); 36 memcpy(&lgid->gid_ext, (u8 *)&uuid + sizeof(lgid->gid), 37 sizeof(lgid->gid_ext)); 38 39 ldev->chid = SMC_LO_RESERVED_CHID; 40 } 41 42 static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid, 43 u32 vid_valid, u32 vid) 44 { 45 struct smc_lo_dev *ldev = smcd->priv; 46 47 /* rgid should be the same as lgid */ 48 if (!ldev || rgid->gid != ldev->local_gid.gid || 49 rgid->gid_ext != ldev->local_gid.gid_ext) 50 return -ENETUNREACH; 51 return 0; 52 } 53 54 static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb, 55 void *client_priv) 56 { 57 struct smc_lo_dmb_node *dmb_node, *tmp_node; 58 struct smc_lo_dev *ldev = smcd->priv; 59 int sba_idx, rc; 60 61 /* check space for new dmb */ 62 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LO_MAX_DMBS) { 63 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) 64 break; 65 } 66 if (sba_idx == SMC_LO_MAX_DMBS) 67 return -ENOSPC; 68 69 dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL); 70 if (!dmb_node) { 71 rc = -ENOMEM; 72 goto err_bit; 73 } 74 75 dmb_node->sba_idx = sba_idx; 76 dmb_node->len = dmb->dmb_len; 77 dmb_node->cpu_addr = kzalloc(dmb_node->len, GFP_KERNEL | 78 __GFP_NOWARN | __GFP_NORETRY | 79 __GFP_NOMEMALLOC); 80 if (!dmb_node->cpu_addr) { 81 rc = -ENOMEM; 82 goto err_node; 83 } 84 dmb_node->dma_addr = SMC_DMA_ADDR_INVALID; 85 refcount_set(&dmb_node->refcnt, 1); 86 87 again: 88 /* add new dmb into hash table */ 89 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); 90 write_lock_bh(&ldev->dmb_ht_lock); 91 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { 92 if (tmp_node->token == dmb_node->token) { 93 write_unlock_bh(&ldev->dmb_ht_lock); 94 goto again; 95 } 96 } 97 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); 98 write_unlock_bh(&ldev->dmb_ht_lock); 99 atomic_inc(&ldev->dmb_cnt); 100 101 dmb->sba_idx = dmb_node->sba_idx; 102 dmb->dmb_tok = dmb_node->token; 103 dmb->cpu_addr = dmb_node->cpu_addr; 104 dmb->dma_addr = dmb_node->dma_addr; 105 dmb->dmb_len = dmb_node->len; 106 107 return 0; 108 109 err_node: 110 kfree(dmb_node); 111 err_bit: 112 clear_bit(sba_idx, ldev->sba_idx_mask); 113 return rc; 114 } 115 116 static void __smc_lo_unregister_dmb(struct smc_lo_dev *ldev, 117 struct smc_lo_dmb_node *dmb_node) 118 { 119 /* remove dmb from hash table */ 120 write_lock_bh(&ldev->dmb_ht_lock); 121 hash_del(&dmb_node->list); 122 write_unlock_bh(&ldev->dmb_ht_lock); 123 124 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); 125 kvfree(dmb_node->cpu_addr); 126 kfree(dmb_node); 127 128 if (atomic_dec_and_test(&ldev->dmb_cnt)) 129 wake_up(&ldev->ldev_release); 130 } 131 132 static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) 133 { 134 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 135 struct smc_lo_dev *ldev = smcd->priv; 136 137 /* find dmb from hash table */ 138 read_lock_bh(&ldev->dmb_ht_lock); 139 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 140 if (tmp_node->token == dmb->dmb_tok) { 141 dmb_node = tmp_node; 142 break; 143 } 144 } 145 if (!dmb_node) { 146 read_unlock_bh(&ldev->dmb_ht_lock); 147 return -EINVAL; 148 } 149 read_unlock_bh(&ldev->dmb_ht_lock); 150 151 if (refcount_dec_and_test(&dmb_node->refcnt)) 152 __smc_lo_unregister_dmb(ldev, dmb_node); 153 return 0; 154 } 155 156 static int smc_lo_support_dmb_nocopy(struct smcd_dev *smcd) 157 { 158 return SMC_LO_SUPPORT_NOCOPY; 159 } 160 161 static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) 162 { 163 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 164 struct smc_lo_dev *ldev = smcd->priv; 165 166 /* find dmb_node according to dmb->dmb_tok */ 167 read_lock_bh(&ldev->dmb_ht_lock); 168 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 169 if (tmp_node->token == dmb->dmb_tok) { 170 dmb_node = tmp_node; 171 break; 172 } 173 } 174 if (!dmb_node) { 175 read_unlock_bh(&ldev->dmb_ht_lock); 176 return -EINVAL; 177 } 178 read_unlock_bh(&ldev->dmb_ht_lock); 179 180 if (!refcount_inc_not_zero(&dmb_node->refcnt)) 181 /* the dmb is being unregistered, but has 182 * not been removed from the hash table. 183 */ 184 return -EINVAL; 185 186 /* provide dmb information */ 187 dmb->sba_idx = dmb_node->sba_idx; 188 dmb->dmb_tok = dmb_node->token; 189 dmb->cpu_addr = dmb_node->cpu_addr; 190 dmb->dma_addr = dmb_node->dma_addr; 191 dmb->dmb_len = dmb_node->len; 192 return 0; 193 } 194 195 static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token) 196 { 197 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 198 struct smc_lo_dev *ldev = smcd->priv; 199 200 /* find dmb_node according to dmb->dmb_tok */ 201 read_lock_bh(&ldev->dmb_ht_lock); 202 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { 203 if (tmp_node->token == token) { 204 dmb_node = tmp_node; 205 break; 206 } 207 } 208 if (!dmb_node) { 209 read_unlock_bh(&ldev->dmb_ht_lock); 210 return -EINVAL; 211 } 212 read_unlock_bh(&ldev->dmb_ht_lock); 213 214 if (refcount_dec_and_test(&dmb_node->refcnt)) 215 __smc_lo_unregister_dmb(ldev, dmb_node); 216 return 0; 217 } 218 219 static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok, 220 unsigned int idx, bool sf, unsigned int offset, 221 void *data, unsigned int size) 222 { 223 struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node; 224 struct smc_lo_dev *ldev = smcd->priv; 225 struct smc_connection *conn; 226 227 if (!sf) 228 /* since sndbuf is merged with peer DMB, there is 229 * no need to copy data from sndbuf to peer DMB. 230 */ 231 return 0; 232 233 read_lock_bh(&ldev->dmb_ht_lock); 234 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) { 235 if (tmp_node->token == dmb_tok) { 236 rmb_node = tmp_node; 237 break; 238 } 239 } 240 if (!rmb_node) { 241 read_unlock_bh(&ldev->dmb_ht_lock); 242 return -EINVAL; 243 } 244 memcpy((char *)rmb_node->cpu_addr + offset, data, size); 245 read_unlock_bh(&ldev->dmb_ht_lock); 246 247 conn = smcd->conn[rmb_node->sba_idx]; 248 if (!conn || conn->killed) 249 return -EPIPE; 250 tasklet_schedule(&conn->rx_tsklet); 251 return 0; 252 } 253 254 static int smc_lo_supports_v2(void) 255 { 256 return SMC_LO_V2_CAPABLE; 257 } 258 259 static void smc_lo_get_local_gid(struct smcd_dev *smcd, 260 struct smcd_gid *smcd_gid) 261 { 262 struct smc_lo_dev *ldev = smcd->priv; 263 264 smcd_gid->gid = ldev->local_gid.gid; 265 smcd_gid->gid_ext = ldev->local_gid.gid_ext; 266 } 267 268 static u16 smc_lo_get_chid(struct smcd_dev *smcd) 269 { 270 return ((struct smc_lo_dev *)smcd->priv)->chid; 271 } 272 273 static struct device *smc_lo_get_dev(struct smcd_dev *smcd) 274 { 275 return &((struct smc_lo_dev *)smcd->priv)->dev; 276 } 277 278 static const struct smcd_ops lo_ops = { 279 .query_remote_gid = smc_lo_query_rgid, 280 .register_dmb = smc_lo_register_dmb, 281 .unregister_dmb = smc_lo_unregister_dmb, 282 .support_dmb_nocopy = smc_lo_support_dmb_nocopy, 283 .attach_dmb = smc_lo_attach_dmb, 284 .detach_dmb = smc_lo_detach_dmb, 285 .add_vlan_id = NULL, 286 .del_vlan_id = NULL, 287 .set_vlan_required = NULL, 288 .reset_vlan_required = NULL, 289 .signal_event = NULL, 290 .move_data = smc_lo_move_data, 291 .supports_v2 = smc_lo_supports_v2, 292 .get_local_gid = smc_lo_get_local_gid, 293 .get_chid = smc_lo_get_chid, 294 .get_dev = smc_lo_get_dev, 295 }; 296 297 static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops, 298 int max_dmbs) 299 { 300 struct smcd_dev *smcd; 301 302 smcd = kzalloc(sizeof(*smcd), GFP_KERNEL); 303 if (!smcd) 304 return NULL; 305 306 smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *), 307 GFP_KERNEL); 308 if (!smcd->conn) 309 goto out_smcd; 310 311 smcd->ops = ops; 312 313 spin_lock_init(&smcd->lock); 314 spin_lock_init(&smcd->lgr_lock); 315 INIT_LIST_HEAD(&smcd->vlan); 316 INIT_LIST_HEAD(&smcd->lgr_list); 317 init_waitqueue_head(&smcd->lgrs_deleted); 318 return smcd; 319 320 out_smcd: 321 kfree(smcd); 322 return NULL; 323 } 324 325 static int smcd_lo_register_dev(struct smc_lo_dev *ldev) 326 { 327 struct smcd_dev *smcd; 328 329 smcd = smcd_lo_alloc_dev(&lo_ops, SMC_LO_MAX_DMBS); 330 if (!smcd) 331 return -ENOMEM; 332 ldev->smcd = smcd; 333 smcd->priv = ldev; 334 smc_ism_set_v2_capable(); 335 mutex_lock(&smcd_dev_list.mutex); 336 list_add(&smcd->list, &smcd_dev_list.list); 337 mutex_unlock(&smcd_dev_list.mutex); 338 pr_warn_ratelimited("smc: adding smcd device %s\n", 339 dev_name(&ldev->dev)); 340 return 0; 341 } 342 343 static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev) 344 { 345 struct smcd_dev *smcd = ldev->smcd; 346 347 pr_warn_ratelimited("smc: removing smcd device %s\n", 348 dev_name(&ldev->dev)); 349 smcd->going_away = 1; 350 smc_smcd_terminate_all(smcd); 351 mutex_lock(&smcd_dev_list.mutex); 352 list_del_init(&smcd->list); 353 mutex_unlock(&smcd_dev_list.mutex); 354 kfree(smcd->conn); 355 kfree(smcd); 356 } 357 358 static int smc_lo_dev_init(struct smc_lo_dev *ldev) 359 { 360 smc_lo_generate_ids(ldev); 361 rwlock_init(&ldev->dmb_ht_lock); 362 hash_init(ldev->dmb_ht); 363 atomic_set(&ldev->dmb_cnt, 0); 364 init_waitqueue_head(&ldev->ldev_release); 365 366 return smcd_lo_register_dev(ldev); 367 } 368 369 static void smc_lo_dev_exit(struct smc_lo_dev *ldev) 370 { 371 smcd_lo_unregister_dev(ldev); 372 if (atomic_read(&ldev->dmb_cnt)) 373 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt)); 374 } 375 376 static void smc_lo_dev_release(struct device *dev) 377 { 378 struct smc_lo_dev *ldev = 379 container_of(dev, struct smc_lo_dev, dev); 380 381 kfree(ldev); 382 } 383 384 static int smc_lo_dev_probe(void) 385 { 386 struct smc_lo_dev *ldev; 387 int ret; 388 389 ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); 390 if (!ldev) 391 return -ENOMEM; 392 393 ldev->dev.parent = NULL; 394 ldev->dev.release = smc_lo_dev_release; 395 device_initialize(&ldev->dev); 396 dev_set_name(&ldev->dev, smc_lo_dev_name); 397 398 ret = smc_lo_dev_init(ldev); 399 if (ret) 400 goto free_dev; 401 402 lo_dev = ldev; /* global loopback device */ 403 return 0; 404 405 free_dev: 406 put_device(&ldev->dev); 407 return ret; 408 } 409 410 static void smc_lo_dev_remove(void) 411 { 412 if (!lo_dev) 413 return; 414 415 smc_lo_dev_exit(lo_dev); 416 put_device(&lo_dev->dev); /* device_initialize in smc_lo_dev_probe */ 417 } 418 419 int smc_loopback_init(void) 420 { 421 return smc_lo_dev_probe(); 422 } 423 424 void smc_loopback_exit(void) 425 { 426 smc_lo_dev_remove(); 427 } 428