1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications Direct over loopback-ism device. 4 * 5 * Functions for loopback-ism device. 6 * 7 * Copyright (c) 2024, Alibaba Inc. 8 * 9 * Author: Wen Gu <guwen@linux.alibaba.com> 10 * Tony Lu <tonylu@linux.alibaba.com> 11 * 12 */ 13 14 #include <linux/device.h> 15 #include <linux/types.h> 16 #include <net/smc.h> 17 18 #include "smc_cdc.h" 19 #include "smc_ism.h" 20 #include "smc_loopback.h" 21 22 #define SMC_LO_V2_CAPABLE 0x1 /* loopback-ism acts as ISMv2 */ 23 #define SMC_LO_SUPPORT_NOCOPY 0x1 24 #define SMC_DMA_ADDR_INVALID (~(dma_addr_t)0) 25 26 static const char smc_lo_dev_name[] = "loopback-ism"; 27 static struct smc_lo_dev *lo_dev; 28 29 static void smc_lo_generate_ids(struct smc_lo_dev *ldev) 30 { 31 struct smcd_gid *lgid = &ldev->local_gid; 32 uuid_t uuid; 33 34 uuid_gen(&uuid); 35 memcpy(&lgid->gid, &uuid, sizeof(lgid->gid)); 36 memcpy(&lgid->gid_ext, (u8 *)&uuid + sizeof(lgid->gid), 37 sizeof(lgid->gid_ext)); 38 39 ldev->chid = SMC_LO_RESERVED_CHID; 40 } 41 42 static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid, 43 u32 vid_valid, u32 vid) 44 { 45 struct smc_lo_dev *ldev = smcd->priv; 46 47 /* rgid should be the same as lgid */ 48 if (!ldev || rgid->gid != ldev->local_gid.gid || 49 rgid->gid_ext != ldev->local_gid.gid_ext) 50 return -ENETUNREACH; 51 return 0; 52 } 53 54 static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb, 55 void *client_priv) 56 { 57 struct smc_lo_dmb_node *dmb_node, *tmp_node; 58 struct smc_lo_dev *ldev = smcd->priv; 59 struct folio *folio; 60 int sba_idx, rc; 61 62 /* check space for new dmb */ 63 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LO_MAX_DMBS) { 64 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) 65 break; 66 } 67 if (sba_idx == SMC_LO_MAX_DMBS) 68 return -ENOSPC; 69 70 dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL); 71 if (!dmb_node) { 72 rc = -ENOMEM; 73 goto err_bit; 74 } 75 76 dmb_node->sba_idx = sba_idx; 77 dmb_node->len = dmb->dmb_len; 78 79 /* not critical; fail under memory pressure and fallback to TCP */ 80 folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | 81 __GFP_NORETRY | __GFP_ZERO, 82 get_order(dmb_node->len)); 83 if (!folio) { 84 rc = -ENOMEM; 85 goto err_node; 86 } 87 dmb_node->cpu_addr = folio_address(folio); 88 dmb_node->dma_addr = SMC_DMA_ADDR_INVALID; 89 refcount_set(&dmb_node->refcnt, 1); 90 91 again: 92 /* add new dmb into hash table */ 93 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); 94 write_lock_bh(&ldev->dmb_ht_lock); 95 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { 96 if (tmp_node->token == dmb_node->token) { 97 write_unlock_bh(&ldev->dmb_ht_lock); 98 goto again; 99 } 100 } 101 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); 102 write_unlock_bh(&ldev->dmb_ht_lock); 103 atomic_inc(&ldev->dmb_cnt); 104 105 dmb->sba_idx = dmb_node->sba_idx; 106 dmb->dmb_tok = dmb_node->token; 107 dmb->cpu_addr = dmb_node->cpu_addr; 108 dmb->dma_addr = dmb_node->dma_addr; 109 dmb->dmb_len = dmb_node->len; 110 111 return 0; 112 113 err_node: 114 kfree(dmb_node); 115 err_bit: 116 clear_bit(sba_idx, ldev->sba_idx_mask); 117 return rc; 118 } 119 120 static void __smc_lo_unregister_dmb(struct smc_lo_dev *ldev, 121 struct smc_lo_dmb_node *dmb_node) 122 { 123 /* remove dmb from hash table */ 124 write_lock_bh(&ldev->dmb_ht_lock); 125 hash_del(&dmb_node->list); 126 write_unlock_bh(&ldev->dmb_ht_lock); 127 128 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); 129 folio_put(virt_to_folio(dmb_node->cpu_addr)); 130 kfree(dmb_node); 131 132 if (atomic_dec_and_test(&ldev->dmb_cnt)) 133 wake_up(&ldev->ldev_release); 134 } 135 136 static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) 137 { 138 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 139 struct smc_lo_dev *ldev = smcd->priv; 140 141 /* find dmb from hash table */ 142 read_lock_bh(&ldev->dmb_ht_lock); 143 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 144 if (tmp_node->token == dmb->dmb_tok) { 145 dmb_node = tmp_node; 146 break; 147 } 148 } 149 if (!dmb_node) { 150 read_unlock_bh(&ldev->dmb_ht_lock); 151 return -EINVAL; 152 } 153 read_unlock_bh(&ldev->dmb_ht_lock); 154 155 if (refcount_dec_and_test(&dmb_node->refcnt)) 156 __smc_lo_unregister_dmb(ldev, dmb_node); 157 return 0; 158 } 159 160 static int smc_lo_support_dmb_nocopy(struct smcd_dev *smcd) 161 { 162 return SMC_LO_SUPPORT_NOCOPY; 163 } 164 165 static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) 166 { 167 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 168 struct smc_lo_dev *ldev = smcd->priv; 169 170 /* find dmb_node according to dmb->dmb_tok */ 171 read_lock_bh(&ldev->dmb_ht_lock); 172 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 173 if (tmp_node->token == dmb->dmb_tok) { 174 dmb_node = tmp_node; 175 break; 176 } 177 } 178 if (!dmb_node) { 179 read_unlock_bh(&ldev->dmb_ht_lock); 180 return -EINVAL; 181 } 182 read_unlock_bh(&ldev->dmb_ht_lock); 183 184 if (!refcount_inc_not_zero(&dmb_node->refcnt)) 185 /* the dmb is being unregistered, but has 186 * not been removed from the hash table. 187 */ 188 return -EINVAL; 189 190 /* provide dmb information */ 191 dmb->sba_idx = dmb_node->sba_idx; 192 dmb->dmb_tok = dmb_node->token; 193 dmb->cpu_addr = dmb_node->cpu_addr; 194 dmb->dma_addr = dmb_node->dma_addr; 195 dmb->dmb_len = dmb_node->len; 196 return 0; 197 } 198 199 static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token) 200 { 201 struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; 202 struct smc_lo_dev *ldev = smcd->priv; 203 204 /* find dmb_node according to dmb->dmb_tok */ 205 read_lock_bh(&ldev->dmb_ht_lock); 206 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { 207 if (tmp_node->token == token) { 208 dmb_node = tmp_node; 209 break; 210 } 211 } 212 if (!dmb_node) { 213 read_unlock_bh(&ldev->dmb_ht_lock); 214 return -EINVAL; 215 } 216 read_unlock_bh(&ldev->dmb_ht_lock); 217 218 if (refcount_dec_and_test(&dmb_node->refcnt)) 219 __smc_lo_unregister_dmb(ldev, dmb_node); 220 return 0; 221 } 222 223 static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok, 224 unsigned int idx, bool sf, unsigned int offset, 225 void *data, unsigned int size) 226 { 227 struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node; 228 struct smc_lo_dev *ldev = smcd->priv; 229 struct smc_connection *conn; 230 231 if (!sf) 232 /* since sndbuf is merged with peer DMB, there is 233 * no need to copy data from sndbuf to peer DMB. 234 */ 235 return 0; 236 237 read_lock_bh(&ldev->dmb_ht_lock); 238 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) { 239 if (tmp_node->token == dmb_tok) { 240 rmb_node = tmp_node; 241 break; 242 } 243 } 244 if (!rmb_node) { 245 read_unlock_bh(&ldev->dmb_ht_lock); 246 return -EINVAL; 247 } 248 memcpy((char *)rmb_node->cpu_addr + offset, data, size); 249 read_unlock_bh(&ldev->dmb_ht_lock); 250 251 conn = smcd->conn[rmb_node->sba_idx]; 252 if (!conn || conn->killed) 253 return -EPIPE; 254 tasklet_schedule(&conn->rx_tsklet); 255 return 0; 256 } 257 258 static void smc_lo_get_local_gid(struct smcd_dev *smcd, 259 struct smcd_gid *smcd_gid) 260 { 261 struct smc_lo_dev *ldev = smcd->priv; 262 263 smcd_gid->gid = ldev->local_gid.gid; 264 smcd_gid->gid_ext = ldev->local_gid.gid_ext; 265 } 266 267 static u16 smc_lo_get_chid(struct smcd_dev *smcd) 268 { 269 return ((struct smc_lo_dev *)smcd->priv)->chid; 270 } 271 272 static struct device *smc_lo_get_dev(struct smcd_dev *smcd) 273 { 274 return &((struct smc_lo_dev *)smcd->priv)->dev; 275 } 276 277 static const struct smcd_ops lo_ops = { 278 .query_remote_gid = smc_lo_query_rgid, 279 .register_dmb = smc_lo_register_dmb, 280 .unregister_dmb = smc_lo_unregister_dmb, 281 .support_dmb_nocopy = smc_lo_support_dmb_nocopy, 282 .attach_dmb = smc_lo_attach_dmb, 283 .detach_dmb = smc_lo_detach_dmb, 284 .add_vlan_id = NULL, 285 .del_vlan_id = NULL, 286 .set_vlan_required = NULL, 287 .reset_vlan_required = NULL, 288 .signal_event = NULL, 289 .move_data = smc_lo_move_data, 290 .get_local_gid = smc_lo_get_local_gid, 291 .get_chid = smc_lo_get_chid, 292 .get_dev = smc_lo_get_dev, 293 }; 294 295 static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops, 296 int max_dmbs) 297 { 298 struct smcd_dev *smcd; 299 300 smcd = kzalloc(sizeof(*smcd), GFP_KERNEL); 301 if (!smcd) 302 return NULL; 303 304 smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *), 305 GFP_KERNEL); 306 if (!smcd->conn) 307 goto out_smcd; 308 309 smcd->ops = ops; 310 311 spin_lock_init(&smcd->lock); 312 spin_lock_init(&smcd->lgr_lock); 313 INIT_LIST_HEAD(&smcd->vlan); 314 INIT_LIST_HEAD(&smcd->lgr_list); 315 init_waitqueue_head(&smcd->lgrs_deleted); 316 return smcd; 317 318 out_smcd: 319 kfree(smcd); 320 return NULL; 321 } 322 323 static int smcd_lo_register_dev(struct smc_lo_dev *ldev) 324 { 325 struct smcd_dev *smcd; 326 327 smcd = smcd_lo_alloc_dev(&lo_ops, SMC_LO_MAX_DMBS); 328 if (!smcd) 329 return -ENOMEM; 330 ldev->smcd = smcd; 331 smcd->priv = ldev; 332 smc_ism_set_v2_capable(); 333 mutex_lock(&smcd_dev_list.mutex); 334 list_add(&smcd->list, &smcd_dev_list.list); 335 mutex_unlock(&smcd_dev_list.mutex); 336 pr_warn_ratelimited("smc: adding smcd device %s\n", 337 dev_name(&ldev->dev)); 338 return 0; 339 } 340 341 static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev) 342 { 343 struct smcd_dev *smcd = ldev->smcd; 344 345 pr_warn_ratelimited("smc: removing smcd device %s\n", 346 dev_name(&ldev->dev)); 347 smcd->going_away = 1; 348 smc_smcd_terminate_all(smcd); 349 mutex_lock(&smcd_dev_list.mutex); 350 list_del_init(&smcd->list); 351 mutex_unlock(&smcd_dev_list.mutex); 352 kfree(smcd->conn); 353 kfree(smcd); 354 } 355 356 static int smc_lo_dev_init(struct smc_lo_dev *ldev) 357 { 358 smc_lo_generate_ids(ldev); 359 rwlock_init(&ldev->dmb_ht_lock); 360 hash_init(ldev->dmb_ht); 361 atomic_set(&ldev->dmb_cnt, 0); 362 init_waitqueue_head(&ldev->ldev_release); 363 364 return smcd_lo_register_dev(ldev); 365 } 366 367 static void smc_lo_dev_exit(struct smc_lo_dev *ldev) 368 { 369 smcd_lo_unregister_dev(ldev); 370 if (atomic_read(&ldev->dmb_cnt)) 371 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt)); 372 } 373 374 static void smc_lo_dev_release(struct device *dev) 375 { 376 struct smc_lo_dev *ldev = 377 container_of(dev, struct smc_lo_dev, dev); 378 379 kfree(ldev); 380 } 381 382 static int smc_lo_dev_probe(void) 383 { 384 struct smc_lo_dev *ldev; 385 int ret; 386 387 ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); 388 if (!ldev) 389 return -ENOMEM; 390 391 ldev->dev.parent = NULL; 392 ldev->dev.release = smc_lo_dev_release; 393 device_initialize(&ldev->dev); 394 dev_set_name(&ldev->dev, smc_lo_dev_name); 395 396 ret = smc_lo_dev_init(ldev); 397 if (ret) 398 goto free_dev; 399 400 lo_dev = ldev; /* global loopback device */ 401 return 0; 402 403 free_dev: 404 put_device(&ldev->dev); 405 return ret; 406 } 407 408 static void smc_lo_dev_remove(void) 409 { 410 if (!lo_dev) 411 return; 412 413 smc_lo_dev_exit(lo_dev); 414 put_device(&lo_dev->dev); /* device_initialize in smc_lo_dev_probe */ 415 } 416 417 int smc_loopback_init(void) 418 { 419 return smc_lo_dev_probe(); 420 } 421 422 void smc_loopback_exit(void) 423 { 424 smc_lo_dev_remove(); 425 } 426