1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions for dibs loopback/loopback-ism device. 4 * 5 * Copyright (c) 2024, Alibaba Inc. 6 * 7 * Author: Wen Gu <guwen@linux.alibaba.com> 8 * Tony Lu <tonylu@linux.alibaba.com> 9 * 10 */ 11 12 #include <linux/bitops.h> 13 #include <linux/device.h> 14 #include <linux/dibs.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/types.h> 19 20 #include "dibs_loopback.h" 21 22 #define DIBS_LO_SUPPORT_NOCOPY 0x1 23 #define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0) 24 25 static const char dibs_lo_dev_name[] = "lo"; 26 /* global loopback device */ 27 static struct dibs_lo_dev *lo_dev; 28 29 static u16 dibs_lo_get_fabric_id(struct dibs_dev *dibs) 30 { 31 return DIBS_LOOPBACK_FABRIC; 32 } 33 34 static int dibs_lo_query_rgid(struct dibs_dev *dibs, const uuid_t *rgid, 35 u32 vid_valid, u32 vid) 36 { 37 /* rgid should be the same as lgid */ 38 if (!uuid_equal(rgid, &dibs->gid)) 39 return -ENETUNREACH; 40 return 0; 41 } 42 43 static int dibs_lo_max_dmbs(void) 44 { 45 return DIBS_LO_MAX_DMBS; 46 } 47 48 static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb, 49 struct dibs_client *client) 50 { 51 struct dibs_lo_dmb_node *dmb_node, *tmp_node; 52 struct dibs_lo_dev *ldev; 53 struct folio *folio; 54 unsigned long flags; 55 int sba_idx, rc; 56 57 ldev = dibs->drv_priv; 58 sba_idx = dmb->idx; 59 /* check space for new dmb */ 60 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) { 61 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) 62 break; 63 } 64 if (sba_idx == DIBS_LO_MAX_DMBS) 65 return -ENOSPC; 66 67 dmb_node = kzalloc_obj(*dmb_node); 68 if (!dmb_node) { 69 rc = -ENOMEM; 70 goto err_bit; 71 } 72 73 dmb_node->sba_idx = sba_idx; 74 dmb_node->len = dmb->dmb_len; 75 76 /* not critical; fail under memory pressure and fallback to TCP */ 77 folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | 78 __GFP_NORETRY | __GFP_ZERO, 79 get_order(dmb_node->len)); 80 if (!folio) { 81 rc = -ENOMEM; 82 goto err_node; 83 } 84 dmb_node->cpu_addr = folio_address(folio); 85 dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID; 86 refcount_set(&dmb_node->refcnt, 1); 87 88 again: 89 /* add new dmb into hash table */ 90 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); 91 write_lock_bh(&ldev->dmb_ht_lock); 92 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { 93 if (tmp_node->token == dmb_node->token) { 94 write_unlock_bh(&ldev->dmb_ht_lock); 95 goto again; 96 } 97 } 98 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); 99 write_unlock_bh(&ldev->dmb_ht_lock); 100 atomic_inc(&ldev->dmb_cnt); 101 102 dmb->idx = dmb_node->sba_idx; 103 dmb->dmb_tok = dmb_node->token; 104 dmb->cpu_addr = dmb_node->cpu_addr; 105 dmb->dma_addr = dmb_node->dma_addr; 106 dmb->dmb_len = dmb_node->len; 107 108 spin_lock_irqsave(&dibs->lock, flags); 109 dibs->dmb_clientid_arr[sba_idx] = client->id; 110 spin_unlock_irqrestore(&dibs->lock, flags); 111 112 return 0; 113 114 err_node: 115 kfree(dmb_node); 116 err_bit: 117 clear_bit(sba_idx, ldev->sba_idx_mask); 118 return rc; 119 } 120 121 static void __dibs_lo_unregister_dmb(struct dibs_lo_dev *ldev, 122 struct dibs_lo_dmb_node *dmb_node) 123 { 124 /* remove dmb from hash table */ 125 write_lock_bh(&ldev->dmb_ht_lock); 126 hash_del(&dmb_node->list); 127 write_unlock_bh(&ldev->dmb_ht_lock); 128 129 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); 130 folio_put(virt_to_folio(dmb_node->cpu_addr)); 131 kfree(dmb_node); 132 133 if (atomic_dec_and_test(&ldev->dmb_cnt)) 134 wake_up(&ldev->ldev_release); 135 } 136 137 static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) 138 { 139 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 140 struct dibs_lo_dev *ldev; 141 unsigned long flags; 142 143 ldev = dibs->drv_priv; 144 145 /* find dmb from hash table */ 146 read_lock_bh(&ldev->dmb_ht_lock); 147 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 148 if (tmp_node->token == dmb->dmb_tok) { 149 dmb_node = tmp_node; 150 break; 151 } 152 } 153 read_unlock_bh(&ldev->dmb_ht_lock); 154 if (!dmb_node) 155 return -EINVAL; 156 157 if (refcount_dec_and_test(&dmb_node->refcnt)) { 158 spin_lock_irqsave(&dibs->lock, flags); 159 dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT; 160 spin_unlock_irqrestore(&dibs->lock, flags); 161 162 __dibs_lo_unregister_dmb(ldev, dmb_node); 163 } 164 return 0; 165 } 166 167 static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs) 168 { 169 return DIBS_LO_SUPPORT_NOCOPY; 170 } 171 172 static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) 173 { 174 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 175 struct dibs_lo_dev *ldev; 176 177 ldev = dibs->drv_priv; 178 179 /* find dmb_node according to dmb->dmb_tok */ 180 read_lock_bh(&ldev->dmb_ht_lock); 181 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 182 if (tmp_node->token == dmb->dmb_tok) { 183 dmb_node = tmp_node; 184 break; 185 } 186 } 187 if (!dmb_node) { 188 read_unlock_bh(&ldev->dmb_ht_lock); 189 return -EINVAL; 190 } 191 read_unlock_bh(&ldev->dmb_ht_lock); 192 193 if (!refcount_inc_not_zero(&dmb_node->refcnt)) 194 /* the dmb is being unregistered, but has 195 * not been removed from the hash table. 196 */ 197 return -EINVAL; 198 199 /* provide dmb information */ 200 dmb->idx = dmb_node->sba_idx; 201 dmb->dmb_tok = dmb_node->token; 202 dmb->cpu_addr = dmb_node->cpu_addr; 203 dmb->dma_addr = dmb_node->dma_addr; 204 dmb->dmb_len = dmb_node->len; 205 return 0; 206 } 207 208 static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token) 209 { 210 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 211 struct dibs_lo_dev *ldev; 212 213 ldev = dibs->drv_priv; 214 215 /* find dmb_node according to dmb->dmb_tok */ 216 read_lock_bh(&ldev->dmb_ht_lock); 217 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { 218 if (tmp_node->token == token) { 219 dmb_node = tmp_node; 220 break; 221 } 222 } 223 if (!dmb_node) { 224 read_unlock_bh(&ldev->dmb_ht_lock); 225 return -EINVAL; 226 } 227 read_unlock_bh(&ldev->dmb_ht_lock); 228 229 if (refcount_dec_and_test(&dmb_node->refcnt)) 230 __dibs_lo_unregister_dmb(ldev, dmb_node); 231 return 0; 232 } 233 234 static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok, 235 unsigned int idx, bool sf, unsigned int offset, 236 void *data, unsigned int size) 237 { 238 struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node; 239 struct dibs_lo_dev *ldev; 240 u16 s_mask; 241 u8 client_id; 242 u32 sba_idx; 243 244 ldev = dibs->drv_priv; 245 246 read_lock_bh(&ldev->dmb_ht_lock); 247 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) { 248 if (tmp_node->token == dmb_tok) { 249 rmb_node = tmp_node; 250 break; 251 } 252 } 253 if (!rmb_node) { 254 read_unlock_bh(&ldev->dmb_ht_lock); 255 return -EINVAL; 256 } 257 if ((u64)offset + size > rmb_node->len) { 258 read_unlock_bh(&ldev->dmb_ht_lock); 259 return -EINVAL; 260 } 261 262 memcpy((char *)rmb_node->cpu_addr + offset, data, size); 263 sba_idx = rmb_node->sba_idx; 264 read_unlock_bh(&ldev->dmb_ht_lock); 265 266 if (!sf) 267 return 0; 268 269 spin_lock(&dibs->lock); 270 client_id = dibs->dmb_clientid_arr[sba_idx]; 271 s_mask = ror16(0x1000, idx); 272 if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id])) 273 dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask); 274 spin_unlock(&dibs->lock); 275 276 return 0; 277 } 278 279 static const struct dibs_dev_ops dibs_lo_ops = { 280 .get_fabric_id = dibs_lo_get_fabric_id, 281 .query_remote_gid = dibs_lo_query_rgid, 282 .max_dmbs = dibs_lo_max_dmbs, 283 .register_dmb = dibs_lo_register_dmb, 284 .unregister_dmb = dibs_lo_unregister_dmb, 285 .move_data = dibs_lo_move_data, 286 .support_mmapped_rdmb = dibs_lo_support_dmb_nocopy, 287 .attach_dmb = dibs_lo_attach_dmb, 288 .detach_dmb = dibs_lo_detach_dmb, 289 }; 290 291 static void dibs_lo_dev_init(struct dibs_lo_dev *ldev) 292 { 293 rwlock_init(&ldev->dmb_ht_lock); 294 hash_init(ldev->dmb_ht); 295 atomic_set(&ldev->dmb_cnt, 0); 296 init_waitqueue_head(&ldev->ldev_release); 297 } 298 299 static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev) 300 { 301 if (atomic_read(&ldev->dmb_cnt)) 302 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt)); 303 } 304 305 static int dibs_lo_dev_probe(void) 306 { 307 struct dibs_lo_dev *ldev; 308 struct dibs_dev *dibs; 309 int ret; 310 311 ldev = kzalloc_obj(*ldev); 312 if (!ldev) 313 return -ENOMEM; 314 315 dibs = dibs_dev_alloc(); 316 if (!dibs) { 317 kfree(ldev); 318 return -ENOMEM; 319 } 320 321 ldev->dibs = dibs; 322 dibs->drv_priv = ldev; 323 dibs_lo_dev_init(ldev); 324 uuid_gen(&dibs->gid); 325 dibs->ops = &dibs_lo_ops; 326 327 dibs->dev.parent = NULL; 328 dev_set_name(&dibs->dev, "%s", dibs_lo_dev_name); 329 330 ret = dibs_dev_add(dibs); 331 if (ret) 332 goto err_reg; 333 lo_dev = ldev; 334 return 0; 335 336 err_reg: 337 kfree(dibs->dmb_clientid_arr); 338 /* pairs with dibs_dev_alloc() */ 339 put_device(&dibs->dev); 340 kfree(ldev); 341 342 return ret; 343 } 344 345 static void dibs_lo_dev_remove(void) 346 { 347 if (!lo_dev) 348 return; 349 350 dibs_dev_del(lo_dev->dibs); 351 dibs_lo_dev_exit(lo_dev); 352 /* pairs with dibs_dev_alloc() */ 353 put_device(&lo_dev->dibs->dev); 354 kfree(lo_dev); 355 lo_dev = NULL; 356 } 357 358 int dibs_loopback_init(void) 359 { 360 return dibs_lo_dev_probe(); 361 } 362 363 void dibs_loopback_exit(void) 364 { 365 dibs_lo_dev_remove(); 366 } 367