1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions for dibs loopback/loopback-ism device. 4 * 5 * Copyright (c) 2024, Alibaba Inc. 6 * 7 * Author: Wen Gu <guwen@linux.alibaba.com> 8 * Tony Lu <tonylu@linux.alibaba.com> 9 * 10 */ 11 12 #include <linux/bitops.h> 13 #include <linux/device.h> 14 #include <linux/dibs.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/types.h> 19 20 #include "dibs_loopback.h" 21 22 #define DIBS_LO_SUPPORT_NOCOPY 0x1 23 #define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0) 24 25 static const char dibs_lo_dev_name[] = "lo"; 26 /* global loopback device */ 27 static struct dibs_lo_dev *lo_dev; 28 29 static u16 dibs_lo_get_fabric_id(struct dibs_dev *dibs) 30 { 31 return DIBS_LOOPBACK_FABRIC; 32 } 33 34 static int dibs_lo_query_rgid(struct dibs_dev *dibs, const uuid_t *rgid, 35 u32 vid_valid, u32 vid) 36 { 37 /* rgid should be the same as lgid */ 38 if (!uuid_equal(rgid, &dibs->gid)) 39 return -ENETUNREACH; 40 return 0; 41 } 42 43 static int dibs_lo_max_dmbs(void) 44 { 45 return DIBS_LO_MAX_DMBS; 46 } 47 48 static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb, 49 struct dibs_client *client) 50 { 51 struct dibs_lo_dmb_node *dmb_node, *tmp_node; 52 struct dibs_lo_dev *ldev; 53 struct folio *folio; 54 unsigned long flags; 55 int sba_idx, rc; 56 57 ldev = dibs->drv_priv; 58 sba_idx = dmb->idx; 59 /* check space for new dmb */ 60 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) { 61 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) 62 break; 63 } 64 if (sba_idx == DIBS_LO_MAX_DMBS) 65 return -ENOSPC; 66 67 dmb_node = kzalloc_obj(*dmb_node); 68 if (!dmb_node) { 69 rc = -ENOMEM; 70 goto err_bit; 71 } 72 73 dmb_node->sba_idx = sba_idx; 74 dmb_node->len = dmb->dmb_len; 75 76 /* not critical; fail under memory pressure and fallback to TCP */ 77 folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | 78 __GFP_NORETRY | __GFP_ZERO, 79 get_order(dmb_node->len)); 80 if (!folio) { 81 rc = -ENOMEM; 82 goto err_node; 83 } 84 dmb_node->cpu_addr = folio_address(folio); 85 dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID; 86 refcount_set(&dmb_node->refcnt, 1); 87 88 again: 89 /* add new dmb into hash table */ 90 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); 91 write_lock_bh(&ldev->dmb_ht_lock); 92 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { 93 if (tmp_node->token == dmb_node->token) { 94 write_unlock_bh(&ldev->dmb_ht_lock); 95 goto again; 96 } 97 } 98 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); 99 write_unlock_bh(&ldev->dmb_ht_lock); 100 atomic_inc(&ldev->dmb_cnt); 101 102 dmb->idx = dmb_node->sba_idx; 103 dmb->dmb_tok = dmb_node->token; 104 dmb->cpu_addr = dmb_node->cpu_addr; 105 dmb->dma_addr = dmb_node->dma_addr; 106 dmb->dmb_len = dmb_node->len; 107 108 spin_lock_irqsave(&dibs->lock, flags); 109 dibs->dmb_clientid_arr[sba_idx] = client->id; 110 spin_unlock_irqrestore(&dibs->lock, flags); 111 112 return 0; 113 114 err_node: 115 kfree(dmb_node); 116 err_bit: 117 clear_bit(sba_idx, ldev->sba_idx_mask); 118 return rc; 119 } 120 121 static void dibs_lo_free_dmb(struct dibs_lo_dev *ldev, 122 struct dibs_lo_dmb_node *dmb_node) 123 { 124 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); 125 folio_put(virt_to_folio(dmb_node->cpu_addr)); 126 kfree(dmb_node); 127 128 if (atomic_dec_and_test(&ldev->dmb_cnt)) 129 wake_up(&ldev->ldev_release); 130 } 131 132 static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) 133 { 134 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 135 struct dibs_lo_dev *ldev; 136 unsigned long flags; 137 bool last; 138 139 ldev = dibs->drv_priv; 140 141 /* find dmb from hash table */ 142 write_lock_bh(&ldev->dmb_ht_lock); 143 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 144 if (tmp_node->token == dmb->dmb_tok) { 145 dmb_node = tmp_node; 146 break; 147 } 148 } 149 if (!dmb_node) { 150 write_unlock_bh(&ldev->dmb_ht_lock); 151 return -EINVAL; 152 } 153 last = refcount_dec_and_test(&dmb_node->refcnt); 154 if (last) 155 hash_del(&dmb_node->list); 156 write_unlock_bh(&ldev->dmb_ht_lock); 157 158 if (last) { 159 spin_lock_irqsave(&dibs->lock, flags); 160 dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT; 161 spin_unlock_irqrestore(&dibs->lock, flags); 162 163 dibs_lo_free_dmb(ldev, dmb_node); 164 } 165 return 0; 166 } 167 168 static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs) 169 { 170 return DIBS_LO_SUPPORT_NOCOPY; 171 } 172 173 static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) 174 { 175 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 176 struct dibs_lo_dev *ldev; 177 178 ldev = dibs->drv_priv; 179 180 /* find dmb_node according to dmb->dmb_tok */ 181 read_lock_bh(&ldev->dmb_ht_lock); 182 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { 183 if (tmp_node->token == dmb->dmb_tok) { 184 dmb_node = tmp_node; 185 break; 186 } 187 } 188 if (!dmb_node) { 189 read_unlock_bh(&ldev->dmb_ht_lock); 190 return -EINVAL; 191 } 192 refcount_inc(&dmb_node->refcnt); 193 read_unlock_bh(&ldev->dmb_ht_lock); 194 195 /* provide dmb information */ 196 dmb->idx = dmb_node->sba_idx; 197 dmb->dmb_tok = dmb_node->token; 198 dmb->cpu_addr = dmb_node->cpu_addr; 199 dmb->dma_addr = dmb_node->dma_addr; 200 dmb->dmb_len = dmb_node->len; 201 return 0; 202 } 203 204 static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token) 205 { 206 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; 207 struct dibs_lo_dev *ldev; 208 bool last; 209 210 ldev = dibs->drv_priv; 211 212 /* find dmb_node according to dmb->dmb_tok */ 213 write_lock_bh(&ldev->dmb_ht_lock); 214 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { 215 if (tmp_node->token == token) { 216 dmb_node = tmp_node; 217 break; 218 } 219 } 220 if (!dmb_node) { 221 write_unlock_bh(&ldev->dmb_ht_lock); 222 return -EINVAL; 223 } 224 last = refcount_dec_and_test(&dmb_node->refcnt); 225 if (last) 226 hash_del(&dmb_node->list); 227 write_unlock_bh(&ldev->dmb_ht_lock); 228 229 if (last) 230 dibs_lo_free_dmb(ldev, dmb_node); 231 232 return 0; 233 } 234 235 static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok, 236 unsigned int idx, bool sf, unsigned int offset, 237 void *data, unsigned int size) 238 { 239 struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node; 240 struct dibs_lo_dev *ldev; 241 u16 s_mask; 242 u8 client_id; 243 u32 sba_idx; 244 245 ldev = dibs->drv_priv; 246 247 read_lock_bh(&ldev->dmb_ht_lock); 248 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) { 249 if (tmp_node->token == dmb_tok) { 250 rmb_node = tmp_node; 251 break; 252 } 253 } 254 if (!rmb_node) { 255 read_unlock_bh(&ldev->dmb_ht_lock); 256 return -EINVAL; 257 } 258 if ((u64)offset + size > rmb_node->len) { 259 read_unlock_bh(&ldev->dmb_ht_lock); 260 return -EINVAL; 261 } 262 263 memcpy((char *)rmb_node->cpu_addr + offset, data, size); 264 sba_idx = rmb_node->sba_idx; 265 read_unlock_bh(&ldev->dmb_ht_lock); 266 267 if (!sf) 268 return 0; 269 270 spin_lock(&dibs->lock); 271 client_id = dibs->dmb_clientid_arr[sba_idx]; 272 s_mask = ror16(0x1000, idx); 273 if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id])) 274 dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask); 275 spin_unlock(&dibs->lock); 276 277 return 0; 278 } 279 280 static const struct dibs_dev_ops dibs_lo_ops = { 281 .get_fabric_id = dibs_lo_get_fabric_id, 282 .query_remote_gid = dibs_lo_query_rgid, 283 .max_dmbs = dibs_lo_max_dmbs, 284 .register_dmb = dibs_lo_register_dmb, 285 .unregister_dmb = dibs_lo_unregister_dmb, 286 .move_data = dibs_lo_move_data, 287 .support_mmapped_rdmb = dibs_lo_support_dmb_nocopy, 288 .attach_dmb = dibs_lo_attach_dmb, 289 .detach_dmb = dibs_lo_detach_dmb, 290 }; 291 292 static void dibs_lo_dev_init(struct dibs_lo_dev *ldev) 293 { 294 rwlock_init(&ldev->dmb_ht_lock); 295 hash_init(ldev->dmb_ht); 296 atomic_set(&ldev->dmb_cnt, 0); 297 init_waitqueue_head(&ldev->ldev_release); 298 } 299 300 static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev) 301 { 302 if (atomic_read(&ldev->dmb_cnt)) 303 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt)); 304 } 305 306 static int dibs_lo_dev_probe(void) 307 { 308 struct dibs_lo_dev *ldev; 309 struct dibs_dev *dibs; 310 int ret; 311 312 ldev = kzalloc_obj(*ldev); 313 if (!ldev) 314 return -ENOMEM; 315 316 dibs = dibs_dev_alloc(); 317 if (!dibs) { 318 kfree(ldev); 319 return -ENOMEM; 320 } 321 322 ldev->dibs = dibs; 323 dibs->drv_priv = ldev; 324 dibs_lo_dev_init(ldev); 325 uuid_gen(&dibs->gid); 326 dibs->ops = &dibs_lo_ops; 327 328 dibs->dev.parent = NULL; 329 dev_set_name(&dibs->dev, "%s", dibs_lo_dev_name); 330 331 ret = dibs_dev_add(dibs); 332 if (ret) 333 goto err_reg; 334 lo_dev = ldev; 335 return 0; 336 337 err_reg: 338 /* pairs with dibs_dev_alloc() */ 339 put_device(&dibs->dev); 340 kfree(ldev); 341 342 return ret; 343 } 344 345 static void dibs_lo_dev_remove(void) 346 { 347 if (!lo_dev) 348 return; 349 350 dibs_dev_del(lo_dev->dibs); 351 dibs_lo_dev_exit(lo_dev); 352 /* pairs with dibs_dev_alloc() */ 353 put_device(&lo_dev->dibs->dev); 354 kfree(lo_dev); 355 lo_dev = NULL; 356 } 357 358 int dibs_loopback_init(void) 359 { 360 return dibs_lo_dev_probe(); 361 } 362 363 void dibs_loopback_exit(void) 364 { 365 dibs_lo_dev_remove(); 366 } 367