1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Functions for dibs loopback/loopback-ism device.
4 *
5 * Copyright (c) 2024, Alibaba Inc.
6 *
7 * Author: Wen Gu <guwen@linux.alibaba.com>
8 * Tony Lu <tonylu@linux.alibaba.com>
9 *
10 */
11
12 #include <linux/bitops.h>
13 #include <linux/device.h>
14 #include <linux/dibs.h>
15 #include <linux/mm.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/types.h>
19
20 #include "dibs_loopback.h"
21
22 #define DIBS_LO_SUPPORT_NOCOPY 0x1
23 #define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0)
24
25 static const char dibs_lo_dev_name[] = "lo";
26 /* global loopback device */
27 static struct dibs_lo_dev *lo_dev;
28
dibs_lo_get_fabric_id(struct dibs_dev * dibs)29 static u16 dibs_lo_get_fabric_id(struct dibs_dev *dibs)
30 {
31 return DIBS_LOOPBACK_FABRIC;
32 }
33
dibs_lo_query_rgid(struct dibs_dev * dibs,const uuid_t * rgid,u32 vid_valid,u32 vid)34 static int dibs_lo_query_rgid(struct dibs_dev *dibs, const uuid_t *rgid,
35 u32 vid_valid, u32 vid)
36 {
37 /* rgid should be the same as lgid */
38 if (!uuid_equal(rgid, &dibs->gid))
39 return -ENETUNREACH;
40 return 0;
41 }
42
dibs_lo_max_dmbs(void)43 static int dibs_lo_max_dmbs(void)
44 {
45 return DIBS_LO_MAX_DMBS;
46 }
47
dibs_lo_register_dmb(struct dibs_dev * dibs,struct dibs_dmb * dmb,struct dibs_client * client)48 static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
49 struct dibs_client *client)
50 {
51 struct dibs_lo_dmb_node *dmb_node, *tmp_node;
52 struct dibs_lo_dev *ldev;
53 struct folio *folio;
54 unsigned long flags;
55 int sba_idx, rc;
56
57 ldev = dibs->drv_priv;
58 sba_idx = dmb->idx;
59 /* check space for new dmb */
60 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) {
61 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
62 break;
63 }
64 if (sba_idx == DIBS_LO_MAX_DMBS)
65 return -ENOSPC;
66
67 dmb_node = kzalloc_obj(*dmb_node);
68 if (!dmb_node) {
69 rc = -ENOMEM;
70 goto err_bit;
71 }
72
73 dmb_node->sba_idx = sba_idx;
74 dmb_node->len = dmb->dmb_len;
75
76 /* not critical; fail under memory pressure and fallback to TCP */
77 folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC |
78 __GFP_NORETRY | __GFP_ZERO,
79 get_order(dmb_node->len));
80 if (!folio) {
81 rc = -ENOMEM;
82 goto err_node;
83 }
84 dmb_node->cpu_addr = folio_address(folio);
85 dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID;
86 refcount_set(&dmb_node->refcnt, 1);
87
88 again:
89 /* add new dmb into hash table */
90 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
91 write_lock_bh(&ldev->dmb_ht_lock);
92 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
93 if (tmp_node->token == dmb_node->token) {
94 write_unlock_bh(&ldev->dmb_ht_lock);
95 goto again;
96 }
97 }
98 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
99 write_unlock_bh(&ldev->dmb_ht_lock);
100 atomic_inc(&ldev->dmb_cnt);
101
102 dmb->idx = dmb_node->sba_idx;
103 dmb->dmb_tok = dmb_node->token;
104 dmb->cpu_addr = dmb_node->cpu_addr;
105 dmb->dma_addr = dmb_node->dma_addr;
106 dmb->dmb_len = dmb_node->len;
107
108 spin_lock_irqsave(&dibs->lock, flags);
109 dibs->dmb_clientid_arr[sba_idx] = client->id;
110 spin_unlock_irqrestore(&dibs->lock, flags);
111
112 return 0;
113
114 err_node:
115 kfree(dmb_node);
116 err_bit:
117 clear_bit(sba_idx, ldev->sba_idx_mask);
118 return rc;
119 }
120
__dibs_lo_unregister_dmb(struct dibs_lo_dev * ldev,struct dibs_lo_dmb_node * dmb_node)121 static void __dibs_lo_unregister_dmb(struct dibs_lo_dev *ldev,
122 struct dibs_lo_dmb_node *dmb_node)
123 {
124 /* remove dmb from hash table */
125 write_lock_bh(&ldev->dmb_ht_lock);
126 hash_del(&dmb_node->list);
127 write_unlock_bh(&ldev->dmb_ht_lock);
128
129 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
130 folio_put(virt_to_folio(dmb_node->cpu_addr));
131 kfree(dmb_node);
132
133 if (atomic_dec_and_test(&ldev->dmb_cnt))
134 wake_up(&ldev->ldev_release);
135 }
136
dibs_lo_unregister_dmb(struct dibs_dev * dibs,struct dibs_dmb * dmb)137 static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
138 {
139 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
140 struct dibs_lo_dev *ldev;
141 unsigned long flags;
142
143 ldev = dibs->drv_priv;
144
145 /* find dmb from hash table */
146 read_lock_bh(&ldev->dmb_ht_lock);
147 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
148 if (tmp_node->token == dmb->dmb_tok) {
149 dmb_node = tmp_node;
150 break;
151 }
152 }
153 read_unlock_bh(&ldev->dmb_ht_lock);
154 if (!dmb_node)
155 return -EINVAL;
156
157 if (refcount_dec_and_test(&dmb_node->refcnt)) {
158 spin_lock_irqsave(&dibs->lock, flags);
159 dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT;
160 spin_unlock_irqrestore(&dibs->lock, flags);
161
162 __dibs_lo_unregister_dmb(ldev, dmb_node);
163 }
164 return 0;
165 }
166
dibs_lo_support_dmb_nocopy(struct dibs_dev * dibs)167 static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs)
168 {
169 return DIBS_LO_SUPPORT_NOCOPY;
170 }
171
dibs_lo_attach_dmb(struct dibs_dev * dibs,struct dibs_dmb * dmb)172 static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
173 {
174 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
175 struct dibs_lo_dev *ldev;
176
177 ldev = dibs->drv_priv;
178
179 /* find dmb_node according to dmb->dmb_tok */
180 read_lock_bh(&ldev->dmb_ht_lock);
181 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
182 if (tmp_node->token == dmb->dmb_tok) {
183 dmb_node = tmp_node;
184 break;
185 }
186 }
187 if (!dmb_node) {
188 read_unlock_bh(&ldev->dmb_ht_lock);
189 return -EINVAL;
190 }
191 read_unlock_bh(&ldev->dmb_ht_lock);
192
193 if (!refcount_inc_not_zero(&dmb_node->refcnt))
194 /* the dmb is being unregistered, but has
195 * not been removed from the hash table.
196 */
197 return -EINVAL;
198
199 /* provide dmb information */
200 dmb->idx = dmb_node->sba_idx;
201 dmb->dmb_tok = dmb_node->token;
202 dmb->cpu_addr = dmb_node->cpu_addr;
203 dmb->dma_addr = dmb_node->dma_addr;
204 dmb->dmb_len = dmb_node->len;
205 return 0;
206 }
207
dibs_lo_detach_dmb(struct dibs_dev * dibs,u64 token)208 static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token)
209 {
210 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
211 struct dibs_lo_dev *ldev;
212
213 ldev = dibs->drv_priv;
214
215 /* find dmb_node according to dmb->dmb_tok */
216 read_lock_bh(&ldev->dmb_ht_lock);
217 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
218 if (tmp_node->token == token) {
219 dmb_node = tmp_node;
220 break;
221 }
222 }
223 if (!dmb_node) {
224 read_unlock_bh(&ldev->dmb_ht_lock);
225 return -EINVAL;
226 }
227 read_unlock_bh(&ldev->dmb_ht_lock);
228
229 if (refcount_dec_and_test(&dmb_node->refcnt))
230 __dibs_lo_unregister_dmb(ldev, dmb_node);
231 return 0;
232 }
233
dibs_lo_move_data(struct dibs_dev * dibs,u64 dmb_tok,unsigned int idx,bool sf,unsigned int offset,void * data,unsigned int size)234 static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok,
235 unsigned int idx, bool sf, unsigned int offset,
236 void *data, unsigned int size)
237 {
238 struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node;
239 struct dibs_lo_dev *ldev;
240 u16 s_mask;
241 u8 client_id;
242 u32 sba_idx;
243
244 ldev = dibs->drv_priv;
245
246 read_lock_bh(&ldev->dmb_ht_lock);
247 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
248 if (tmp_node->token == dmb_tok) {
249 rmb_node = tmp_node;
250 break;
251 }
252 }
253 if (!rmb_node) {
254 read_unlock_bh(&ldev->dmb_ht_lock);
255 return -EINVAL;
256 }
257 memcpy((char *)rmb_node->cpu_addr + offset, data, size);
258 sba_idx = rmb_node->sba_idx;
259 read_unlock_bh(&ldev->dmb_ht_lock);
260
261 if (!sf)
262 return 0;
263
264 spin_lock(&dibs->lock);
265 client_id = dibs->dmb_clientid_arr[sba_idx];
266 s_mask = ror16(0x1000, idx);
267 if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id]))
268 dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask);
269 spin_unlock(&dibs->lock);
270
271 return 0;
272 }
273
274 static const struct dibs_dev_ops dibs_lo_ops = {
275 .get_fabric_id = dibs_lo_get_fabric_id,
276 .query_remote_gid = dibs_lo_query_rgid,
277 .max_dmbs = dibs_lo_max_dmbs,
278 .register_dmb = dibs_lo_register_dmb,
279 .unregister_dmb = dibs_lo_unregister_dmb,
280 .move_data = dibs_lo_move_data,
281 .support_mmapped_rdmb = dibs_lo_support_dmb_nocopy,
282 .attach_dmb = dibs_lo_attach_dmb,
283 .detach_dmb = dibs_lo_detach_dmb,
284 };
285
dibs_lo_dev_init(struct dibs_lo_dev * ldev)286 static void dibs_lo_dev_init(struct dibs_lo_dev *ldev)
287 {
288 rwlock_init(&ldev->dmb_ht_lock);
289 hash_init(ldev->dmb_ht);
290 atomic_set(&ldev->dmb_cnt, 0);
291 init_waitqueue_head(&ldev->ldev_release);
292 }
293
dibs_lo_dev_exit(struct dibs_lo_dev * ldev)294 static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev)
295 {
296 if (atomic_read(&ldev->dmb_cnt))
297 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
298 }
299
dibs_lo_dev_probe(void)300 static int dibs_lo_dev_probe(void)
301 {
302 struct dibs_lo_dev *ldev;
303 struct dibs_dev *dibs;
304 int ret;
305
306 ldev = kzalloc_obj(*ldev);
307 if (!ldev)
308 return -ENOMEM;
309
310 dibs = dibs_dev_alloc();
311 if (!dibs) {
312 kfree(ldev);
313 return -ENOMEM;
314 }
315
316 ldev->dibs = dibs;
317 dibs->drv_priv = ldev;
318 dibs_lo_dev_init(ldev);
319 uuid_gen(&dibs->gid);
320 dibs->ops = &dibs_lo_ops;
321
322 dibs->dev.parent = NULL;
323 dev_set_name(&dibs->dev, "%s", dibs_lo_dev_name);
324
325 ret = dibs_dev_add(dibs);
326 if (ret)
327 goto err_reg;
328 lo_dev = ldev;
329 return 0;
330
331 err_reg:
332 kfree(dibs->dmb_clientid_arr);
333 /* pairs with dibs_dev_alloc() */
334 put_device(&dibs->dev);
335 kfree(ldev);
336
337 return ret;
338 }
339
dibs_lo_dev_remove(void)340 static void dibs_lo_dev_remove(void)
341 {
342 if (!lo_dev)
343 return;
344
345 dibs_dev_del(lo_dev->dibs);
346 dibs_lo_dev_exit(lo_dev);
347 /* pairs with dibs_dev_alloc() */
348 put_device(&lo_dev->dibs->dev);
349 kfree(lo_dev);
350 lo_dev = NULL;
351 }
352
dibs_loopback_init(void)353 int dibs_loopback_init(void)
354 {
355 return dibs_lo_dev_probe();
356 }
357
dibs_loopback_exit(void)358 void dibs_loopback_exit(void)
359 {
360 dibs_lo_dev_remove();
361 }
362