1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #include <linux/libnvdimm.h>
8
9 #include "rxe.h"
10 #include "rxe_loc.h"
11
12 /* Return a random 8 bit key value that is
13 * different than the last_key. Set last_key to -1
14 * if this is the first key for an MR or MW
15 */
rxe_get_next_key(u32 last_key)16 u8 rxe_get_next_key(u32 last_key)
17 {
18 u8 key;
19
20 do {
21 get_random_bytes(&key, 1);
22 } while (key == last_key);
23
24 return key;
25 }
26
mr_check_range(struct rxe_mr * mr,u64 iova,size_t length)27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
28 {
29 switch (mr->ibmr.type) {
30 case IB_MR_TYPE_DMA:
31 return 0;
32
33 case IB_MR_TYPE_USER:
34 case IB_MR_TYPE_MEM_REG:
35 if (iova < mr->ibmr.iova ||
36 length > mr->ibmr.length ||
37 iova - mr->ibmr.iova > mr->ibmr.length - length) {
38 rxe_dbg_mr(mr, "iova/length out of range\n");
39 return -EINVAL;
40 }
41 return 0;
42
43 default:
44 rxe_dbg_mr(mr, "mr type not supported\n");
45 return -EINVAL;
46 }
47 }
48
rxe_mr_init(int access,struct rxe_mr * mr)49 void rxe_mr_init(int access, struct rxe_mr *mr)
50 {
51 u32 key = mr->elem.index << 8 | rxe_get_next_key(-1);
52
53 /* set ibmr->l/rkey and also copy into private l/rkey
54 * for user MRs these will always be the same
55 * for cases where caller 'owns' the key portion
56 * they may be different until REG_MR WQE is executed.
57 */
58 mr->lkey = mr->ibmr.lkey = key;
59 mr->rkey = mr->ibmr.rkey = key;
60
61 mr->access = access;
62 mr->ibmr.page_size = PAGE_SIZE;
63 mr->page_mask = PAGE_MASK;
64 mr->page_shift = PAGE_SHIFT;
65 mr->state = RXE_MR_STATE_INVALID;
66 }
67
rxe_mr_init_dma(int access,struct rxe_mr * mr)68 void rxe_mr_init_dma(int access, struct rxe_mr *mr)
69 {
70 rxe_mr_init(access, mr);
71
72 mr->state = RXE_MR_STATE_VALID;
73 mr->ibmr.type = IB_MR_TYPE_DMA;
74 }
75
76 /*
77 * Convert iova to page_info index. The page_info stores pages of size
78 * PAGE_SIZE, but MRs can have different page sizes. This function
79 * handles the conversion for all cases:
80 *
81 * 1. mr->page_size > PAGE_SIZE:
82 * The MR's iova may not be aligned to mr->page_size. We use the
83 * aligned base (iova & page_mask) as reference, then calculate
84 * which PAGE_SIZE sub-page the iova falls into.
85 *
86 * 2. mr->page_size <= PAGE_SIZE:
87 * Use simple shift arithmetic since each page_info entry corresponds
88 * to one or more MR pages.
89 */
rxe_mr_iova_to_index(struct rxe_mr * mr,u64 iova)90 static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
91 {
92 int idx;
93
94 if (mr_page_size(mr) > PAGE_SIZE)
95 idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT;
96 else
97 idx = (iova >> mr->page_shift) -
98 (mr->ibmr.iova >> mr->page_shift);
99
100 WARN_ON(idx >= mr->nbuf);
101 return idx;
102 }
103
104 /*
105 * Convert iova to offset within the page_info entry.
106 *
107 * For mr_page_size > PAGE_SIZE, the offset is within the system page.
108 * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size.
109 */
rxe_mr_iova_to_page_offset(struct rxe_mr * mr,u64 iova)110 static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
111 {
112 if (mr_page_size(mr) > PAGE_SIZE)
113 return iova & (PAGE_SIZE - 1);
114 else
115 return iova & (mr_page_size(mr) - 1);
116 }
117
is_pmem_page(struct page * pg)118 static bool is_pmem_page(struct page *pg)
119 {
120 unsigned long paddr = page_to_phys(pg);
121
122 return REGION_INTERSECTS ==
123 region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
124 IORES_DESC_PERSISTENT_MEMORY);
125 }
126
rxe_mr_fill_pages_from_sgt(struct rxe_mr * mr,struct sg_table * sgt)127 static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
128 {
129 struct sg_page_iter sg_iter;
130 struct page *page;
131 bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
132
133 WARN_ON(mr_page_size(mr) != PAGE_SIZE);
134
135 __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
136 if (!__sg_page_iter_next(&sg_iter))
137 return 0;
138
139 while (true) {
140 page = sg_page_iter_page(&sg_iter);
141
142 if (persistent && !is_pmem_page(page)) {
143 rxe_dbg_mr(mr, "Page can't be persistent\n");
144 return -EINVAL;
145 }
146
147 mr->page_info[mr->nbuf].page = page;
148 mr->page_info[mr->nbuf].offset = 0;
149 mr->nbuf++;
150
151 if (!__sg_page_iter_next(&sg_iter))
152 break;
153 }
154
155 return 0;
156 }
157
__alloc_mr_page_info(struct rxe_mr * mr,int num_pages)158 static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages)
159 {
160 mr->page_info = kzalloc_objs(struct rxe_mr_page, num_pages);
161 if (!mr->page_info)
162 return -ENOMEM;
163
164 mr->max_allowed_buf = num_pages;
165 mr->nbuf = 0;
166
167 return 0;
168 }
169
alloc_mr_page_info(struct rxe_mr * mr,int num_pages)170 static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages)
171 {
172 int ret;
173
174 WARN_ON(mr->num_buf);
175 ret = __alloc_mr_page_info(mr, num_pages);
176 if (ret)
177 return ret;
178
179 mr->num_buf = num_pages;
180
181 return 0;
182 }
183
free_mr_page_info(struct rxe_mr * mr)184 static void free_mr_page_info(struct rxe_mr *mr)
185 {
186 if (!mr->page_info)
187 return;
188
189 kfree(mr->page_info);
190 mr->page_info = NULL;
191 }
192
rxe_mr_init_user(struct rxe_dev * rxe,u64 start,u64 length,int access,struct rxe_mr * mr)193 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
194 int access, struct rxe_mr *mr)
195 {
196 struct ib_umem *umem;
197 int err;
198
199 rxe_mr_init(access, mr);
200
201 umem = ib_umem_get_va(&rxe->ib_dev, start, length, access);
202 if (IS_ERR(umem)) {
203 rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
204 (int)PTR_ERR(umem));
205 return PTR_ERR(umem);
206 }
207
208 err = alloc_mr_page_info(mr, ib_umem_num_pages(umem));
209 if (err)
210 goto err2;
211
212 err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
213 if (err)
214 goto err1;
215
216 mr->umem = umem;
217 mr->ibmr.type = IB_MR_TYPE_USER;
218 mr->state = RXE_MR_STATE_VALID;
219
220 return 0;
221 err1:
222 free_mr_page_info(mr);
223 err2:
224 ib_umem_release(umem);
225 return err;
226 }
227
rxe_mr_init_fast(int max_pages,struct rxe_mr * mr)228 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
229 {
230 int err;
231
232 /* always allow remote access for FMRs */
233 rxe_mr_init(RXE_ACCESS_REMOTE, mr);
234
235 err = alloc_mr_page_info(mr, max_pages);
236 if (err)
237 goto err1;
238
239 mr->state = RXE_MR_STATE_FREE;
240 mr->ibmr.type = IB_MR_TYPE_MEM_REG;
241
242 return 0;
243
244 err1:
245 return err;
246 }
247
248 /*
249 * I) MRs with page_size >= PAGE_SIZE,
250 * Split a large MR page (mr->page_size) into multiple PAGE_SIZE
251 * sub-pages and store them in page_info, offset is always 0.
252 *
253 * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page()
254 * represents one mr->page_size region, which we must split into
255 * (mr->page_size >> PAGE_SHIFT) individual pages.
256 *
257 * II) MRs with page_size < PAGE_SIZE,
258 * Save each PAGE_SIZE page and its offset within the system page in page_info.
259 */
rxe_set_page(struct ib_mr * ibmr,u64 dma_addr)260 static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr)
261 {
262 struct rxe_mr *mr = to_rmr(ibmr);
263 bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
264 u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT;
265
266 pages_per_mr = MAX(1, pages_per_mr);
267
268 for (i = 0; i < pages_per_mr; i++) {
269 u64 addr = dma_addr + i * PAGE_SIZE;
270 struct page *sub_page = ib_virt_dma_to_page(addr);
271
272 if (unlikely(mr->nbuf >= mr->max_allowed_buf))
273 return -ENOMEM;
274
275 if (persistent && !is_pmem_page(sub_page)) {
276 rxe_dbg_mr(mr, "Page cannot be persistent\n");
277 return -EINVAL;
278 }
279
280 mr->page_info[mr->nbuf].page = sub_page;
281 mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1);
282 mr->nbuf++;
283 }
284
285 return 0;
286 }
287
rxe_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sgl,int sg_nents,unsigned int * sg_offset)288 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,
289 int sg_nents, unsigned int *sg_offset)
290 {
291 struct rxe_mr *mr = to_rmr(ibmr);
292 unsigned int page_size = mr_page_size(mr);
293
294 /*
295 * Ensure page_size and PAGE_SIZE are compatible for mapping.
296 * We require one to be a multiple of the other for correct
297 * iova-to-page conversion.
298 */
299 if (!IS_ALIGNED(page_size, PAGE_SIZE) &&
300 !IS_ALIGNED(PAGE_SIZE, page_size)) {
301 rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n",
302 page_size, PAGE_SIZE);
303 return -EINVAL;
304 }
305
306 if (mr_page_size(mr) > PAGE_SIZE) {
307 /* resize page_info if needed */
308 u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf;
309
310 if (map_mr_pages > mr->max_allowed_buf) {
311 rxe_dbg_mr(mr, "requested pages %u exceed max %u\n",
312 map_mr_pages, mr->max_allowed_buf);
313 free_mr_page_info(mr);
314 if (__alloc_mr_page_info(mr, map_mr_pages))
315 return -ENOMEM;
316 }
317 }
318
319 mr->nbuf = 0;
320 mr->page_shift = ilog2(page_size);
321 mr->page_mask = ~((u64)page_size - 1);
322
323 return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
324 }
325
rxe_mr_copy_xarray(struct rxe_mr * mr,u64 iova,void * addr,unsigned int length,enum rxe_mr_copy_dir dir)326 static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
327 unsigned int length, enum rxe_mr_copy_dir dir)
328 {
329 unsigned int bytes;
330 u8 *va;
331
332 while (length) {
333 unsigned long index = rxe_mr_iova_to_index(mr, iova);
334 struct rxe_mr_page *info = &mr->page_info[index];
335 unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
336
337 if (!info->page)
338 return -EFAULT;
339
340 page_offset += info->offset;
341 bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset);
342 va = kmap_local_page(info->page);
343
344 if (dir == RXE_FROM_MR_OBJ)
345 memcpy(addr, va + page_offset, bytes);
346 else
347 memcpy(va + page_offset, addr, bytes);
348 kunmap_local(va);
349
350 addr += bytes;
351 iova += bytes;
352 length -= bytes;
353 }
354
355 return 0;
356 }
357
rxe_mr_copy_dma(struct rxe_mr * mr,u64 dma_addr,void * addr,unsigned int length,enum rxe_mr_copy_dir dir)358 static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr,
359 unsigned int length, enum rxe_mr_copy_dir dir)
360 {
361 unsigned int page_offset = dma_addr & (PAGE_SIZE - 1);
362 unsigned int bytes;
363 struct page *page;
364 u8 *va;
365
366 while (length) {
367 page = ib_virt_dma_to_page(dma_addr);
368 bytes = min_t(unsigned int, length,
369 PAGE_SIZE - page_offset);
370 va = kmap_local_page(page);
371
372 if (dir == RXE_TO_MR_OBJ)
373 memcpy(va + page_offset, addr, bytes);
374 else
375 memcpy(addr, va + page_offset, bytes);
376
377 kunmap_local(va);
378 page_offset = 0;
379 dma_addr += bytes;
380 addr += bytes;
381 length -= bytes;
382 }
383 }
384
rxe_mr_copy(struct rxe_mr * mr,u64 iova,void * addr,unsigned int length,enum rxe_mr_copy_dir dir)385 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
386 unsigned int length, enum rxe_mr_copy_dir dir)
387 {
388 int err;
389
390 if (length == 0)
391 return 0;
392
393 if (WARN_ON(!mr))
394 return -EINVAL;
395
396 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
397 rxe_mr_copy_dma(mr, iova, addr, length, dir);
398 return 0;
399 }
400
401 err = mr_check_range(mr, iova, length);
402 if (unlikely(err)) {
403 rxe_dbg_mr(mr, "iova out of range\n");
404 return err;
405 }
406
407 if (is_odp_mr(mr))
408 return rxe_odp_mr_copy(mr, iova, addr, length, dir);
409 else
410 return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
411 }
412
413 /* copy data in or out of a wqe, i.e. sg list
414 * under the control of a dma descriptor
415 */
copy_data(struct rxe_pd * pd,int access,struct rxe_dma_info * dma,void * addr,int length,enum rxe_mr_copy_dir dir)416 int copy_data(
417 struct rxe_pd *pd,
418 int access,
419 struct rxe_dma_info *dma,
420 void *addr,
421 int length,
422 enum rxe_mr_copy_dir dir)
423 {
424 int bytes;
425 struct rxe_sge *sge = &dma->sge[dma->cur_sge];
426 int offset = dma->sge_offset;
427 int resid = dma->resid;
428 struct rxe_mr *mr = NULL;
429 u64 iova;
430 int err;
431
432 if (length == 0)
433 return 0;
434
435 if (length > resid) {
436 err = -EINVAL;
437 goto err2;
438 }
439
440 if (sge->length && (offset < sge->length)) {
441 mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);
442 if (!mr) {
443 err = -EINVAL;
444 goto err1;
445 }
446 }
447
448 while (length > 0) {
449 bytes = length;
450
451 if (offset >= sge->length) {
452 if (mr) {
453 rxe_put(mr);
454 mr = NULL;
455 }
456 sge++;
457 dma->cur_sge++;
458 offset = 0;
459
460 if (dma->cur_sge >= dma->num_sge) {
461 err = -ENOSPC;
462 goto err2;
463 }
464
465 if (sge->length) {
466 mr = lookup_mr(pd, access, sge->lkey,
467 RXE_LOOKUP_LOCAL);
468 if (!mr) {
469 err = -EINVAL;
470 goto err1;
471 }
472 } else {
473 continue;
474 }
475 }
476
477 if (bytes > sge->length - offset)
478 bytes = sge->length - offset;
479
480 if (bytes > 0) {
481 iova = sge->addr + offset;
482 err = rxe_mr_copy(mr, iova, addr, bytes, dir);
483 if (err)
484 goto err2;
485
486 offset += bytes;
487 resid -= bytes;
488 length -= bytes;
489 addr += bytes;
490 }
491 }
492
493 dma->sge_offset = offset;
494 dma->resid = resid;
495
496 if (mr)
497 rxe_put(mr);
498
499 return 0;
500
501 err2:
502 if (mr)
503 rxe_put(mr);
504 err1:
505 return err;
506 }
507
rxe_mr_flush_pmem_iova(struct rxe_mr * mr,u64 iova,unsigned int length)508 static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
509 {
510 unsigned int bytes;
511 int err;
512 u8 *va;
513
514 err = mr_check_range(mr, iova, length);
515 if (err)
516 return err;
517
518 while (length > 0) {
519 unsigned long index = rxe_mr_iova_to_index(mr, iova);
520 struct rxe_mr_page *info = &mr->page_info[index];
521 unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
522
523 if (!info->page)
524 return -EFAULT;
525
526 page_offset += info->offset;
527 bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset);
528
529 va = kmap_local_page(info->page);
530 arch_wb_cache_pmem(va + page_offset, bytes);
531 kunmap_local(va);
532
533 length -= bytes;
534 iova += bytes;
535 }
536
537 return 0;
538 }
539
rxe_flush_pmem_iova(struct rxe_mr * mr,u64 start,unsigned int length)540 int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 start, unsigned int length)
541 {
542 int err;
543
544 /* mr must be valid even if length is zero */
545 if (WARN_ON(!mr))
546 return -EINVAL;
547
548 if (length == 0)
549 return 0;
550
551 if (mr->ibmr.type == IB_MR_TYPE_DMA)
552 return -EFAULT;
553
554 if (is_odp_mr(mr))
555 err = rxe_odp_flush_pmem_iova(mr, start, length);
556 else
557 err = rxe_mr_flush_pmem_iova(mr, start, length);
558
559 return err;
560 }
561
562 /* Guarantee atomicity of atomic operations at the machine level. */
563 DEFINE_SPINLOCK(atomic_ops_lock);
564
rxe_mr_do_atomic_op(struct rxe_mr * mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 * orig_val)565 enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
566 u64 compare, u64 swap_add, u64 *orig_val)
567 {
568 unsigned int page_offset;
569 struct page *page;
570 u64 value;
571 u64 *va;
572
573 if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
574 rxe_dbg_mr(mr, "mr not in valid state\n");
575 return RESPST_ERR_RKEY_VIOLATION;
576 }
577
578 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
579 page_offset = iova & (PAGE_SIZE - 1);
580 page = ib_virt_dma_to_page(iova);
581 } else {
582 unsigned long index;
583 int err;
584 struct rxe_mr_page *info;
585
586 err = mr_check_range(mr, iova, sizeof(value));
587 if (err) {
588 rxe_dbg_mr(mr, "iova out of range\n");
589 return RESPST_ERR_RKEY_VIOLATION;
590 }
591 page_offset = rxe_mr_iova_to_page_offset(mr, iova);
592 index = rxe_mr_iova_to_index(mr, iova);
593 info = &mr->page_info[index];
594 if (!info->page)
595 return RESPST_ERR_RKEY_VIOLATION;
596
597 page_offset += info->offset;
598 page = info->page;
599 }
600
601 if (unlikely(page_offset & 0x7)) {
602 rxe_dbg_mr(mr, "iova not aligned\n");
603 return RESPST_ERR_MISALIGNED_ATOMIC;
604 }
605
606 va = kmap_local_page(page);
607
608 spin_lock_bh(&atomic_ops_lock);
609 value = *orig_val = va[page_offset >> 3];
610
611 if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
612 if (value == compare)
613 va[page_offset >> 3] = swap_add;
614 } else {
615 value += swap_add;
616 va[page_offset >> 3] = value;
617 }
618 spin_unlock_bh(&atomic_ops_lock);
619
620 kunmap_local(va);
621
622 return RESPST_NONE;
623 }
624
rxe_mr_do_atomic_write(struct rxe_mr * mr,u64 iova,u64 value)625 enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
626 {
627 unsigned int page_offset;
628 struct page *page;
629 u64 *va;
630
631 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
632 page_offset = iova & (PAGE_SIZE - 1);
633 page = ib_virt_dma_to_page(iova);
634 } else {
635 unsigned long index;
636 int err;
637 struct rxe_mr_page *info;
638
639 /* See IBA oA19-28 */
640 err = mr_check_range(mr, iova, sizeof(value));
641 if (unlikely(err)) {
642 rxe_dbg_mr(mr, "iova out of range\n");
643 return RESPST_ERR_RKEY_VIOLATION;
644 }
645 page_offset = rxe_mr_iova_to_page_offset(mr, iova);
646 index = rxe_mr_iova_to_index(mr, iova);
647 info = &mr->page_info[index];
648 if (!info->page)
649 return RESPST_ERR_RKEY_VIOLATION;
650
651 page_offset += info->offset;
652 page = info->page;
653 }
654
655 /* See IBA A19.4.2 */
656 if (unlikely(page_offset & 0x7)) {
657 rxe_dbg_mr(mr, "misaligned address\n");
658 return RESPST_ERR_MISALIGNED_ATOMIC;
659 }
660
661 va = kmap_local_page(page);
662 /* Do atomic write after all prior operations have completed */
663 smp_store_release(&va[page_offset >> 3], value);
664 kunmap_local(va);
665
666 return RESPST_NONE;
667 }
668
advance_dma_data(struct rxe_dma_info * dma,unsigned int length)669 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
670 {
671 struct rxe_sge *sge = &dma->sge[dma->cur_sge];
672 int offset = dma->sge_offset;
673 int resid = dma->resid;
674
675 while (length) {
676 unsigned int bytes;
677
678 if (offset >= sge->length) {
679 sge++;
680 dma->cur_sge++;
681 offset = 0;
682 if (dma->cur_sge >= dma->num_sge)
683 return -ENOSPC;
684 }
685
686 bytes = length;
687
688 if (bytes > sge->length - offset)
689 bytes = sge->length - offset;
690
691 offset += bytes;
692 resid -= bytes;
693 length -= bytes;
694 }
695
696 dma->sge_offset = offset;
697 dma->resid = resid;
698
699 return 0;
700 }
701
lookup_mr(struct rxe_pd * pd,int access,u32 key,enum rxe_mr_lookup_type type)702 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
703 enum rxe_mr_lookup_type type)
704 {
705 struct rxe_mr *mr;
706 struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
707 int index = key >> 8;
708
709 mr = rxe_pool_get_index(&rxe->mr_pool, index);
710 if (!mr)
711 return NULL;
712
713 if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
714 (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
715 mr_pd(mr) != pd || ((access & mr->access) != access) ||
716 mr->state != RXE_MR_STATE_VALID)) {
717 rxe_put(mr);
718 mr = NULL;
719 }
720
721 return mr;
722 }
723
rxe_invalidate_mr(struct rxe_qp * qp,u32 key)724 int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
725 {
726 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
727 struct rxe_mr *mr;
728 int remote;
729 int ret;
730
731 mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
732 if (!mr) {
733 rxe_dbg_qp(qp, "No MR for key %#x\n", key);
734 ret = -EINVAL;
735 goto err;
736 }
737
738 remote = mr->access & RXE_ACCESS_REMOTE;
739 if (remote ? (key != mr->rkey) : (key != mr->lkey)) {
740 rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",
741 key, (remote ? mr->rkey : mr->lkey));
742 ret = -EINVAL;
743 goto err_drop_ref;
744 }
745
746 if (atomic_read(&mr->num_mw) > 0) {
747 rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");
748 ret = -EINVAL;
749 goto err_drop_ref;
750 }
751
752 if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {
753 rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);
754 ret = -EINVAL;
755 goto err_drop_ref;
756 }
757
758 mr->state = RXE_MR_STATE_FREE;
759 ret = 0;
760
761 err_drop_ref:
762 rxe_put(mr);
763 err:
764 return ret;
765 }
766
767 /* user can (re)register fast MR by executing a REG_MR WQE.
768 * user is expected to hold a reference on the ib mr until the
769 * WQE completes.
770 * Once a fast MR is created this is the only way to change the
771 * private keys. It is the responsibility of the user to maintain
772 * the ib mr keys in sync with rxe mr keys.
773 */
rxe_reg_fast_mr(struct rxe_qp * qp,struct rxe_send_wqe * wqe)774 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
775 {
776 struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);
777 u32 key = wqe->wr.wr.reg.key;
778 u32 access = wqe->wr.wr.reg.access;
779
780 /* user can only register MR in free state */
781 if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
782 rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);
783 return -EINVAL;
784 }
785
786 /* user can only register mr with qp in same protection domain */
787 if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
788 rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");
789 return -EINVAL;
790 }
791
792 /* user is only allowed to change key portion of l/rkey */
793 if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
794 rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",
795 key, mr->lkey);
796 return -EINVAL;
797 }
798
799 mr->access = access;
800 mr->lkey = key;
801 mr->rkey = key;
802 mr->ibmr.iova = wqe->wr.wr.reg.mr->iova;
803 mr->state = RXE_MR_STATE_VALID;
804
805 return 0;
806 }
807
rxe_mr_cleanup(struct rxe_pool_elem * elem)808 void rxe_mr_cleanup(struct rxe_pool_elem *elem)
809 {
810 struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
811
812 rxe_put(mr_pd(mr));
813 ib_umem_release(mr->umem);
814
815 if (mr->ibmr.type != IB_MR_TYPE_DMA)
816 free_mr_page_info(mr);
817 }
818