1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ 2 /* 3 * Copyright (c) 2017 Hisilicon Limited. 4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 5 */ 6 7 #include <rdma/ib_umem.h> 8 #include "hns_roce_device.h" 9 10 int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, 11 struct hns_roce_db *db) 12 { 13 unsigned long page_addr = virt & PAGE_MASK; 14 struct hns_roce_user_db_page *page; 15 unsigned int offset; 16 int ret = 0; 17 18 mutex_lock(&context->page_mutex); 19 20 list_for_each_entry(page, &context->page_list, list) 21 if (page->user_virt == page_addr) 22 goto found; 23 24 page = kmalloc(sizeof(*page), GFP_KERNEL); 25 if (!page) { 26 ret = -ENOMEM; 27 goto out; 28 } 29 30 refcount_set(&page->refcount, 1); 31 page->user_virt = page_addr; 32 page->umem = ib_umem_get(context->ibucontext.device, page_addr, 33 PAGE_SIZE, 0); 34 if (IS_ERR(page->umem)) { 35 ret = PTR_ERR(page->umem); 36 kfree(page); 37 goto out; 38 } 39 40 list_add(&page->list, &context->page_list); 41 42 found: 43 offset = virt - page_addr; 44 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset; 45 db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset; 46 db->u.user_page = page; 47 refcount_inc(&page->refcount); 48 49 out: 50 mutex_unlock(&context->page_mutex); 51 52 return ret; 53 } 54 55 void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, 56 struct hns_roce_db *db) 57 { 58 mutex_lock(&context->page_mutex); 59 60 refcount_dec(&db->u.user_page->refcount); 61 if (refcount_dec_if_one(&db->u.user_page->refcount)) { 62 list_del(&db->u.user_page->list); 63 ib_umem_release(db->u.user_page->umem); 64 kfree(db->u.user_page); 65 } 66 67 mutex_unlock(&context->page_mutex); 68 } 69 70 static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( 71 struct device *dma_device) 72 { 73 struct hns_roce_db_pgdir *pgdir; 74 75 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 76 if (!pgdir) 77 return NULL; 78 79 bitmap_fill(pgdir->order1, 80 HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); 81 pgdir->bits[0] = pgdir->order0; 82 pgdir->bits[1] = pgdir->order1; 83 pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, 84 &pgdir->db_dma, GFP_KERNEL); 85 if (!pgdir->page) { 86 kfree(pgdir); 87 return NULL; 88 } 89 90 return pgdir; 91 } 92 93 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, 94 struct hns_roce_db *db, int order) 95 { 96 unsigned long o; 97 unsigned long i; 98 99 for (o = order; o <= 1; ++o) { 100 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); 101 if (i < HNS_ROCE_DB_PER_PAGE >> o) 102 goto found; 103 } 104 105 return -ENOMEM; 106 107 found: 108 clear_bit(i, pgdir->bits[o]); 109 110 i <<= o; 111 112 if (o > order) 113 set_bit(i ^ 1, pgdir->bits[order]); 114 115 db->u.pgdir = pgdir; 116 db->index = i; 117 db->db_record = pgdir->page + db->index; 118 db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; 119 db->order = order; 120 121 return 0; 122 } 123 124 int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, 125 int order) 126 { 127 struct hns_roce_db_pgdir *pgdir; 128 int ret = 0; 129 130 mutex_lock(&hr_dev->pgdir_mutex); 131 132 list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) 133 if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) 134 goto out; 135 136 pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); 137 if (!pgdir) { 138 ret = -ENOMEM; 139 goto out; 140 } 141 142 list_add(&pgdir->list, &hr_dev->pgdir_list); 143 144 /* This should never fail -- we just allocated an empty page: */ 145 WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); 146 147 out: 148 mutex_unlock(&hr_dev->pgdir_mutex); 149 150 return ret; 151 } 152 153 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) 154 { 155 unsigned long o; 156 unsigned long i; 157 158 mutex_lock(&hr_dev->pgdir_mutex); 159 160 o = db->order; 161 i = db->index; 162 163 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 164 clear_bit(i ^ 1, db->u.pgdir->order0); 165 ++o; 166 } 167 168 i >>= o; 169 set_bit(i, db->u.pgdir->bits[o]); 170 171 if (bitmap_full(db->u.pgdir->order1, 172 HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { 173 dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, 174 db->u.pgdir->db_dma); 175 list_del(&db->u.pgdir->list); 176 kfree(db->u.pgdir); 177 } 178 179 mutex_unlock(&hr_dev->pgdir_mutex); 180 } 181