1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ 2 /* 3 * Copyright (c) 2017 Hisilicon Limited. 4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 5 */ 6 7 #include <linux/platform_device.h> 8 #include <rdma/ib_umem.h> 9 #include "hns_roce_device.h" 10 11 int hns_roce_db_map_user(struct hns_roce_ucontext *context, 12 struct ib_udata *udata, unsigned long virt, 13 struct hns_roce_db *db) 14 { 15 struct hns_roce_user_db_page *page; 16 int ret = 0; 17 18 mutex_lock(&context->page_mutex); 19 20 list_for_each_entry(page, &context->page_list, list) 21 if (page->user_virt == (virt & PAGE_MASK)) 22 goto found; 23 24 page = kmalloc(sizeof(*page), GFP_KERNEL); 25 if (!page) { 26 ret = -ENOMEM; 27 goto out; 28 } 29 30 refcount_set(&page->refcount, 1); 31 page->user_virt = (virt & PAGE_MASK); 32 page->umem = ib_umem_get(udata, virt & PAGE_MASK, PAGE_SIZE, 0, 0); 33 if (IS_ERR(page->umem)) { 34 ret = PTR_ERR(page->umem); 35 kfree(page); 36 goto out; 37 } 38 39 list_add(&page->list, &context->page_list); 40 41 found: 42 db->dma = sg_dma_address(page->umem->sg_head.sgl) + 43 (virt & ~PAGE_MASK); 44 page->umem->sg_head.sgl->offset = virt & ~PAGE_MASK; 45 db->virt_addr = sg_virt(page->umem->sg_head.sgl); 46 db->u.user_page = page; 47 refcount_inc(&page->refcount); 48 49 out: 50 mutex_unlock(&context->page_mutex); 51 52 return ret; 53 } 54 EXPORT_SYMBOL(hns_roce_db_map_user); 55 56 void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, 57 struct hns_roce_db *db) 58 { 59 mutex_lock(&context->page_mutex); 60 61 refcount_dec(&db->u.user_page->refcount); 62 if (refcount_dec_if_one(&db->u.user_page->refcount)) { 63 list_del(&db->u.user_page->list); 64 ib_umem_release(db->u.user_page->umem); 65 kfree(db->u.user_page); 66 } 67 68 mutex_unlock(&context->page_mutex); 69 } 70 EXPORT_SYMBOL(hns_roce_db_unmap_user); 71 72 static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( 73 struct device *dma_device) 74 { 75 struct hns_roce_db_pgdir *pgdir; 76 77 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 78 if (!pgdir) 79 return NULL; 80 81 bitmap_fill(pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2); 82 pgdir->bits[0] = pgdir->order0; 83 pgdir->bits[1] = pgdir->order1; 84 pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, 85 &pgdir->db_dma, GFP_KERNEL); 86 if (!pgdir->page) { 87 kfree(pgdir); 88 return NULL; 89 } 90 91 return pgdir; 92 } 93 94 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, 95 struct hns_roce_db *db, int order) 96 { 97 int o; 98 int i; 99 100 for (o = order; o <= 1; ++o) { 101 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); 102 if (i < HNS_ROCE_DB_PER_PAGE >> o) 103 goto found; 104 } 105 106 return -ENOMEM; 107 108 found: 109 clear_bit(i, pgdir->bits[o]); 110 111 i <<= o; 112 113 if (o > order) 114 set_bit(i ^ 1, pgdir->bits[order]); 115 116 db->u.pgdir = pgdir; 117 db->index = i; 118 db->db_record = pgdir->page + db->index; 119 db->dma = pgdir->db_dma + db->index * 4; 120 db->order = order; 121 122 return 0; 123 } 124 125 int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, 126 int order) 127 { 128 struct hns_roce_db_pgdir *pgdir; 129 int ret = 0; 130 131 mutex_lock(&hr_dev->pgdir_mutex); 132 133 list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) 134 if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) 135 goto out; 136 137 pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); 138 if (!pgdir) { 139 ret = -ENOMEM; 140 goto out; 141 } 142 143 list_add(&pgdir->list, &hr_dev->pgdir_list); 144 145 /* This should never fail -- we just allocated an empty page: */ 146 WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); 147 148 out: 149 mutex_unlock(&hr_dev->pgdir_mutex); 150 151 return ret; 152 } 153 EXPORT_SYMBOL_GPL(hns_roce_alloc_db); 154 155 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) 156 { 157 int o; 158 int i; 159 160 mutex_lock(&hr_dev->pgdir_mutex); 161 162 o = db->order; 163 i = db->index; 164 165 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 166 clear_bit(i ^ 1, db->u.pgdir->order0); 167 ++o; 168 } 169 170 i >>= o; 171 set_bit(i, db->u.pgdir->bits[o]); 172 173 if (bitmap_full(db->u.pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2)) { 174 dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, 175 db->u.pgdir->db_dma); 176 list_del(&db->u.pgdir->list); 177 kfree(db->u.pgdir); 178 } 179 180 mutex_unlock(&hr_dev->pgdir_mutex); 181 } 182 EXPORT_SYMBOL_GPL(hns_roce_free_db); 183