1 /* 2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of EITHER the GNU General Public License 6 * version 2 as published by the Free Software Foundation or the BSD 7 * 2-Clause License. This program is distributed in the hope that it 8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED 9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. 10 * See the GNU General Public License version 2 for more details at 11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program available in the file COPYING in the main 15 * directory of this source tree. 16 * 17 * The BSD 2-Clause License 18 * 19 * Redistribution and use in source and binary forms, with or 20 * without modification, are permitted provided that the following 21 * conditions are met: 22 * 23 * - Redistributions of source code must retain the above 24 * copyright notice, this list of conditions and the following 25 * disclaimer. 26 * 27 * - Redistributions in binary form must reproduce the above 28 * copyright notice, this list of conditions and the following 29 * disclaimer in the documentation and/or other materials 30 * provided with the distribution. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 43 * OF THE POSSIBILITY OF SUCH DAMAGE. 44 */ 45 46 #include <linux/errno.h> 47 #include <linux/slab.h> 48 #include <linux/bitmap.h> 49 50 #include "pvrdma.h" 51 52 int pvrdma_page_dir_init(struct pvrdma_dev *dev, struct pvrdma_page_dir *pdir, 53 u64 npages, bool alloc_pages) 54 { 55 u64 i; 56 57 if (npages > PVRDMA_PAGE_DIR_MAX_PAGES) 58 return -EINVAL; 59 60 memset(pdir, 0, sizeof(*pdir)); 61 62 pdir->dir = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 63 &pdir->dir_dma, GFP_KERNEL); 64 if (!pdir->dir) 65 goto err; 66 67 pdir->ntables = PVRDMA_PAGE_DIR_TABLE(npages - 1) + 1; 68 pdir->tables = kcalloc(pdir->ntables, sizeof(*pdir->tables), 69 GFP_KERNEL); 70 if (!pdir->tables) 71 goto err; 72 73 for (i = 0; i < pdir->ntables; i++) { 74 pdir->tables[i] = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 75 (dma_addr_t *)&pdir->dir[i], 76 GFP_KERNEL); 77 if (!pdir->tables[i]) 78 goto err; 79 } 80 81 pdir->npages = npages; 82 83 if (alloc_pages) { 84 pdir->pages = kcalloc(npages, sizeof(*pdir->pages), 85 GFP_KERNEL); 86 if (!pdir->pages) 87 goto err; 88 89 for (i = 0; i < pdir->npages; i++) { 90 dma_addr_t page_dma; 91 92 pdir->pages[i] = dma_alloc_coherent(&dev->pdev->dev, 93 PAGE_SIZE, 94 &page_dma, 95 GFP_KERNEL); 96 if (!pdir->pages[i]) 97 goto err; 98 99 pvrdma_page_dir_insert_dma(pdir, i, page_dma); 100 } 101 } 102 103 return 0; 104 105 err: 106 pvrdma_page_dir_cleanup(dev, pdir); 107 108 return -ENOMEM; 109 } 110 111 static u64 *pvrdma_page_dir_table(struct pvrdma_page_dir *pdir, u64 idx) 112 { 113 return pdir->tables[PVRDMA_PAGE_DIR_TABLE(idx)]; 114 } 115 116 dma_addr_t pvrdma_page_dir_get_dma(struct pvrdma_page_dir *pdir, u64 idx) 117 { 118 return pvrdma_page_dir_table(pdir, idx)[PVRDMA_PAGE_DIR_PAGE(idx)]; 119 } 120 121 static void pvrdma_page_dir_cleanup_pages(struct pvrdma_dev *dev, 122 struct pvrdma_page_dir *pdir) 123 { 124 if (pdir->pages) { 125 u64 i; 126 127 for (i = 0; i < pdir->npages && pdir->pages[i]; i++) { 128 dma_addr_t page_dma = pvrdma_page_dir_get_dma(pdir, i); 129 130 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 131 pdir->pages[i], page_dma); 132 } 133 134 kfree(pdir->pages); 135 } 136 } 137 138 static void pvrdma_page_dir_cleanup_tables(struct pvrdma_dev *dev, 139 struct pvrdma_page_dir *pdir) 140 { 141 if (pdir->tables) { 142 int i; 143 144 pvrdma_page_dir_cleanup_pages(dev, pdir); 145 146 for (i = 0; i < pdir->ntables; i++) { 147 u64 *table = pdir->tables[i]; 148 149 if (table) 150 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 151 table, pdir->dir[i]); 152 } 153 154 kfree(pdir->tables); 155 } 156 } 157 158 void pvrdma_page_dir_cleanup(struct pvrdma_dev *dev, 159 struct pvrdma_page_dir *pdir) 160 { 161 if (pdir->dir) { 162 pvrdma_page_dir_cleanup_tables(dev, pdir); 163 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 164 pdir->dir, pdir->dir_dma); 165 } 166 } 167 168 int pvrdma_page_dir_insert_dma(struct pvrdma_page_dir *pdir, u64 idx, 169 dma_addr_t daddr) 170 { 171 u64 *table; 172 173 if (idx >= pdir->npages) 174 return -EINVAL; 175 176 table = pvrdma_page_dir_table(pdir, idx); 177 table[PVRDMA_PAGE_DIR_PAGE(idx)] = daddr; 178 179 return 0; 180 } 181 182 int pvrdma_page_dir_insert_umem(struct pvrdma_page_dir *pdir, 183 struct ib_umem *umem, u64 offset) 184 { 185 struct ib_block_iter biter; 186 u64 i = offset; 187 int ret = 0; 188 189 if (offset >= pdir->npages) 190 return -EINVAL; 191 192 rdma_umem_for_each_dma_block (umem, &biter, PAGE_SIZE) { 193 ret = pvrdma_page_dir_insert_dma( 194 pdir, i, rdma_block_iter_dma_address(&biter)); 195 if (ret) 196 goto exit; 197 198 i++; 199 } 200 201 exit: 202 return ret; 203 } 204 205 int pvrdma_page_dir_insert_page_list(struct pvrdma_page_dir *pdir, 206 u64 *page_list, 207 int num_pages) 208 { 209 int i; 210 int ret; 211 212 if (num_pages > pdir->npages) 213 return -EINVAL; 214 215 for (i = 0; i < num_pages; i++) { 216 ret = pvrdma_page_dir_insert_dma(pdir, i, page_list[i]); 217 if (ret) 218 return ret; 219 } 220 221 return 0; 222 } 223 224 void pvrdma_qp_cap_to_ib(struct ib_qp_cap *dst, const struct pvrdma_qp_cap *src) 225 { 226 dst->max_send_wr = src->max_send_wr; 227 dst->max_recv_wr = src->max_recv_wr; 228 dst->max_send_sge = src->max_send_sge; 229 dst->max_recv_sge = src->max_recv_sge; 230 dst->max_inline_data = src->max_inline_data; 231 } 232 233 void ib_qp_cap_to_pvrdma(struct pvrdma_qp_cap *dst, const struct ib_qp_cap *src) 234 { 235 dst->max_send_wr = src->max_send_wr; 236 dst->max_recv_wr = src->max_recv_wr; 237 dst->max_send_sge = src->max_send_sge; 238 dst->max_recv_sge = src->max_recv_sge; 239 dst->max_inline_data = src->max_inline_data; 240 } 241 242 void pvrdma_gid_to_ib(union ib_gid *dst, const union pvrdma_gid *src) 243 { 244 BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid)); 245 memcpy(dst, src, sizeof(*src)); 246 } 247 248 void ib_gid_to_pvrdma(union pvrdma_gid *dst, const union ib_gid *src) 249 { 250 BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid)); 251 memcpy(dst, src, sizeof(*src)); 252 } 253 254 void pvrdma_global_route_to_ib(struct ib_global_route *dst, 255 const struct pvrdma_global_route *src) 256 { 257 pvrdma_gid_to_ib(&dst->dgid, &src->dgid); 258 dst->flow_label = src->flow_label; 259 dst->sgid_index = src->sgid_index; 260 dst->hop_limit = src->hop_limit; 261 dst->traffic_class = src->traffic_class; 262 } 263 264 void ib_global_route_to_pvrdma(struct pvrdma_global_route *dst, 265 const struct ib_global_route *src) 266 { 267 ib_gid_to_pvrdma(&dst->dgid, &src->dgid); 268 dst->flow_label = src->flow_label; 269 dst->sgid_index = src->sgid_index; 270 dst->hop_limit = src->hop_limit; 271 dst->traffic_class = src->traffic_class; 272 } 273 274 void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst, 275 const struct pvrdma_ah_attr *src) 276 { 277 dst->type = RDMA_AH_ATTR_TYPE_ROCE; 278 pvrdma_global_route_to_ib(rdma_ah_retrieve_grh(dst), &src->grh); 279 rdma_ah_set_dlid(dst, src->dlid); 280 rdma_ah_set_sl(dst, src->sl); 281 rdma_ah_set_path_bits(dst, src->src_path_bits); 282 rdma_ah_set_static_rate(dst, src->static_rate); 283 rdma_ah_set_ah_flags(dst, src->ah_flags); 284 rdma_ah_set_port_num(dst, src->port_num); 285 memcpy(dst->roce.dmac, &src->dmac, ETH_ALEN); 286 } 287 288 void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst, 289 const struct rdma_ah_attr *src) 290 { 291 ib_global_route_to_pvrdma(&dst->grh, rdma_ah_read_grh(src)); 292 dst->dlid = rdma_ah_get_dlid(src); 293 dst->sl = rdma_ah_get_sl(src); 294 dst->src_path_bits = rdma_ah_get_path_bits(src); 295 dst->static_rate = rdma_ah_get_static_rate(src); 296 dst->ah_flags = rdma_ah_get_ah_flags(src); 297 dst->port_num = rdma_ah_get_port_num(src); 298 memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac)); 299 } 300 301 u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type) 302 { 303 return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 304 PVRDMA_GID_TYPE_FLAG_ROCE_V2 : 305 PVRDMA_GID_TYPE_FLAG_ROCE_V1; 306 } 307