1 /*- 2 * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/module.h> 29 #include <rdma/ib_umem.h> 30 #include <rdma/ib_umem_odp.h> 31 #include "mlx5_ib.h" 32 33 /* @umem: umem object to scan 34 * @addr: ib virtual address requested by the user 35 * @max_page_shift: high limit for page_shift - 0 means no limit 36 * @count: number of PAGE_SIZE pages covered by umem 37 * @shift: page shift for the compound pages found in the region 38 * @ncont: number of compund pages 39 * @order: log2 of the number of compound pages 40 */ 41 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, 42 unsigned long max_page_shift, 43 int *count, int *shift, 44 int *ncont, int *order) 45 { 46 unsigned long tmp; 47 unsigned long m; 48 u64 base = ~0, p = 0; 49 u64 len, pfn; 50 int i = 0; 51 struct scatterlist *sg; 52 int entry; 53 54 addr = addr >> PAGE_SHIFT; 55 tmp = (unsigned long)addr; 56 m = find_first_bit(&tmp, BITS_PER_LONG); 57 if (max_page_shift) 58 m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m); 59 60 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 61 len = sg_dma_len(sg) >> PAGE_SHIFT; 62 pfn = sg_dma_address(sg) >> PAGE_SHIFT; 63 if (base + p != pfn) { 64 /* If either the offset or the new 65 * base are unaligned update m 66 */ 67 tmp = (unsigned long)(pfn | p); 68 if (!IS_ALIGNED(tmp, 1 << m)) 69 m = find_first_bit(&tmp, BITS_PER_LONG); 70 71 base = pfn; 72 p = 0; 73 } 74 75 p += len; 76 i += len; 77 } 78 79 if (i) { 80 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); 81 82 if (order) 83 *order = ilog2(roundup_pow_of_two(i) >> m); 84 85 *ncont = DIV_ROUND_UP(i, (1 << m)); 86 } else { 87 m = 0; 88 89 if (order) 90 *order = 0; 91 92 *ncont = 0; 93 } 94 *shift = PAGE_SHIFT + m; 95 *count = i; 96 } 97 98 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 99 static u64 umem_dma_to_mtt(dma_addr_t umem_dma) 100 { 101 u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; 102 103 if (umem_dma & ODP_READ_ALLOWED_BIT) 104 mtt_entry |= MLX5_IB_MTT_READ; 105 if (umem_dma & ODP_WRITE_ALLOWED_BIT) 106 mtt_entry |= MLX5_IB_MTT_WRITE; 107 108 return mtt_entry; 109 } 110 #endif 111 112 /* 113 * Populate the given array with bus addresses from the umem. 114 * 115 * dev - mlx5_ib device 116 * umem - umem to use to fill the pages 117 * page_shift - determines the page size used in the resulting array 118 * offset - offset into the umem to start from, 119 * only implemented for ODP umems 120 * num_pages - total number of pages to fill 121 * pas - bus addresses array to fill 122 * access_flags - access flags to set on all present pages. 123 use enum mlx5_ib_mtt_access_flags for this. 124 */ 125 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 126 int page_shift, size_t offset, size_t num_pages, 127 __be64 *pas, int access_flags) 128 { 129 unsigned long umem_page_shift = ilog2(umem->page_size); 130 int shift = page_shift - umem_page_shift; 131 int mask = (1 << shift) - 1; 132 int i, k; 133 u64 cur = 0; 134 u64 base; 135 int len; 136 struct scatterlist *sg; 137 int entry; 138 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 139 const bool odp = umem->odp_data != NULL; 140 141 if (odp) { 142 WARN_ON(shift != 0); 143 WARN_ON(access_flags != (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)); 144 145 for (i = 0; i < num_pages; ++i) { 146 dma_addr_t pa = umem->odp_data->dma_list[offset + i]; 147 148 pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); 149 } 150 return; 151 } 152 #endif 153 154 i = 0; 155 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 156 len = sg_dma_len(sg) >> umem_page_shift; 157 base = sg_dma_address(sg); 158 for (k = 0; k < len; k++) { 159 if (!(i & mask)) { 160 cur = base + (k << umem_page_shift); 161 cur |= access_flags; 162 163 pas[i >> shift] = cpu_to_be64(cur); 164 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n", 165 i >> shift, (long long)be64_to_cpu(pas[i >> shift])); 166 } else 167 mlx5_ib_dbg(dev, "=====> 0x%llx\n", 168 (long long)(base + (k << umem_page_shift))); 169 i++; 170 } 171 } 172 } 173 174 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 175 int page_shift, __be64 *pas, int access_flags) 176 { 177 return __mlx5_ib_populate_pas(dev, umem, page_shift, 0, 178 ib_umem_num_pages(umem), pas, 179 access_flags); 180 } 181 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) 182 { 183 u64 page_size; 184 u64 page_mask; 185 u64 off_size; 186 u64 off_mask; 187 u64 buf_off; 188 189 page_size = (u64)1 << page_shift; 190 page_mask = page_size - 1; 191 buf_off = addr & page_mask; 192 off_size = page_size >> 6; 193 off_mask = off_size - 1; 194 195 if (buf_off & off_mask) 196 return -EINVAL; 197 198 *offset = buf_off >> ilog2(off_size); 199 return 0; 200 } 201