1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved. 4 */ 5 6 /* 7 * The rdma_rxe driver supports type 1 or type 2B memory windows. 8 * Type 1 MWs are created by ibv_alloc_mw() verbs calls and bound by 9 * ibv_bind_mw() calls. Type 2 MWs are also created by ibv_alloc_mw() 10 * but bound by bind_mw work requests. The ibv_bind_mw() call is converted 11 * by libibverbs to a bind_mw work request. 12 */ 13 14 #include "rxe.h" 15 16 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 17 { 18 struct rxe_mw *mw = to_rmw(ibmw); 19 struct rxe_pd *pd = to_rpd(ibmw->pd); 20 struct rxe_dev *rxe = to_rdev(ibmw->device); 21 int ret; 22 23 rxe_get(pd); 24 25 ret = rxe_add_to_pool(&rxe->mw_pool, mw); 26 if (ret) { 27 rxe_put(pd); 28 return ret; 29 } 30 31 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1); 32 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ? 33 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID; 34 spin_lock_init(&mw->lock); 35 36 rxe_finalize(mw); 37 38 return 0; 39 } 40 41 int rxe_dealloc_mw(struct ib_mw *ibmw) 42 { 43 struct rxe_mw *mw = to_rmw(ibmw); 44 45 rxe_cleanup(mw); 46 47 return 0; 48 } 49 50 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 51 struct rxe_mw *mw, struct rxe_mr *mr, int access) 52 { 53 if (mw->ibmw.type == IB_MW_TYPE_1) { 54 if (unlikely(mw->state != RXE_MW_STATE_VALID)) { 55 rxe_dbg_mw(mw, 56 "attempt to bind a type 1 MW not in the valid state\n"); 57 return -EINVAL; 58 } 59 60 /* o10-36.2.2 */ 61 if (unlikely((access & IB_ZERO_BASED))) { 62 rxe_dbg_mw(mw, "attempt to bind a zero based type 1 MW\n"); 63 return -EINVAL; 64 } 65 } 66 67 if (mw->ibmw.type == IB_MW_TYPE_2) { 68 /* o10-37.2.30 */ 69 if (unlikely(mw->state != RXE_MW_STATE_FREE)) { 70 rxe_dbg_mw(mw, 71 "attempt to bind a type 2 MW not in the free state\n"); 72 return -EINVAL; 73 } 74 75 /* C10-72 */ 76 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) { 77 rxe_dbg_mw(mw, 78 "attempt to bind type 2 MW with qp with different PD\n"); 79 return -EINVAL; 80 } 81 82 /* o10-37.2.40 */ 83 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) { 84 rxe_dbg_mw(mw, 85 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n"); 86 return -EINVAL; 87 } 88 } 89 90 /* remaining checks only apply to a nonzero MR */ 91 if (!mr) 92 return 0; 93 94 if (unlikely(mr->access & IB_ZERO_BASED)) { 95 rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n"); 96 return -EINVAL; 97 } 98 99 /* C10-73 */ 100 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) { 101 rxe_dbg_mw(mw, 102 "attempt to bind an MW to an MR without bind access\n"); 103 return -EINVAL; 104 } 105 106 /* C10-74 */ 107 if (unlikely((access & 108 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) && 109 !(mr->access & IB_ACCESS_LOCAL_WRITE))) { 110 rxe_dbg_mw(mw, 111 "attempt to bind an Writable MW to an MR without local write access\n"); 112 return -EINVAL; 113 } 114 115 /* C10-75 */ 116 if (access & IB_ZERO_BASED) { 117 if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) { 118 rxe_dbg_mw(mw, 119 "attempt to bind a ZB MW outside of the MR\n"); 120 return -EINVAL; 121 } 122 } else { 123 if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) || 124 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) > 125 (mr->ibmr.iova + mr->ibmr.length)))) { 126 rxe_dbg_mw(mw, 127 "attempt to bind a VA MW outside of the MR\n"); 128 return -EINVAL; 129 } 130 } 131 132 return 0; 133 } 134 135 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 136 struct rxe_mw *mw, struct rxe_mr *mr, int access) 137 { 138 u32 key = wqe->wr.wr.mw.rkey & 0xff; 139 140 mw->rkey = (mw->rkey & ~0xff) | key; 141 mw->access = access; 142 mw->state = RXE_MW_STATE_VALID; 143 mw->addr = wqe->wr.wr.mw.addr; 144 mw->length = wqe->wr.wr.mw.length; 145 146 if (mw->mr) { 147 rxe_put(mw->mr); 148 atomic_dec(&mw->mr->num_mw); 149 mw->mr = NULL; 150 } 151 152 if (mw->length) { 153 mw->mr = mr; 154 atomic_inc(&mr->num_mw); 155 rxe_get(mr); 156 } 157 158 if (mw->ibmw.type == IB_MW_TYPE_2) { 159 rxe_get(qp); 160 mw->qp = qp; 161 } 162 } 163 164 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) 165 { 166 int ret; 167 struct rxe_mw *mw; 168 struct rxe_mr *mr; 169 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 170 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey; 171 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey; 172 int access = wqe->wr.wr.mw.access; 173 174 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8); 175 if (unlikely(!mw)) { 176 ret = -EINVAL; 177 goto err; 178 } 179 180 if (unlikely(mw->rkey != mw_rkey)) { 181 ret = -EINVAL; 182 goto err_drop_mw; 183 } 184 185 if (likely(wqe->wr.wr.mw.length)) { 186 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8); 187 if (unlikely(!mr)) { 188 ret = -EINVAL; 189 goto err_drop_mw; 190 } 191 192 if (unlikely(mr->lkey != mr_lkey)) { 193 ret = -EINVAL; 194 goto err_drop_mr; 195 } 196 } else { 197 mr = NULL; 198 } 199 200 if (access & ~RXE_ACCESS_SUPPORTED_MW) { 201 rxe_err_mw(mw, "access %#x not supported\n", access); 202 ret = -EOPNOTSUPP; 203 goto err_drop_mr; 204 } 205 206 spin_lock_bh(&mw->lock); 207 208 ret = rxe_check_bind_mw(qp, wqe, mw, mr, access); 209 if (ret) 210 goto err_unlock; 211 212 rxe_do_bind_mw(qp, wqe, mw, mr, access); 213 err_unlock: 214 spin_unlock_bh(&mw->lock); 215 err_drop_mr: 216 if (mr) 217 rxe_put(mr); 218 err_drop_mw: 219 rxe_put(mw); 220 err: 221 return ret; 222 } 223 224 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw) 225 { 226 if (unlikely(mw->state == RXE_MW_STATE_INVALID)) 227 return -EINVAL; 228 229 /* o10-37.2.26 */ 230 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1)) 231 return -EINVAL; 232 233 return 0; 234 } 235 236 static void rxe_do_invalidate_mw(struct rxe_mw *mw) 237 { 238 struct rxe_qp *qp; 239 struct rxe_mr *mr; 240 241 /* valid type 2 MW will always have a QP pointer */ 242 qp = mw->qp; 243 mw->qp = NULL; 244 rxe_put(qp); 245 246 /* valid type 2 MW will always have an MR pointer */ 247 mr = mw->mr; 248 mw->mr = NULL; 249 atomic_dec(&mr->num_mw); 250 rxe_put(mr); 251 252 mw->access = 0; 253 mw->addr = 0; 254 mw->length = 0; 255 mw->state = RXE_MW_STATE_FREE; 256 } 257 258 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) 259 { 260 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 261 struct rxe_mw *mw; 262 int ret; 263 264 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8); 265 if (!mw) { 266 ret = -EINVAL; 267 goto err; 268 } 269 270 if (rkey != mw->rkey) { 271 ret = -EINVAL; 272 goto err_drop_ref; 273 } 274 275 spin_lock_bh(&mw->lock); 276 277 ret = rxe_check_invalidate_mw(qp, mw); 278 if (ret) 279 goto err_unlock; 280 281 rxe_do_invalidate_mw(mw); 282 err_unlock: 283 spin_unlock_bh(&mw->lock); 284 err_drop_ref: 285 rxe_put(mw); 286 err: 287 return ret; 288 } 289 290 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) 291 { 292 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 293 struct rxe_pd *pd = to_rpd(qp->ibqp.pd); 294 struct rxe_mw *mw; 295 int index = rkey >> 8; 296 297 mw = rxe_pool_get_index(&rxe->mw_pool, index); 298 if (!mw) 299 return NULL; 300 301 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd || 302 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) || 303 (mw->length == 0) || ((access & mw->access) != access) || 304 mw->state != RXE_MW_STATE_VALID)) { 305 rxe_put(mw); 306 return NULL; 307 } 308 309 return mw; 310 } 311 312 void rxe_mw_cleanup(struct rxe_pool_elem *elem) 313 { 314 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem); 315 struct rxe_pd *pd = to_rpd(mw->ibmw.pd); 316 317 rxe_put(pd); 318 319 if (mw->mr) { 320 struct rxe_mr *mr = mw->mr; 321 322 mw->mr = NULL; 323 atomic_dec(&mr->num_mw); 324 rxe_put(mr); 325 } 326 327 if (mw->qp) { 328 struct rxe_qp *qp = mw->qp; 329 330 mw->qp = NULL; 331 rxe_put(qp); 332 } 333 334 mw->access = 0; 335 mw->addr = 0; 336 mw->length = 0; 337 mw->state = RXE_MW_STATE_INVALID; 338 } 339