1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 /* 3 * Copyright(c) 2023 - Cornelis Networks, Inc. 4 */ 5 6 #include <linux/types.h> 7 8 #include "hfi.h" 9 #include "common.h" 10 #include "device.h" 11 #include "pinning.h" 12 #include "mmu_rb.h" 13 #include "user_sdma.h" 14 #include "trace.h" 15 16 struct sdma_mmu_node { 17 struct mmu_rb_node rb; 18 struct hfi1_user_sdma_pkt_q *pq; 19 struct page **pages; 20 unsigned int npages; 21 }; 22 23 static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr, 24 unsigned long len); 25 static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, void *arg2, 26 bool *stop); 27 static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode); 28 29 static struct mmu_rb_ops sdma_rb_ops = { 30 .filter = sdma_rb_filter, 31 .evict = sdma_rb_evict, 32 .remove = sdma_rb_remove, 33 }; 34 35 int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq) 36 { 37 struct hfi1_devdata *dd = pq->dd; 38 int ret; 39 40 ret = hfi1_mmu_rb_register(pq, &sdma_rb_ops, dd->pport->hfi1_wq, 41 &pq->handler); 42 if (ret) 43 dd_dev_err(dd, 44 "[%u:%u] Failed to register system memory DMA support with MMU: %d\n", 45 pq->ctxt, pq->subctxt, ret); 46 return ret; 47 } 48 49 void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq) 50 { 51 if (pq->handler) 52 hfi1_mmu_rb_unregister(pq->handler); 53 } 54 55 static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages) 56 { 57 struct evict_data evict_data; 58 59 evict_data.cleared = 0; 60 evict_data.target = npages; 61 hfi1_mmu_rb_evict(pq->handler, &evict_data); 62 return evict_data.cleared; 63 } 64 65 static void unpin_vector_pages(struct mm_struct *mm, struct page **pages, 66 unsigned int start, unsigned int npages) 67 { 68 hfi1_release_user_pages(mm, pages + start, npages, false); 69 kfree(pages); 70 } 71 72 static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node) 73 { 74 return node->rb.handler->mn.mm; 75 } 76 77 static void free_system_node(struct sdma_mmu_node *node) 78 { 79 if (node->npages) { 80 unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0, 81 node->npages); 82 atomic_sub(node->npages, &node->pq->n_locked); 83 } 84 kfree(node); 85 } 86 87 /* 88 * kref_get()'s an additional kref on the returned rb_node to prevent rb_node 89 * from being released until after rb_node is assigned to an SDMA descriptor 90 * (struct sdma_desc) under add_system_iovec_to_sdma_packet(), even if the 91 * virtual address range for rb_node is invalidated between now and then. 92 */ 93 static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler, 94 unsigned long start, 95 unsigned long end) 96 { 97 struct mmu_rb_node *rb_node; 98 unsigned long flags; 99 100 spin_lock_irqsave(&handler->lock, flags); 101 rb_node = hfi1_mmu_rb_get_first(handler, start, (end - start)); 102 if (!rb_node) { 103 spin_unlock_irqrestore(&handler->lock, flags); 104 return NULL; 105 } 106 107 /* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */ 108 kref_get(&rb_node->refcount); 109 spin_unlock_irqrestore(&handler->lock, flags); 110 111 return container_of(rb_node, struct sdma_mmu_node, rb); 112 } 113 114 static int pin_system_pages(struct user_sdma_request *req, 115 uintptr_t start_address, size_t length, 116 struct sdma_mmu_node *node, int npages) 117 { 118 struct hfi1_user_sdma_pkt_q *pq = req->pq; 119 int pinned, cleared; 120 struct page **pages; 121 122 pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL); 123 if (!pages) 124 return -ENOMEM; 125 126 retry: 127 if (!hfi1_can_pin_pages(pq->dd, current->mm, atomic_read(&pq->n_locked), 128 npages)) { 129 SDMA_DBG(req, "Evicting: nlocked %u npages %u", 130 atomic_read(&pq->n_locked), npages); 131 cleared = sdma_cache_evict(pq, npages); 132 if (cleared >= npages) 133 goto retry; 134 } 135 136 SDMA_DBG(req, "Acquire user pages start_address %lx node->npages %u npages %u", 137 start_address, node->npages, npages); 138 pinned = hfi1_acquire_user_pages(current->mm, start_address, npages, 0, 139 pages); 140 141 if (pinned < 0) { 142 kfree(pages); 143 SDMA_DBG(req, "pinned %d", pinned); 144 return pinned; 145 } 146 if (pinned != npages) { 147 unpin_vector_pages(current->mm, pages, node->npages, pinned); 148 SDMA_DBG(req, "npages %u pinned %d", npages, pinned); 149 return -EFAULT; 150 } 151 node->rb.addr = start_address; 152 node->rb.len = length; 153 node->pages = pages; 154 node->npages = npages; 155 atomic_add(pinned, &pq->n_locked); 156 SDMA_DBG(req, "done. pinned %d", pinned); 157 return 0; 158 } 159 160 /* 161 * kref refcount on *node_p will be 2 on successful addition: one kref from 162 * kref_init() for mmu_rb_handler and one kref to prevent *node_p from being 163 * released until after *node_p is assigned to an SDMA descriptor (struct 164 * sdma_desc) under add_system_iovec_to_sdma_packet(), even if the virtual 165 * address range for *node_p is invalidated between now and then. 166 */ 167 static int add_system_pinning(struct user_sdma_request *req, 168 struct sdma_mmu_node **node_p, 169 unsigned long start, unsigned long len) 170 171 { 172 struct hfi1_user_sdma_pkt_q *pq = req->pq; 173 struct sdma_mmu_node *node; 174 int ret; 175 176 node = kzalloc(sizeof(*node), GFP_KERNEL); 177 if (!node) 178 return -ENOMEM; 179 180 /* First kref "moves" to mmu_rb_handler */ 181 kref_init(&node->rb.refcount); 182 183 /* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */ 184 kref_get(&node->rb.refcount); 185 186 node->pq = pq; 187 ret = pin_system_pages(req, start, len, node, PFN_DOWN(len)); 188 if (ret == 0) { 189 ret = hfi1_mmu_rb_insert(pq->handler, &node->rb); 190 if (ret) 191 free_system_node(node); 192 else 193 *node_p = node; 194 195 return ret; 196 } 197 198 kfree(node); 199 return ret; 200 } 201 202 static int get_system_cache_entry(struct user_sdma_request *req, 203 struct sdma_mmu_node **node_p, 204 size_t req_start, size_t req_len) 205 { 206 struct hfi1_user_sdma_pkt_q *pq = req->pq; 207 u64 start = ALIGN_DOWN(req_start, PAGE_SIZE); 208 u64 end = PFN_ALIGN(req_start + req_len); 209 int ret; 210 211 if ((end - start) == 0) { 212 SDMA_DBG(req, 213 "Request for empty cache entry req_start %lx req_len %lx start %llx end %llx", 214 req_start, req_len, start, end); 215 return -EINVAL; 216 } 217 218 SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len); 219 220 while (1) { 221 struct sdma_mmu_node *node = 222 find_system_node(pq->handler, start, end); 223 u64 prepend_len = 0; 224 225 SDMA_DBG(req, "node %p start %llx end %llu", node, start, end); 226 if (!node) { 227 ret = add_system_pinning(req, node_p, start, 228 end - start); 229 if (ret == -EEXIST) { 230 /* 231 * Another execution context has inserted a 232 * conficting entry first. 233 */ 234 continue; 235 } 236 return ret; 237 } 238 239 if (node->rb.addr <= start) { 240 /* 241 * This entry covers at least part of the region. If it doesn't extend 242 * to the end, then this will be called again for the next segment. 243 */ 244 *node_p = node; 245 return 0; 246 } 247 248 SDMA_DBG(req, "prepend: node->rb.addr %lx, node->rb.refcount %d", 249 node->rb.addr, kref_read(&node->rb.refcount)); 250 prepend_len = node->rb.addr - start; 251 252 /* 253 * This node will not be returned, instead a new node 254 * will be. So release the reference. 255 */ 256 kref_put(&node->rb.refcount, hfi1_mmu_rb_release); 257 258 /* Prepend a node to cover the beginning of the allocation */ 259 ret = add_system_pinning(req, node_p, start, prepend_len); 260 if (ret == -EEXIST) { 261 /* Another execution context has inserted a conficting entry first. */ 262 continue; 263 } 264 return ret; 265 } 266 } 267 268 static void sdma_mmu_rb_node_get(void *ctx) 269 { 270 struct mmu_rb_node *node = ctx; 271 272 kref_get(&node->refcount); 273 } 274 275 static void sdma_mmu_rb_node_put(void *ctx) 276 { 277 struct sdma_mmu_node *node = ctx; 278 279 kref_put(&node->rb.refcount, hfi1_mmu_rb_release); 280 } 281 282 static int add_mapping_to_sdma_packet(struct user_sdma_request *req, 283 struct user_sdma_txreq *tx, 284 struct sdma_mmu_node *cache_entry, 285 size_t start, 286 size_t from_this_cache_entry) 287 { 288 struct hfi1_user_sdma_pkt_q *pq = req->pq; 289 unsigned int page_offset; 290 unsigned int from_this_page; 291 size_t page_index; 292 void *ctx; 293 int ret; 294 295 /* 296 * Because the cache may be more fragmented than the memory that is being accessed, 297 * it's not strictly necessary to have a descriptor per cache entry. 298 */ 299 300 while (from_this_cache_entry) { 301 page_index = PFN_DOWN(start - cache_entry->rb.addr); 302 303 if (page_index >= cache_entry->npages) { 304 SDMA_DBG(req, 305 "Request for page_index %zu >= cache_entry->npages %u", 306 page_index, cache_entry->npages); 307 return -EINVAL; 308 } 309 310 page_offset = start - ALIGN_DOWN(start, PAGE_SIZE); 311 from_this_page = PAGE_SIZE - page_offset; 312 313 if (from_this_page < from_this_cache_entry) { 314 ctx = NULL; 315 } else { 316 /* 317 * In the case they are equal the next line has no practical effect, 318 * but it's better to do a register to register copy than a conditional 319 * branch. 320 */ 321 from_this_page = from_this_cache_entry; 322 ctx = cache_entry; 323 } 324 325 ret = sdma_txadd_page(pq->dd, &tx->txreq, 326 cache_entry->pages[page_index], 327 page_offset, from_this_page, 328 ctx, 329 sdma_mmu_rb_node_get, 330 sdma_mmu_rb_node_put); 331 if (ret) { 332 /* 333 * When there's a failure, the entire request is freed by 334 * user_sdma_send_pkts(). 335 */ 336 SDMA_DBG(req, 337 "sdma_txadd_page failed %d page_index %lu page_offset %u from_this_page %u", 338 ret, page_index, page_offset, from_this_page); 339 return ret; 340 } 341 start += from_this_page; 342 from_this_cache_entry -= from_this_page; 343 } 344 return 0; 345 } 346 347 static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req, 348 struct user_sdma_txreq *tx, 349 struct user_sdma_iovec *iovec, 350 size_t from_this_iovec) 351 { 352 while (from_this_iovec > 0) { 353 struct sdma_mmu_node *cache_entry; 354 size_t from_this_cache_entry; 355 size_t start; 356 int ret; 357 358 start = (uintptr_t)iovec->iov.iov_base + iovec->offset; 359 ret = get_system_cache_entry(req, &cache_entry, start, 360 from_this_iovec); 361 if (ret) { 362 SDMA_DBG(req, "pin system segment failed %d", ret); 363 return ret; 364 } 365 366 from_this_cache_entry = cache_entry->rb.len - (start - cache_entry->rb.addr); 367 if (from_this_cache_entry > from_this_iovec) 368 from_this_cache_entry = from_this_iovec; 369 370 ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start, 371 from_this_cache_entry); 372 373 /* 374 * Done adding cache_entry to zero or more sdma_desc. Can 375 * kref_put() the "safety" kref taken under 376 * get_system_cache_entry(). 377 */ 378 kref_put(&cache_entry->rb.refcount, hfi1_mmu_rb_release); 379 380 if (ret) { 381 SDMA_DBG(req, "add system segment failed %d", ret); 382 return ret; 383 } 384 385 iovec->offset += from_this_cache_entry; 386 from_this_iovec -= from_this_cache_entry; 387 } 388 389 return 0; 390 } 391 392 /* 393 * Add up to pkt_data_remaining bytes to the txreq, starting at the current 394 * offset in the given iovec entry and continuing until all data has been added 395 * to the iovec or the iovec entry type changes. 396 * 397 * On success, prior to returning, adjust pkt_data_remaining, req->iov_idx, and 398 * the offset value in req->iov[req->iov_idx] to reflect the data that has been 399 * consumed. 400 */ 401 int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req, 402 struct user_sdma_txreq *tx, 403 struct user_sdma_iovec *iovec, 404 u32 *pkt_data_remaining) 405 { 406 size_t remaining_to_add = *pkt_data_remaining; 407 /* 408 * Walk through iovec entries, ensure the associated pages 409 * are pinned and mapped, add data to the packet until no more 410 * data remains to be added or the iovec entry type changes. 411 */ 412 while (remaining_to_add > 0) { 413 struct user_sdma_iovec *cur_iovec; 414 size_t from_this_iovec; 415 int ret; 416 417 cur_iovec = iovec; 418 from_this_iovec = iovec->iov.iov_len - iovec->offset; 419 420 if (from_this_iovec > remaining_to_add) { 421 from_this_iovec = remaining_to_add; 422 } else { 423 /* The current iovec entry will be consumed by this pass. */ 424 req->iov_idx++; 425 iovec++; 426 } 427 428 ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec, 429 from_this_iovec); 430 if (ret) 431 return ret; 432 433 remaining_to_add -= from_this_iovec; 434 } 435 *pkt_data_remaining = remaining_to_add; 436 437 return 0; 438 } 439 440 static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr, 441 unsigned long len) 442 { 443 return (bool)(node->addr == addr); 444 } 445 446 /* 447 * Return 1 to remove the node from the rb tree and call the remove op. 448 * 449 * Called with the rb tree lock held. 450 */ 451 static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, 452 void *evict_arg, bool *stop) 453 { 454 struct sdma_mmu_node *node = 455 container_of(mnode, struct sdma_mmu_node, rb); 456 struct evict_data *evict_data = evict_arg; 457 458 /* this node will be evicted, add its pages to our count */ 459 evict_data->cleared += node->npages; 460 461 /* have enough pages been cleared? */ 462 if (evict_data->cleared >= evict_data->target) 463 *stop = true; 464 465 return 1; /* remove this node */ 466 } 467 468 static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode) 469 { 470 struct sdma_mmu_node *node = 471 container_of(mnode, struct sdma_mmu_node, rb); 472 473 free_system_node(node); 474 } 475