1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/anon_inodes.h> 3 #include <linux/uio.h> 4 #include "internal.h" 5 6 struct ondemand_anon_file { 7 struct file *file; 8 int fd; 9 }; 10 11 static inline void cachefiles_req_put(struct cachefiles_req *req) 12 { 13 if (refcount_dec_and_test(&req->ref)) 14 kfree(req); 15 } 16 17 static int cachefiles_ondemand_fd_release(struct inode *inode, 18 struct file *file) 19 { 20 struct cachefiles_object *object = file->private_data; 21 struct cachefiles_cache *cache; 22 struct cachefiles_ondemand_info *info; 23 int object_id; 24 struct cachefiles_req *req; 25 XA_STATE(xas, NULL, 0); 26 27 if (!object) 28 return 0; 29 30 info = object->ondemand; 31 cache = object->volume->cache; 32 xas.xa = &cache->reqs; 33 34 xa_lock(&cache->reqs); 35 spin_lock(&info->lock); 36 object_id = info->ondemand_id; 37 info->ondemand_id = CACHEFILES_ONDEMAND_ID_CLOSED; 38 cachefiles_ondemand_set_object_close(object); 39 spin_unlock(&info->lock); 40 41 /* Only flush CACHEFILES_REQ_NEW marked req to avoid race with daemon_read */ 42 xas_for_each_marked(&xas, req, ULONG_MAX, CACHEFILES_REQ_NEW) { 43 if (req->msg.object_id == object_id && 44 req->msg.opcode == CACHEFILES_OP_CLOSE) { 45 complete(&req->done); 46 xas_store(&xas, NULL); 47 } 48 } 49 xa_unlock(&cache->reqs); 50 51 xa_erase(&cache->ondemand_ids, object_id); 52 trace_cachefiles_ondemand_fd_release(object, object_id); 53 cachefiles_put_object(object, cachefiles_obj_put_ondemand_fd); 54 cachefiles_put_unbind_pincount(cache); 55 return 0; 56 } 57 58 static ssize_t cachefiles_ondemand_fd_write_iter(struct kiocb *kiocb, 59 struct iov_iter *iter) 60 { 61 struct cachefiles_object *object = kiocb->ki_filp->private_data; 62 struct cachefiles_cache *cache = object->volume->cache; 63 struct file *file = object->file; 64 size_t len = iter->count; 65 loff_t pos = kiocb->ki_pos; 66 const struct cred *saved_cred; 67 int ret; 68 69 if (!file) 70 return -ENOBUFS; 71 72 cachefiles_begin_secure(cache, &saved_cred); 73 ret = __cachefiles_prepare_write(object, file, &pos, &len, len, true); 74 cachefiles_end_secure(cache, saved_cred); 75 if (ret < 0) 76 return ret; 77 78 trace_cachefiles_ondemand_fd_write(object, file_inode(file), pos, len); 79 ret = __cachefiles_write(object, file, pos, iter, NULL, NULL); 80 if (!ret) 81 ret = len; 82 83 return ret; 84 } 85 86 static loff_t cachefiles_ondemand_fd_llseek(struct file *filp, loff_t pos, 87 int whence) 88 { 89 struct cachefiles_object *object = filp->private_data; 90 struct file *file = object->file; 91 92 if (!file) 93 return -ENOBUFS; 94 95 return vfs_llseek(file, pos, whence); 96 } 97 98 static long cachefiles_ondemand_fd_ioctl(struct file *filp, unsigned int ioctl, 99 unsigned long id) 100 { 101 struct cachefiles_object *object = filp->private_data; 102 struct cachefiles_cache *cache = object->volume->cache; 103 struct cachefiles_req *req; 104 XA_STATE(xas, &cache->reqs, id); 105 106 if (ioctl != CACHEFILES_IOC_READ_COMPLETE) 107 return -EINVAL; 108 109 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags)) 110 return -EOPNOTSUPP; 111 112 xa_lock(&cache->reqs); 113 req = xas_load(&xas); 114 if (!req || req->msg.opcode != CACHEFILES_OP_READ || 115 req->object != object) { 116 xa_unlock(&cache->reqs); 117 return -EINVAL; 118 } 119 xas_store(&xas, NULL); 120 xa_unlock(&cache->reqs); 121 122 trace_cachefiles_ondemand_cread(object, id); 123 complete(&req->done); 124 return 0; 125 } 126 127 static const struct file_operations cachefiles_ondemand_fd_fops = { 128 .owner = THIS_MODULE, 129 .release = cachefiles_ondemand_fd_release, 130 .write_iter = cachefiles_ondemand_fd_write_iter, 131 .llseek = cachefiles_ondemand_fd_llseek, 132 .unlocked_ioctl = cachefiles_ondemand_fd_ioctl, 133 }; 134 135 /* 136 * OPEN request Completion (copen) 137 * - command: "copen <id>,<cache_size>" 138 * <cache_size> indicates the object size if >=0, error code if negative 139 */ 140 int cachefiles_ondemand_copen(struct cachefiles_cache *cache, char *args) 141 { 142 struct cachefiles_req *req; 143 struct fscache_cookie *cookie; 144 struct cachefiles_ondemand_info *info; 145 char *pid, *psize; 146 unsigned long id; 147 long size; 148 int ret; 149 XA_STATE(xas, &cache->reqs, 0); 150 151 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags)) 152 return -EOPNOTSUPP; 153 154 if (!*args) { 155 pr_err("Empty id specified\n"); 156 return -EINVAL; 157 } 158 159 pid = args; 160 psize = strchr(args, ','); 161 if (!psize) { 162 pr_err("Cache size is not specified\n"); 163 return -EINVAL; 164 } 165 166 *psize = 0; 167 psize++; 168 169 ret = kstrtoul(pid, 0, &id); 170 if (ret) 171 return ret; 172 173 xa_lock(&cache->reqs); 174 xas.xa_index = id; 175 req = xas_load(&xas); 176 if (!req || req->msg.opcode != CACHEFILES_OP_OPEN || 177 !req->object->ondemand->ondemand_id) { 178 xa_unlock(&cache->reqs); 179 return -EINVAL; 180 } 181 xas_store(&xas, NULL); 182 xa_unlock(&cache->reqs); 183 184 info = req->object->ondemand; 185 /* fail OPEN request if copen format is invalid */ 186 ret = kstrtol(psize, 0, &size); 187 if (ret) { 188 req->error = ret; 189 goto out; 190 } 191 192 /* fail OPEN request if daemon reports an error */ 193 if (size < 0) { 194 if (!IS_ERR_VALUE(size)) { 195 req->error = -EINVAL; 196 ret = -EINVAL; 197 } else { 198 req->error = size; 199 ret = 0; 200 } 201 goto out; 202 } 203 204 spin_lock(&info->lock); 205 /* 206 * The anonymous fd was closed before copen ? Fail the request. 207 * 208 * t1 | t2 209 * --------------------------------------------------------- 210 * cachefiles_ondemand_copen 211 * req = xa_erase(&cache->reqs, id) 212 * // Anon fd is maliciously closed. 213 * cachefiles_ondemand_fd_release 214 * xa_lock(&cache->reqs) 215 * cachefiles_ondemand_set_object_close(object) 216 * xa_unlock(&cache->reqs) 217 * cachefiles_ondemand_set_object_open 218 * // No one will ever close it again. 219 * cachefiles_ondemand_daemon_read 220 * cachefiles_ondemand_select_req 221 * 222 * Get a read req but its fd is already closed. The daemon can't 223 * issue a cread ioctl with an closed fd, then hung. 224 */ 225 if (info->ondemand_id == CACHEFILES_ONDEMAND_ID_CLOSED) { 226 spin_unlock(&info->lock); 227 req->error = -EBADFD; 228 goto out; 229 } 230 cookie = req->object->cookie; 231 cookie->object_size = size; 232 if (size) 233 clear_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); 234 else 235 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); 236 trace_cachefiles_ondemand_copen(req->object, id, size); 237 238 cachefiles_ondemand_set_object_open(req->object); 239 spin_unlock(&info->lock); 240 wake_up_all(&cache->daemon_pollwq); 241 242 out: 243 spin_lock(&info->lock); 244 /* Need to set object close to avoid reopen status continuing */ 245 if (info->ondemand_id == CACHEFILES_ONDEMAND_ID_CLOSED) 246 cachefiles_ondemand_set_object_close(req->object); 247 spin_unlock(&info->lock); 248 complete(&req->done); 249 return ret; 250 } 251 252 int cachefiles_ondemand_restore(struct cachefiles_cache *cache, char *args) 253 { 254 struct cachefiles_req *req; 255 256 XA_STATE(xas, &cache->reqs, 0); 257 258 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags)) 259 return -EOPNOTSUPP; 260 261 /* 262 * Reset the requests to CACHEFILES_REQ_NEW state, so that the 263 * requests have been processed halfway before the crash of the 264 * user daemon could be reprocessed after the recovery. 265 */ 266 xas_lock(&xas); 267 xas_for_each(&xas, req, ULONG_MAX) 268 xas_set_mark(&xas, CACHEFILES_REQ_NEW); 269 xas_unlock(&xas); 270 271 wake_up_all(&cache->daemon_pollwq); 272 return 0; 273 } 274 275 static int cachefiles_ondemand_get_fd(struct cachefiles_req *req, 276 struct ondemand_anon_file *anon_file) 277 { 278 struct cachefiles_object *object; 279 struct cachefiles_cache *cache; 280 struct cachefiles_open *load; 281 u32 object_id; 282 int ret; 283 284 object = cachefiles_grab_object(req->object, 285 cachefiles_obj_get_ondemand_fd); 286 cache = object->volume->cache; 287 288 ret = xa_alloc_cyclic(&cache->ondemand_ids, &object_id, NULL, 289 XA_LIMIT(1, INT_MAX), 290 &cache->ondemand_id_next, GFP_KERNEL); 291 if (ret < 0) 292 goto err; 293 294 anon_file->fd = get_unused_fd_flags(O_WRONLY); 295 if (anon_file->fd < 0) { 296 ret = anon_file->fd; 297 goto err_free_id; 298 } 299 300 anon_file->file = anon_inode_getfile("[cachefiles]", 301 &cachefiles_ondemand_fd_fops, object, O_WRONLY); 302 if (IS_ERR(anon_file->file)) { 303 ret = PTR_ERR(anon_file->file); 304 goto err_put_fd; 305 } 306 307 spin_lock(&object->ondemand->lock); 308 if (object->ondemand->ondemand_id > 0) { 309 spin_unlock(&object->ondemand->lock); 310 /* Pair with check in cachefiles_ondemand_fd_release(). */ 311 anon_file->file->private_data = NULL; 312 ret = -EEXIST; 313 goto err_put_file; 314 } 315 316 anon_file->file->f_mode |= FMODE_PWRITE | FMODE_LSEEK; 317 318 load = (void *)req->msg.data; 319 load->fd = anon_file->fd; 320 object->ondemand->ondemand_id = object_id; 321 spin_unlock(&object->ondemand->lock); 322 323 cachefiles_get_unbind_pincount(cache); 324 trace_cachefiles_ondemand_open(object, &req->msg, load); 325 return 0; 326 327 err_put_file: 328 fput(anon_file->file); 329 anon_file->file = NULL; 330 err_put_fd: 331 put_unused_fd(anon_file->fd); 332 anon_file->fd = ret; 333 err_free_id: 334 xa_erase(&cache->ondemand_ids, object_id); 335 err: 336 spin_lock(&object->ondemand->lock); 337 /* Avoid marking an opened object as closed. */ 338 if (object->ondemand->ondemand_id <= 0) 339 cachefiles_ondemand_set_object_close(object); 340 spin_unlock(&object->ondemand->lock); 341 cachefiles_put_object(object, cachefiles_obj_put_ondemand_fd); 342 return ret; 343 } 344 345 static void ondemand_object_worker(struct work_struct *work) 346 { 347 struct cachefiles_ondemand_info *info = 348 container_of(work, struct cachefiles_ondemand_info, ondemand_work); 349 350 cachefiles_ondemand_init_object(info->object); 351 } 352 353 /* 354 * If there are any inflight or subsequent READ requests on the 355 * closed object, reopen it. 356 * Skip read requests whose related object is reopening. 357 */ 358 static struct cachefiles_req *cachefiles_ondemand_select_req(struct xa_state *xas, 359 unsigned long xa_max) 360 { 361 struct cachefiles_req *req; 362 struct cachefiles_object *object; 363 struct cachefiles_ondemand_info *info; 364 365 xas_for_each_marked(xas, req, xa_max, CACHEFILES_REQ_NEW) { 366 if (req->msg.opcode != CACHEFILES_OP_READ) 367 return req; 368 object = req->object; 369 info = object->ondemand; 370 if (cachefiles_ondemand_object_is_close(object)) { 371 cachefiles_ondemand_set_object_reopening(object); 372 queue_work(fscache_wq, &info->ondemand_work); 373 continue; 374 } 375 if (cachefiles_ondemand_object_is_reopening(object)) 376 continue; 377 return req; 378 } 379 return NULL; 380 } 381 382 static inline bool cachefiles_ondemand_finish_req(struct cachefiles_req *req, 383 struct xa_state *xas, int err) 384 { 385 if (unlikely(!xas || !req)) 386 return false; 387 388 if (xa_cmpxchg(xas->xa, xas->xa_index, req, NULL, 0) != req) 389 return false; 390 391 req->error = err; 392 complete(&req->done); 393 return true; 394 } 395 396 ssize_t cachefiles_ondemand_daemon_read(struct cachefiles_cache *cache, 397 char __user *_buffer, size_t buflen) 398 { 399 struct cachefiles_req *req; 400 struct cachefiles_msg *msg; 401 size_t n; 402 int ret = 0; 403 struct ondemand_anon_file anon_file; 404 XA_STATE(xas, &cache->reqs, cache->req_id_next); 405 406 xa_lock(&cache->reqs); 407 /* 408 * Cyclically search for a request that has not ever been processed, 409 * to prevent requests from being processed repeatedly, and make 410 * request distribution fair. 411 */ 412 req = cachefiles_ondemand_select_req(&xas, ULONG_MAX); 413 if (!req && cache->req_id_next > 0) { 414 xas_set(&xas, 0); 415 req = cachefiles_ondemand_select_req(&xas, cache->req_id_next - 1); 416 } 417 if (!req) { 418 xa_unlock(&cache->reqs); 419 return 0; 420 } 421 422 msg = &req->msg; 423 n = msg->len; 424 425 if (n > buflen) { 426 xa_unlock(&cache->reqs); 427 return -EMSGSIZE; 428 } 429 430 xas_clear_mark(&xas, CACHEFILES_REQ_NEW); 431 cache->req_id_next = xas.xa_index + 1; 432 refcount_inc(&req->ref); 433 cachefiles_grab_object(req->object, cachefiles_obj_get_read_req); 434 xa_unlock(&cache->reqs); 435 436 if (msg->opcode == CACHEFILES_OP_OPEN) { 437 ret = cachefiles_ondemand_get_fd(req, &anon_file); 438 if (ret) 439 goto out; 440 } 441 442 msg->msg_id = xas.xa_index; 443 msg->object_id = req->object->ondemand->ondemand_id; 444 445 if (copy_to_user(_buffer, msg, n) != 0) 446 ret = -EFAULT; 447 448 if (msg->opcode == CACHEFILES_OP_OPEN) { 449 if (ret < 0) { 450 fput(anon_file.file); 451 put_unused_fd(anon_file.fd); 452 goto out; 453 } 454 fd_install(anon_file.fd, anon_file.file); 455 } 456 out: 457 cachefiles_put_object(req->object, cachefiles_obj_put_read_req); 458 /* Remove error request and CLOSE request has no reply */ 459 if (ret || msg->opcode == CACHEFILES_OP_CLOSE) 460 cachefiles_ondemand_finish_req(req, &xas, ret); 461 cachefiles_req_put(req); 462 return ret ? ret : n; 463 } 464 465 typedef int (*init_req_fn)(struct cachefiles_req *req, void *private); 466 467 static int cachefiles_ondemand_send_req(struct cachefiles_object *object, 468 enum cachefiles_opcode opcode, 469 size_t data_len, 470 init_req_fn init_req, 471 void *private) 472 { 473 struct cachefiles_cache *cache = object->volume->cache; 474 struct cachefiles_req *req = NULL; 475 XA_STATE(xas, &cache->reqs, 0); 476 int ret; 477 478 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags)) 479 return 0; 480 481 if (test_bit(CACHEFILES_DEAD, &cache->flags)) { 482 ret = -EIO; 483 goto out; 484 } 485 486 req = kzalloc(sizeof(*req) + data_len, GFP_KERNEL); 487 if (!req) { 488 ret = -ENOMEM; 489 goto out; 490 } 491 492 refcount_set(&req->ref, 1); 493 req->object = object; 494 init_completion(&req->done); 495 req->msg.opcode = opcode; 496 req->msg.len = sizeof(struct cachefiles_msg) + data_len; 497 498 ret = init_req(req, private); 499 if (ret) 500 goto out; 501 502 do { 503 /* 504 * Stop enqueuing the request when daemon is dying. The 505 * following two operations need to be atomic as a whole. 506 * 1) check cache state, and 507 * 2) enqueue request if cache is alive. 508 * Otherwise the request may be enqueued after xarray has been 509 * flushed, leaving the orphan request never being completed. 510 * 511 * CPU 1 CPU 2 512 * ===== ===== 513 * test CACHEFILES_DEAD bit 514 * set CACHEFILES_DEAD bit 515 * flush requests in the xarray 516 * enqueue the request 517 */ 518 xas_lock(&xas); 519 520 if (test_bit(CACHEFILES_DEAD, &cache->flags)) { 521 xas_unlock(&xas); 522 ret = -EIO; 523 goto out; 524 } 525 526 /* coupled with the barrier in cachefiles_flush_reqs() */ 527 smp_mb(); 528 529 if (opcode == CACHEFILES_OP_CLOSE && 530 !cachefiles_ondemand_object_is_open(object)) { 531 WARN_ON_ONCE(object->ondemand->ondemand_id == 0); 532 xas_unlock(&xas); 533 ret = -EIO; 534 goto out; 535 } 536 537 xas.xa_index = 0; 538 xas_find_marked(&xas, UINT_MAX, XA_FREE_MARK); 539 if (xas.xa_node == XAS_RESTART) 540 xas_set_err(&xas, -EBUSY); 541 xas_store(&xas, req); 542 xas_clear_mark(&xas, XA_FREE_MARK); 543 xas_set_mark(&xas, CACHEFILES_REQ_NEW); 544 xas_unlock(&xas); 545 } while (xas_nomem(&xas, GFP_KERNEL)); 546 547 ret = xas_error(&xas); 548 if (ret) 549 goto out; 550 551 wake_up_all(&cache->daemon_pollwq); 552 wait: 553 ret = wait_for_completion_killable(&req->done); 554 if (!ret) { 555 ret = req->error; 556 } else { 557 ret = -EINTR; 558 if (!cachefiles_ondemand_finish_req(req, &xas, ret)) { 559 /* Someone will complete it soon. */ 560 cpu_relax(); 561 goto wait; 562 } 563 } 564 cachefiles_req_put(req); 565 return ret; 566 out: 567 /* Reset the object to close state in error handling path. 568 * If error occurs after creating the anonymous fd, 569 * cachefiles_ondemand_fd_release() will set object to close. 570 */ 571 if (opcode == CACHEFILES_OP_OPEN) 572 cachefiles_ondemand_set_object_close(object); 573 kfree(req); 574 return ret; 575 } 576 577 static int cachefiles_ondemand_init_open_req(struct cachefiles_req *req, 578 void *private) 579 { 580 struct cachefiles_object *object = req->object; 581 struct fscache_cookie *cookie = object->cookie; 582 struct fscache_volume *volume = object->volume->vcookie; 583 struct cachefiles_open *load = (void *)req->msg.data; 584 size_t volume_key_size, cookie_key_size; 585 void *volume_key, *cookie_key; 586 587 /* 588 * Volume key is a NUL-terminated string. key[0] stores strlen() of the 589 * string, followed by the content of the string (excluding '\0'). 590 */ 591 volume_key_size = volume->key[0] + 1; 592 volume_key = volume->key + 1; 593 594 /* Cookie key is binary data, which is netfs specific. */ 595 cookie_key_size = cookie->key_len; 596 cookie_key = fscache_get_key(cookie); 597 598 if (!(object->cookie->advice & FSCACHE_ADV_WANT_CACHE_SIZE)) { 599 pr_err("WANT_CACHE_SIZE is needed for on-demand mode\n"); 600 return -EINVAL; 601 } 602 603 load->volume_key_size = volume_key_size; 604 load->cookie_key_size = cookie_key_size; 605 memcpy(load->data, volume_key, volume_key_size); 606 memcpy(load->data + volume_key_size, cookie_key, cookie_key_size); 607 608 return 0; 609 } 610 611 static int cachefiles_ondemand_init_close_req(struct cachefiles_req *req, 612 void *private) 613 { 614 struct cachefiles_object *object = req->object; 615 616 if (!cachefiles_ondemand_object_is_open(object)) 617 return -ENOENT; 618 619 trace_cachefiles_ondemand_close(object, &req->msg); 620 return 0; 621 } 622 623 struct cachefiles_read_ctx { 624 loff_t off; 625 size_t len; 626 }; 627 628 static int cachefiles_ondemand_init_read_req(struct cachefiles_req *req, 629 void *private) 630 { 631 struct cachefiles_object *object = req->object; 632 struct cachefiles_read *load = (void *)req->msg.data; 633 struct cachefiles_read_ctx *read_ctx = private; 634 635 load->off = read_ctx->off; 636 load->len = read_ctx->len; 637 trace_cachefiles_ondemand_read(object, &req->msg, load); 638 return 0; 639 } 640 641 int cachefiles_ondemand_init_object(struct cachefiles_object *object) 642 { 643 struct fscache_cookie *cookie = object->cookie; 644 struct fscache_volume *volume = object->volume->vcookie; 645 size_t volume_key_size, cookie_key_size, data_len; 646 647 if (!object->ondemand) 648 return 0; 649 650 /* 651 * CacheFiles will firstly check the cache file under the root cache 652 * directory. If the coherency check failed, it will fallback to 653 * creating a new tmpfile as the cache file. Reuse the previously 654 * allocated object ID if any. 655 */ 656 if (cachefiles_ondemand_object_is_open(object)) 657 return 0; 658 659 volume_key_size = volume->key[0] + 1; 660 cookie_key_size = cookie->key_len; 661 data_len = sizeof(struct cachefiles_open) + 662 volume_key_size + cookie_key_size; 663 664 return cachefiles_ondemand_send_req(object, CACHEFILES_OP_OPEN, 665 data_len, cachefiles_ondemand_init_open_req, NULL); 666 } 667 668 void cachefiles_ondemand_clean_object(struct cachefiles_object *object) 669 { 670 cachefiles_ondemand_send_req(object, CACHEFILES_OP_CLOSE, 0, 671 cachefiles_ondemand_init_close_req, NULL); 672 } 673 674 int cachefiles_ondemand_init_obj_info(struct cachefiles_object *object, 675 struct cachefiles_volume *volume) 676 { 677 if (!cachefiles_in_ondemand_mode(volume->cache)) 678 return 0; 679 680 object->ondemand = kzalloc(sizeof(struct cachefiles_ondemand_info), 681 GFP_KERNEL); 682 if (!object->ondemand) 683 return -ENOMEM; 684 685 object->ondemand->object = object; 686 spin_lock_init(&object->ondemand->lock); 687 INIT_WORK(&object->ondemand->ondemand_work, ondemand_object_worker); 688 return 0; 689 } 690 691 void cachefiles_ondemand_deinit_obj_info(struct cachefiles_object *object) 692 { 693 kfree(object->ondemand); 694 object->ondemand = NULL; 695 } 696 697 int cachefiles_ondemand_read(struct cachefiles_object *object, 698 loff_t pos, size_t len) 699 { 700 struct cachefiles_read_ctx read_ctx = {pos, len}; 701 702 return cachefiles_ondemand_send_req(object, CACHEFILES_OP_READ, 703 sizeof(struct cachefiles_read), 704 cachefiles_ondemand_init_read_req, &read_ctx); 705 } 706