1 /* 2 * Network block device - make block devices work over TCP 3 * 4 * Note that you can not swap over this thing, yet. Seems to work but 5 * deadlocks sometimes - you can not swap over TCP in general. 6 * 7 * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> 8 * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> 9 * 10 * This file is released under GPLv2 or later. 11 * 12 * (part of code stolen from loop.c) 13 */ 14 15 #include <linux/major.h> 16 17 #include <linux/blkdev.h> 18 #include <linux/module.h> 19 #include <linux/init.h> 20 #include <linux/sched.h> 21 #include <linux/fs.h> 22 #include <linux/bio.h> 23 #include <linux/stat.h> 24 #include <linux/errno.h> 25 #include <linux/file.h> 26 #include <linux/ioctl.h> 27 #include <linux/mutex.h> 28 #include <linux/compiler.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/slab.h> 32 #include <net/sock.h> 33 #include <linux/net.h> 34 #include <linux/kthread.h> 35 #include <linux/types.h> 36 #include <linux/debugfs.h> 37 #include <linux/blk-mq.h> 38 39 #include <linux/uaccess.h> 40 #include <asm/types.h> 41 42 #include <linux/nbd.h> 43 44 static DEFINE_IDR(nbd_index_idr); 45 static DEFINE_MUTEX(nbd_index_mutex); 46 47 struct nbd_sock { 48 struct socket *sock; 49 struct mutex tx_lock; 50 }; 51 52 #define NBD_TIMEDOUT 0 53 #define NBD_DISCONNECT_REQUESTED 1 54 #define NBD_DISCONNECTED 2 55 #define NBD_RUNNING 3 56 57 struct nbd_device { 58 u32 flags; 59 unsigned long runtime_flags; 60 struct nbd_sock **socks; 61 int magic; 62 63 struct blk_mq_tag_set tag_set; 64 65 struct mutex config_lock; 66 struct gendisk *disk; 67 int num_connections; 68 atomic_t recv_threads; 69 wait_queue_head_t recv_wq; 70 loff_t blksize; 71 loff_t bytesize; 72 73 struct task_struct *task_recv; 74 struct task_struct *task_setup; 75 76 #if IS_ENABLED(CONFIG_DEBUG_FS) 77 struct dentry *dbg_dir; 78 #endif 79 }; 80 81 struct nbd_cmd { 82 struct nbd_device *nbd; 83 struct completion send_complete; 84 }; 85 86 #if IS_ENABLED(CONFIG_DEBUG_FS) 87 static struct dentry *nbd_dbg_dir; 88 #endif 89 90 #define nbd_name(nbd) ((nbd)->disk->disk_name) 91 92 #define NBD_MAGIC 0x68797548 93 94 static unsigned int nbds_max = 16; 95 static int max_part; 96 static struct workqueue_struct *recv_workqueue; 97 static int part_shift; 98 99 static inline struct device *nbd_to_dev(struct nbd_device *nbd) 100 { 101 return disk_to_dev(nbd->disk); 102 } 103 104 static bool nbd_is_connected(struct nbd_device *nbd) 105 { 106 return !!nbd->task_recv; 107 } 108 109 static const char *nbdcmd_to_ascii(int cmd) 110 { 111 switch (cmd) { 112 case NBD_CMD_READ: return "read"; 113 case NBD_CMD_WRITE: return "write"; 114 case NBD_CMD_DISC: return "disconnect"; 115 case NBD_CMD_FLUSH: return "flush"; 116 case NBD_CMD_TRIM: return "trim/discard"; 117 } 118 return "invalid"; 119 } 120 121 static int nbd_size_clear(struct nbd_device *nbd, struct block_device *bdev) 122 { 123 bdev->bd_inode->i_size = 0; 124 set_capacity(nbd->disk, 0); 125 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 126 127 return 0; 128 } 129 130 static void nbd_size_update(struct nbd_device *nbd, struct block_device *bdev) 131 { 132 if (!nbd_is_connected(nbd)) 133 return; 134 135 bdev->bd_inode->i_size = nbd->bytesize; 136 set_capacity(nbd->disk, nbd->bytesize >> 9); 137 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 138 } 139 140 static int nbd_size_set(struct nbd_device *nbd, struct block_device *bdev, 141 loff_t blocksize, loff_t nr_blocks) 142 { 143 int ret; 144 145 ret = set_blocksize(bdev, blocksize); 146 if (ret) 147 return ret; 148 149 nbd->blksize = blocksize; 150 nbd->bytesize = blocksize * nr_blocks; 151 152 nbd_size_update(nbd, bdev); 153 154 return 0; 155 } 156 157 static void nbd_end_request(struct nbd_cmd *cmd) 158 { 159 struct nbd_device *nbd = cmd->nbd; 160 struct request *req = blk_mq_rq_from_pdu(cmd); 161 int error = req->errors ? -EIO : 0; 162 163 dev_dbg(nbd_to_dev(nbd), "request %p: %s\n", cmd, 164 error ? "failed" : "done"); 165 166 blk_mq_complete_request(req, error); 167 } 168 169 /* 170 * Forcibly shutdown the socket causing all listeners to error 171 */ 172 static void sock_shutdown(struct nbd_device *nbd) 173 { 174 int i; 175 176 if (nbd->num_connections == 0) 177 return; 178 if (test_and_set_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) 179 return; 180 181 for (i = 0; i < nbd->num_connections; i++) { 182 struct nbd_sock *nsock = nbd->socks[i]; 183 mutex_lock(&nsock->tx_lock); 184 kernel_sock_shutdown(nsock->sock, SHUT_RDWR); 185 mutex_unlock(&nsock->tx_lock); 186 } 187 dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); 188 } 189 190 static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, 191 bool reserved) 192 { 193 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 194 struct nbd_device *nbd = cmd->nbd; 195 196 dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n"); 197 set_bit(NBD_TIMEDOUT, &nbd->runtime_flags); 198 req->errors++; 199 200 mutex_lock(&nbd->config_lock); 201 sock_shutdown(nbd); 202 mutex_unlock(&nbd->config_lock); 203 return BLK_EH_HANDLED; 204 } 205 206 /* 207 * Send or receive packet. 208 */ 209 static int sock_xmit(struct nbd_device *nbd, int index, int send, void *buf, 210 int size, int msg_flags) 211 { 212 struct socket *sock = nbd->socks[index]->sock; 213 int result; 214 struct msghdr msg; 215 struct kvec iov; 216 unsigned long pflags = current->flags; 217 218 if (unlikely(!sock)) { 219 dev_err_ratelimited(disk_to_dev(nbd->disk), 220 "Attempted %s on closed socket in sock_xmit\n", 221 (send ? "send" : "recv")); 222 return -EINVAL; 223 } 224 225 current->flags |= PF_MEMALLOC; 226 do { 227 sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; 228 iov.iov_base = buf; 229 iov.iov_len = size; 230 msg.msg_name = NULL; 231 msg.msg_namelen = 0; 232 msg.msg_control = NULL; 233 msg.msg_controllen = 0; 234 msg.msg_flags = msg_flags | MSG_NOSIGNAL; 235 236 if (send) 237 result = kernel_sendmsg(sock, &msg, &iov, 1, size); 238 else 239 result = kernel_recvmsg(sock, &msg, &iov, 1, size, 240 msg.msg_flags); 241 242 if (result <= 0) { 243 if (result == 0) 244 result = -EPIPE; /* short read */ 245 break; 246 } 247 size -= result; 248 buf += result; 249 } while (size > 0); 250 251 tsk_restore_flags(current, pflags, PF_MEMALLOC); 252 253 return result; 254 } 255 256 static inline int sock_send_bvec(struct nbd_device *nbd, int index, 257 struct bio_vec *bvec, int flags) 258 { 259 int result; 260 void *kaddr = kmap(bvec->bv_page); 261 result = sock_xmit(nbd, index, 1, kaddr + bvec->bv_offset, 262 bvec->bv_len, flags); 263 kunmap(bvec->bv_page); 264 return result; 265 } 266 267 /* always call with the tx_lock held */ 268 static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index) 269 { 270 struct request *req = blk_mq_rq_from_pdu(cmd); 271 int result; 272 struct nbd_request request; 273 unsigned long size = blk_rq_bytes(req); 274 struct bio *bio; 275 u32 type; 276 u32 tag = blk_mq_unique_tag(req); 277 278 switch (req_op(req)) { 279 case REQ_OP_DISCARD: 280 type = NBD_CMD_TRIM; 281 break; 282 case REQ_OP_FLUSH: 283 type = NBD_CMD_FLUSH; 284 break; 285 case REQ_OP_WRITE: 286 type = NBD_CMD_WRITE; 287 break; 288 case REQ_OP_READ: 289 type = NBD_CMD_READ; 290 break; 291 default: 292 return -EIO; 293 } 294 295 if (rq_data_dir(req) == WRITE && 296 (nbd->flags & NBD_FLAG_READ_ONLY)) { 297 dev_err_ratelimited(disk_to_dev(nbd->disk), 298 "Write on read-only\n"); 299 return -EIO; 300 } 301 302 memset(&request, 0, sizeof(request)); 303 request.magic = htonl(NBD_REQUEST_MAGIC); 304 request.type = htonl(type); 305 if (type != NBD_CMD_FLUSH) { 306 request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); 307 request.len = htonl(size); 308 } 309 memcpy(request.handle, &tag, sizeof(tag)); 310 311 dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", 312 cmd, nbdcmd_to_ascii(type), 313 (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); 314 result = sock_xmit(nbd, index, 1, &request, sizeof(request), 315 (type == NBD_CMD_WRITE) ? MSG_MORE : 0); 316 if (result <= 0) { 317 dev_err_ratelimited(disk_to_dev(nbd->disk), 318 "Send control failed (result %d)\n", result); 319 return -EIO; 320 } 321 322 if (type != NBD_CMD_WRITE) 323 return 0; 324 325 bio = req->bio; 326 while (bio) { 327 struct bio *next = bio->bi_next; 328 struct bvec_iter iter; 329 struct bio_vec bvec; 330 331 bio_for_each_segment(bvec, bio, iter) { 332 bool is_last = !next && bio_iter_last(bvec, iter); 333 int flags = is_last ? 0 : MSG_MORE; 334 335 dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", 336 cmd, bvec.bv_len); 337 result = sock_send_bvec(nbd, index, &bvec, flags); 338 if (result <= 0) { 339 dev_err(disk_to_dev(nbd->disk), 340 "Send data failed (result %d)\n", 341 result); 342 return -EIO; 343 } 344 /* 345 * The completion might already have come in, 346 * so break for the last one instead of letting 347 * the iterator do it. This prevents use-after-free 348 * of the bio. 349 */ 350 if (is_last) 351 break; 352 } 353 bio = next; 354 } 355 return 0; 356 } 357 358 static inline int sock_recv_bvec(struct nbd_device *nbd, int index, 359 struct bio_vec *bvec) 360 { 361 int result; 362 void *kaddr = kmap(bvec->bv_page); 363 result = sock_xmit(nbd, index, 0, kaddr + bvec->bv_offset, 364 bvec->bv_len, MSG_WAITALL); 365 kunmap(bvec->bv_page); 366 return result; 367 } 368 369 /* NULL returned = something went wrong, inform userspace */ 370 static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) 371 { 372 int result; 373 struct nbd_reply reply; 374 struct nbd_cmd *cmd; 375 struct request *req = NULL; 376 u16 hwq; 377 u32 tag; 378 379 reply.magic = 0; 380 result = sock_xmit(nbd, index, 0, &reply, sizeof(reply), MSG_WAITALL); 381 if (result <= 0) { 382 if (!test_bit(NBD_DISCONNECTED, &nbd->runtime_flags) && 383 !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 384 dev_err(disk_to_dev(nbd->disk), 385 "Receive control failed (result %d)\n", result); 386 return ERR_PTR(result); 387 } 388 389 if (ntohl(reply.magic) != NBD_REPLY_MAGIC) { 390 dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", 391 (unsigned long)ntohl(reply.magic)); 392 return ERR_PTR(-EPROTO); 393 } 394 395 memcpy(&tag, reply.handle, sizeof(u32)); 396 397 hwq = blk_mq_unique_tag_to_hwq(tag); 398 if (hwq < nbd->tag_set.nr_hw_queues) 399 req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq], 400 blk_mq_unique_tag_to_tag(tag)); 401 if (!req || !blk_mq_request_started(req)) { 402 dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n", 403 tag, req); 404 return ERR_PTR(-ENOENT); 405 } 406 cmd = blk_mq_rq_to_pdu(req); 407 if (ntohl(reply.error)) { 408 dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n", 409 ntohl(reply.error)); 410 req->errors++; 411 return cmd; 412 } 413 414 dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", cmd); 415 if (rq_data_dir(req) != WRITE) { 416 struct req_iterator iter; 417 struct bio_vec bvec; 418 419 rq_for_each_segment(bvec, req, iter) { 420 result = sock_recv_bvec(nbd, index, &bvec); 421 if (result <= 0) { 422 dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n", 423 result); 424 req->errors++; 425 return cmd; 426 } 427 dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n", 428 cmd, bvec.bv_len); 429 } 430 } else { 431 /* See the comment in nbd_queue_rq. */ 432 wait_for_completion(&cmd->send_complete); 433 } 434 return cmd; 435 } 436 437 static ssize_t pid_show(struct device *dev, 438 struct device_attribute *attr, char *buf) 439 { 440 struct gendisk *disk = dev_to_disk(dev); 441 struct nbd_device *nbd = (struct nbd_device *)disk->private_data; 442 443 return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv)); 444 } 445 446 static struct device_attribute pid_attr = { 447 .attr = { .name = "pid", .mode = S_IRUGO}, 448 .show = pid_show, 449 }; 450 451 struct recv_thread_args { 452 struct work_struct work; 453 struct nbd_device *nbd; 454 int index; 455 }; 456 457 static void recv_work(struct work_struct *work) 458 { 459 struct recv_thread_args *args = container_of(work, 460 struct recv_thread_args, 461 work); 462 struct nbd_device *nbd = args->nbd; 463 struct nbd_cmd *cmd; 464 int ret = 0; 465 466 BUG_ON(nbd->magic != NBD_MAGIC); 467 while (1) { 468 cmd = nbd_read_stat(nbd, args->index); 469 if (IS_ERR(cmd)) { 470 ret = PTR_ERR(cmd); 471 break; 472 } 473 474 nbd_end_request(cmd); 475 } 476 477 /* 478 * We got an error, shut everybody down if this wasn't the result of a 479 * disconnect request. 480 */ 481 if (ret && !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 482 sock_shutdown(nbd); 483 atomic_dec(&nbd->recv_threads); 484 wake_up(&nbd->recv_wq); 485 } 486 487 static void nbd_clear_req(struct request *req, void *data, bool reserved) 488 { 489 struct nbd_cmd *cmd; 490 491 if (!blk_mq_request_started(req)) 492 return; 493 cmd = blk_mq_rq_to_pdu(req); 494 req->errors++; 495 nbd_end_request(cmd); 496 } 497 498 static void nbd_clear_que(struct nbd_device *nbd) 499 { 500 BUG_ON(nbd->magic != NBD_MAGIC); 501 502 blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL); 503 dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n"); 504 } 505 506 507 static void nbd_handle_cmd(struct nbd_cmd *cmd, int index) 508 { 509 struct request *req = blk_mq_rq_from_pdu(cmd); 510 struct nbd_device *nbd = cmd->nbd; 511 struct nbd_sock *nsock; 512 513 if (index >= nbd->num_connections) { 514 dev_err_ratelimited(disk_to_dev(nbd->disk), 515 "Attempted send on invalid socket\n"); 516 goto error_out; 517 } 518 519 if (test_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) { 520 dev_err_ratelimited(disk_to_dev(nbd->disk), 521 "Attempted send on closed socket\n"); 522 goto error_out; 523 } 524 525 req->errors = 0; 526 527 nsock = nbd->socks[index]; 528 mutex_lock(&nsock->tx_lock); 529 if (unlikely(!nsock->sock)) { 530 mutex_unlock(&nsock->tx_lock); 531 dev_err_ratelimited(disk_to_dev(nbd->disk), 532 "Attempted send on closed socket\n"); 533 goto error_out; 534 } 535 536 if (nbd_send_cmd(nbd, cmd, index) != 0) { 537 dev_err_ratelimited(disk_to_dev(nbd->disk), 538 "Request send failed\n"); 539 req->errors++; 540 nbd_end_request(cmd); 541 } 542 543 mutex_unlock(&nsock->tx_lock); 544 545 return; 546 547 error_out: 548 req->errors++; 549 nbd_end_request(cmd); 550 } 551 552 static int nbd_queue_rq(struct blk_mq_hw_ctx *hctx, 553 const struct blk_mq_queue_data *bd) 554 { 555 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); 556 557 /* 558 * Since we look at the bio's to send the request over the network we 559 * need to make sure the completion work doesn't mark this request done 560 * before we are done doing our send. This keeps us from dereferencing 561 * freed data if we have particularly fast completions (ie we get the 562 * completion before we exit sock_xmit on the last bvec) or in the case 563 * that the server is misbehaving (or there was an error) before we're 564 * done sending everything over the wire. 565 */ 566 init_completion(&cmd->send_complete); 567 blk_mq_start_request(bd->rq); 568 nbd_handle_cmd(cmd, hctx->queue_num); 569 complete(&cmd->send_complete); 570 571 return BLK_MQ_RQ_QUEUE_OK; 572 } 573 574 static int nbd_add_socket(struct nbd_device *nbd, struct socket *sock) 575 { 576 struct nbd_sock **socks; 577 struct nbd_sock *nsock; 578 579 if (!nbd->task_setup) 580 nbd->task_setup = current; 581 if (nbd->task_setup != current) { 582 dev_err(disk_to_dev(nbd->disk), 583 "Device being setup by another task"); 584 return -EINVAL; 585 } 586 587 socks = krealloc(nbd->socks, (nbd->num_connections + 1) * 588 sizeof(struct nbd_sock *), GFP_KERNEL); 589 if (!socks) 590 return -ENOMEM; 591 nsock = kzalloc(sizeof(struct nbd_sock), GFP_KERNEL); 592 if (!nsock) 593 return -ENOMEM; 594 595 nbd->socks = socks; 596 597 mutex_init(&nsock->tx_lock); 598 nsock->sock = sock; 599 socks[nbd->num_connections++] = nsock; 600 601 return 0; 602 } 603 604 /* Reset all properties of an NBD device */ 605 static void nbd_reset(struct nbd_device *nbd) 606 { 607 int i; 608 609 for (i = 0; i < nbd->num_connections; i++) 610 kfree(nbd->socks[i]); 611 kfree(nbd->socks); 612 nbd->socks = NULL; 613 nbd->runtime_flags = 0; 614 nbd->blksize = 1024; 615 nbd->bytesize = 0; 616 set_capacity(nbd->disk, 0); 617 nbd->flags = 0; 618 nbd->tag_set.timeout = 0; 619 nbd->num_connections = 0; 620 nbd->task_setup = NULL; 621 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue); 622 } 623 624 static void nbd_bdev_reset(struct block_device *bdev) 625 { 626 set_device_ro(bdev, false); 627 bdev->bd_inode->i_size = 0; 628 if (max_part > 0) { 629 blkdev_reread_part(bdev); 630 bdev->bd_invalidated = 1; 631 } 632 } 633 634 static void nbd_parse_flags(struct nbd_device *nbd, struct block_device *bdev) 635 { 636 if (nbd->flags & NBD_FLAG_READ_ONLY) 637 set_device_ro(bdev, true); 638 if (nbd->flags & NBD_FLAG_SEND_TRIM) 639 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue); 640 if (nbd->flags & NBD_FLAG_SEND_FLUSH) 641 blk_queue_write_cache(nbd->disk->queue, true, false); 642 else 643 blk_queue_write_cache(nbd->disk->queue, false, false); 644 } 645 646 static void send_disconnects(struct nbd_device *nbd) 647 { 648 struct nbd_request request = {}; 649 int i, ret; 650 651 request.magic = htonl(NBD_REQUEST_MAGIC); 652 request.type = htonl(NBD_CMD_DISC); 653 654 for (i = 0; i < nbd->num_connections; i++) { 655 ret = sock_xmit(nbd, i, 1, &request, sizeof(request), 0); 656 if (ret <= 0) 657 dev_err(disk_to_dev(nbd->disk), 658 "Send disconnect failed %d\n", ret); 659 } 660 } 661 662 static int nbd_dev_dbg_init(struct nbd_device *nbd); 663 static void nbd_dev_dbg_close(struct nbd_device *nbd); 664 665 /* Must be called with config_lock held */ 666 static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, 667 unsigned int cmd, unsigned long arg) 668 { 669 switch (cmd) { 670 case NBD_DISCONNECT: { 671 dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n"); 672 if (!nbd->socks) 673 return -EINVAL; 674 675 mutex_unlock(&nbd->config_lock); 676 fsync_bdev(bdev); 677 mutex_lock(&nbd->config_lock); 678 679 /* Check again after getting mutex back. */ 680 if (!nbd->socks) 681 return -EINVAL; 682 683 if (!test_and_set_bit(NBD_DISCONNECT_REQUESTED, 684 &nbd->runtime_flags)) 685 send_disconnects(nbd); 686 return 0; 687 } 688 689 case NBD_CLEAR_SOCK: 690 sock_shutdown(nbd); 691 nbd_clear_que(nbd); 692 kill_bdev(bdev); 693 nbd_bdev_reset(bdev); 694 /* 695 * We want to give the run thread a chance to wait for everybody 696 * to clean up and then do it's own cleanup. 697 */ 698 if (!test_bit(NBD_RUNNING, &nbd->runtime_flags)) { 699 int i; 700 701 for (i = 0; i < nbd->num_connections; i++) 702 kfree(nbd->socks[i]); 703 kfree(nbd->socks); 704 nbd->socks = NULL; 705 nbd->num_connections = 0; 706 nbd->task_setup = NULL; 707 } 708 return 0; 709 710 case NBD_SET_SOCK: { 711 int err; 712 struct socket *sock = sockfd_lookup(arg, &err); 713 714 if (!sock) 715 return err; 716 717 err = nbd_add_socket(nbd, sock); 718 if (!err && max_part) 719 bdev->bd_invalidated = 1; 720 721 return err; 722 } 723 724 case NBD_SET_BLKSIZE: { 725 loff_t bsize = div_s64(nbd->bytesize, arg); 726 727 return nbd_size_set(nbd, bdev, arg, bsize); 728 } 729 730 case NBD_SET_SIZE: 731 return nbd_size_set(nbd, bdev, nbd->blksize, 732 div_s64(arg, nbd->blksize)); 733 734 case NBD_SET_SIZE_BLOCKS: 735 return nbd_size_set(nbd, bdev, nbd->blksize, arg); 736 737 case NBD_SET_TIMEOUT: 738 nbd->tag_set.timeout = arg * HZ; 739 return 0; 740 741 case NBD_SET_FLAGS: 742 nbd->flags = arg; 743 return 0; 744 745 case NBD_DO_IT: { 746 struct recv_thread_args *args; 747 int num_connections = nbd->num_connections; 748 int error = 0, i; 749 750 if (nbd->task_recv) 751 return -EBUSY; 752 if (!nbd->socks) 753 return -EINVAL; 754 if (num_connections > 1 && 755 !(nbd->flags & NBD_FLAG_CAN_MULTI_CONN)) { 756 dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n"); 757 error = -EINVAL; 758 goto out_err; 759 } 760 761 set_bit(NBD_RUNNING, &nbd->runtime_flags); 762 blk_mq_update_nr_hw_queues(&nbd->tag_set, nbd->num_connections); 763 args = kcalloc(num_connections, sizeof(*args), GFP_KERNEL); 764 if (!args) { 765 error = -ENOMEM; 766 goto out_err; 767 } 768 nbd->task_recv = current; 769 mutex_unlock(&nbd->config_lock); 770 771 nbd_parse_flags(nbd, bdev); 772 773 error = device_create_file(disk_to_dev(nbd->disk), &pid_attr); 774 if (error) { 775 dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n"); 776 goto out_recv; 777 } 778 779 nbd_size_update(nbd, bdev); 780 781 nbd_dev_dbg_init(nbd); 782 for (i = 0; i < num_connections; i++) { 783 sk_set_memalloc(nbd->socks[i]->sock->sk); 784 atomic_inc(&nbd->recv_threads); 785 INIT_WORK(&args[i].work, recv_work); 786 args[i].nbd = nbd; 787 args[i].index = i; 788 queue_work(recv_workqueue, &args[i].work); 789 } 790 wait_event_interruptible(nbd->recv_wq, 791 atomic_read(&nbd->recv_threads) == 0); 792 for (i = 0; i < num_connections; i++) 793 flush_work(&args[i].work); 794 nbd_dev_dbg_close(nbd); 795 nbd_size_clear(nbd, bdev); 796 device_remove_file(disk_to_dev(nbd->disk), &pid_attr); 797 out_recv: 798 mutex_lock(&nbd->config_lock); 799 nbd->task_recv = NULL; 800 out_err: 801 sock_shutdown(nbd); 802 nbd_clear_que(nbd); 803 kill_bdev(bdev); 804 nbd_bdev_reset(bdev); 805 806 /* user requested, ignore socket errors */ 807 if (test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 808 error = 0; 809 if (test_bit(NBD_TIMEDOUT, &nbd->runtime_flags)) 810 error = -ETIMEDOUT; 811 812 nbd_reset(nbd); 813 return error; 814 } 815 816 case NBD_CLEAR_QUE: 817 /* 818 * This is for compatibility only. The queue is always cleared 819 * by NBD_DO_IT or NBD_CLEAR_SOCK. 820 */ 821 return 0; 822 823 case NBD_PRINT_DEBUG: 824 /* 825 * For compatibility only, we no longer keep a list of 826 * outstanding requests. 827 */ 828 return 0; 829 } 830 return -ENOTTY; 831 } 832 833 static int nbd_ioctl(struct block_device *bdev, fmode_t mode, 834 unsigned int cmd, unsigned long arg) 835 { 836 struct nbd_device *nbd = bdev->bd_disk->private_data; 837 int error; 838 839 if (!capable(CAP_SYS_ADMIN)) 840 return -EPERM; 841 842 BUG_ON(nbd->magic != NBD_MAGIC); 843 844 mutex_lock(&nbd->config_lock); 845 error = __nbd_ioctl(bdev, nbd, cmd, arg); 846 mutex_unlock(&nbd->config_lock); 847 848 return error; 849 } 850 851 static const struct block_device_operations nbd_fops = 852 { 853 .owner = THIS_MODULE, 854 .ioctl = nbd_ioctl, 855 .compat_ioctl = nbd_ioctl, 856 }; 857 858 #if IS_ENABLED(CONFIG_DEBUG_FS) 859 860 static int nbd_dbg_tasks_show(struct seq_file *s, void *unused) 861 { 862 struct nbd_device *nbd = s->private; 863 864 if (nbd->task_recv) 865 seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv)); 866 867 return 0; 868 } 869 870 static int nbd_dbg_tasks_open(struct inode *inode, struct file *file) 871 { 872 return single_open(file, nbd_dbg_tasks_show, inode->i_private); 873 } 874 875 static const struct file_operations nbd_dbg_tasks_ops = { 876 .open = nbd_dbg_tasks_open, 877 .read = seq_read, 878 .llseek = seq_lseek, 879 .release = single_release, 880 }; 881 882 static int nbd_dbg_flags_show(struct seq_file *s, void *unused) 883 { 884 struct nbd_device *nbd = s->private; 885 u32 flags = nbd->flags; 886 887 seq_printf(s, "Hex: 0x%08x\n\n", flags); 888 889 seq_puts(s, "Known flags:\n"); 890 891 if (flags & NBD_FLAG_HAS_FLAGS) 892 seq_puts(s, "NBD_FLAG_HAS_FLAGS\n"); 893 if (flags & NBD_FLAG_READ_ONLY) 894 seq_puts(s, "NBD_FLAG_READ_ONLY\n"); 895 if (flags & NBD_FLAG_SEND_FLUSH) 896 seq_puts(s, "NBD_FLAG_SEND_FLUSH\n"); 897 if (flags & NBD_FLAG_SEND_TRIM) 898 seq_puts(s, "NBD_FLAG_SEND_TRIM\n"); 899 900 return 0; 901 } 902 903 static int nbd_dbg_flags_open(struct inode *inode, struct file *file) 904 { 905 return single_open(file, nbd_dbg_flags_show, inode->i_private); 906 } 907 908 static const struct file_operations nbd_dbg_flags_ops = { 909 .open = nbd_dbg_flags_open, 910 .read = seq_read, 911 .llseek = seq_lseek, 912 .release = single_release, 913 }; 914 915 static int nbd_dev_dbg_init(struct nbd_device *nbd) 916 { 917 struct dentry *dir; 918 919 if (!nbd_dbg_dir) 920 return -EIO; 921 922 dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir); 923 if (!dir) { 924 dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n", 925 nbd_name(nbd)); 926 return -EIO; 927 } 928 nbd->dbg_dir = dir; 929 930 debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops); 931 debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize); 932 debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout); 933 debugfs_create_u64("blocksize", 0444, dir, &nbd->blksize); 934 debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_ops); 935 936 return 0; 937 } 938 939 static void nbd_dev_dbg_close(struct nbd_device *nbd) 940 { 941 debugfs_remove_recursive(nbd->dbg_dir); 942 } 943 944 static int nbd_dbg_init(void) 945 { 946 struct dentry *dbg_dir; 947 948 dbg_dir = debugfs_create_dir("nbd", NULL); 949 if (!dbg_dir) 950 return -EIO; 951 952 nbd_dbg_dir = dbg_dir; 953 954 return 0; 955 } 956 957 static void nbd_dbg_close(void) 958 { 959 debugfs_remove_recursive(nbd_dbg_dir); 960 } 961 962 #else /* IS_ENABLED(CONFIG_DEBUG_FS) */ 963 964 static int nbd_dev_dbg_init(struct nbd_device *nbd) 965 { 966 return 0; 967 } 968 969 static void nbd_dev_dbg_close(struct nbd_device *nbd) 970 { 971 } 972 973 static int nbd_dbg_init(void) 974 { 975 return 0; 976 } 977 978 static void nbd_dbg_close(void) 979 { 980 } 981 982 #endif 983 984 static int nbd_init_request(void *data, struct request *rq, 985 unsigned int hctx_idx, unsigned int request_idx, 986 unsigned int numa_node) 987 { 988 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq); 989 cmd->nbd = data; 990 return 0; 991 } 992 993 static struct blk_mq_ops nbd_mq_ops = { 994 .queue_rq = nbd_queue_rq, 995 .init_request = nbd_init_request, 996 .timeout = nbd_xmit_timeout, 997 }; 998 999 static void nbd_dev_remove(struct nbd_device *nbd) 1000 { 1001 struct gendisk *disk = nbd->disk; 1002 nbd->magic = 0; 1003 if (disk) { 1004 del_gendisk(disk); 1005 blk_cleanup_queue(disk->queue); 1006 blk_mq_free_tag_set(&nbd->tag_set); 1007 put_disk(disk); 1008 } 1009 kfree(nbd); 1010 } 1011 1012 static int nbd_dev_add(int index) 1013 { 1014 struct nbd_device *nbd; 1015 struct gendisk *disk; 1016 struct request_queue *q; 1017 int err = -ENOMEM; 1018 1019 nbd = kzalloc(sizeof(struct nbd_device), GFP_KERNEL); 1020 if (!nbd) 1021 goto out; 1022 1023 disk = alloc_disk(1 << part_shift); 1024 if (!disk) 1025 goto out_free_nbd; 1026 1027 if (index >= 0) { 1028 err = idr_alloc(&nbd_index_idr, nbd, index, index + 1, 1029 GFP_KERNEL); 1030 if (err == -ENOSPC) 1031 err = -EEXIST; 1032 } else { 1033 err = idr_alloc(&nbd_index_idr, nbd, 0, 0, GFP_KERNEL); 1034 if (err >= 0) 1035 index = err; 1036 } 1037 if (err < 0) 1038 goto out_free_disk; 1039 1040 nbd->disk = disk; 1041 nbd->tag_set.ops = &nbd_mq_ops; 1042 nbd->tag_set.nr_hw_queues = 1; 1043 nbd->tag_set.queue_depth = 128; 1044 nbd->tag_set.numa_node = NUMA_NO_NODE; 1045 nbd->tag_set.cmd_size = sizeof(struct nbd_cmd); 1046 nbd->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | 1047 BLK_MQ_F_SG_MERGE | BLK_MQ_F_BLOCKING; 1048 nbd->tag_set.driver_data = nbd; 1049 1050 err = blk_mq_alloc_tag_set(&nbd->tag_set); 1051 if (err) 1052 goto out_free_idr; 1053 1054 q = blk_mq_init_queue(&nbd->tag_set); 1055 if (IS_ERR(q)) { 1056 err = PTR_ERR(q); 1057 goto out_free_tags; 1058 } 1059 disk->queue = q; 1060 1061 /* 1062 * Tell the block layer that we are not a rotational device 1063 */ 1064 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); 1065 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); 1066 disk->queue->limits.discard_granularity = 512; 1067 blk_queue_max_discard_sectors(disk->queue, UINT_MAX); 1068 disk->queue->limits.discard_zeroes_data = 0; 1069 blk_queue_max_hw_sectors(disk->queue, 65536); 1070 disk->queue->limits.max_sectors = 256; 1071 1072 nbd->magic = NBD_MAGIC; 1073 mutex_init(&nbd->config_lock); 1074 disk->major = NBD_MAJOR; 1075 disk->first_minor = index << part_shift; 1076 disk->fops = &nbd_fops; 1077 disk->private_data = nbd; 1078 sprintf(disk->disk_name, "nbd%d", index); 1079 init_waitqueue_head(&nbd->recv_wq); 1080 nbd_reset(nbd); 1081 add_disk(disk); 1082 return index; 1083 1084 out_free_tags: 1085 blk_mq_free_tag_set(&nbd->tag_set); 1086 out_free_idr: 1087 idr_remove(&nbd_index_idr, index); 1088 out_free_disk: 1089 put_disk(disk); 1090 out_free_nbd: 1091 kfree(nbd); 1092 out: 1093 return err; 1094 } 1095 1096 /* 1097 * And here should be modules and kernel interface 1098 * (Just smiley confuses emacs :-) 1099 */ 1100 1101 static int __init nbd_init(void) 1102 { 1103 int i; 1104 1105 BUILD_BUG_ON(sizeof(struct nbd_request) != 28); 1106 1107 if (max_part < 0) { 1108 printk(KERN_ERR "nbd: max_part must be >= 0\n"); 1109 return -EINVAL; 1110 } 1111 1112 part_shift = 0; 1113 if (max_part > 0) { 1114 part_shift = fls(max_part); 1115 1116 /* 1117 * Adjust max_part according to part_shift as it is exported 1118 * to user space so that user can know the max number of 1119 * partition kernel should be able to manage. 1120 * 1121 * Note that -1 is required because partition 0 is reserved 1122 * for the whole disk. 1123 */ 1124 max_part = (1UL << part_shift) - 1; 1125 } 1126 1127 if ((1UL << part_shift) > DISK_MAX_PARTS) 1128 return -EINVAL; 1129 1130 if (nbds_max > 1UL << (MINORBITS - part_shift)) 1131 return -EINVAL; 1132 recv_workqueue = alloc_workqueue("knbd-recv", 1133 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1134 if (!recv_workqueue) 1135 return -ENOMEM; 1136 1137 if (register_blkdev(NBD_MAJOR, "nbd")) 1138 return -EIO; 1139 1140 nbd_dbg_init(); 1141 1142 mutex_lock(&nbd_index_mutex); 1143 for (i = 0; i < nbds_max; i++) 1144 nbd_dev_add(i); 1145 mutex_unlock(&nbd_index_mutex); 1146 return 0; 1147 } 1148 1149 static int nbd_exit_cb(int id, void *ptr, void *data) 1150 { 1151 struct nbd_device *nbd = ptr; 1152 nbd_dev_remove(nbd); 1153 return 0; 1154 } 1155 1156 static void __exit nbd_cleanup(void) 1157 { 1158 nbd_dbg_close(); 1159 1160 idr_for_each(&nbd_index_idr, &nbd_exit_cb, NULL); 1161 idr_destroy(&nbd_index_idr); 1162 destroy_workqueue(recv_workqueue); 1163 unregister_blkdev(NBD_MAJOR, "nbd"); 1164 } 1165 1166 module_init(nbd_init); 1167 module_exit(nbd_cleanup); 1168 1169 MODULE_DESCRIPTION("Network Block Device"); 1170 MODULE_LICENSE("GPL"); 1171 1172 module_param(nbds_max, int, 0444); 1173 MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)"); 1174 module_param(max_part, int, 0444); 1175 MODULE_PARM_DESC(max_part, "number of partitions per device (default: 0)"); 1176