1 /* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-path-selector.h" 10 #include "dm-hw-handler.h" 11 #include "dm-bio-list.h" 12 #include "dm-bio-record.h" 13 14 #include <linux/ctype.h> 15 #include <linux/init.h> 16 #include <linux/mempool.h> 17 #include <linux/module.h> 18 #include <linux/pagemap.h> 19 #include <linux/slab.h> 20 #include <linux/time.h> 21 #include <linux/workqueue.h> 22 #include <asm/atomic.h> 23 24 #define MESG_STR(x) x, sizeof(x) 25 26 /* Path properties */ 27 struct pgpath { 28 struct list_head list; 29 30 struct priority_group *pg; /* Owning PG */ 31 unsigned fail_count; /* Cumulative failure count */ 32 33 struct path path; 34 }; 35 36 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 37 38 /* 39 * Paths are grouped into Priority Groups and numbered from 1 upwards. 40 * Each has a path selector which controls which path gets used. 41 */ 42 struct priority_group { 43 struct list_head list; 44 45 struct multipath *m; /* Owning multipath instance */ 46 struct path_selector ps; 47 48 unsigned pg_num; /* Reference number */ 49 unsigned bypassed; /* Temporarily bypass this PG? */ 50 51 unsigned nr_pgpaths; /* Number of paths in PG */ 52 struct list_head pgpaths; 53 }; 54 55 /* Multipath context */ 56 struct multipath { 57 struct list_head list; 58 struct dm_target *ti; 59 60 spinlock_t lock; 61 62 struct hw_handler hw_handler; 63 unsigned nr_priority_groups; 64 struct list_head priority_groups; 65 unsigned pg_init_required; /* pg_init needs calling? */ 66 67 unsigned nr_valid_paths; /* Total number of usable paths */ 68 struct pgpath *current_pgpath; 69 struct priority_group *current_pg; 70 struct priority_group *next_pg; /* Switch to this PG if set */ 71 unsigned repeat_count; /* I/Os left before calling PS again */ 72 73 unsigned queue_io; /* Must we queue all I/O? */ 74 unsigned queue_if_no_path; /* Queue I/O if last path fails? */ 75 unsigned suspended; /* Has dm core suspended our I/O? */ 76 77 struct work_struct process_queued_ios; 78 struct bio_list queued_ios; 79 unsigned queue_size; 80 81 struct work_struct trigger_event; 82 83 /* 84 * We must use a mempool of mpath_io structs so that we 85 * can resubmit bios on error. 86 */ 87 mempool_t *mpio_pool; 88 }; 89 90 /* 91 * Context information attached to each bio we process. 92 */ 93 struct mpath_io { 94 struct pgpath *pgpath; 95 struct dm_bio_details details; 96 }; 97 98 typedef int (*action_fn) (struct pgpath *pgpath); 99 100 #define MIN_IOS 256 /* Mempool size */ 101 102 static kmem_cache_t *_mpio_cache; 103 104 struct workqueue_struct *kmultipathd; 105 static void process_queued_ios(void *data); 106 static void trigger_event(void *data); 107 108 109 /*----------------------------------------------- 110 * Allocation routines 111 *-----------------------------------------------*/ 112 113 static struct pgpath *alloc_pgpath(void) 114 { 115 struct pgpath *pgpath = kmalloc(sizeof(*pgpath), GFP_KERNEL); 116 117 if (pgpath) { 118 memset(pgpath, 0, sizeof(*pgpath)); 119 pgpath->path.is_active = 1; 120 } 121 122 return pgpath; 123 } 124 125 static inline void free_pgpath(struct pgpath *pgpath) 126 { 127 kfree(pgpath); 128 } 129 130 static struct priority_group *alloc_priority_group(void) 131 { 132 struct priority_group *pg; 133 134 pg = kmalloc(sizeof(*pg), GFP_KERNEL); 135 if (!pg) 136 return NULL; 137 138 memset(pg, 0, sizeof(*pg)); 139 INIT_LIST_HEAD(&pg->pgpaths); 140 141 return pg; 142 } 143 144 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 145 { 146 struct pgpath *pgpath, *tmp; 147 148 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 149 list_del(&pgpath->list); 150 dm_put_device(ti, pgpath->path.dev); 151 free_pgpath(pgpath); 152 } 153 } 154 155 static void free_priority_group(struct priority_group *pg, 156 struct dm_target *ti) 157 { 158 struct path_selector *ps = &pg->ps; 159 160 if (ps->type) { 161 ps->type->destroy(ps); 162 dm_put_path_selector(ps->type); 163 } 164 165 free_pgpaths(&pg->pgpaths, ti); 166 kfree(pg); 167 } 168 169 static struct multipath *alloc_multipath(void) 170 { 171 struct multipath *m; 172 173 m = kmalloc(sizeof(*m), GFP_KERNEL); 174 if (m) { 175 memset(m, 0, sizeof(*m)); 176 INIT_LIST_HEAD(&m->priority_groups); 177 spin_lock_init(&m->lock); 178 m->queue_io = 1; 179 INIT_WORK(&m->process_queued_ios, process_queued_ios, m); 180 INIT_WORK(&m->trigger_event, trigger_event, m); 181 m->mpio_pool = mempool_create(MIN_IOS, mempool_alloc_slab, 182 mempool_free_slab, _mpio_cache); 183 if (!m->mpio_pool) { 184 kfree(m); 185 return NULL; 186 } 187 } 188 189 return m; 190 } 191 192 static void free_multipath(struct multipath *m) 193 { 194 struct priority_group *pg, *tmp; 195 struct hw_handler *hwh = &m->hw_handler; 196 197 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 198 list_del(&pg->list); 199 free_priority_group(pg, m->ti); 200 } 201 202 if (hwh->type) { 203 hwh->type->destroy(hwh); 204 dm_put_hw_handler(hwh->type); 205 } 206 207 mempool_destroy(m->mpio_pool); 208 kfree(m); 209 } 210 211 212 /*----------------------------------------------- 213 * Path selection 214 *-----------------------------------------------*/ 215 216 static void __switch_pg(struct multipath *m, struct pgpath *pgpath) 217 { 218 struct hw_handler *hwh = &m->hw_handler; 219 220 m->current_pg = pgpath->pg; 221 222 /* Must we initialise the PG first, and queue I/O till it's ready? */ 223 if (hwh->type && hwh->type->pg_init) { 224 m->pg_init_required = 1; 225 m->queue_io = 1; 226 } else { 227 m->pg_init_required = 0; 228 m->queue_io = 0; 229 } 230 } 231 232 static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg) 233 { 234 struct path *path; 235 236 path = pg->ps.type->select_path(&pg->ps, &m->repeat_count); 237 if (!path) 238 return -ENXIO; 239 240 m->current_pgpath = path_to_pgpath(path); 241 242 if (m->current_pg != pg) 243 __switch_pg(m, m->current_pgpath); 244 245 return 0; 246 } 247 248 static void __choose_pgpath(struct multipath *m) 249 { 250 struct priority_group *pg; 251 unsigned bypassed = 1; 252 253 if (!m->nr_valid_paths) 254 goto failed; 255 256 /* Were we instructed to switch PG? */ 257 if (m->next_pg) { 258 pg = m->next_pg; 259 m->next_pg = NULL; 260 if (!__choose_path_in_pg(m, pg)) 261 return; 262 } 263 264 /* Don't change PG until it has no remaining paths */ 265 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg)) 266 return; 267 268 /* 269 * Loop through priority groups until we find a valid path. 270 * First time we skip PGs marked 'bypassed'. 271 * Second time we only try the ones we skipped. 272 */ 273 do { 274 list_for_each_entry(pg, &m->priority_groups, list) { 275 if (pg->bypassed == bypassed) 276 continue; 277 if (!__choose_path_in_pg(m, pg)) 278 return; 279 } 280 } while (bypassed--); 281 282 failed: 283 m->current_pgpath = NULL; 284 m->current_pg = NULL; 285 } 286 287 static int map_io(struct multipath *m, struct bio *bio, struct mpath_io *mpio, 288 unsigned was_queued) 289 { 290 int r = 1; 291 unsigned long flags; 292 struct pgpath *pgpath; 293 294 spin_lock_irqsave(&m->lock, flags); 295 296 /* Do we need to select a new pgpath? */ 297 if (!m->current_pgpath || 298 (!m->queue_io && (m->repeat_count && --m->repeat_count == 0))) 299 __choose_pgpath(m); 300 301 pgpath = m->current_pgpath; 302 303 if (was_queued) 304 m->queue_size--; 305 306 if ((pgpath && m->queue_io) || 307 (!pgpath && m->queue_if_no_path && !m->suspended)) { 308 /* Queue for the daemon to resubmit */ 309 bio_list_add(&m->queued_ios, bio); 310 m->queue_size++; 311 if (m->pg_init_required || !m->queue_io) 312 queue_work(kmultipathd, &m->process_queued_ios); 313 pgpath = NULL; 314 r = 0; 315 } else if (!pgpath) 316 r = -EIO; /* Failed */ 317 else 318 bio->bi_bdev = pgpath->path.dev->bdev; 319 320 mpio->pgpath = pgpath; 321 322 spin_unlock_irqrestore(&m->lock, flags); 323 324 return r; 325 } 326 327 /* 328 * If we run out of usable paths, should we queue I/O or error it? 329 */ 330 static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path) 331 { 332 unsigned long flags; 333 334 spin_lock_irqsave(&m->lock, flags); 335 336 m->queue_if_no_path = queue_if_no_path; 337 if (!m->queue_if_no_path) 338 queue_work(kmultipathd, &m->process_queued_ios); 339 340 spin_unlock_irqrestore(&m->lock, flags); 341 342 return 0; 343 } 344 345 /*----------------------------------------------------------------- 346 * The multipath daemon is responsible for resubmitting queued ios. 347 *---------------------------------------------------------------*/ 348 349 static void dispatch_queued_ios(struct multipath *m) 350 { 351 int r; 352 unsigned long flags; 353 struct bio *bio = NULL, *next; 354 struct mpath_io *mpio; 355 union map_info *info; 356 357 spin_lock_irqsave(&m->lock, flags); 358 bio = bio_list_get(&m->queued_ios); 359 spin_unlock_irqrestore(&m->lock, flags); 360 361 while (bio) { 362 next = bio->bi_next; 363 bio->bi_next = NULL; 364 365 info = dm_get_mapinfo(bio); 366 mpio = info->ptr; 367 368 r = map_io(m, bio, mpio, 1); 369 if (r < 0) 370 bio_endio(bio, bio->bi_size, r); 371 else if (r == 1) 372 generic_make_request(bio); 373 374 bio = next; 375 } 376 } 377 378 static void process_queued_ios(void *data) 379 { 380 struct multipath *m = (struct multipath *) data; 381 struct hw_handler *hwh = &m->hw_handler; 382 struct pgpath *pgpath; 383 unsigned init_required, must_queue = 0; 384 unsigned long flags; 385 386 spin_lock_irqsave(&m->lock, flags); 387 388 if (!m->current_pgpath) 389 __choose_pgpath(m); 390 391 pgpath = m->current_pgpath; 392 393 if ((pgpath && m->queue_io) || 394 (!pgpath && m->queue_if_no_path && !m->suspended)) 395 must_queue = 1; 396 397 init_required = m->pg_init_required; 398 if (init_required) 399 m->pg_init_required = 0; 400 401 spin_unlock_irqrestore(&m->lock, flags); 402 403 if (init_required) 404 hwh->type->pg_init(hwh, pgpath->pg->bypassed, &pgpath->path); 405 406 if (!must_queue) 407 dispatch_queued_ios(m); 408 } 409 410 /* 411 * An event is triggered whenever a path is taken out of use. 412 * Includes path failure and PG bypass. 413 */ 414 static void trigger_event(void *data) 415 { 416 struct multipath *m = (struct multipath *) data; 417 418 dm_table_event(m->ti->table); 419 } 420 421 /*----------------------------------------------------------------- 422 * Constructor/argument parsing: 423 * <#multipath feature args> [<arg>]* 424 * <#hw_handler args> [hw_handler [<arg>]*] 425 * <#priority groups> 426 * <initial priority group> 427 * [<selector> <#selector args> [<arg>]* 428 * <#paths> <#per-path selector args> 429 * [<path> [<arg>]* ]+ ]+ 430 *---------------------------------------------------------------*/ 431 struct param { 432 unsigned min; 433 unsigned max; 434 char *error; 435 }; 436 437 #define ESTR(s) ("dm-multipath: " s) 438 439 static int read_param(struct param *param, char *str, unsigned *v, char **error) 440 { 441 if (!str || 442 (sscanf(str, "%u", v) != 1) || 443 (*v < param->min) || 444 (*v > param->max)) { 445 *error = param->error; 446 return -EINVAL; 447 } 448 449 return 0; 450 } 451 452 struct arg_set { 453 unsigned argc; 454 char **argv; 455 }; 456 457 static char *shift(struct arg_set *as) 458 { 459 char *r; 460 461 if (as->argc) { 462 as->argc--; 463 r = *as->argv; 464 as->argv++; 465 return r; 466 } 467 468 return NULL; 469 } 470 471 static void consume(struct arg_set *as, unsigned n) 472 { 473 BUG_ON (as->argc < n); 474 as->argc -= n; 475 as->argv += n; 476 } 477 478 static int parse_path_selector(struct arg_set *as, struct priority_group *pg, 479 struct dm_target *ti) 480 { 481 int r; 482 struct path_selector_type *pst; 483 unsigned ps_argc; 484 485 static struct param _params[] = { 486 {0, 1024, ESTR("invalid number of path selector args")}, 487 }; 488 489 pst = dm_get_path_selector(shift(as)); 490 if (!pst) { 491 ti->error = ESTR("unknown path selector type"); 492 return -EINVAL; 493 } 494 495 r = read_param(_params, shift(as), &ps_argc, &ti->error); 496 if (r) 497 return -EINVAL; 498 499 r = pst->create(&pg->ps, ps_argc, as->argv); 500 if (r) { 501 dm_put_path_selector(pst); 502 ti->error = ESTR("path selector constructor failed"); 503 return r; 504 } 505 506 pg->ps.type = pst; 507 consume(as, ps_argc); 508 509 return 0; 510 } 511 512 static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps, 513 struct dm_target *ti) 514 { 515 int r; 516 struct pgpath *p; 517 518 /* we need at least a path arg */ 519 if (as->argc < 1) { 520 ti->error = ESTR("no device given"); 521 return NULL; 522 } 523 524 p = alloc_pgpath(); 525 if (!p) 526 return NULL; 527 528 r = dm_get_device(ti, shift(as), ti->begin, ti->len, 529 dm_table_get_mode(ti->table), &p->path.dev); 530 if (r) { 531 ti->error = ESTR("error getting device"); 532 goto bad; 533 } 534 535 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 536 if (r) { 537 dm_put_device(ti, p->path.dev); 538 goto bad; 539 } 540 541 return p; 542 543 bad: 544 free_pgpath(p); 545 return NULL; 546 } 547 548 static struct priority_group *parse_priority_group(struct arg_set *as, 549 struct multipath *m, 550 struct dm_target *ti) 551 { 552 static struct param _params[] = { 553 {1, 1024, ESTR("invalid number of paths")}, 554 {0, 1024, ESTR("invalid number of selector args")} 555 }; 556 557 int r; 558 unsigned i, nr_selector_args, nr_params; 559 struct priority_group *pg; 560 561 if (as->argc < 2) { 562 as->argc = 0; 563 ti->error = ESTR("not enough priority group aruments"); 564 return NULL; 565 } 566 567 pg = alloc_priority_group(); 568 if (!pg) { 569 ti->error = ESTR("couldn't allocate priority group"); 570 return NULL; 571 } 572 pg->m = m; 573 574 r = parse_path_selector(as, pg, ti); 575 if (r) 576 goto bad; 577 578 /* 579 * read the paths 580 */ 581 r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error); 582 if (r) 583 goto bad; 584 585 r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error); 586 if (r) 587 goto bad; 588 589 nr_params = 1 + nr_selector_args; 590 for (i = 0; i < pg->nr_pgpaths; i++) { 591 struct pgpath *pgpath; 592 struct arg_set path_args; 593 594 if (as->argc < nr_params) 595 goto bad; 596 597 path_args.argc = nr_params; 598 path_args.argv = as->argv; 599 600 pgpath = parse_path(&path_args, &pg->ps, ti); 601 if (!pgpath) 602 goto bad; 603 604 pgpath->pg = pg; 605 list_add_tail(&pgpath->list, &pg->pgpaths); 606 consume(as, nr_params); 607 } 608 609 return pg; 610 611 bad: 612 free_priority_group(pg, ti); 613 return NULL; 614 } 615 616 static int parse_hw_handler(struct arg_set *as, struct multipath *m, 617 struct dm_target *ti) 618 { 619 int r; 620 struct hw_handler_type *hwht; 621 unsigned hw_argc; 622 623 static struct param _params[] = { 624 {0, 1024, ESTR("invalid number of hardware handler args")}, 625 }; 626 627 r = read_param(_params, shift(as), &hw_argc, &ti->error); 628 if (r) 629 return -EINVAL; 630 631 if (!hw_argc) 632 return 0; 633 634 hwht = dm_get_hw_handler(shift(as)); 635 if (!hwht) { 636 ti->error = ESTR("unknown hardware handler type"); 637 return -EINVAL; 638 } 639 640 r = hwht->create(&m->hw_handler, hw_argc - 1, as->argv); 641 if (r) { 642 dm_put_hw_handler(hwht); 643 ti->error = ESTR("hardware handler constructor failed"); 644 return r; 645 } 646 647 m->hw_handler.type = hwht; 648 consume(as, hw_argc - 1); 649 650 return 0; 651 } 652 653 static int parse_features(struct arg_set *as, struct multipath *m, 654 struct dm_target *ti) 655 { 656 int r; 657 unsigned argc; 658 659 static struct param _params[] = { 660 {0, 1, ESTR("invalid number of feature args")}, 661 }; 662 663 r = read_param(_params, shift(as), &argc, &ti->error); 664 if (r) 665 return -EINVAL; 666 667 if (!argc) 668 return 0; 669 670 if (!strnicmp(shift(as), MESG_STR("queue_if_no_path"))) 671 return queue_if_no_path(m, 1); 672 else { 673 ti->error = "Unrecognised multipath feature request"; 674 return -EINVAL; 675 } 676 } 677 678 static int multipath_ctr(struct dm_target *ti, unsigned int argc, 679 char **argv) 680 { 681 /* target parameters */ 682 static struct param _params[] = { 683 {1, 1024, ESTR("invalid number of priority groups")}, 684 {1, 1024, ESTR("invalid initial priority group number")}, 685 }; 686 687 int r; 688 struct multipath *m; 689 struct arg_set as; 690 unsigned pg_count = 0; 691 unsigned next_pg_num; 692 693 as.argc = argc; 694 as.argv = argv; 695 696 m = alloc_multipath(); 697 if (!m) { 698 ti->error = ESTR("can't allocate multipath"); 699 return -EINVAL; 700 } 701 702 r = parse_features(&as, m, ti); 703 if (r) 704 goto bad; 705 706 r = parse_hw_handler(&as, m, ti); 707 if (r) 708 goto bad; 709 710 r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error); 711 if (r) 712 goto bad; 713 714 r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error); 715 if (r) 716 goto bad; 717 718 /* parse the priority groups */ 719 while (as.argc) { 720 struct priority_group *pg; 721 722 pg = parse_priority_group(&as, m, ti); 723 if (!pg) { 724 r = -EINVAL; 725 goto bad; 726 } 727 728 m->nr_valid_paths += pg->nr_pgpaths; 729 list_add_tail(&pg->list, &m->priority_groups); 730 pg_count++; 731 pg->pg_num = pg_count; 732 if (!--next_pg_num) 733 m->next_pg = pg; 734 } 735 736 if (pg_count != m->nr_priority_groups) { 737 ti->error = ESTR("priority group count mismatch"); 738 r = -EINVAL; 739 goto bad; 740 } 741 742 ti->private = m; 743 m->ti = ti; 744 745 return 0; 746 747 bad: 748 free_multipath(m); 749 return r; 750 } 751 752 static void multipath_dtr(struct dm_target *ti) 753 { 754 struct multipath *m = (struct multipath *) ti->private; 755 free_multipath(m); 756 } 757 758 /* 759 * Map bios, recording original fields for later in case we have to resubmit 760 */ 761 static int multipath_map(struct dm_target *ti, struct bio *bio, 762 union map_info *map_context) 763 { 764 int r; 765 struct mpath_io *mpio; 766 struct multipath *m = (struct multipath *) ti->private; 767 768 mpio = mempool_alloc(m->mpio_pool, GFP_NOIO); 769 dm_bio_record(&mpio->details, bio); 770 771 map_context->ptr = mpio; 772 bio->bi_rw |= (1 << BIO_RW_FAILFAST); 773 r = map_io(m, bio, mpio, 0); 774 if (r < 0) 775 mempool_free(mpio, m->mpio_pool); 776 777 return r; 778 } 779 780 /* 781 * Take a path out of use. 782 */ 783 static int fail_path(struct pgpath *pgpath) 784 { 785 unsigned long flags; 786 struct multipath *m = pgpath->pg->m; 787 788 spin_lock_irqsave(&m->lock, flags); 789 790 if (!pgpath->path.is_active) 791 goto out; 792 793 DMWARN("dm-multipath: Failing path %s.", pgpath->path.dev->name); 794 795 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 796 pgpath->path.is_active = 0; 797 pgpath->fail_count++; 798 799 m->nr_valid_paths--; 800 801 if (pgpath == m->current_pgpath) 802 m->current_pgpath = NULL; 803 804 queue_work(kmultipathd, &m->trigger_event); 805 806 out: 807 spin_unlock_irqrestore(&m->lock, flags); 808 809 return 0; 810 } 811 812 /* 813 * Reinstate a previously-failed path 814 */ 815 static int reinstate_path(struct pgpath *pgpath) 816 { 817 int r = 0; 818 unsigned long flags; 819 struct multipath *m = pgpath->pg->m; 820 821 spin_lock_irqsave(&m->lock, flags); 822 823 if (pgpath->path.is_active) 824 goto out; 825 826 if (!pgpath->pg->ps.type) { 827 DMWARN("Reinstate path not supported by path selector %s", 828 pgpath->pg->ps.type->name); 829 r = -EINVAL; 830 goto out; 831 } 832 833 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 834 if (r) 835 goto out; 836 837 pgpath->path.is_active = 1; 838 839 m->current_pgpath = NULL; 840 if (!m->nr_valid_paths++) 841 queue_work(kmultipathd, &m->process_queued_ios); 842 843 queue_work(kmultipathd, &m->trigger_event); 844 845 out: 846 spin_unlock_irqrestore(&m->lock, flags); 847 848 return r; 849 } 850 851 /* 852 * Fail or reinstate all paths that match the provided struct dm_dev. 853 */ 854 static int action_dev(struct multipath *m, struct dm_dev *dev, 855 action_fn action) 856 { 857 int r = 0; 858 struct pgpath *pgpath; 859 struct priority_group *pg; 860 861 list_for_each_entry(pg, &m->priority_groups, list) { 862 list_for_each_entry(pgpath, &pg->pgpaths, list) { 863 if (pgpath->path.dev == dev) 864 r = action(pgpath); 865 } 866 } 867 868 return r; 869 } 870 871 /* 872 * Temporarily try to avoid having to use the specified PG 873 */ 874 static void bypass_pg(struct multipath *m, struct priority_group *pg, 875 int bypassed) 876 { 877 unsigned long flags; 878 879 spin_lock_irqsave(&m->lock, flags); 880 881 pg->bypassed = bypassed; 882 m->current_pgpath = NULL; 883 m->current_pg = NULL; 884 885 spin_unlock_irqrestore(&m->lock, flags); 886 887 queue_work(kmultipathd, &m->trigger_event); 888 } 889 890 /* 891 * Switch to using the specified PG from the next I/O that gets mapped 892 */ 893 static int switch_pg_num(struct multipath *m, const char *pgstr) 894 { 895 struct priority_group *pg; 896 unsigned pgnum; 897 unsigned long flags; 898 899 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum || 900 (pgnum > m->nr_priority_groups)) { 901 DMWARN("invalid PG number supplied to switch_pg_num"); 902 return -EINVAL; 903 } 904 905 spin_lock_irqsave(&m->lock, flags); 906 list_for_each_entry(pg, &m->priority_groups, list) { 907 pg->bypassed = 0; 908 if (--pgnum) 909 continue; 910 911 m->current_pgpath = NULL; 912 m->current_pg = NULL; 913 m->next_pg = pg; 914 } 915 spin_unlock_irqrestore(&m->lock, flags); 916 917 queue_work(kmultipathd, &m->trigger_event); 918 return 0; 919 } 920 921 /* 922 * Set/clear bypassed status of a PG. 923 * PGs are numbered upwards from 1 in the order they were declared. 924 */ 925 static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed) 926 { 927 struct priority_group *pg; 928 unsigned pgnum; 929 930 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum || 931 (pgnum > m->nr_priority_groups)) { 932 DMWARN("invalid PG number supplied to bypass_pg"); 933 return -EINVAL; 934 } 935 936 list_for_each_entry(pg, &m->priority_groups, list) { 937 if (!--pgnum) 938 break; 939 } 940 941 bypass_pg(m, pg, bypassed); 942 return 0; 943 } 944 945 /* 946 * pg_init must call this when it has completed its initialisation 947 */ 948 void dm_pg_init_complete(struct path *path, unsigned err_flags) 949 { 950 struct pgpath *pgpath = path_to_pgpath(path); 951 struct priority_group *pg = pgpath->pg; 952 struct multipath *m = pg->m; 953 unsigned long flags; 954 955 /* We insist on failing the path if the PG is already bypassed. */ 956 if (err_flags && pg->bypassed) 957 err_flags |= MP_FAIL_PATH; 958 959 if (err_flags & MP_FAIL_PATH) 960 fail_path(pgpath); 961 962 if (err_flags & MP_BYPASS_PG) 963 bypass_pg(m, pg, 1); 964 965 spin_lock_irqsave(&m->lock, flags); 966 if (!err_flags) 967 m->queue_io = 0; 968 else { 969 m->current_pgpath = NULL; 970 m->current_pg = NULL; 971 } 972 queue_work(kmultipathd, &m->process_queued_ios); 973 spin_unlock_irqrestore(&m->lock, flags); 974 } 975 976 /* 977 * end_io handling 978 */ 979 static int do_end_io(struct multipath *m, struct bio *bio, 980 int error, struct mpath_io *mpio) 981 { 982 struct hw_handler *hwh = &m->hw_handler; 983 unsigned err_flags = MP_FAIL_PATH; /* Default behavior */ 984 985 if (!error) 986 return 0; /* I/O complete */ 987 988 if ((error == -EWOULDBLOCK) && bio_rw_ahead(bio)) 989 return error; 990 991 spin_lock(&m->lock); 992 if (!m->nr_valid_paths) { 993 if (!m->queue_if_no_path || m->suspended) { 994 spin_unlock(&m->lock); 995 return -EIO; 996 } else { 997 spin_unlock(&m->lock); 998 goto requeue; 999 } 1000 } 1001 spin_unlock(&m->lock); 1002 1003 if (hwh->type && hwh->type->error) 1004 err_flags = hwh->type->error(hwh, bio); 1005 1006 if (mpio->pgpath) { 1007 if (err_flags & MP_FAIL_PATH) 1008 fail_path(mpio->pgpath); 1009 1010 if (err_flags & MP_BYPASS_PG) 1011 bypass_pg(m, mpio->pgpath->pg, 1); 1012 } 1013 1014 if (err_flags & MP_ERROR_IO) 1015 return -EIO; 1016 1017 requeue: 1018 dm_bio_restore(&mpio->details, bio); 1019 1020 /* queue for the daemon to resubmit or fail */ 1021 spin_lock(&m->lock); 1022 bio_list_add(&m->queued_ios, bio); 1023 m->queue_size++; 1024 if (!m->queue_io) 1025 queue_work(kmultipathd, &m->process_queued_ios); 1026 spin_unlock(&m->lock); 1027 1028 return 1; /* io not complete */ 1029 } 1030 1031 static int multipath_end_io(struct dm_target *ti, struct bio *bio, 1032 int error, union map_info *map_context) 1033 { 1034 struct multipath *m = (struct multipath *) ti->private; 1035 struct mpath_io *mpio = (struct mpath_io *) map_context->ptr; 1036 struct pgpath *pgpath = mpio->pgpath; 1037 struct path_selector *ps; 1038 int r; 1039 1040 r = do_end_io(m, bio, error, mpio); 1041 if (pgpath) { 1042 ps = &pgpath->pg->ps; 1043 if (ps->type->end_io) 1044 ps->type->end_io(ps, &pgpath->path); 1045 } 1046 if (r <= 0) 1047 mempool_free(mpio, m->mpio_pool); 1048 1049 return r; 1050 } 1051 1052 /* 1053 * Suspend can't complete until all the I/O is processed so if 1054 * the last path failed we will now error any queued I/O. 1055 */ 1056 static void multipath_presuspend(struct dm_target *ti) 1057 { 1058 struct multipath *m = (struct multipath *) ti->private; 1059 unsigned long flags; 1060 1061 spin_lock_irqsave(&m->lock, flags); 1062 m->suspended = 1; 1063 if (m->queue_if_no_path) 1064 queue_work(kmultipathd, &m->process_queued_ios); 1065 spin_unlock_irqrestore(&m->lock, flags); 1066 } 1067 1068 static void multipath_resume(struct dm_target *ti) 1069 { 1070 struct multipath *m = (struct multipath *) ti->private; 1071 unsigned long flags; 1072 1073 spin_lock_irqsave(&m->lock, flags); 1074 m->suspended = 0; 1075 spin_unlock_irqrestore(&m->lock, flags); 1076 } 1077 1078 /* 1079 * Info output has the following format: 1080 * num_multipath_feature_args [multipath_feature_args]* 1081 * num_handler_status_args [handler_status_args]* 1082 * num_groups init_group_number 1083 * [A|D|E num_ps_status_args [ps_status_args]* 1084 * num_paths num_selector_args 1085 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1086 * 1087 * Table output has the following format (identical to the constructor string): 1088 * num_feature_args [features_args]* 1089 * num_handler_args hw_handler [hw_handler_args]* 1090 * num_groups init_group_number 1091 * [priority selector-name num_ps_args [ps_args]* 1092 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1093 */ 1094 static int multipath_status(struct dm_target *ti, status_type_t type, 1095 char *result, unsigned int maxlen) 1096 { 1097 int sz = 0; 1098 unsigned long flags; 1099 struct multipath *m = (struct multipath *) ti->private; 1100 struct hw_handler *hwh = &m->hw_handler; 1101 struct priority_group *pg; 1102 struct pgpath *p; 1103 unsigned pg_num; 1104 char state; 1105 1106 spin_lock_irqsave(&m->lock, flags); 1107 1108 /* Features */ 1109 if (type == STATUSTYPE_INFO) 1110 DMEMIT("1 %u ", m->queue_size); 1111 else if (m->queue_if_no_path) 1112 DMEMIT("1 queue_if_no_path "); 1113 else 1114 DMEMIT("0 "); 1115 1116 if (hwh->type && hwh->type->status) 1117 sz += hwh->type->status(hwh, type, result + sz, maxlen - sz); 1118 else if (!hwh->type || type == STATUSTYPE_INFO) 1119 DMEMIT("0 "); 1120 else 1121 DMEMIT("1 %s ", hwh->type->name); 1122 1123 DMEMIT("%u ", m->nr_priority_groups); 1124 1125 if (m->next_pg) 1126 pg_num = m->next_pg->pg_num; 1127 else if (m->current_pg) 1128 pg_num = m->current_pg->pg_num; 1129 else 1130 pg_num = 1; 1131 1132 DMEMIT("%u ", pg_num); 1133 1134 switch (type) { 1135 case STATUSTYPE_INFO: 1136 list_for_each_entry(pg, &m->priority_groups, list) { 1137 if (pg->bypassed) 1138 state = 'D'; /* Disabled */ 1139 else if (pg == m->current_pg) 1140 state = 'A'; /* Currently Active */ 1141 else 1142 state = 'E'; /* Enabled */ 1143 1144 DMEMIT("%c ", state); 1145 1146 if (pg->ps.type->status) 1147 sz += pg->ps.type->status(&pg->ps, NULL, type, 1148 result + sz, 1149 maxlen - sz); 1150 else 1151 DMEMIT("0 "); 1152 1153 DMEMIT("%u %u ", pg->nr_pgpaths, 1154 pg->ps.type->info_args); 1155 1156 list_for_each_entry(p, &pg->pgpaths, list) { 1157 DMEMIT("%s %s %u ", p->path.dev->name, 1158 p->path.is_active ? "A" : "F", 1159 p->fail_count); 1160 if (pg->ps.type->status) 1161 sz += pg->ps.type->status(&pg->ps, 1162 &p->path, type, result + sz, 1163 maxlen - sz); 1164 } 1165 } 1166 break; 1167 1168 case STATUSTYPE_TABLE: 1169 list_for_each_entry(pg, &m->priority_groups, list) { 1170 DMEMIT("%s ", pg->ps.type->name); 1171 1172 if (pg->ps.type->status) 1173 sz += pg->ps.type->status(&pg->ps, NULL, type, 1174 result + sz, 1175 maxlen - sz); 1176 else 1177 DMEMIT("0 "); 1178 1179 DMEMIT("%u %u ", pg->nr_pgpaths, 1180 pg->ps.type->table_args); 1181 1182 list_for_each_entry(p, &pg->pgpaths, list) { 1183 DMEMIT("%s ", p->path.dev->name); 1184 if (pg->ps.type->status) 1185 sz += pg->ps.type->status(&pg->ps, 1186 &p->path, type, result + sz, 1187 maxlen - sz); 1188 } 1189 } 1190 break; 1191 } 1192 1193 spin_unlock_irqrestore(&m->lock, flags); 1194 1195 return 0; 1196 } 1197 1198 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 1199 { 1200 int r; 1201 struct dm_dev *dev; 1202 struct multipath *m = (struct multipath *) ti->private; 1203 action_fn action; 1204 1205 if (argc == 1) { 1206 if (!strnicmp(argv[0], MESG_STR("queue_if_no_path"))) 1207 return queue_if_no_path(m, 1); 1208 else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path"))) 1209 return queue_if_no_path(m, 0); 1210 } 1211 1212 if (argc != 2) 1213 goto error; 1214 1215 if (!strnicmp(argv[0], MESG_STR("disable_group"))) 1216 return bypass_pg_num(m, argv[1], 1); 1217 else if (!strnicmp(argv[0], MESG_STR("enable_group"))) 1218 return bypass_pg_num(m, argv[1], 0); 1219 else if (!strnicmp(argv[0], MESG_STR("switch_group"))) 1220 return switch_pg_num(m, argv[1]); 1221 else if (!strnicmp(argv[0], MESG_STR("reinstate_path"))) 1222 action = reinstate_path; 1223 else if (!strnicmp(argv[0], MESG_STR("fail_path"))) 1224 action = fail_path; 1225 else 1226 goto error; 1227 1228 r = dm_get_device(ti, argv[1], ti->begin, ti->len, 1229 dm_table_get_mode(ti->table), &dev); 1230 if (r) { 1231 DMWARN("dm-multipath message: error getting device %s", 1232 argv[1]); 1233 return -EINVAL; 1234 } 1235 1236 r = action_dev(m, dev, action); 1237 1238 dm_put_device(ti, dev); 1239 1240 return r; 1241 1242 error: 1243 DMWARN("Unrecognised multipath message received."); 1244 return -EINVAL; 1245 } 1246 1247 /*----------------------------------------------------------------- 1248 * Module setup 1249 *---------------------------------------------------------------*/ 1250 static struct target_type multipath_target = { 1251 .name = "multipath", 1252 .version = {1, 0, 4}, 1253 .module = THIS_MODULE, 1254 .ctr = multipath_ctr, 1255 .dtr = multipath_dtr, 1256 .map = multipath_map, 1257 .end_io = multipath_end_io, 1258 .presuspend = multipath_presuspend, 1259 .resume = multipath_resume, 1260 .status = multipath_status, 1261 .message = multipath_message, 1262 }; 1263 1264 static int __init dm_multipath_init(void) 1265 { 1266 int r; 1267 1268 /* allocate a slab for the dm_ios */ 1269 _mpio_cache = kmem_cache_create("dm_mpath", sizeof(struct mpath_io), 1270 0, 0, NULL, NULL); 1271 if (!_mpio_cache) 1272 return -ENOMEM; 1273 1274 r = dm_register_target(&multipath_target); 1275 if (r < 0) { 1276 DMERR("%s: register failed %d", multipath_target.name, r); 1277 kmem_cache_destroy(_mpio_cache); 1278 return -EINVAL; 1279 } 1280 1281 kmultipathd = create_workqueue("kmpathd"); 1282 if (!kmultipathd) { 1283 DMERR("%s: failed to create workqueue kmpathd", 1284 multipath_target.name); 1285 dm_unregister_target(&multipath_target); 1286 kmem_cache_destroy(_mpio_cache); 1287 return -ENOMEM; 1288 } 1289 1290 DMINFO("dm-multipath version %u.%u.%u loaded", 1291 multipath_target.version[0], multipath_target.version[1], 1292 multipath_target.version[2]); 1293 1294 return r; 1295 } 1296 1297 static void __exit dm_multipath_exit(void) 1298 { 1299 int r; 1300 1301 destroy_workqueue(kmultipathd); 1302 1303 r = dm_unregister_target(&multipath_target); 1304 if (r < 0) 1305 DMERR("%s: target unregister failed %d", 1306 multipath_target.name, r); 1307 kmem_cache_destroy(_mpio_cache); 1308 } 1309 1310 EXPORT_SYMBOL_GPL(dm_pg_init_complete); 1311 1312 module_init(dm_multipath_init); 1313 module_exit(dm_multipath_exit); 1314 1315 MODULE_DESCRIPTION(DM_NAME " multipath target"); 1316 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 1317 MODULE_LICENSE("GPL"); 1318