Lines Matching refs:mg
416 metaslab_group_t *mg; in metaslab_class_validate() local
425 if ((mg = mc->mc_rotor) == NULL) in metaslab_class_validate()
429 vd = mg->mg_vd; in metaslab_class_validate()
432 ASSERT3P(mg->mg_class, ==, mc); in metaslab_class_validate()
434 } while ((mg = mg->mg_next) != mc->mc_rotor); in metaslab_class_validate()
489 metaslab_group_t *mg = tvd->vdev_mg; in metaslab_class_histogram_verify() local
496 mg->mg_class != mc) { in metaslab_class_histogram_verify()
501 mc_hist[i] += mg->mg_histogram[i]; in metaslab_class_histogram_verify()
527 metaslab_group_t *mg = tvd->vdev_mg; in metaslab_class_fragmentation() local
534 mg->mg_class != mc) { in metaslab_class_fragmentation()
542 if (mg->mg_fragmentation == ZFS_FRAG_INVALID) { in metaslab_class_fragmentation()
551 fragmentation += mg->mg_fragmentation * in metaslab_class_fragmentation()
552 metaslab_group_get_space(mg); in metaslab_class_fragmentation()
577 metaslab_group_t *mg = tvd->vdev_mg; in metaslab_class_expandable_space() local
580 mg->mg_class != mc) { in metaslab_class_expandable_space()
698 metaslab_group_alloc_update(metaslab_group_t *mg) in metaslab_group_alloc_update() argument
700 vdev_t *vd = mg->mg_vd; in metaslab_group_alloc_update()
701 metaslab_class_t *mc = mg->mg_class; in metaslab_group_alloc_update()
710 mutex_enter(&mg->mg_lock); in metaslab_group_alloc_update()
711 was_allocatable = mg->mg_allocatable; in metaslab_group_alloc_update()
712 was_initialized = mg->mg_initialized; in metaslab_group_alloc_update()
714 mg->mg_free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) / in metaslab_group_alloc_update()
727 mg->mg_initialized = metaslab_group_initialized(mg); in metaslab_group_alloc_update()
728 if (!was_initialized && mg->mg_initialized) { in metaslab_group_alloc_update()
730 } else if (was_initialized && !mg->mg_initialized) { in metaslab_group_alloc_update()
734 if (mg->mg_initialized) in metaslab_group_alloc_update()
735 mg->mg_no_free_space = B_FALSE; in metaslab_group_alloc_update()
743 mg->mg_allocatable = (mg->mg_activation_count > 0 && in metaslab_group_alloc_update()
744 mg->mg_free_capacity > zfs_mg_noalloc_threshold && in metaslab_group_alloc_update()
745 (mg->mg_fragmentation == ZFS_FRAG_INVALID || in metaslab_group_alloc_update()
746 mg->mg_fragmentation <= zfs_mg_fragmentation_threshold)); in metaslab_group_alloc_update()
763 if (was_allocatable && !mg->mg_allocatable) in metaslab_group_alloc_update()
765 else if (!was_allocatable && mg->mg_allocatable) in metaslab_group_alloc_update()
769 mutex_exit(&mg->mg_lock); in metaslab_group_alloc_update()
794 metaslab_group_t *mg; in metaslab_group_create() local
796 mg = kmem_zalloc(sizeof (metaslab_group_t), KM_SLEEP); in metaslab_group_create()
797 mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); in metaslab_group_create()
798 mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL); in metaslab_group_create()
799 cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL); in metaslab_group_create()
800 mg->mg_primaries = kmem_zalloc(allocators * sizeof (metaslab_t *), in metaslab_group_create()
802 mg->mg_secondaries = kmem_zalloc(allocators * sizeof (metaslab_t *), in metaslab_group_create()
804 avl_create(&mg->mg_metaslab_tree, metaslab_compare, in metaslab_group_create()
806 mg->mg_vd = vd; in metaslab_group_create()
807 mg->mg_class = mc; in metaslab_group_create()
808 mg->mg_activation_count = 0; in metaslab_group_create()
809 mg->mg_initialized = B_FALSE; in metaslab_group_create()
810 mg->mg_no_free_space = B_TRUE; in metaslab_group_create()
811 mg->mg_allocators = allocators; in metaslab_group_create()
813 mg->mg_alloc_queue_depth = kmem_zalloc(allocators * in metaslab_group_create()
815 mg->mg_cur_max_alloc_queue_depth = kmem_zalloc(allocators * in metaslab_group_create()
818 zfs_refcount_create_tracked(&mg->mg_alloc_queue_depth[i]); in metaslab_group_create()
819 mg->mg_cur_max_alloc_queue_depth[i] = 0; in metaslab_group_create()
822 mg->mg_taskq = taskq_create("metaslab_group_taskq", metaslab_load_pct, in metaslab_group_create()
825 return (mg); in metaslab_group_create()
829 metaslab_group_destroy(metaslab_group_t *mg) in metaslab_group_destroy() argument
831 ASSERT(mg->mg_prev == NULL); in metaslab_group_destroy()
832 ASSERT(mg->mg_next == NULL); in metaslab_group_destroy()
838 ASSERT(mg->mg_activation_count <= 0); in metaslab_group_destroy()
840 taskq_destroy(mg->mg_taskq); in metaslab_group_destroy()
841 avl_destroy(&mg->mg_metaslab_tree); in metaslab_group_destroy()
842 kmem_free(mg->mg_primaries, mg->mg_allocators * sizeof (metaslab_t *)); in metaslab_group_destroy()
843 kmem_free(mg->mg_secondaries, mg->mg_allocators * in metaslab_group_destroy()
845 mutex_destroy(&mg->mg_lock); in metaslab_group_destroy()
846 mutex_destroy(&mg->mg_ms_disabled_lock); in metaslab_group_destroy()
847 cv_destroy(&mg->mg_ms_disabled_cv); in metaslab_group_destroy()
849 for (int i = 0; i < mg->mg_allocators; i++) { in metaslab_group_destroy()
850 zfs_refcount_destroy(&mg->mg_alloc_queue_depth[i]); in metaslab_group_destroy()
851 mg->mg_cur_max_alloc_queue_depth[i] = 0; in metaslab_group_destroy()
853 kmem_free(mg->mg_alloc_queue_depth, mg->mg_allocators * in metaslab_group_destroy()
855 kmem_free(mg->mg_cur_max_alloc_queue_depth, mg->mg_allocators * in metaslab_group_destroy()
858 kmem_free(mg, sizeof (metaslab_group_t)); in metaslab_group_destroy()
862 metaslab_group_activate(metaslab_group_t *mg) in metaslab_group_activate() argument
864 metaslab_class_t *mc = mg->mg_class; in metaslab_group_activate()
869 ASSERT(mc->mc_rotor != mg); in metaslab_group_activate()
870 ASSERT(mg->mg_prev == NULL); in metaslab_group_activate()
871 ASSERT(mg->mg_next == NULL); in metaslab_group_activate()
872 ASSERT(mg->mg_activation_count <= 0); in metaslab_group_activate()
874 if (++mg->mg_activation_count <= 0) in metaslab_group_activate()
877 mg->mg_aliquot = metaslab_aliquot * MAX(1, mg->mg_vd->vdev_children); in metaslab_group_activate()
878 metaslab_group_alloc_update(mg); in metaslab_group_activate()
881 mg->mg_prev = mg; in metaslab_group_activate()
882 mg->mg_next = mg; in metaslab_group_activate()
885 mg->mg_prev = mgprev; in metaslab_group_activate()
886 mg->mg_next = mgnext; in metaslab_group_activate()
887 mgprev->mg_next = mg; in metaslab_group_activate()
888 mgnext->mg_prev = mg; in metaslab_group_activate()
890 mc->mc_rotor = mg; in metaslab_group_activate()
900 metaslab_group_passivate(metaslab_group_t *mg) in metaslab_group_passivate() argument
902 metaslab_class_t *mc = mg->mg_class; in metaslab_group_passivate()
910 if (--mg->mg_activation_count != 0) { in metaslab_group_passivate()
911 ASSERT(mc->mc_rotor != mg); in metaslab_group_passivate()
912 ASSERT(mg->mg_prev == NULL); in metaslab_group_passivate()
913 ASSERT(mg->mg_next == NULL); in metaslab_group_passivate()
914 ASSERT(mg->mg_activation_count < 0); in metaslab_group_passivate()
933 taskq_wait(mg->mg_taskq); in metaslab_group_passivate()
935 metaslab_group_alloc_update(mg); in metaslab_group_passivate()
936 for (int i = 0; i < mg->mg_allocators; i++) { in metaslab_group_passivate()
937 metaslab_t *msp = mg->mg_primaries[i]; in metaslab_group_passivate()
944 msp = mg->mg_secondaries[i]; in metaslab_group_passivate()
953 mgprev = mg->mg_prev; in metaslab_group_passivate()
954 mgnext = mg->mg_next; in metaslab_group_passivate()
956 if (mg == mgnext) { in metaslab_group_passivate()
964 mg->mg_prev = NULL; in metaslab_group_passivate()
965 mg->mg_next = NULL; in metaslab_group_passivate()
969 metaslab_group_initialized(metaslab_group_t *mg) in metaslab_group_initialized() argument
971 vdev_t *vd = mg->mg_vd; in metaslab_group_initialized()
974 return (vs->vs_space != 0 && mg->mg_activation_count > 0); in metaslab_group_initialized()
978 metaslab_group_get_space(metaslab_group_t *mg) in metaslab_group_get_space() argument
980 return ((1ULL << mg->mg_vd->vdev_ms_shift) * mg->mg_vd->vdev_ms_count); in metaslab_group_get_space()
984 metaslab_group_histogram_verify(metaslab_group_t *mg) in metaslab_group_histogram_verify() argument
987 vdev_t *vd = mg->mg_vd; in metaslab_group_histogram_verify()
1004 if (msp->ms_sm == NULL || msp->ms_group != mg) in metaslab_group_histogram_verify()
1013 VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]); in metaslab_group_histogram_verify()
1019 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_histogram_add() argument
1021 metaslab_class_t *mc = mg->mg_class; in metaslab_group_histogram_add()
1022 uint64_t ashift = mg->mg_vd->vdev_ashift; in metaslab_group_histogram_add()
1028 mutex_enter(&mg->mg_lock); in metaslab_group_histogram_add()
1030 mg->mg_histogram[i + ashift] += in metaslab_group_histogram_add()
1035 mutex_exit(&mg->mg_lock); in metaslab_group_histogram_add()
1039 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_histogram_remove() argument
1041 metaslab_class_t *mc = mg->mg_class; in metaslab_group_histogram_remove()
1042 uint64_t ashift = mg->mg_vd->vdev_ashift; in metaslab_group_histogram_remove()
1048 mutex_enter(&mg->mg_lock); in metaslab_group_histogram_remove()
1050 ASSERT3U(mg->mg_histogram[i + ashift], >=, in metaslab_group_histogram_remove()
1055 mg->mg_histogram[i + ashift] -= in metaslab_group_histogram_remove()
1060 mutex_exit(&mg->mg_lock); in metaslab_group_histogram_remove()
1064 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_add() argument
1067 mutex_enter(&mg->mg_lock); in metaslab_group_add()
1068 msp->ms_group = mg; in metaslab_group_add()
1070 avl_add(&mg->mg_metaslab_tree, msp); in metaslab_group_add()
1071 mutex_exit(&mg->mg_lock); in metaslab_group_add()
1074 metaslab_group_histogram_add(mg, msp); in metaslab_group_add()
1079 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_remove() argument
1082 metaslab_group_histogram_remove(mg, msp); in metaslab_group_remove()
1085 mutex_enter(&mg->mg_lock); in metaslab_group_remove()
1086 ASSERT(msp->ms_group == mg); in metaslab_group_remove()
1087 avl_remove(&mg->mg_metaslab_tree, msp); in metaslab_group_remove()
1097 mutex_exit(&mg->mg_lock); in metaslab_group_remove()
1101 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) in metaslab_group_sort_impl() argument
1104 ASSERT(MUTEX_HELD(&mg->mg_lock)); in metaslab_group_sort_impl()
1105 ASSERT(msp->ms_group == mg); in metaslab_group_sort_impl()
1107 avl_remove(&mg->mg_metaslab_tree, msp); in metaslab_group_sort_impl()
1109 avl_add(&mg->mg_metaslab_tree, msp); in metaslab_group_sort_impl()
1114 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) in metaslab_group_sort() argument
1123 mutex_enter(&mg->mg_lock); in metaslab_group_sort()
1124 metaslab_group_sort_impl(mg, msp, weight); in metaslab_group_sort()
1125 mutex_exit(&mg->mg_lock); in metaslab_group_sort()
1136 metaslab_group_fragmentation(metaslab_group_t *mg) in metaslab_group_fragmentation() argument
1138 vdev_t *vd = mg->mg_vd; in metaslab_group_fragmentation()
1147 if (msp->ms_group != mg) in metaslab_group_fragmentation()
1154 if (valid_ms <= mg->mg_vd->vdev_ms_count / 2) in metaslab_group_fragmentation()
1173 metaslab_group_allocatable(metaslab_group_t *mg, metaslab_group_t *rotor, in metaslab_group_allocatable() argument
1176 spa_t *spa = mg->mg_vd->vdev_spa; in metaslab_group_allocatable()
1177 metaslab_class_t *mc = mg->mg_class; in metaslab_group_allocatable()
1203 if (mg->mg_allocatable) { in metaslab_group_allocatable()
1206 uint64_t qmax = mg->mg_cur_max_alloc_queue_depth[allocator]; in metaslab_group_allocatable()
1215 if (mg->mg_no_free_space) in metaslab_group_allocatable()
1226 &mg->mg_alloc_queue_depth[allocator]); in metaslab_group_allocatable()
1244 for (mgp = mg->mg_next; mgp != rotor; mgp = mgp->mg_next) { in metaslab_group_allocatable()
2554 metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, in metaslab_init() argument
2557 vdev_t *vd = mg->mg_vd; in metaslab_init()
2617 metaslab_group_add(mg, ms); in metaslab_init()
2631 metaslab_space_update(vd, mg->mg_class, in metaslab_init()
2676 metaslab_group_t *mg = msp->ms_group; in metaslab_fini() local
2677 vdev_t *vd = mg->mg_vd; in metaslab_fini()
2682 metaslab_group_remove(mg, msp); in metaslab_fini()
2686 metaslab_space_update(vd, mg->mg_class, in metaslab_fini()
2860 metaslab_group_t *mg = msp->ms_group; in metaslab_space_weight() local
2861 vdev_t *vd = mg->mg_vd; in metaslab_space_weight()
3020 metaslab_group_t *mg = msp->ms_group; in metaslab_segment_weight() local
3022 uint8_t shift = mg->mg_vd->vdev_ashift; in metaslab_segment_weight()
3169 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, in metaslab_activate_allocator() argument
3181 metaslab_group_sort(mg, msp, msp->ms_weight | in metaslab_activate_allocator()
3187 mg->mg_primaries : mg->mg_secondaries); in metaslab_activate_allocator()
3189 mutex_enter(&mg->mg_lock); in metaslab_activate_allocator()
3191 mutex_exit(&mg->mg_lock); in metaslab_activate_allocator()
3202 metaslab_group_sort_impl(mg, msp, in metaslab_activate_allocator()
3205 mutex_exit(&mg->mg_lock); in metaslab_activate_allocator()
3288 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp, in metaslab_passivate_allocator() argument
3295 metaslab_group_sort(mg, msp, weight); in metaslab_passivate_allocator()
3299 mutex_enter(&mg->mg_lock); in metaslab_passivate_allocator()
3300 ASSERT3P(msp->ms_group, ==, mg); in metaslab_passivate_allocator()
3302 ASSERT3U(msp->ms_allocator, <, mg->mg_allocators); in metaslab_passivate_allocator()
3305 ASSERT3P(mg->mg_primaries[msp->ms_allocator], ==, msp); in metaslab_passivate_allocator()
3307 mg->mg_primaries[msp->ms_allocator] = NULL; in metaslab_passivate_allocator()
3309 ASSERT3P(mg->mg_secondaries[msp->ms_allocator], ==, msp); in metaslab_passivate_allocator()
3311 mg->mg_secondaries[msp->ms_allocator] = NULL; in metaslab_passivate_allocator()
3314 metaslab_group_sort_impl(mg, msp, weight); in metaslab_passivate_allocator()
3315 mutex_exit(&mg->mg_lock); in metaslab_passivate_allocator()
3386 metaslab_group_preload(metaslab_group_t *mg) in metaslab_group_preload() argument
3388 spa_t *spa = mg->mg_vd->vdev_spa; in metaslab_group_preload()
3390 avl_tree_t *t = &mg->mg_metaslab_tree; in metaslab_group_preload()
3394 taskq_wait(mg->mg_taskq); in metaslab_group_preload()
3398 mutex_enter(&mg->mg_lock); in metaslab_group_preload()
3404 ASSERT3P(msp->ms_group, ==, mg); in metaslab_group_preload()
3416 VERIFY(taskq_dispatch(mg->mg_taskq, metaslab_preload, in metaslab_group_preload()
3419 mutex_exit(&mg->mg_lock); in metaslab_group_preload()
3634 metaslab_group_t *mg = msp->ms_group; in metaslab_flush_update() local
3635 spa_t *spa = mg->mg_vd->vdev_spa; in metaslab_flush_update()
3733 metaslab_group_t *mg = msp->ms_group; in metaslab_flush() local
3740 metaslab_group_histogram_verify(mg); in metaslab_flush()
3741 metaslab_class_histogram_verify(mg->mg_class); in metaslab_flush()
3742 metaslab_group_histogram_remove(mg, msp); in metaslab_flush()
3755 metaslab_group_histogram_add(mg, msp); in metaslab_flush()
3756 metaslab_group_histogram_verify(mg); in metaslab_flush()
3757 metaslab_class_histogram_verify(mg->mg_class); in metaslab_flush()
3819 metaslab_group_t *mg = msp->ms_group; in metaslab_sync() local
3820 vdev_t *vd = mg->mg_vd; in metaslab_sync()
3949 metaslab_group_histogram_verify(mg); in metaslab_sync()
3950 metaslab_class_histogram_verify(mg->mg_class); in metaslab_sync()
3951 metaslab_group_histogram_remove(mg, msp); in metaslab_sync()
4065 metaslab_group_histogram_add(mg, msp); in metaslab_sync()
4066 metaslab_group_histogram_verify(mg); in metaslab_sync()
4067 metaslab_class_histogram_verify(mg->mg_class); in metaslab_sync()
4134 metaslab_group_t *mg = msp->ms_group; in metaslab_sync_done() local
4135 vdev_t *vd = mg->mg_vd; in metaslab_sync_done()
4191 metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size); in metaslab_sync_done()
4214 metaslab_space_update(vd, mg->mg_class, alloc_delta + defer_delta, in metaslab_sync_done()
4280 mutex_enter(&mg->mg_lock); in metaslab_sync_done()
4281 mg->mg_ms_ready++; in metaslab_sync_done()
4282 mutex_exit(&mg->mg_lock); in metaslab_sync_done()
4322 metaslab_sync_reassess(metaslab_group_t *mg) in metaslab_sync_reassess() argument
4324 spa_t *spa = mg->mg_class->mc_spa; in metaslab_sync_reassess()
4327 metaslab_group_alloc_update(mg); in metaslab_sync_reassess()
4328 mg->mg_fragmentation = metaslab_group_fragmentation(mg); in metaslab_sync_reassess()
4337 if (mg->mg_activation_count > 0) { in metaslab_sync_reassess()
4338 metaslab_group_preload(mg); in metaslab_sync_reassess()
4374 metaslab_trace_add(zio_alloc_list_t *zal, metaslab_group_t *mg, in metaslab_trace_add() argument
4403 mat->mat_mg = mg; in metaslab_trace_add()
4457 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; in metaslab_group_alloc_increment() local
4458 if (!mg->mg_class->mc_alloc_throttle_enabled) in metaslab_group_alloc_increment()
4461 (void) zfs_refcount_add(&mg->mg_alloc_queue_depth[allocator], tag); in metaslab_group_alloc_increment()
4465 metaslab_group_increment_qdepth(metaslab_group_t *mg, int allocator) in metaslab_group_increment_qdepth() argument
4467 uint64_t max = mg->mg_max_alloc_queue_depth; in metaslab_group_increment_qdepth()
4468 uint64_t cur = mg->mg_cur_max_alloc_queue_depth[allocator]; in metaslab_group_increment_qdepth()
4470 if (atomic_cas_64(&mg->mg_cur_max_alloc_queue_depth[allocator], in metaslab_group_increment_qdepth()
4473 &mg->mg_class->mc_alloc_max_slots[allocator]); in metaslab_group_increment_qdepth()
4476 cur = mg->mg_cur_max_alloc_queue_depth[allocator]; in metaslab_group_increment_qdepth()
4488 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; in metaslab_group_alloc_decrement() local
4489 if (!mg->mg_class->mc_alloc_throttle_enabled) in metaslab_group_alloc_decrement()
4492 (void) zfs_refcount_remove(&mg->mg_alloc_queue_depth[allocator], tag); in metaslab_group_alloc_decrement()
4494 metaslab_group_increment_qdepth(mg, allocator); in metaslab_group_alloc_decrement()
4507 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; in metaslab_group_alloc_verify() local
4509 &mg->mg_alloc_queue_depth[allocator], tag)); in metaslab_group_alloc_verify()
4527 metaslab_group_t *mg = msp->ms_group; in metaslab_block_alloc() local
4528 vdev_t *vd = mg->mg_vd; in metaslab_block_alloc()
4537 vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg); in metaslab_block_alloc()
4567 find_valid_metaslab(metaslab_group_t *mg, uint64_t activation_weight, in find_valid_metaslab() argument
4573 avl_tree_t *t = &mg->mg_metaslab_tree; in find_valid_metaslab()
4581 metaslab_trace_add(zal, mg, msp, asize, d, in find_valid_metaslab()
4659 metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal, in metaslab_group_alloc_normal() argument
4669 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { in metaslab_group_alloc_normal()
4672 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { in metaslab_group_alloc_normal()
4682 if (mg->mg_ms_ready < mg->mg_allocators * 3) in metaslab_group_alloc_normal()
4685 ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2); in metaslab_group_alloc_normal()
4702 mutex_enter(&mg->mg_lock); in metaslab_group_alloc_normal()
4705 mg->mg_primaries[allocator] != NULL) { in metaslab_group_alloc_normal()
4706 msp = mg->mg_primaries[allocator]; in metaslab_group_alloc_normal()
4721 mg->mg_secondaries[allocator] != NULL) { in metaslab_group_alloc_normal()
4722 msp = mg->mg_secondaries[allocator]; in metaslab_group_alloc_normal()
4735 msp = find_valid_metaslab(mg, activation_weight, dva, d, in metaslab_group_alloc_normal()
4740 mutex_exit(&mg->mg_lock); in metaslab_group_alloc_normal()
4846 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc_normal()
4859 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc_normal()
4868 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc_normal()
4879 metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator); in metaslab_group_alloc_normal()
4944 metaslab_group_sort(mg, msp, weight); in metaslab_group_alloc_normal()
4964 metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal, in metaslab_group_alloc() argument
4969 ASSERT(mg->mg_initialized); in metaslab_group_alloc()
4971 offset = metaslab_group_alloc_normal(mg, zal, asize, txg, want_unique, in metaslab_group_alloc()
4974 mutex_enter(&mg->mg_lock); in metaslab_group_alloc()
4976 mg->mg_failed_allocations++; in metaslab_group_alloc()
4977 metaslab_trace_add(zal, mg, NULL, asize, d, in metaslab_group_alloc()
4991 mg->mg_no_free_space = B_TRUE; in metaslab_group_alloc()
4994 mg->mg_allocations++; in metaslab_group_alloc()
4995 mutex_exit(&mg->mg_lock); in metaslab_group_alloc()
5007 metaslab_group_t *mg, *rotor; in metaslab_alloc_dva() local
5055 mg = vd->vdev_mg; in metaslab_alloc_dva()
5058 mg->mg_next != NULL) in metaslab_alloc_dva()
5059 mg = mg->mg_next; in metaslab_alloc_dva()
5061 mg = mc->mc_rotor; in metaslab_alloc_dva()
5065 mg = vd->vdev_mg->mg_next; in metaslab_alloc_dva()
5068 mg = mc->mc_rotor; in metaslab_alloc_dva()
5075 if (mg->mg_class != mc || mg->mg_activation_count <= 0) in metaslab_alloc_dva()
5076 mg = mc->mc_rotor; in metaslab_alloc_dva()
5078 rotor = mg; in metaslab_alloc_dva()
5083 ASSERT(mg->mg_activation_count == 1); in metaslab_alloc_dva()
5084 vd = mg->mg_vd; in metaslab_alloc_dva()
5105 allocatable = metaslab_group_allocatable(mg, rotor, in metaslab_alloc_dva()
5110 metaslab_trace_add(zal, mg, NULL, psize, d, in metaslab_alloc_dva()
5115 ASSERT(mg->mg_initialized); in metaslab_alloc_dva()
5125 metaslab_trace_add(zal, mg, NULL, psize, d, in metaslab_alloc_dva()
5130 ASSERT(mg->mg_class == mc); in metaslab_alloc_dva()
5141 uint64_t offset = metaslab_group_alloc(mg, zal, asize, txg, in metaslab_alloc_dva()
5171 mg->mg_bias = ((cu - vu) * in metaslab_alloc_dva()
5172 (int64_t)mg->mg_aliquot) / 100; in metaslab_alloc_dva()
5174 mg->mg_bias = 0; in metaslab_alloc_dva()
5178 mg->mg_aliquot + mg->mg_bias) { in metaslab_alloc_dva()
5179 mc->mc_rotor = mg->mg_next; in metaslab_alloc_dva()
5191 mc->mc_rotor = mg->mg_next; in metaslab_alloc_dva()
5193 } while ((mg = mg->mg_next) != rotor); in metaslab_alloc_dva()
5906 metaslab_group_disable_wait(metaslab_group_t *mg) in metaslab_group_disable_wait() argument
5908 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); in metaslab_group_disable_wait()
5909 while (mg->mg_disabled_updating) { in metaslab_group_disable_wait()
5910 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); in metaslab_group_disable_wait()
5915 metaslab_group_disabled_increment(metaslab_group_t *mg) in metaslab_group_disabled_increment() argument
5917 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); in metaslab_group_disabled_increment()
5918 ASSERT(mg->mg_disabled_updating); in metaslab_group_disabled_increment()
5920 while (mg->mg_ms_disabled >= max_disabled_ms) { in metaslab_group_disabled_increment()
5921 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); in metaslab_group_disabled_increment()
5923 mg->mg_ms_disabled++; in metaslab_group_disabled_increment()
5924 ASSERT3U(mg->mg_ms_disabled, <=, max_disabled_ms); in metaslab_group_disabled_increment()
5937 metaslab_group_t *mg = msp->ms_group; in metaslab_disable() local
5939 mutex_enter(&mg->mg_ms_disabled_lock); in metaslab_disable()
5950 metaslab_group_disable_wait(mg); in metaslab_disable()
5951 mg->mg_disabled_updating = B_TRUE; in metaslab_disable()
5953 metaslab_group_disabled_increment(mg); in metaslab_disable()
5959 mg->mg_disabled_updating = B_FALSE; in metaslab_disable()
5960 cv_broadcast(&mg->mg_ms_disabled_cv); in metaslab_disable()
5961 mutex_exit(&mg->mg_ms_disabled_lock); in metaslab_disable()
5967 metaslab_group_t *mg = msp->ms_group; in metaslab_enable() local
5968 spa_t *spa = mg->mg_vd->vdev_spa; in metaslab_enable()
5978 mutex_enter(&mg->mg_ms_disabled_lock); in metaslab_enable()
5981 mg->mg_ms_disabled--; in metaslab_enable()
5982 cv_broadcast(&mg->mg_ms_disabled_cv); in metaslab_enable()
5987 mutex_exit(&mg->mg_ms_disabled_lock); in metaslab_enable()