217 	metaslab_group_t *mg;  in metaslab_class_validate()  local
226 	if ((mg = mc->mc_rotor) == NULL)  in metaslab_class_validate()
230 		vd = mg->mg_vd;  in metaslab_class_validate()
233 		ASSERT3P(mg->mg_class, ==, mc);  in metaslab_class_validate()
235 	} while ((mg = mg->mg_next) != mc->mc_rotor);  in metaslab_class_validate()
289 		metaslab_group_t *mg = tvd->vdev_mg;  in metaslab_class_histogram_verify()  local
296 		    mg->mg_class != mc) {  in metaslab_class_histogram_verify()
301 			mc_hist[i] += mg->mg_histogram[i];  in metaslab_class_histogram_verify()
327 		metaslab_group_t *mg = tvd->vdev_mg;  in metaslab_class_fragmentation()  local
334 		    mg->mg_class != mc) {  in metaslab_class_fragmentation()
342 		if (mg->mg_fragmentation == ZFS_FRAG_INVALID) {  in metaslab_class_fragmentation()
351 		fragmentation += mg->mg_fragmentation *  in metaslab_class_fragmentation()
352 		    metaslab_group_get_space(mg);  in metaslab_class_fragmentation()
376 		metaslab_group_t *mg = tvd->vdev_mg;  in metaslab_class_expandable_space()  local
379 		    mg->mg_class != mc) {  in metaslab_class_expandable_space()
426 metaslab_group_alloc_update(metaslab_group_t *mg)  in metaslab_group_alloc_update()  argument
428 	vdev_t *vd = mg->mg_vd;  in metaslab_group_alloc_update()
429 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_alloc_update()
435 	mutex_enter(&mg->mg_lock);  in metaslab_group_alloc_update()
436 	was_allocatable = mg->mg_allocatable;  in metaslab_group_alloc_update()
438 	mg->mg_free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) /  in metaslab_group_alloc_update()
447 	mg->mg_allocatable = (mg->mg_free_capacity > zfs_mg_noalloc_threshold &&  in metaslab_group_alloc_update()
448 	    (mg->mg_fragmentation == ZFS_FRAG_INVALID ||  in metaslab_group_alloc_update()
449 	    mg->mg_fragmentation <= zfs_mg_fragmentation_threshold));  in metaslab_group_alloc_update()
466 	if (was_allocatable && !mg->mg_allocatable)  in metaslab_group_alloc_update()
468 	else if (!was_allocatable && mg->mg_allocatable)  in metaslab_group_alloc_update()
471 	mutex_exit(&mg->mg_lock);  in metaslab_group_alloc_update()
477 	metaslab_group_t *mg;  in metaslab_group_create()  local
479 	mg = kmem_zalloc(sizeof (metaslab_group_t), KM_SLEEP);  in metaslab_group_create()
480 	mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL);  in metaslab_group_create()
481 	avl_create(&mg->mg_metaslab_tree, metaslab_compare,  in metaslab_group_create()
483 	mg->mg_vd = vd;  in metaslab_group_create()
484 	mg->mg_class = mc;  in metaslab_group_create()
485 	mg->mg_activation_count = 0;  in metaslab_group_create()
487 	mg->mg_taskq = taskq_create("metaslab_group_taskq", metaslab_load_pct,  in metaslab_group_create()
490 	return (mg);  in metaslab_group_create()
494 metaslab_group_destroy(metaslab_group_t *mg)  in metaslab_group_destroy()  argument
496 	ASSERT(mg->mg_prev == NULL);  in metaslab_group_destroy()
497 	ASSERT(mg->mg_next == NULL);  in metaslab_group_destroy()
503 	ASSERT(mg->mg_activation_count <= 0);  in metaslab_group_destroy()
505 	taskq_destroy(mg->mg_taskq);  in metaslab_group_destroy()
506 	avl_destroy(&mg->mg_metaslab_tree);  in metaslab_group_destroy()
507 	mutex_destroy(&mg->mg_lock);  in metaslab_group_destroy()
508 	kmem_free(mg, sizeof (metaslab_group_t));  in metaslab_group_destroy()
512 metaslab_group_activate(metaslab_group_t *mg)  in metaslab_group_activate()  argument
514 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_activate()
519 	ASSERT(mc->mc_rotor != mg);  in metaslab_group_activate()
520 	ASSERT(mg->mg_prev == NULL);  in metaslab_group_activate()
521 	ASSERT(mg->mg_next == NULL);  in metaslab_group_activate()
522 	ASSERT(mg->mg_activation_count <= 0);  in metaslab_group_activate()
524 	if (++mg->mg_activation_count <= 0)  in metaslab_group_activate()
527 	mg->mg_aliquot = metaslab_aliquot * MAX(1, mg->mg_vd->vdev_children);  in metaslab_group_activate()
528 	metaslab_group_alloc_update(mg);  in metaslab_group_activate()
531 		mg->mg_prev = mg;  in metaslab_group_activate()
532 		mg->mg_next = mg;  in metaslab_group_activate()
535 		mg->mg_prev = mgprev;  in metaslab_group_activate()
536 		mg->mg_next = mgnext;  in metaslab_group_activate()
537 		mgprev->mg_next = mg;  in metaslab_group_activate()
538 		mgnext->mg_prev = mg;  in metaslab_group_activate()
540 	mc->mc_rotor = mg;  in metaslab_group_activate()
544 metaslab_group_passivate(metaslab_group_t *mg)  in metaslab_group_passivate()  argument
546 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_passivate()
551 	if (--mg->mg_activation_count != 0) {  in metaslab_group_passivate()
552 		ASSERT(mc->mc_rotor != mg);  in metaslab_group_passivate()
553 		ASSERT(mg->mg_prev == NULL);  in metaslab_group_passivate()
554 		ASSERT(mg->mg_next == NULL);  in metaslab_group_passivate()
555 		ASSERT(mg->mg_activation_count < 0);  in metaslab_group_passivate()
559 	taskq_wait(mg->mg_taskq);  in metaslab_group_passivate()
560 	metaslab_group_alloc_update(mg);  in metaslab_group_passivate()
562 	mgprev = mg->mg_prev;  in metaslab_group_passivate()
563 	mgnext = mg->mg_next;  in metaslab_group_passivate()
565 	if (mg == mgnext) {  in metaslab_group_passivate()
573 	mg->mg_prev = NULL;  in metaslab_group_passivate()
574 	mg->mg_next = NULL;  in metaslab_group_passivate()
578 metaslab_group_get_space(metaslab_group_t *mg)  in metaslab_group_get_space()  argument
580 	return ((1ULL << mg->mg_vd->vdev_ms_shift) * mg->mg_vd->vdev_ms_count);  in metaslab_group_get_space()
584 metaslab_group_histogram_verify(metaslab_group_t *mg)  in metaslab_group_histogram_verify()  argument
587 	vdev_t *vd = mg->mg_vd;  in metaslab_group_histogram_verify()
612 		VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]);  in metaslab_group_histogram_verify()
618 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp)  in metaslab_group_histogram_add()  argument
620 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_histogram_add()
621 	uint64_t ashift = mg->mg_vd->vdev_ashift;  in metaslab_group_histogram_add()
627 	mutex_enter(&mg->mg_lock);  in metaslab_group_histogram_add()
629 		mg->mg_histogram[i + ashift] +=  in metaslab_group_histogram_add()
634 	mutex_exit(&mg->mg_lock);  in metaslab_group_histogram_add()
638 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp)  in metaslab_group_histogram_remove()  argument
640 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_histogram_remove()
641 	uint64_t ashift = mg->mg_vd->vdev_ashift;  in metaslab_group_histogram_remove()
647 	mutex_enter(&mg->mg_lock);  in metaslab_group_histogram_remove()
649 		ASSERT3U(mg->mg_histogram[i + ashift], >=,  in metaslab_group_histogram_remove()
654 		mg->mg_histogram[i + ashift] -=  in metaslab_group_histogram_remove()
659 	mutex_exit(&mg->mg_lock);  in metaslab_group_histogram_remove()
663 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp)  in metaslab_group_add()  argument
666 	mutex_enter(&mg->mg_lock);  in metaslab_group_add()
667 	msp->ms_group = mg;  in metaslab_group_add()
669 	avl_add(&mg->mg_metaslab_tree, msp);  in metaslab_group_add()
670 	mutex_exit(&mg->mg_lock);  in metaslab_group_add()
673 	metaslab_group_histogram_add(mg, msp);  in metaslab_group_add()
678 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp)  in metaslab_group_remove()  argument
681 	metaslab_group_histogram_remove(mg, msp);  in metaslab_group_remove()
684 	mutex_enter(&mg->mg_lock);  in metaslab_group_remove()
685 	ASSERT(msp->ms_group == mg);  in metaslab_group_remove()
686 	avl_remove(&mg->mg_metaslab_tree, msp);  in metaslab_group_remove()
688 	mutex_exit(&mg->mg_lock);  in metaslab_group_remove()
692 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)  in metaslab_group_sort()  argument
701 	mutex_enter(&mg->mg_lock);  in metaslab_group_sort()
702 	ASSERT(msp->ms_group == mg);  in metaslab_group_sort()
703 	avl_remove(&mg->mg_metaslab_tree, msp);  in metaslab_group_sort()
705 	avl_add(&mg->mg_metaslab_tree, msp);  in metaslab_group_sort()
706 	mutex_exit(&mg->mg_lock);  in metaslab_group_sort()
717 metaslab_group_fragmentation(metaslab_group_t *mg)  in metaslab_group_fragmentation()  argument
719 	vdev_t *vd = mg->mg_vd;  in metaslab_group_fragmentation()
749 metaslab_group_allocatable(metaslab_group_t *mg)  in metaslab_group_allocatable()  argument
751 	vdev_t *vd = mg->mg_vd;  in metaslab_group_allocatable()
753 	metaslab_class_t *mc = mg->mg_class;  in metaslab_group_allocatable()
766 	return ((mg->mg_free_capacity > zfs_mg_noalloc_threshold &&  in metaslab_group_allocatable()
767 	    (mg->mg_fragmentation == ZFS_FRAG_INVALID ||  in metaslab_group_allocatable()
768 	    mg->mg_fragmentation <= zfs_mg_fragmentation_threshold)) ||  in metaslab_group_allocatable()
1212 metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg,  in metaslab_init()  argument
1215 	vdev_t *vd = mg->mg_vd;  in metaslab_init()
1251 	metaslab_group_add(mg, ms);  in metaslab_init()
1254 	ms->ms_ops = mg->mg_class->mc_ops;  in metaslab_init()
1289 	metaslab_group_t *mg = msp->ms_group;  in metaslab_fini()  local
1291 	metaslab_group_remove(mg, msp);  in metaslab_fini()
1296 	vdev_space_update(mg->mg_vd, -space_map_allocated(msp->ms_sm),  in metaslab_fini()
1433 	metaslab_group_t *mg = msp->ms_group;  in metaslab_weight()  local
1434 	vdev_t *vd = mg->mg_vd;  in metaslab_weight()
1564 metaslab_group_preload(metaslab_group_t *mg)  in metaslab_group_preload()  argument
1566 	spa_t *spa = mg->mg_vd->vdev_spa;  in metaslab_group_preload()
1568 	avl_tree_t *t = &mg->mg_metaslab_tree;  in metaslab_group_preload()
1572 		taskq_wait(mg->mg_taskq);  in metaslab_group_preload()
1576 	mutex_enter(&mg->mg_lock);  in metaslab_group_preload()
1607 		mutex_exit(&mg->mg_lock);  in metaslab_group_preload()
1608 		VERIFY(taskq_dispatch(mg->mg_taskq, metaslab_preload,  in metaslab_group_preload()
1610 		mutex_enter(&mg->mg_lock);  in metaslab_group_preload()
1613 	mutex_exit(&mg->mg_lock);  in metaslab_group_preload()
1782 	metaslab_group_t *mg = msp->ms_group;  in metaslab_sync()  local
1783 	vdev_t *vd = mg->mg_vd;  in metaslab_sync()
1848 	metaslab_group_histogram_verify(mg);  in metaslab_sync()
1849 	metaslab_class_histogram_verify(mg->mg_class);  in metaslab_sync()
1850 	metaslab_group_histogram_remove(mg, msp);  in metaslab_sync()
1879 	metaslab_group_histogram_add(mg, msp);  in metaslab_sync()
1880 	metaslab_group_histogram_verify(mg);  in metaslab_sync()
1881 	metaslab_class_histogram_verify(mg->mg_class);  in metaslab_sync()
1916 	metaslab_group_t *mg = msp->ms_group;  in metaslab_sync_done()  local
1917 	vdev_t *vd = mg->mg_vd;  in metaslab_sync_done()
2003 	metaslab_group_sort(mg, msp, metaslab_weight(msp));  in metaslab_sync_done()
2008 metaslab_sync_reassess(metaslab_group_t *mg)  in metaslab_sync_reassess()  argument
2010 	metaslab_group_alloc_update(mg);  in metaslab_sync_reassess()
2011 	mg->mg_fragmentation = metaslab_group_fragmentation(mg);  in metaslab_sync_reassess()
2016 	metaslab_group_preload(mg);  in metaslab_sync_reassess()
2037 metaslab_group_alloc(metaslab_group_t *mg, uint64_t psize, uint64_t asize,  in metaslab_group_alloc()  argument
2040 	spa_t *spa = mg->mg_vd->vdev_spa;  in metaslab_group_alloc()
2043 	avl_tree_t *t = &mg->mg_metaslab_tree;  in metaslab_group_alloc()
2050 		if (DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) {  in metaslab_group_alloc()
2059 		mutex_enter(&mg->mg_lock);  in metaslab_group_alloc()
2066 				    mg->mg_vd->vdev_id, txg,  in metaslab_group_alloc()
2067 				    mg, msp, psize, asize, msp->ms_weight);  in metaslab_group_alloc()
2068 				mutex_exit(&mg->mg_lock);  in metaslab_group_alloc()
2093 		mutex_exit(&mg->mg_lock);  in metaslab_group_alloc()
2143 		vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg);  in metaslab_group_alloc()
2160 	metaslab_group_t *mg, *rotor;  in metaslab_alloc_dva()  local
2209 			mg = vd->vdev_mg;  in metaslab_alloc_dva()
2212 			    mg->mg_next != NULL)  in metaslab_alloc_dva()
2213 				mg = mg->mg_next;  in metaslab_alloc_dva()
2215 			mg = mc->mc_rotor;  in metaslab_alloc_dva()
2219 		mg = vd->vdev_mg->mg_next;  in metaslab_alloc_dva()
2221 		mg = mc->mc_rotor;  in metaslab_alloc_dva()
2228 	if (mg->mg_class != mc || mg->mg_activation_count <= 0)  in metaslab_alloc_dva()
2229 		mg = mc->mc_rotor;  in metaslab_alloc_dva()
2231 	rotor = mg;  in metaslab_alloc_dva()
2235 		ASSERT(mg->mg_activation_count == 1);  in metaslab_alloc_dva()
2237 		vd = mg->mg_vd;  in metaslab_alloc_dva()
2262 			allocatable = metaslab_group_allocatable(mg);  in metaslab_alloc_dva()
2278 		ASSERT(mg->mg_class == mc);  in metaslab_alloc_dva()
2289 		offset = metaslab_group_alloc(mg, psize, asize, txg, distance,  in metaslab_alloc_dva()
2318 				mg->mg_bias = ((cu - vu) *  in metaslab_alloc_dva()
2319 				    (int64_t)mg->mg_aliquot) / 100;  in metaslab_alloc_dva()
2321 				mg->mg_bias = 0;  in metaslab_alloc_dva()
2325 			    mg->mg_aliquot + mg->mg_bias) {  in metaslab_alloc_dva()
2326 				mc->mc_rotor = mg->mg_next;  in metaslab_alloc_dva()
2338 		mc->mc_rotor = mg->mg_next;  in metaslab_alloc_dva()
2340 	} while ((mg = mg->mg_next) != rotor);  in metaslab_alloc_dva()