Lines Matching refs:t

60 static inline sector_t *get_node(struct dm_table *t,  in get_node()  argument
63 return t->index[l] + (n * KEYS_PER_NODE); in get_node()
70 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) in high() argument
72 for (; l < t->depth - 1; l++) in high()
75 if (n >= t->counts[l]) in high()
78 return get_node(t, l, n)[KEYS_PER_NODE - 1]; in high()
85 static int setup_btree_index(unsigned int l, struct dm_table *t) in setup_btree_index() argument
90 for (n = 0U; n < t->counts[l]; n++) { in setup_btree_index()
91 node = get_node(t, l, n); in setup_btree_index()
94 node[k] = high(t, l + 1, get_child(n, k)); in setup_btree_index()
104 static int alloc_targets(struct dm_table *t, unsigned int num) in alloc_targets() argument
121 t->num_allocated = num; in alloc_targets()
122 t->highs = n_highs; in alloc_targets()
123 t->targets = n_targets; in alloc_targets()
131 struct dm_table *t; in dm_table_create() local
136 t = kzalloc(sizeof(*t), GFP_KERNEL); in dm_table_create()
138 if (!t) in dm_table_create()
141 INIT_LIST_HEAD(&t->devices); in dm_table_create()
142 init_rwsem(&t->devices_lock); in dm_table_create()
150 kfree(t); in dm_table_create()
154 if (alloc_targets(t, num_targets)) { in dm_table_create()
155 kfree(t); in dm_table_create()
159 t->type = DM_TYPE_NONE; in dm_table_create()
160 t->mode = mode; in dm_table_create()
161 t->md = md; in dm_table_create()
162 t->flush_bypasses_map = true; in dm_table_create()
163 *result = t; in dm_table_create()
181 static void dm_table_destroy_crypto_profile(struct dm_table *t);
183 void dm_table_destroy(struct dm_table *t) in dm_table_destroy() argument
185 if (!t) in dm_table_destroy()
189 if (t->depth >= 2) in dm_table_destroy()
190 kvfree(t->index[t->depth - 2]); in dm_table_destroy()
193 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_destroy()
194 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_destroy()
202 kvfree(t->highs); in dm_table_destroy()
205 free_devices(&t->devices, t->md); in dm_table_destroy()
207 dm_free_md_mempools(t->mempools); in dm_table_destroy()
209 dm_table_destroy_crypto_profile(t); in dm_table_destroy()
211 kfree(t); in dm_table_destroy()
372 struct dm_table *t = ti->table; in dm_get_device() local
374 BUG_ON(!t); in dm_get_device()
380 if (dev == disk_devt(t->md->disk)) in dm_get_device()
383 down_write(&t->devices_lock); in dm_get_device()
385 dd = find_device(&t->devices, dev); in dm_get_device()
393 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); in dm_get_device()
400 list_add(&dd->list, &t->devices); in dm_get_device()
404 r = upgrade_mode(dd, mode, t->md); in dm_get_device()
410 up_write(&t->devices_lock); in dm_get_device()
415 up_write(&t->devices_lock); in dm_get_device()
467 struct dm_table *t = ti->table; in dm_put_device() local
468 struct list_head *devices = &t->devices; in dm_put_device()
471 down_write(&t->devices_lock); in dm_put_device()
481 dm_device_name(t->md), d->name); in dm_put_device()
485 dm_put_table_device(t->md, d); in dm_put_device()
491 up_write(&t->devices_lock); in dm_put_device()
498 static int adjoin(struct dm_table *t, struct dm_target *ti) in adjoin() argument
502 if (!t->num_targets) in adjoin()
505 prev = &t->targets[t->num_targets - 1]; in adjoin()
619 static int validate_hardware_logical_block_alignment(struct dm_table *t, in validate_hardware_logical_block_alignment() argument
647 for (i = 0; i < t->num_targets; i++) { in validate_hardware_logical_block_alignment()
648 ti = dm_table_get_target(t, i); in validate_hardware_logical_block_alignment()
676 dm_device_name(t->md), i, in validate_hardware_logical_block_alignment()
686 int dm_table_add_target(struct dm_table *t, const char *type, in dm_table_add_target() argument
693 if (t->singleton) { in dm_table_add_target()
695 dm_device_name(t->md), t->targets->type->name); in dm_table_add_target()
699 BUG_ON(t->num_targets >= t->num_allocated); in dm_table_add_target()
701 ti = t->targets + t->num_targets; in dm_table_add_target()
705 DMERR("%s: zero-length target", dm_device_name(t->md)); in dm_table_add_target()
709 DMERR("%s: too large device", dm_device_name(t->md)); in dm_table_add_target()
715 DMERR("%s: %s: unknown target type", dm_device_name(t->md), type); in dm_table_add_target()
720 if (t->num_targets) { in dm_table_add_target()
724 t->singleton = true; in dm_table_add_target()
728 !(t->mode & BLK_OPEN_WRITE)) { in dm_table_add_target()
733 if (t->immutable_target_type) { in dm_table_add_target()
734 if (t->immutable_target_type != ti->type) { in dm_table_add_target()
739 if (t->num_targets) { in dm_table_add_target()
743 t->immutable_target_type = ti->type; in dm_table_add_target()
746 ti->table = t; in dm_table_add_target()
754 if (!adjoin(t, ti)) { in dm_table_add_target()
770 t->highs[t->num_targets++] = ti->begin + ti->len - 1; in dm_table_add_target()
774 dm_device_name(t->md), type); in dm_table_add_target()
780 t->flush_bypasses_map = false; in dm_table_add_target()
785 DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r)); in dm_table_add_target()
859 void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type) in dm_table_set_type() argument
861 t->type = type; in dm_table_set_type()
883 static bool dm_table_supports_dax(struct dm_table *t, in dm_table_supports_dax() argument
887 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_dax()
888 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_dax()
915 static int dm_table_determine_type(struct dm_table *t) in dm_table_determine_type() argument
919 struct list_head *devices = dm_table_get_devices(t); in dm_table_determine_type()
920 enum dm_queue_mode live_md_type = dm_get_md_type(t->md); in dm_table_determine_type()
922 if (t->type != DM_TYPE_NONE) { in dm_table_determine_type()
924 if (t->type == DM_TYPE_BIO_BASED) { in dm_table_determine_type()
928 BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED); in dm_table_determine_type()
932 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_determine_type()
933 ti = dm_table_get_target(t, i); in dm_table_determine_type()
962 t->type = DM_TYPE_BIO_BASED; in dm_table_determine_type()
963 if (dm_table_supports_dax(t, device_not_dax_capable) || in dm_table_determine_type()
965 t->type = DM_TYPE_DAX_BIO_BASED; in dm_table_determine_type()
972 t->type = DM_TYPE_REQUEST_BASED; in dm_table_determine_type()
981 if (t->num_targets > 1) { in dm_table_determine_type()
988 struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx); in dm_table_determine_type()
992 t->type = live_table->type; in dm_table_determine_type()
993 dm_put_live_table(t->md, srcu_idx); in dm_table_determine_type()
997 ti = dm_table_get_immutable_target(t); in dm_table_determine_type()
1016 enum dm_queue_mode dm_table_get_type(struct dm_table *t) in dm_table_get_type() argument
1018 return t->type; in dm_table_get_type()
1021 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t) in dm_table_get_immutable_target_type() argument
1023 return t->immutable_target_type; in dm_table_get_immutable_target_type()
1026 struct dm_target *dm_table_get_immutable_target(struct dm_table *t) in dm_table_get_immutable_target() argument
1029 if (t->num_targets > 1 || in dm_table_get_immutable_target()
1030 !dm_target_is_immutable(t->targets[0].type)) in dm_table_get_immutable_target()
1033 return t->targets; in dm_table_get_immutable_target()
1036 struct dm_target *dm_table_get_wildcard_target(struct dm_table *t) in dm_table_get_wildcard_target() argument
1038 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_get_wildcard_target()
1039 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_get_wildcard_target()
1048 bool dm_table_request_based(struct dm_table *t) in dm_table_request_based() argument
1050 return __table_type_request_based(dm_table_get_type(t)); in dm_table_request_based()
1053 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md) in dm_table_alloc_md_mempools() argument
1055 enum dm_queue_mode type = dm_table_get_type(t); in dm_table_alloc_md_mempools()
1079 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_alloc_md_mempools()
1080 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_alloc_md_mempools()
1097 t->mempools = pools; in dm_table_alloc_md_mempools()
1105 static int setup_indexes(struct dm_table *t) in setup_indexes() argument
1112 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1113 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); in setup_indexes()
1114 total += t->counts[i]; in setup_indexes()
1122 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1123 t->index[i] = indexes; in setup_indexes()
1124 indexes += (KEYS_PER_NODE * t->counts[i]); in setup_indexes()
1125 setup_btree_index(i, t); in setup_indexes()
1134 static int dm_table_build_index(struct dm_table *t) in dm_table_build_index() argument
1140 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); in dm_table_build_index()
1141 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); in dm_table_build_index()
1144 t->counts[t->depth - 1] = leaf_nodes; in dm_table_build_index()
1145 t->index[t->depth - 1] = t->highs; in dm_table_build_index()
1147 if (t->depth >= 2) in dm_table_build_index()
1148 r = setup_indexes(t); in dm_table_build_index()
1178 struct dm_table *t; in dm_keyslot_evict() local
1181 t = dm_get_live_table(md, &srcu_idx); in dm_keyslot_evict()
1182 if (!t) in dm_keyslot_evict()
1185 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_keyslot_evict()
1186 struct dm_target *ti = dm_table_get_target(t, i); in dm_keyslot_evict()
1280 struct dm_table *t; in dm_exec_wrappedkey_op() local
1286 t = dm_get_live_table(md, &srcu_idx); in dm_exec_wrappedkey_op()
1287 if (!t) in dm_exec_wrappedkey_op()
1299 for (i = 0; i < t->num_targets; i++) { in dm_exec_wrappedkey_op()
1300 ti = dm_table_get_target(t, i); in dm_exec_wrappedkey_op()
1394 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1396 dm_destroy_crypto_profile(t->crypto_profile); in dm_table_destroy_crypto_profile()
1397 t->crypto_profile = NULL; in dm_table_destroy_crypto_profile()
1409 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1419 dmcp->md = t->md; in dm_table_construct_crypto_profile()
1429 for (i = 0; i < t->num_targets; i++) { in dm_table_construct_crypto_profile()
1430 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_construct_crypto_profile()
1450 if (t->md->queue && in dm_table_construct_crypto_profile()
1452 t->md->queue->crypto_profile)) { in dm_table_construct_crypto_profile()
1479 t->crypto_profile = profile; in dm_table_construct_crypto_profile()
1485 struct dm_table *t) in dm_update_crypto_profile() argument
1487 if (!t->crypto_profile) in dm_update_crypto_profile()
1492 blk_crypto_register(t->crypto_profile, q); in dm_update_crypto_profile()
1495 t->crypto_profile); in dm_update_crypto_profile()
1496 dm_destroy_crypto_profile(t->crypto_profile); in dm_update_crypto_profile()
1498 t->crypto_profile = NULL; in dm_update_crypto_profile()
1503 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1512 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1517 struct dm_table *t) in dm_update_crypto_profile() argument
1527 int dm_table_complete(struct dm_table *t) in dm_table_complete() argument
1531 r = dm_table_determine_type(t); in dm_table_complete()
1537 r = dm_table_build_index(t); in dm_table_complete()
1543 r = dm_table_construct_crypto_profile(t); in dm_table_complete()
1549 r = dm_table_alloc_md_mempools(t, t->md); in dm_table_complete()
1557 void dm_table_event_callback(struct dm_table *t, in dm_table_event_callback() argument
1561 t->event_fn = fn; in dm_table_event_callback()
1562 t->event_context = context; in dm_table_event_callback()
1566 void dm_table_event(struct dm_table *t) in dm_table_event() argument
1569 if (t->event_fn) in dm_table_event()
1570 t->event_fn(t->event_context); in dm_table_event()
1575 inline sector_t dm_table_get_size(struct dm_table *t) in dm_table_get_size() argument
1577 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; in dm_table_get_size()
1587 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) in dm_table_find_target() argument
1592 if (unlikely(sector >= dm_table_get_size(t))) in dm_table_find_target()
1595 for (l = 0; l < t->depth; l++) { in dm_table_find_target()
1597 node = get_node(t, l, n); in dm_table_find_target()
1604 return &t->targets[(KEYS_PER_NODE * n) + k]; in dm_table_find_target()
1630 static bool dm_table_any_dev_attr(struct dm_table *t, in dm_table_any_dev_attr() argument
1633 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_any_dev_attr()
1634 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_any_dev_attr()
1660 bool dm_table_has_no_data_devices(struct dm_table *t) in dm_table_has_no_data_devices() argument
1662 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_has_no_data_devices()
1663 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_has_no_data_devices()
1677 bool dm_table_is_wildcard(struct dm_table *t) in dm_table_is_wildcard() argument
1679 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_is_wildcard()
1680 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_is_wildcard()
1710 static bool dm_table_supports_zoned(struct dm_table *t, bool zoned) in dm_table_supports_zoned() argument
1712 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_zoned()
1713 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_zoned()
1756 static int validate_hardware_zoned(struct dm_table *t, bool zoned, in validate_hardware_zoned() argument
1762 if (!dm_table_supports_zoned(t, zoned)) { in validate_hardware_zoned()
1764 dm_device_name(t->md)); in validate_hardware_zoned()
1772 if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) { in validate_hardware_zoned()
1774 dm_device_name(t->md)); in validate_hardware_zoned()
1784 int dm_calculate_queue_limits(struct dm_table *t, in dm_calculate_queue_limits() argument
1793 t->integrity_supported = true; in dm_calculate_queue_limits()
1794 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_calculate_queue_limits()
1795 struct dm_target *ti = dm_table_get_target(t, i); in dm_calculate_queue_limits()
1798 t->integrity_supported = false; in dm_calculate_queue_limits()
1801 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_calculate_queue_limits()
1802 struct dm_target *ti = dm_table_get_target(t, i); in dm_calculate_queue_limits()
1848 dm_device_name(t->md), in dm_calculate_queue_limits()
1852 if (t->integrity_supported || in dm_calculate_queue_limits()
1857 dm_device_name(t->md), in dm_calculate_queue_limits()
1860 t->integrity_supported = false; in dm_calculate_queue_limits()
1880 if (validate_hardware_zoned(t, zoned, zone_sectors)) in dm_calculate_queue_limits()
1883 return validate_hardware_logical_block_alignment(t, limits); in dm_calculate_queue_limits()
1890 static bool dm_table_supports_flush(struct dm_table *t) in dm_table_supports_flush() argument
1892 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_flush()
1893 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_flush()
1928 static bool dm_table_supports_write_zeroes(struct dm_table *t) in dm_table_supports_write_zeroes() argument
1930 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_write_zeroes()
1931 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_write_zeroes()
1944 static bool dm_table_supports_nowait(struct dm_table *t) in dm_table_supports_nowait() argument
1946 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_nowait()
1947 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_nowait()
1962 static bool dm_table_supports_discards(struct dm_table *t) in dm_table_supports_discards() argument
1964 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_discards()
1965 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_discards()
1991 static bool dm_table_supports_secure_erase(struct dm_table *t) in dm_table_supports_secure_erase() argument
1993 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_secure_erase()
1994 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_secure_erase()
2014 static bool dm_table_supports_atomic_writes(struct dm_table *t) in dm_table_supports_atomic_writes() argument
2016 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_atomic_writes()
2017 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_atomic_writes()
2033 bool dm_table_supports_size_change(struct dm_table *t, sector_t old_size, in dm_table_supports_size_change() argument
2036 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && dm_has_zone_plugs(t->md) && in dm_table_supports_size_change()
2040 dm_device_name(t->md)); in dm_table_supports_size_change()
2050 int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, in dm_table_set_restrictions() argument
2056 if (!dm_table_supports_nowait(t)) in dm_table_set_restrictions()
2063 if (!__table_type_bio_based(t->type)) in dm_table_set_restrictions()
2066 if (!dm_table_supports_discards(t)) { in dm_table_set_restrictions()
2072 if (!dm_table_supports_write_zeroes(t)) { in dm_table_set_restrictions()
2077 if (!dm_table_supports_secure_erase(t)) in dm_table_set_restrictions()
2080 if (dm_table_supports_flush(t)) in dm_table_set_restrictions()
2083 if (dm_table_supports_dax(t, device_not_dax_capable)) in dm_table_set_restrictions()
2091 r = dm_set_zones_restrictions(t, q, limits); in dm_table_set_restrictions()
2094 } else if (dm_has_zone_plugs(t->md)) { in dm_table_set_restrictions()
2097 dm_device_name(t->md)); in dm_table_set_restrictions()
2102 if (dm_table_supports_atomic_writes(t)) in dm_table_set_restrictions()
2116 r = dm_revalidate_zones(t, q); in dm_table_set_restrictions()
2124 dm_finalize_zone_settings(t, limits); in dm_table_set_restrictions()
2126 if (dm_table_supports_dax(t, device_not_dax_synchronous_capable)) in dm_table_set_restrictions()
2127 set_dax_synchronous(t->md->dax_dev); in dm_table_set_restrictions()
2129 if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL)) in dm_table_set_restrictions()
2130 dax_write_cache(t->md->dax_dev, true); in dm_table_set_restrictions()
2132 dm_update_crypto_profile(q, t); in dm_table_set_restrictions()
2136 struct list_head *dm_table_get_devices(struct dm_table *t) in dm_table_get_devices() argument
2138 return &t->devices; in dm_table_get_devices()
2141 blk_mode_t dm_table_get_mode(struct dm_table *t) in dm_table_get_mode() argument
2143 return t->mode; in dm_table_get_mode()
2153 static void suspend_targets(struct dm_table *t, enum suspend_mode mode) in suspend_targets() argument
2155 lockdep_assert_held(&t->md->suspend_lock); in suspend_targets()
2157 for (unsigned int i = 0; i < t->num_targets; i++) { in suspend_targets()
2158 struct dm_target *ti = dm_table_get_target(t, i); in suspend_targets()
2177 void dm_table_presuspend_targets(struct dm_table *t) in dm_table_presuspend_targets() argument
2179 if (!t) in dm_table_presuspend_targets()
2182 suspend_targets(t, PRESUSPEND); in dm_table_presuspend_targets()
2185 void dm_table_presuspend_undo_targets(struct dm_table *t) in dm_table_presuspend_undo_targets() argument
2187 if (!t) in dm_table_presuspend_undo_targets()
2190 suspend_targets(t, PRESUSPEND_UNDO); in dm_table_presuspend_undo_targets()
2193 void dm_table_postsuspend_targets(struct dm_table *t) in dm_table_postsuspend_targets() argument
2195 if (!t) in dm_table_postsuspend_targets()
2198 suspend_targets(t, POSTSUSPEND); in dm_table_postsuspend_targets()
2201 int dm_table_resume_targets(struct dm_table *t) in dm_table_resume_targets() argument
2206 lockdep_assert_held(&t->md->suspend_lock); in dm_table_resume_targets()
2208 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2209 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2217 dm_device_name(t->md), ti->type->name, r); in dm_table_resume_targets()
2222 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2223 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2232 struct mapped_device *dm_table_get_md(struct dm_table *t) in dm_table_get_md() argument
2234 return t->md; in dm_table_get_md()
2238 const char *dm_table_device_name(struct dm_table *t) in dm_table_device_name() argument
2240 return dm_device_name(t->md); in dm_table_device_name()
2244 void dm_table_run_md_queue_async(struct dm_table *t) in dm_table_run_md_queue_async() argument
2246 if (!dm_table_request_based(t)) in dm_table_run_md_queue_async()
2249 if (t->md->queue) in dm_table_run_md_queue_async()
2250 blk_mq_run_hw_queues(t->md->queue, true); in dm_table_run_md_queue_async()