1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Internal header file _only_ for device mapper core 4 * 5 * Copyright (C) 2016 Red Hat, Inc. All rights reserved. 6 * 7 * This file is released under the LGPL. 8 */ 9 10 #ifndef DM_CORE_INTERNAL_H 11 #define DM_CORE_INTERNAL_H 12 13 #include <linux/kthread.h> 14 #include <linux/ktime.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-crypto-profile.h> 17 #include <linux/jump_label.h> 18 19 #include <trace/events/block.h> 20 21 #include "dm.h" 22 #include "dm-ima.h" 23 24 #define DM_RESERVED_MAX_IOS 1024 25 #define DM_MAX_TARGETS 1048576 26 #define DM_MAX_TARGET_PARAMS 1024 27 28 struct dm_io; 29 30 struct dm_kobject_holder { 31 struct kobject kobj; 32 struct completion completion; 33 }; 34 35 /* 36 * DM core internal structures used directly by dm.c, dm-rq.c and dm-table.c. 37 * DM targets must _not_ deference a mapped_device or dm_table to directly 38 * access their members! 39 */ 40 41 /* 42 * For mempools pre-allocation at the table loading time. 43 */ 44 struct dm_md_mempools { 45 struct bio_set bs; 46 struct bio_set io_bs; 47 }; 48 49 struct mapped_device { 50 struct mutex suspend_lock; 51 52 struct mutex table_devices_lock; 53 struct list_head table_devices; 54 55 /* 56 * The current mapping (struct dm_table *). 57 * Use dm_get_live_table{_fast} or take suspend_lock for 58 * dereference. 59 */ 60 void __rcu *map; 61 62 unsigned long flags; 63 64 /* Protect queue and type against concurrent access. */ 65 struct mutex type_lock; 66 enum dm_queue_mode type; 67 68 int numa_node_id; 69 struct request_queue *queue; 70 71 atomic_t holders; 72 atomic_t open_count; 73 74 struct dm_target *immutable_target; 75 struct target_type *immutable_target_type; 76 77 char name[16]; 78 struct gendisk *disk; 79 struct dax_device *dax_dev; 80 81 wait_queue_head_t wait; 82 unsigned long __percpu *pending_io; 83 84 /* forced geometry settings */ 85 struct hd_geometry geometry; 86 87 /* 88 * Processing queue (flush) 89 */ 90 struct workqueue_struct *wq; 91 92 /* 93 * A list of ios that arrived while we were suspended. 94 */ 95 struct work_struct work; 96 spinlock_t deferred_lock; 97 struct bio_list deferred; 98 99 /* 100 * requeue work context is needed for cloning one new bio 101 * to represent the dm_io to be requeued, since each 102 * dm_io may point to the original bio from FS. 103 */ 104 struct work_struct requeue_work; 105 struct dm_io *requeue_list; 106 107 void *interface_ptr; 108 109 /* 110 * Event handling. 111 */ 112 wait_queue_head_t eventq; 113 atomic_t event_nr; 114 atomic_t uevent_seq; 115 struct list_head uevent_list; 116 spinlock_t uevent_lock; /* Protect access to uevent_list */ 117 118 /* for blk-mq request-based DM support */ 119 bool init_tio_pdu:1; 120 struct blk_mq_tag_set *tag_set; 121 122 struct dm_stats stats; 123 124 /* the number of internal suspends */ 125 unsigned int internal_suspend_count; 126 127 int swap_bios; 128 struct semaphore swap_bios_semaphore; 129 struct mutex swap_bios_lock; 130 131 /* 132 * io objects are allocated from here. 133 */ 134 struct dm_md_mempools *mempools; 135 136 /* kobject and completion */ 137 struct dm_kobject_holder kobj_holder; 138 139 struct srcu_struct io_barrier; 140 141 #ifdef CONFIG_BLK_DEV_ZONED 142 unsigned int nr_zones; 143 void *zone_revalidate_map; 144 #endif 145 146 #ifdef CONFIG_IMA 147 struct dm_ima_measurements ima; 148 #endif 149 }; 150 151 /* 152 * Bits for the flags field of struct mapped_device. 153 */ 154 #define DMF_BLOCK_IO_FOR_SUSPEND 0 155 #define DMF_SUSPENDED 1 156 #define DMF_FROZEN 2 157 #define DMF_FREEING 3 158 #define DMF_DELETING 4 159 #define DMF_NOFLUSH_SUSPENDING 5 160 #define DMF_DEFERRED_REMOVE 6 161 #define DMF_SUSPENDED_INTERNALLY 7 162 #define DMF_POST_SUSPENDING 8 163 #define DMF_EMULATE_ZONE_APPEND 9 164 165 void disable_discard(struct mapped_device *md); 166 void disable_write_zeroes(struct mapped_device *md); 167 168 static inline sector_t dm_get_size(struct mapped_device *md) 169 { 170 return get_capacity(md->disk); 171 } 172 173 static inline struct dm_stats *dm_get_stats(struct mapped_device *md) 174 { 175 return &md->stats; 176 } 177 178 DECLARE_STATIC_KEY_FALSE(stats_enabled); 179 DECLARE_STATIC_KEY_FALSE(swap_bios_enabled); 180 DECLARE_STATIC_KEY_FALSE(zoned_enabled); 181 182 static inline bool dm_emulate_zone_append(struct mapped_device *md) 183 { 184 if (blk_queue_is_zoned(md->queue)) 185 return test_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); 186 return false; 187 } 188 189 #define DM_TABLE_MAX_DEPTH 16 190 191 struct dm_table { 192 struct mapped_device *md; 193 enum dm_queue_mode type; 194 195 /* btree table */ 196 unsigned int depth; 197 unsigned int counts[DM_TABLE_MAX_DEPTH]; /* in nodes */ 198 sector_t *index[DM_TABLE_MAX_DEPTH]; 199 200 unsigned int num_targets; 201 unsigned int num_allocated; 202 sector_t *highs; 203 struct dm_target *targets; 204 205 struct target_type *immutable_target_type; 206 207 bool integrity_supported:1; 208 bool singleton:1; 209 /* set if all the targets in the table have "flush_bypasses_map" set */ 210 bool flush_bypasses_map:1; 211 212 /* 213 * Indicates the rw permissions for the new logical device. This 214 * should be a combination of BLK_OPEN_READ and BLK_OPEN_WRITE. 215 */ 216 blk_mode_t mode; 217 218 /* a list of devices used by this table */ 219 struct list_head devices; 220 struct rw_semaphore devices_lock; 221 222 /* events get handed up using this callback */ 223 void (*event_fn)(void *data); 224 void *event_context; 225 226 struct dm_md_mempools *mempools; 227 228 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 229 struct blk_crypto_profile *crypto_profile; 230 #endif 231 }; 232 233 static inline struct dm_target *dm_table_get_target(struct dm_table *t, 234 unsigned int index) 235 { 236 BUG_ON(index >= t->num_targets); 237 return t->targets + index; 238 } 239 240 /* 241 * One of these is allocated per clone bio. 242 */ 243 #define DM_TIO_MAGIC 28714 244 struct dm_target_io { 245 unsigned short magic; 246 blk_short_t flags; 247 unsigned int target_bio_nr; 248 struct dm_io *io; 249 struct dm_target *ti; 250 unsigned int *len_ptr; 251 sector_t old_sector; 252 struct bio clone; 253 }; 254 #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone)) 255 #define DM_IO_BIO_OFFSET \ 256 (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio)) 257 258 /* 259 * dm_target_io flags 260 */ 261 enum { 262 DM_TIO_INSIDE_DM_IO, 263 DM_TIO_IS_DUPLICATE_BIO 264 }; 265 266 static inline bool dm_tio_flagged(struct dm_target_io *tio, unsigned int bit) 267 { 268 return (tio->flags & (1U << bit)) != 0; 269 } 270 271 static inline void dm_tio_set_flag(struct dm_target_io *tio, unsigned int bit) 272 { 273 tio->flags |= (1U << bit); 274 } 275 276 static inline bool dm_tio_is_normal(struct dm_target_io *tio) 277 { 278 return (dm_tio_flagged(tio, DM_TIO_INSIDE_DM_IO) && 279 !dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); 280 } 281 282 /* 283 * One of these is allocated per original bio. 284 * It contains the first clone used for that original. 285 */ 286 #define DM_IO_MAGIC 19577 287 struct dm_io { 288 unsigned short magic; 289 blk_short_t flags; 290 spinlock_t lock; 291 unsigned long start_time; 292 void *data; 293 struct dm_io *next; 294 struct dm_stats_aux stats_aux; 295 blk_status_t status; 296 atomic_t io_count; 297 struct mapped_device *md; 298 299 /* The three fields represent mapped part of original bio */ 300 struct bio *orig_bio; 301 unsigned int sector_offset; /* offset to end of orig_bio */ 302 unsigned int sectors; 303 304 /* last member of dm_target_io is 'struct bio' */ 305 struct dm_target_io tio; 306 }; 307 308 /* 309 * dm_io flags 310 */ 311 enum { 312 DM_IO_ACCOUNTED, 313 DM_IO_WAS_SPLIT, 314 DM_IO_BLK_STAT 315 }; 316 317 static inline bool dm_io_flagged(struct dm_io *io, unsigned int bit) 318 { 319 return (io->flags & (1U << bit)) != 0; 320 } 321 322 static inline void dm_io_set_flag(struct dm_io *io, unsigned int bit) 323 { 324 io->flags |= (1U << bit); 325 } 326 327 void dm_io_rewind(struct dm_io *io, struct bio_set *bs); 328 329 static inline struct completion *dm_get_completion_from_kobject(struct kobject *kobj) 330 { 331 return &container_of(kobj, struct dm_kobject_holder, kobj)->completion; 332 } 333 334 unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max); 335 336 static inline bool dm_message_test_buffer_overflow(char *result, unsigned int maxlen) 337 { 338 return !maxlen || strlen(result) + 1 >= maxlen; 339 } 340 341 extern atomic_t dm_global_event_nr; 342 extern wait_queue_head_t dm_global_eventq; 343 void dm_issue_global_event(void); 344 345 #endif 346