1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright 2023 Red Hat 4 */ 5 6 #ifndef VDO_H 7 #define VDO_H 8 9 #include <linux/atomic.h> 10 #include <linux/blk_types.h> 11 #include <linux/completion.h> 12 #include <linux/dm-kcopyd.h> 13 #include <linux/list.h> 14 #include <linux/spinlock.h> 15 16 #include "admin-state.h" 17 #include "encodings.h" 18 #include "funnel-workqueue.h" 19 #include "packer.h" 20 #include "physical-zone.h" 21 #include "statistics.h" 22 #include "thread-registry.h" 23 #include "types.h" 24 25 enum notifier_state { 26 /* Notifications are allowed but not in progress */ 27 MAY_NOTIFY, 28 /* A notification is in progress */ 29 NOTIFYING, 30 /* Notifications are not allowed */ 31 MAY_NOT_NOTIFY, 32 /* A notification has completed */ 33 NOTIFIED, 34 }; 35 36 /** 37 * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone 38 * read-only. 39 * @listener: The object to notify. 40 * @parent: The completion to notify in order to acknowledge the notification. 41 */ 42 typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent); 43 44 /* 45 * An object to be notified when the VDO enters read-only mode 46 */ 47 struct read_only_listener { 48 /* The listener */ 49 void *listener; 50 /* The method to call to notify the listener */ 51 vdo_read_only_notification_fn notify; 52 /* A pointer to the next listener */ 53 struct read_only_listener *next; 54 }; 55 56 struct vdo_thread { 57 struct vdo *vdo; 58 thread_id_t thread_id; 59 struct vdo_work_queue *queue; 60 /* 61 * Each thread maintains its own notion of whether the VDO is read-only so that the 62 * read-only state can be checked from any base thread without worrying about 63 * synchronization or thread safety. This does mean that knowledge of the VDO going 64 * read-only does not occur simultaneously across the VDO's threads, but that does not seem 65 * to cause any problems. 66 */ 67 bool is_read_only; 68 /* 69 * A list of objects waiting to be notified on this thread that the VDO has entered 70 * read-only mode. 71 */ 72 struct read_only_listener *listeners; 73 struct registered_thread allocating_thread; 74 }; 75 76 /* Keep struct bio statistics atomically */ 77 struct atomic_bio_stats { 78 atomic64_t read; /* Number of not REQ_WRITE bios */ 79 atomic64_t write; /* Number of REQ_WRITE bios */ 80 atomic64_t discard; /* Number of REQ_DISCARD bios */ 81 atomic64_t flush; /* Number of REQ_FLUSH bios */ 82 atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */ 83 atomic64_t fua; /* Number of REQ_FUA bios */ 84 }; 85 86 /* Counters are atomic since updates can arrive concurrently from arbitrary threads. */ 87 struct atomic_statistics { 88 atomic64_t bios_submitted; 89 atomic64_t bios_completed; 90 atomic64_t flush_out; 91 atomic64_t invalid_advice_pbn_count; 92 atomic64_t no_space_error_count; 93 atomic64_t read_only_error_count; 94 struct atomic_bio_stats bios_in; 95 struct atomic_bio_stats bios_in_partial; 96 struct atomic_bio_stats bios_out; 97 struct atomic_bio_stats bios_out_completed; 98 struct atomic_bio_stats bios_acknowledged; 99 struct atomic_bio_stats bios_acknowledged_partial; 100 struct atomic_bio_stats bios_meta; 101 struct atomic_bio_stats bios_meta_completed; 102 struct atomic_bio_stats bios_journal; 103 struct atomic_bio_stats bios_journal_completed; 104 struct atomic_bio_stats bios_page_cache; 105 struct atomic_bio_stats bios_page_cache_completed; 106 }; 107 108 struct read_only_notifier { 109 /* The completion for entering read-only mode */ 110 struct vdo_completion completion; 111 /* A completion waiting for notifications to be drained or enabled */ 112 struct vdo_completion *waiter; 113 /* Lock to protect the next two fields */ 114 spinlock_t lock; 115 /* The code of the error which put the VDO into read-only mode */ 116 int read_only_error; 117 /* The current state of the notifier (values described above) */ 118 enum notifier_state state; 119 }; 120 121 /* 122 * The thread ID returned when the current thread is not a vdo thread, or can not be determined 123 * (usually due to being at interrupt context). 124 */ 125 #define VDO_INVALID_THREAD_ID ((thread_id_t) -1) 126 127 struct thread_config { 128 zone_count_t logical_zone_count; 129 zone_count_t physical_zone_count; 130 zone_count_t hash_zone_count; 131 thread_count_t bio_thread_count; 132 thread_count_t thread_count; 133 thread_id_t admin_thread; 134 thread_id_t journal_thread; 135 thread_id_t packer_thread; 136 thread_id_t dedupe_thread; 137 thread_id_t bio_ack_thread; 138 thread_id_t cpu_thread; 139 thread_id_t *logical_threads; 140 thread_id_t *physical_threads; 141 thread_id_t *hash_zone_threads; 142 thread_id_t *bio_threads; 143 }; 144 145 struct thread_count_config; 146 147 struct vdo_geometry_block { 148 /* The vio for reading and writing the geometry block to disk */ 149 struct vio vio; 150 /* A buffer to hold the geometry block */ 151 u8 *buffer; 152 }; 153 154 struct vdo_super_block { 155 /* The vio for reading and writing the super block to disk */ 156 struct vio vio; 157 /* A buffer to hold the super block */ 158 u8 *buffer; 159 /* Whether this super block may not be written */ 160 bool unwritable; 161 }; 162 163 struct data_vio_pool; 164 165 struct vdo_administrator { 166 struct vdo_completion completion; 167 struct admin_state state; 168 atomic_t busy; 169 u32 phase; 170 struct completion callback_sync; 171 }; 172 173 struct vdo { 174 char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN]; 175 struct vdo_thread *threads; 176 vdo_action_fn action; 177 struct vdo_completion *completion; 178 struct vio_tracer *vio_tracer; 179 180 /* The atomic version of the state of this vdo */ 181 atomic_t state; 182 /* The full state of all components */ 183 struct vdo_component_states states; 184 /* 185 * A counter value to attach to thread names and log messages to identify the individual 186 * device. 187 */ 188 unsigned int instance; 189 /* The read-only notifier */ 190 struct read_only_notifier read_only_notifier; 191 /* The load-time configuration of this vdo */ 192 struct device_config *device_config; 193 /* The thread mapping */ 194 struct thread_config thread_config; 195 196 /* The geometry block */ 197 struct vdo_geometry_block geometry_block; 198 199 /* The super block */ 200 struct vdo_super_block super_block; 201 202 /* The partitioning of the underlying storage */ 203 struct layout layout; 204 struct layout next_layout; 205 struct dm_kcopyd_client *partition_copier; 206 207 /* The block map */ 208 struct block_map *block_map; 209 210 /* The journal for block map recovery */ 211 struct recovery_journal *recovery_journal; 212 213 /* The slab depot */ 214 struct slab_depot *depot; 215 216 /* The compressed-block packer */ 217 struct packer *packer; 218 /* Whether incoming data should be compressed */ 219 bool compressing; 220 221 /* The handler for flush requests */ 222 struct flusher *flusher; 223 224 /* The state the vdo was in when loaded (primarily for unit tests) */ 225 enum vdo_state load_state; 226 227 /* The logical zones of this vdo */ 228 struct logical_zones *logical_zones; 229 230 /* The physical zones of this vdo */ 231 struct physical_zones *physical_zones; 232 233 /* The hash lock zones of this vdo */ 234 struct hash_zones *hash_zones; 235 236 /* Bio submission manager used for sending bios to the storage device. */ 237 struct io_submitter *io_submitter; 238 239 /* The pool of data_vios for servicing incoming bios */ 240 struct data_vio_pool *data_vio_pool; 241 242 /* The manager for administrative operations */ 243 struct vdo_administrator admin; 244 245 /* Flags controlling administrative operations */ 246 const struct admin_state_code *suspend_type; 247 bool allocations_allowed; 248 bool dump_on_shutdown; 249 bool needs_formatting; 250 atomic_t processing_message; 251 252 /* 253 * Statistics 254 * Atomic stats counters 255 */ 256 struct atomic_statistics stats; 257 /* Used to gather statistics without allocating memory */ 258 struct vdo_statistics stats_buffer; 259 /* Protects the stats_buffer */ 260 struct mutex stats_mutex; 261 262 /* A list of all device_configs referencing this vdo */ 263 struct list_head device_config_list; 264 265 /* This VDO's list entry for the device registry */ 266 struct list_head registration; 267 268 /* Underlying block device info. */ 269 u64 starting_sector_offset; 270 struct volume_geometry geometry; 271 272 /* N blobs of context data for LZ4 code, one per CPU thread. */ 273 char **compression_context; 274 }; 275 276 /** 277 * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue 278 * for acknowledging received and processed bios. 279 * @vdo: The vdo. 280 * 281 * Note that this directly controls the handling of write operations, but the compile-time flag 282 * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations. 283 * 284 * Return: Whether a bio-acknowledgement work queue is in use. 285 */ 286 static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo) 287 { 288 return vdo->device_config->thread_counts.bio_ack_threads > 0; 289 } 290 291 /** 292 * typedef vdo_filter_fn - Method type for vdo matching methods. 293 * @vdo: The vdo to match. 294 * @context: A parameter for the filter to use. 295 * 296 * Return: True if the vdo matches the filter criteria, false if it doesn't. 297 */ 298 typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context); 299 300 void vdo_initialize_device_registry_once(void); 301 struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context); 302 303 int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, 304 const struct vdo_work_queue_type *type, 305 unsigned int queue_count, void *contexts[]); 306 307 static inline int __must_check vdo_make_default_thread(struct vdo *vdo, 308 thread_id_t thread_id) 309 { 310 return vdo_make_thread(vdo, thread_id, NULL, 1, NULL); 311 } 312 313 int __must_check vdo_make(unsigned int instance, struct device_config *config, 314 char **reason, struct vdo **vdo_ptr); 315 316 void vdo_destroy(struct vdo *vdo); 317 318 int __must_check vdo_format_components(struct vdo *vdo); 319 320 void vdo_format_super_block(struct vdo *vdo, struct vdo_completion *parent); 321 322 void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent); 323 324 struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo); 325 326 const char * __must_check vdo_get_device_name(const struct dm_target *target); 327 328 int __must_check vdo_synchronous_flush(struct vdo *vdo); 329 330 const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo); 331 332 bool vdo_set_compressing(struct vdo *vdo, bool enable); 333 334 bool vdo_get_compressing(struct vdo *vdo); 335 336 void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats); 337 338 thread_id_t vdo_get_callback_thread_id(void); 339 340 enum vdo_state __must_check vdo_get_state(const struct vdo *vdo); 341 342 void vdo_set_state(struct vdo *vdo, enum vdo_state state); 343 344 int vdo_clear_layout(struct vdo *vdo); 345 void vdo_save_geometry_block(struct vdo *vdo, struct vdo_completion *parent); 346 void vdo_save_super_block(struct vdo *vdo, struct vdo_completion *parent); 347 348 void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent); 349 350 int vdo_register_read_only_listener(struct vdo *vdo, void *listener, 351 vdo_read_only_notification_fn notification, 352 thread_id_t thread_id); 353 354 int vdo_enable_read_only_entry(struct vdo *vdo); 355 356 void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent); 357 358 void vdo_allow_read_only_mode_entry(struct vdo_completion *parent); 359 360 void vdo_enter_read_only_mode(struct vdo *vdo, int error_code); 361 362 bool __must_check vdo_is_read_only(struct vdo *vdo); 363 364 bool __must_check vdo_in_read_only_mode(const struct vdo *vdo); 365 366 bool __must_check vdo_in_recovery_mode(const struct vdo *vdo); 367 368 void vdo_enter_recovery_mode(struct vdo *vdo); 369 370 void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name); 371 372 void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, 373 const char *name); 374 375 void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone, 376 const char *name); 377 378 int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, 379 struct physical_zone **zone_ptr); 380 381 void vdo_dump_status(const struct vdo *vdo); 382 383 #endif /* VDO_H */ 384