106e932feSMatthew Sakai /* SPDX-License-Identifier: GPL-2.0-only */
206e932feSMatthew Sakai /*
306e932feSMatthew Sakai * Copyright 2023 Red Hat
406e932feSMatthew Sakai */
506e932feSMatthew Sakai
606e932feSMatthew Sakai #ifndef VDO_H
706e932feSMatthew Sakai #define VDO_H
806e932feSMatthew Sakai
906e932feSMatthew Sakai #include <linux/atomic.h>
1006e932feSMatthew Sakai #include <linux/blk_types.h>
1106e932feSMatthew Sakai #include <linux/completion.h>
1206e932feSMatthew Sakai #include <linux/dm-kcopyd.h>
1306e932feSMatthew Sakai #include <linux/list.h>
1406e932feSMatthew Sakai #include <linux/spinlock.h>
1506e932feSMatthew Sakai
1606e932feSMatthew Sakai #include "admin-state.h"
1706e932feSMatthew Sakai #include "encodings.h"
1806e932feSMatthew Sakai #include "funnel-workqueue.h"
1906e932feSMatthew Sakai #include "packer.h"
2006e932feSMatthew Sakai #include "physical-zone.h"
2106e932feSMatthew Sakai #include "statistics.h"
2206e932feSMatthew Sakai #include "thread-registry.h"
2306e932feSMatthew Sakai #include "types.h"
2406e932feSMatthew Sakai
2506e932feSMatthew Sakai enum notifier_state {
26*571eff39SMike Snitzer /* Notifications are allowed but not in progress */
2706e932feSMatthew Sakai MAY_NOTIFY,
28*571eff39SMike Snitzer /* A notification is in progress */
2906e932feSMatthew Sakai NOTIFYING,
30*571eff39SMike Snitzer /* Notifications are not allowed */
3106e932feSMatthew Sakai MAY_NOT_NOTIFY,
32*571eff39SMike Snitzer /* A notification has completed */
3306e932feSMatthew Sakai NOTIFIED,
3406e932feSMatthew Sakai };
3506e932feSMatthew Sakai
3606e932feSMatthew Sakai /**
3706e932feSMatthew Sakai * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone
3806e932feSMatthew Sakai * read-only.
3906e932feSMatthew Sakai * @listener: The object to notify.
4006e932feSMatthew Sakai * @parent: The completion to notify in order to acknowledge the notification.
4106e932feSMatthew Sakai */
4206e932feSMatthew Sakai typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent);
4306e932feSMatthew Sakai
4406e932feSMatthew Sakai /*
4506e932feSMatthew Sakai * An object to be notified when the VDO enters read-only mode
4606e932feSMatthew Sakai */
4706e932feSMatthew Sakai struct read_only_listener {
4806e932feSMatthew Sakai /* The listener */
4906e932feSMatthew Sakai void *listener;
5006e932feSMatthew Sakai /* The method to call to notify the listener */
5106e932feSMatthew Sakai vdo_read_only_notification_fn notify;
5206e932feSMatthew Sakai /* A pointer to the next listener */
5306e932feSMatthew Sakai struct read_only_listener *next;
5406e932feSMatthew Sakai };
5506e932feSMatthew Sakai
5606e932feSMatthew Sakai struct vdo_thread {
5706e932feSMatthew Sakai struct vdo *vdo;
5806e932feSMatthew Sakai thread_id_t thread_id;
5906e932feSMatthew Sakai struct vdo_work_queue *queue;
6006e932feSMatthew Sakai /*
6106e932feSMatthew Sakai * Each thread maintains its own notion of whether the VDO is read-only so that the
6206e932feSMatthew Sakai * read-only state can be checked from any base thread without worrying about
6306e932feSMatthew Sakai * synchronization or thread safety. This does mean that knowledge of the VDO going
6406e932feSMatthew Sakai * read-only does not occur simultaneously across the VDO's threads, but that does not seem
6506e932feSMatthew Sakai * to cause any problems.
6606e932feSMatthew Sakai */
6706e932feSMatthew Sakai bool is_read_only;
6806e932feSMatthew Sakai /*
6906e932feSMatthew Sakai * A list of objects waiting to be notified on this thread that the VDO has entered
7006e932feSMatthew Sakai * read-only mode.
7106e932feSMatthew Sakai */
7206e932feSMatthew Sakai struct read_only_listener *listeners;
7306e932feSMatthew Sakai struct registered_thread allocating_thread;
7406e932feSMatthew Sakai };
7506e932feSMatthew Sakai
7606e932feSMatthew Sakai /* Keep struct bio statistics atomically */
7706e932feSMatthew Sakai struct atomic_bio_stats {
7806e932feSMatthew Sakai atomic64_t read; /* Number of not REQ_WRITE bios */
7906e932feSMatthew Sakai atomic64_t write; /* Number of REQ_WRITE bios */
8006e932feSMatthew Sakai atomic64_t discard; /* Number of REQ_DISCARD bios */
8106e932feSMatthew Sakai atomic64_t flush; /* Number of REQ_FLUSH bios */
8206e932feSMatthew Sakai atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */
8306e932feSMatthew Sakai atomic64_t fua; /* Number of REQ_FUA bios */
8406e932feSMatthew Sakai };
8506e932feSMatthew Sakai
8606e932feSMatthew Sakai /* Counters are atomic since updates can arrive concurrently from arbitrary threads. */
8706e932feSMatthew Sakai struct atomic_statistics {
8806e932feSMatthew Sakai atomic64_t bios_submitted;
8906e932feSMatthew Sakai atomic64_t bios_completed;
9006e932feSMatthew Sakai atomic64_t flush_out;
9106e932feSMatthew Sakai atomic64_t invalid_advice_pbn_count;
9206e932feSMatthew Sakai atomic64_t no_space_error_count;
9306e932feSMatthew Sakai atomic64_t read_only_error_count;
9406e932feSMatthew Sakai struct atomic_bio_stats bios_in;
9506e932feSMatthew Sakai struct atomic_bio_stats bios_in_partial;
9606e932feSMatthew Sakai struct atomic_bio_stats bios_out;
9706e932feSMatthew Sakai struct atomic_bio_stats bios_out_completed;
9806e932feSMatthew Sakai struct atomic_bio_stats bios_acknowledged;
9906e932feSMatthew Sakai struct atomic_bio_stats bios_acknowledged_partial;
10006e932feSMatthew Sakai struct atomic_bio_stats bios_meta;
10106e932feSMatthew Sakai struct atomic_bio_stats bios_meta_completed;
10206e932feSMatthew Sakai struct atomic_bio_stats bios_journal;
10306e932feSMatthew Sakai struct atomic_bio_stats bios_journal_completed;
10406e932feSMatthew Sakai struct atomic_bio_stats bios_page_cache;
10506e932feSMatthew Sakai struct atomic_bio_stats bios_page_cache_completed;
10606e932feSMatthew Sakai };
10706e932feSMatthew Sakai
10806e932feSMatthew Sakai struct read_only_notifier {
10906e932feSMatthew Sakai /* The completion for entering read-only mode */
11006e932feSMatthew Sakai struct vdo_completion completion;
11106e932feSMatthew Sakai /* A completion waiting for notifications to be drained or enabled */
11206e932feSMatthew Sakai struct vdo_completion *waiter;
11306e932feSMatthew Sakai /* Lock to protect the next two fields */
11406e932feSMatthew Sakai spinlock_t lock;
11506e932feSMatthew Sakai /* The code of the error which put the VDO into read-only mode */
11606e932feSMatthew Sakai int read_only_error;
11706e932feSMatthew Sakai /* The current state of the notifier (values described above) */
11806e932feSMatthew Sakai enum notifier_state state;
11906e932feSMatthew Sakai };
12006e932feSMatthew Sakai
12106e932feSMatthew Sakai /*
12206e932feSMatthew Sakai * The thread ID returned when the current thread is not a vdo thread, or can not be determined
12306e932feSMatthew Sakai * (usually due to being at interrupt context).
12406e932feSMatthew Sakai */
12506e932feSMatthew Sakai #define VDO_INVALID_THREAD_ID ((thread_id_t) -1)
12606e932feSMatthew Sakai
12706e932feSMatthew Sakai struct thread_config {
12806e932feSMatthew Sakai zone_count_t logical_zone_count;
12906e932feSMatthew Sakai zone_count_t physical_zone_count;
13006e932feSMatthew Sakai zone_count_t hash_zone_count;
13106e932feSMatthew Sakai thread_count_t bio_thread_count;
13206e932feSMatthew Sakai thread_count_t thread_count;
13306e932feSMatthew Sakai thread_id_t admin_thread;
13406e932feSMatthew Sakai thread_id_t journal_thread;
13506e932feSMatthew Sakai thread_id_t packer_thread;
13606e932feSMatthew Sakai thread_id_t dedupe_thread;
13706e932feSMatthew Sakai thread_id_t bio_ack_thread;
13806e932feSMatthew Sakai thread_id_t cpu_thread;
13906e932feSMatthew Sakai thread_id_t *logical_threads;
14006e932feSMatthew Sakai thread_id_t *physical_threads;
14106e932feSMatthew Sakai thread_id_t *hash_zone_threads;
14206e932feSMatthew Sakai thread_id_t *bio_threads;
14306e932feSMatthew Sakai };
14406e932feSMatthew Sakai
14506e932feSMatthew Sakai struct thread_count_config;
14606e932feSMatthew Sakai
14706e932feSMatthew Sakai struct vdo_super_block {
14806e932feSMatthew Sakai /* The vio for reading and writing the super block to disk */
14906e932feSMatthew Sakai struct vio vio;
15006e932feSMatthew Sakai /* A buffer to hold the super block */
15106e932feSMatthew Sakai u8 *buffer;
15206e932feSMatthew Sakai /* Whether this super block may not be written */
15306e932feSMatthew Sakai bool unwritable;
15406e932feSMatthew Sakai };
15506e932feSMatthew Sakai
15606e932feSMatthew Sakai struct data_vio_pool;
15706e932feSMatthew Sakai
15806e932feSMatthew Sakai struct vdo_administrator {
15906e932feSMatthew Sakai struct vdo_completion completion;
16006e932feSMatthew Sakai struct admin_state state;
16106e932feSMatthew Sakai atomic_t busy;
16206e932feSMatthew Sakai u32 phase;
16306e932feSMatthew Sakai struct completion callback_sync;
16406e932feSMatthew Sakai };
16506e932feSMatthew Sakai
16606e932feSMatthew Sakai struct vdo {
16706e932feSMatthew Sakai char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN];
16806e932feSMatthew Sakai struct vdo_thread *threads;
16906e932feSMatthew Sakai vdo_action_fn action;
17006e932feSMatthew Sakai struct vdo_completion *completion;
17106e932feSMatthew Sakai struct vio_tracer *vio_tracer;
17206e932feSMatthew Sakai
17306e932feSMatthew Sakai /* The atomic version of the state of this vdo */
17406e932feSMatthew Sakai atomic_t state;
17506e932feSMatthew Sakai /* The full state of all components */
17606e932feSMatthew Sakai struct vdo_component_states states;
17706e932feSMatthew Sakai /*
17806e932feSMatthew Sakai * A counter value to attach to thread names and log messages to identify the individual
17906e932feSMatthew Sakai * device.
18006e932feSMatthew Sakai */
18106e932feSMatthew Sakai unsigned int instance;
18206e932feSMatthew Sakai /* The read-only notifier */
18306e932feSMatthew Sakai struct read_only_notifier read_only_notifier;
18406e932feSMatthew Sakai /* The load-time configuration of this vdo */
18506e932feSMatthew Sakai struct device_config *device_config;
18606e932feSMatthew Sakai /* The thread mapping */
18706e932feSMatthew Sakai struct thread_config thread_config;
18806e932feSMatthew Sakai
18906e932feSMatthew Sakai /* The super block */
19006e932feSMatthew Sakai struct vdo_super_block super_block;
19106e932feSMatthew Sakai
19206e932feSMatthew Sakai /* The partitioning of the underlying storage */
19306e932feSMatthew Sakai struct layout layout;
19406e932feSMatthew Sakai struct layout next_layout;
19506e932feSMatthew Sakai struct dm_kcopyd_client *partition_copier;
19606e932feSMatthew Sakai
19706e932feSMatthew Sakai /* The block map */
19806e932feSMatthew Sakai struct block_map *block_map;
19906e932feSMatthew Sakai
20006e932feSMatthew Sakai /* The journal for block map recovery */
20106e932feSMatthew Sakai struct recovery_journal *recovery_journal;
20206e932feSMatthew Sakai
20306e932feSMatthew Sakai /* The slab depot */
20406e932feSMatthew Sakai struct slab_depot *depot;
20506e932feSMatthew Sakai
20606e932feSMatthew Sakai /* The compressed-block packer */
20706e932feSMatthew Sakai struct packer *packer;
20806e932feSMatthew Sakai /* Whether incoming data should be compressed */
20906e932feSMatthew Sakai bool compressing;
21006e932feSMatthew Sakai
21106e932feSMatthew Sakai /* The handler for flush requests */
21206e932feSMatthew Sakai struct flusher *flusher;
21306e932feSMatthew Sakai
21406e932feSMatthew Sakai /* The state the vdo was in when loaded (primarily for unit tests) */
21506e932feSMatthew Sakai enum vdo_state load_state;
21606e932feSMatthew Sakai
21706e932feSMatthew Sakai /* The logical zones of this vdo */
21806e932feSMatthew Sakai struct logical_zones *logical_zones;
21906e932feSMatthew Sakai
22006e932feSMatthew Sakai /* The physical zones of this vdo */
22106e932feSMatthew Sakai struct physical_zones *physical_zones;
22206e932feSMatthew Sakai
22306e932feSMatthew Sakai /* The hash lock zones of this vdo */
22406e932feSMatthew Sakai struct hash_zones *hash_zones;
22506e932feSMatthew Sakai
22606e932feSMatthew Sakai /* Bio submission manager used for sending bios to the storage device. */
22706e932feSMatthew Sakai struct io_submitter *io_submitter;
22806e932feSMatthew Sakai
22906e932feSMatthew Sakai /* The pool of data_vios for servicing incoming bios */
23006e932feSMatthew Sakai struct data_vio_pool *data_vio_pool;
23106e932feSMatthew Sakai
23206e932feSMatthew Sakai /* The manager for administrative operations */
23306e932feSMatthew Sakai struct vdo_administrator admin;
23406e932feSMatthew Sakai
23506e932feSMatthew Sakai /* Flags controlling administrative operations */
23606e932feSMatthew Sakai const struct admin_state_code *suspend_type;
23706e932feSMatthew Sakai bool allocations_allowed;
23806e932feSMatthew Sakai bool dump_on_shutdown;
23906e932feSMatthew Sakai atomic_t processing_message;
24006e932feSMatthew Sakai
24106e932feSMatthew Sakai /*
24206e932feSMatthew Sakai * Statistics
24306e932feSMatthew Sakai * Atomic stats counters
24406e932feSMatthew Sakai */
24506e932feSMatthew Sakai struct atomic_statistics stats;
24606e932feSMatthew Sakai /* Used to gather statistics without allocating memory */
24706e932feSMatthew Sakai struct vdo_statistics stats_buffer;
24806e932feSMatthew Sakai /* Protects the stats_buffer */
24906e932feSMatthew Sakai struct mutex stats_mutex;
25006e932feSMatthew Sakai
25106e932feSMatthew Sakai /* A list of all device_configs referencing this vdo */
25206e932feSMatthew Sakai struct list_head device_config_list;
25306e932feSMatthew Sakai
25406e932feSMatthew Sakai /* This VDO's list entry for the device registry */
25506e932feSMatthew Sakai struct list_head registration;
25606e932feSMatthew Sakai
25706e932feSMatthew Sakai /* Underlying block device info. */
25806e932feSMatthew Sakai u64 starting_sector_offset;
25906e932feSMatthew Sakai struct volume_geometry geometry;
26006e932feSMatthew Sakai
26106e932feSMatthew Sakai /* N blobs of context data for LZ4 code, one per CPU thread. */
26206e932feSMatthew Sakai char **compression_context;
26306e932feSMatthew Sakai };
26406e932feSMatthew Sakai
26506e932feSMatthew Sakai /**
26606e932feSMatthew Sakai * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue
26706e932feSMatthew Sakai * for acknowledging received and processed bios.
26806e932feSMatthew Sakai * @vdo: The vdo.
26906e932feSMatthew Sakai *
27006e932feSMatthew Sakai * Note that this directly controls the handling of write operations, but the compile-time flag
27106e932feSMatthew Sakai * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations.
27206e932feSMatthew Sakai *
27306e932feSMatthew Sakai * Return: Whether a bio-acknowledgement work queue is in use.
27406e932feSMatthew Sakai */
vdo_uses_bio_ack_queue(struct vdo * vdo)27506e932feSMatthew Sakai static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo)
27606e932feSMatthew Sakai {
27706e932feSMatthew Sakai return vdo->device_config->thread_counts.bio_ack_threads > 0;
27806e932feSMatthew Sakai }
27906e932feSMatthew Sakai
28006e932feSMatthew Sakai /**
28106e932feSMatthew Sakai * typedef vdo_filter_fn - Method type for vdo matching methods.
28206e932feSMatthew Sakai *
28306e932feSMatthew Sakai * A filter function returns false if the vdo doesn't match.
28406e932feSMatthew Sakai */
28506e932feSMatthew Sakai typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context);
28606e932feSMatthew Sakai
28706e932feSMatthew Sakai void vdo_initialize_device_registry_once(void);
28806e932feSMatthew Sakai struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context);
28906e932feSMatthew Sakai
29006e932feSMatthew Sakai int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id,
29106e932feSMatthew Sakai const struct vdo_work_queue_type *type,
29206e932feSMatthew Sakai unsigned int queue_count, void *contexts[]);
29306e932feSMatthew Sakai
vdo_make_default_thread(struct vdo * vdo,thread_id_t thread_id)29406e932feSMatthew Sakai static inline int __must_check vdo_make_default_thread(struct vdo *vdo,
29506e932feSMatthew Sakai thread_id_t thread_id)
29606e932feSMatthew Sakai {
29706e932feSMatthew Sakai return vdo_make_thread(vdo, thread_id, NULL, 1, NULL);
29806e932feSMatthew Sakai }
29906e932feSMatthew Sakai
30006e932feSMatthew Sakai int __must_check vdo_make(unsigned int instance, struct device_config *config,
30106e932feSMatthew Sakai char **reason, struct vdo **vdo_ptr);
30206e932feSMatthew Sakai
30306e932feSMatthew Sakai void vdo_destroy(struct vdo *vdo);
30406e932feSMatthew Sakai
30506e932feSMatthew Sakai void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent);
30606e932feSMatthew Sakai
30706e932feSMatthew Sakai struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo);
30806e932feSMatthew Sakai
30906e932feSMatthew Sakai const char * __must_check vdo_get_device_name(const struct dm_target *target);
31006e932feSMatthew Sakai
31106e932feSMatthew Sakai int __must_check vdo_synchronous_flush(struct vdo *vdo);
31206e932feSMatthew Sakai
31306e932feSMatthew Sakai const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo);
31406e932feSMatthew Sakai
31506e932feSMatthew Sakai bool vdo_set_compressing(struct vdo *vdo, bool enable);
31606e932feSMatthew Sakai
31706e932feSMatthew Sakai bool vdo_get_compressing(struct vdo *vdo);
31806e932feSMatthew Sakai
31906e932feSMatthew Sakai void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats);
32006e932feSMatthew Sakai
32106e932feSMatthew Sakai thread_id_t vdo_get_callback_thread_id(void);
32206e932feSMatthew Sakai
32306e932feSMatthew Sakai enum vdo_state __must_check vdo_get_state(const struct vdo *vdo);
32406e932feSMatthew Sakai
32506e932feSMatthew Sakai void vdo_set_state(struct vdo *vdo, enum vdo_state state);
32606e932feSMatthew Sakai
32706e932feSMatthew Sakai void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent);
32806e932feSMatthew Sakai
32906e932feSMatthew Sakai int vdo_register_read_only_listener(struct vdo *vdo, void *listener,
33006e932feSMatthew Sakai vdo_read_only_notification_fn notification,
33106e932feSMatthew Sakai thread_id_t thread_id);
33206e932feSMatthew Sakai
33306e932feSMatthew Sakai int vdo_enable_read_only_entry(struct vdo *vdo);
33406e932feSMatthew Sakai
33506e932feSMatthew Sakai void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent);
33606e932feSMatthew Sakai
33706e932feSMatthew Sakai void vdo_allow_read_only_mode_entry(struct vdo_completion *parent);
33806e932feSMatthew Sakai
33906e932feSMatthew Sakai void vdo_enter_read_only_mode(struct vdo *vdo, int error_code);
34006e932feSMatthew Sakai
34106e932feSMatthew Sakai bool __must_check vdo_is_read_only(struct vdo *vdo);
34206e932feSMatthew Sakai
34306e932feSMatthew Sakai bool __must_check vdo_in_read_only_mode(const struct vdo *vdo);
34406e932feSMatthew Sakai
34506e932feSMatthew Sakai bool __must_check vdo_in_recovery_mode(const struct vdo *vdo);
34606e932feSMatthew Sakai
34706e932feSMatthew Sakai void vdo_enter_recovery_mode(struct vdo *vdo);
34806e932feSMatthew Sakai
34906e932feSMatthew Sakai void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name);
35006e932feSMatthew Sakai
35106e932feSMatthew Sakai void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone,
35206e932feSMatthew Sakai const char *name);
35306e932feSMatthew Sakai
35406e932feSMatthew Sakai void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone,
35506e932feSMatthew Sakai const char *name);
35606e932feSMatthew Sakai
35706e932feSMatthew Sakai int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn,
35806e932feSMatthew Sakai struct physical_zone **zone_ptr);
35906e932feSMatthew Sakai
36006e932feSMatthew Sakai void vdo_dump_status(const struct vdo *vdo);
36106e932feSMatthew Sakai
36206e932feSMatthew Sakai #endif /* VDO_H */
363