1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2023 Red Hat 4 */ 5 6 #include "logical-zone.h" 7 8 #include "logger.h" 9 #include "memory-alloc.h" 10 #include "permassert.h" 11 #include "string-utils.h" 12 13 #include "action-manager.h" 14 #include "admin-state.h" 15 #include "block-map.h" 16 #include "completion.h" 17 #include "constants.h" 18 #include "data-vio.h" 19 #include "flush.h" 20 #include "int-map.h" 21 #include "physical-zone.h" 22 #include "vdo.h" 23 24 #define ALLOCATIONS_PER_ZONE 128 25 26 /** 27 * as_logical_zone() - Convert a generic vdo_completion to a logical_zone. 28 * @completion: The completion to convert. 29 * 30 * Return: The completion as a logical_zone. 31 */ 32 static struct logical_zone *as_logical_zone(struct vdo_completion *completion) 33 { 34 vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION); 35 return container_of(completion, struct logical_zone, completion); 36 } 37 38 /* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */ 39 static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number) 40 { 41 struct logical_zones *zones = context; 42 43 return zones->zones[zone_number].thread_id; 44 } 45 46 /** 47 * initialize_zone() - Initialize a logical zone. 48 * @zones: The logical_zones to which this zone belongs. 49 * @zone_number: The logical_zone's index. 50 */ 51 static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number) 52 { 53 int result; 54 struct vdo *vdo = zones->vdo; 55 struct logical_zone *zone = &zones->zones[zone_number]; 56 zone_count_t allocation_zone_number; 57 58 result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations); 59 if (result != VDO_SUCCESS) 60 return result; 61 62 if (zone_number < vdo->thread_config.logical_zone_count - 1) 63 zone->next = &zones->zones[zone_number + 1]; 64 65 vdo_initialize_completion(&zone->completion, vdo, 66 VDO_GENERATION_FLUSHED_COMPLETION); 67 zone->zones = zones; 68 zone->zone_number = zone_number; 69 zone->thread_id = vdo->thread_config.logical_threads[zone_number]; 70 zone->block_map_zone = &vdo->block_map->zones[zone_number]; 71 INIT_LIST_HEAD(&zone->write_vios); 72 vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); 73 74 allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count; 75 zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number]; 76 77 return vdo_make_default_thread(vdo, zone->thread_id); 78 } 79 80 /** 81 * vdo_make_logical_zones() - Create a set of logical zones. 82 * @vdo: The vdo to which the zones will belong. 83 * @zones_ptr: A pointer to hold the new zones. 84 * 85 * Return: VDO_SUCCESS or an error code. 86 */ 87 int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr) 88 { 89 struct logical_zones *zones; 90 int result; 91 zone_count_t zone; 92 zone_count_t zone_count = vdo->thread_config.logical_zone_count; 93 94 if (zone_count == 0) 95 return VDO_SUCCESS; 96 97 result = vdo_allocate_extended(struct logical_zones, zone_count, 98 struct logical_zone, __func__, &zones); 99 if (result != VDO_SUCCESS) 100 return result; 101 102 zones->vdo = vdo; 103 zones->zone_count = zone_count; 104 for (zone = 0; zone < zone_count; zone++) { 105 result = initialize_zone(zones, zone); 106 if (result != VDO_SUCCESS) { 107 vdo_free_logical_zones(zones); 108 return result; 109 } 110 } 111 112 result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone, 113 vdo->thread_config.admin_thread, zones, NULL, 114 vdo, &zones->manager); 115 if (result != VDO_SUCCESS) { 116 vdo_free_logical_zones(zones); 117 return result; 118 } 119 120 *zones_ptr = zones; 121 return VDO_SUCCESS; 122 } 123 124 /** 125 * vdo_free_logical_zones() - Free a set of logical zones. 126 * @zones: The set of zones to free. 127 */ 128 void vdo_free_logical_zones(struct logical_zones *zones) 129 { 130 zone_count_t index; 131 132 if (zones == NULL) 133 return; 134 135 vdo_free(vdo_forget(zones->manager)); 136 137 for (index = 0; index < zones->zone_count; index++) 138 vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations)); 139 140 vdo_free(zones); 141 } 142 143 static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what) 144 { 145 VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), 146 "%s() called on correct thread", what); 147 } 148 149 /** 150 * check_for_drain_complete() - Check whether this zone has drained. 151 * @zone: The zone to check. 152 */ 153 static void check_for_drain_complete(struct logical_zone *zone) 154 { 155 if (!vdo_is_state_draining(&zone->state) || zone->notifying || 156 !list_empty(&zone->write_vios)) 157 return; 158 159 vdo_finish_draining(&zone->state); 160 } 161 162 /** 163 * initiate_drain() - Initiate a drain. 164 * 165 * Implements vdo_admin_initiator_fn. 166 */ 167 static void initiate_drain(struct admin_state *state) 168 { 169 check_for_drain_complete(container_of(state, struct logical_zone, state)); 170 } 171 172 /** 173 * drain_logical_zone() - Drain a logical zone. 174 * 175 * Implements vdo_zone_action_fn. 176 */ 177 static void drain_logical_zone(void *context, zone_count_t zone_number, 178 struct vdo_completion *parent) 179 { 180 struct logical_zones *zones = context; 181 182 vdo_start_draining(&zones->zones[zone_number].state, 183 vdo_get_current_manager_operation(zones->manager), parent, 184 initiate_drain); 185 } 186 187 void vdo_drain_logical_zones(struct logical_zones *zones, 188 const struct admin_state_code *operation, 189 struct vdo_completion *parent) 190 { 191 vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL, 192 parent); 193 } 194 195 /** 196 * resume_logical_zone() - Resume a logical zone. 197 * 198 * Implements vdo_zone_action_fn. 199 */ 200 static void resume_logical_zone(void *context, zone_count_t zone_number, 201 struct vdo_completion *parent) 202 { 203 struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]); 204 205 vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); 206 } 207 208 /** 209 * vdo_resume_logical_zones() - Resume a set of logical zones. 210 * @zones: The logical zones to resume. 211 * @parent: The object to notify when the zones have resumed. 212 */ 213 void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent) 214 { 215 vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL, 216 resume_logical_zone, NULL, parent); 217 } 218 219 /** 220 * update_oldest_active_generation() - Update the oldest active generation. 221 * @zone: The zone. 222 * 223 * Return: true if the oldest active generation has changed. 224 */ 225 static bool update_oldest_active_generation(struct logical_zone *zone) 226 { 227 struct data_vio *data_vio = 228 list_first_entry_or_null(&zone->write_vios, struct data_vio, 229 write_entry); 230 sequence_number_t oldest = 231 (data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation; 232 233 if (oldest == zone->oldest_active_generation) 234 return false; 235 236 WRITE_ONCE(zone->oldest_active_generation, oldest); 237 return true; 238 } 239 240 /** 241 * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical 242 * zone. 243 * @zone: The logical zone. 244 * @expected_generation: The expected value of the flush generation before the increment. 245 */ 246 void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, 247 sequence_number_t expected_generation) 248 { 249 assert_on_zone_thread(zone, __func__); 250 VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation), 251 "logical zone %u flush generation %llu should be %llu before increment", 252 zone->zone_number, (unsigned long long) zone->flush_generation, 253 (unsigned long long) expected_generation); 254 255 zone->flush_generation++; 256 zone->ios_in_flush_generation = 0; 257 update_oldest_active_generation(zone); 258 } 259 260 /** 261 * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write 262 * data_vio. 263 * @data_vio: The data_vio. 264 */ 265 void vdo_acquire_flush_generation_lock(struct data_vio *data_vio) 266 { 267 struct logical_zone *zone = data_vio->logical.zone; 268 269 assert_on_zone_thread(zone, __func__); 270 VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal"); 271 272 data_vio->flush_generation = zone->flush_generation; 273 list_add_tail(&data_vio->write_entry, &zone->write_vios); 274 zone->ios_in_flush_generation++; 275 } 276 277 static void attempt_generation_complete_notification(struct vdo_completion *completion); 278 279 /** 280 * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs. 281 * @completion: The zone completion. 282 * 283 * This callback is registered in attempt_generation_complete_notification(). 284 */ 285 static void notify_flusher(struct vdo_completion *completion) 286 { 287 struct logical_zone *zone = as_logical_zone(completion); 288 289 vdo_complete_flushes(zone->zones->vdo->flusher); 290 vdo_launch_completion_callback(completion, 291 attempt_generation_complete_notification, 292 zone->thread_id); 293 } 294 295 /** 296 * attempt_generation_complete_notification() - Notify the flusher if some generation no 297 * longer has active VIOs. 298 * @completion: The zone completion. 299 */ 300 static void attempt_generation_complete_notification(struct vdo_completion *completion) 301 { 302 struct logical_zone *zone = as_logical_zone(completion); 303 304 assert_on_zone_thread(zone, __func__); 305 if (zone->oldest_active_generation <= zone->notification_generation) { 306 zone->notifying = false; 307 check_for_drain_complete(zone); 308 return; 309 } 310 311 zone->notifying = true; 312 zone->notification_generation = zone->oldest_active_generation; 313 vdo_launch_completion_callback(&zone->completion, notify_flusher, 314 vdo_get_flusher_thread_id(zone->zones->vdo->flusher)); 315 } 316 317 /** 318 * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a 319 * write data_vio. 320 * @data_vio: The data_vio whose lock is to be released. 321 * 322 * If there are pending flushes, and this data_vio completes the oldest generation active in this 323 * zone, an attempt will be made to finish any flushes which may now be complete. 324 */ 325 void vdo_release_flush_generation_lock(struct data_vio *data_vio) 326 { 327 struct logical_zone *zone = data_vio->logical.zone; 328 329 assert_on_zone_thread(zone, __func__); 330 331 if (!data_vio_has_flush_generation_lock(data_vio)) 332 return; 333 334 list_del_init(&data_vio->write_entry); 335 VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation), 336 "data_vio releasing lock on generation %llu is not older than oldest active generation %llu", 337 (unsigned long long) data_vio->flush_generation, 338 (unsigned long long) zone->oldest_active_generation); 339 340 if (!update_oldest_active_generation(zone) || zone->notifying) 341 return; 342 343 attempt_generation_complete_notification(&zone->completion); 344 } 345 346 struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone) 347 { 348 if (zone->allocation_count == ALLOCATIONS_PER_ZONE) { 349 zone->allocation_count = 0; 350 zone->allocation_zone = zone->allocation_zone->next; 351 } 352 353 zone->allocation_count++; 354 return zone->allocation_zone; 355 } 356 357 /** 358 * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging. 359 * @zone: The zone to dump 360 * 361 * Context: the information is dumped in a thread-unsafe fashion. 362 * 363 */ 364 void vdo_dump_logical_zone(const struct logical_zone *zone) 365 { 366 vdo_log_info("logical_zone %u", zone->zone_number); 367 vdo_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu", 368 (unsigned long long) READ_ONCE(zone->flush_generation), 369 (unsigned long long) READ_ONCE(zone->oldest_active_generation), 370 (unsigned long long) READ_ONCE(zone->notification_generation), 371 vdo_bool_to_string(READ_ONCE(zone->notifying)), 372 (unsigned long long) READ_ONCE(zone->ios_in_flush_generation)); 373 } 374