Lines Matching +full:block +full:- +full:fetch
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include "memory-alloc.h"
15 #include "block-map.h"
19 #include "int-map.h"
20 #include "io-submitter.h"
21 #include "recovery-journal.h"
22 #include "slab-depot.h"
25 #include "wait-queue.h"
28 * An explicitly numbered block mapping. Numbering the mappings allows them to be sorted by logical
29 * block number during repair while still preserving the relative order of journal entries with
30 * the same logical block number.
44 /* Block sequence number */
68 /* The number of entries to be applied to the block map */
70 /* The sequence number of the first valid block for block map recovery */
72 /* The sequence number of the first valid block for slab journal replay */
74 /* The sequence number of the last valid block of the journal (if known) */
77 * The highest sequence number of the journal. During recovery (vs read-only rebuild), not
84 /* The number of block map data blocks known to be allocated */
87 /* These fields are for playing the journal into the block map */
88 /* The entry data for the block map recovery */
92 /* number of pending (non-ready) requests*/
98 * a heap wrapping journal_entries. It re-orders and sorts journal entries in ascending LBN
119 /* These fields are only used during read-only rebuild */
121 /* the number of leaf pages in the block map */
123 /* the last slot of the block map */
127 * The page completions used for playing the journal into the block map, and, during
128 * read-only rebuild, for rebuilding the reference counts from the block map.
147 if (mapping1->block_map_slot.pbn != mapping2->block_map_slot.pbn) in mapping_is_less_than()
148 return mapping1->block_map_slot.pbn < mapping2->block_map_slot.pbn; in mapping_is_less_than()
150 if (mapping1->block_map_slot.slot != mapping2->block_map_slot.slot) in mapping_is_less_than()
151 return mapping1->block_map_slot.slot < mapping2->block_map_slot.slot; in mapping_is_less_than()
153 if (mapping1->number != mapping2->number) in mapping_is_less_than()
154 return mapping1->number < mapping2->number; in mapping_is_less_than()
174 struct replay_heap *heap = &repair->replay_heap; in sort_next_heap_element()
177 if (heap->nr == 0) in sort_next_heap_element()
184 last = &repair->entries[--heap->nr]; in sort_next_heap_element()
185 swap_mappings(heap->data, last, NULL); in sort_next_heap_element()
191 * as_repair_completion() - Convert a generic completion to a repair_completion.
206 struct vdo_completion *completion = &repair->completion; in prepare_repair_completion()
207 const struct thread_config *thread_config = &completion->vdo->thread_config; in prepare_repair_completion()
212 thread_config->logical_threads[0] : in prepare_repair_completion()
213 thread_config->admin_thread); in prepare_repair_completion()
222 vdo_launch_completion(&repair->completion); in launch_repair_completion()
227 while (repair->vio_count > 0) in uninitialize_vios()
228 free_vio_components(&repair->vios[--repair->vio_count]); in uninitialize_vios()
230 vdo_free(vdo_forget(repair->vios)); in uninitialize_vios()
242 repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false; in free_repair_completion()
245 vdo_free(vdo_forget(repair->journal_data)); in free_repair_completion()
246 vdo_free(vdo_forget(repair->entries)); in free_repair_completion()
252 struct vdo_completion *parent = completion->parent; in finish_repair()
253 struct vdo *vdo = completion->vdo; in finish_repair()
258 if (vdo->load_state != VDO_REBUILD_FOR_UPGRADE) in finish_repair()
259 vdo->states.vdo.complete_recoveries++; in finish_repair()
261 vdo_initialize_recovery_journal_post_repair(vdo->recovery_journal, in finish_repair()
262 vdo->states.vdo.complete_recoveries, in finish_repair()
263 repair->highest_tail, in finish_repair()
264 repair->logical_blocks_used, in finish_repair()
265 repair->block_map_data_blocks); in finish_repair()
268 if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { in finish_repair()
269 vdo_log_info("Read-only rebuild complete"); in finish_repair()
281 vdo_continue_completion(parent, vdo_allocate_reference_counters(vdo->depot)); in finish_repair()
285 * abort_repair() - Handle a repair error.
290 struct vdo_completion *parent = completion->parent; in abort_repair()
291 int result = completion->result; in abort_repair()
294 if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) in abort_repair()
295 vdo_log_info("Read-only rebuild aborted"); in abort_repair()
304 * abort_on_error() - Abort a repair if there is an error.
315 vdo_fail_completion(&repair->completion, result); in abort_on_error()
320 * drain_slab_depot() - Flush out all dirty refcounts blocks now that they have been rebuilt or
326 struct vdo *vdo = completion->vdo; in drain_slab_depot()
333 if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { in drain_slab_depot()
338 repair->entries_added_to_slab_journals); in drain_slab_depot()
342 vdo_drain_slab_depot(vdo->depot, operation, completion); in drain_slab_depot()
346 * flush_block_map_updates() - Flush the block map now that all the reference counts are rebuilt.
353 vdo_assert_on_admin_thread(completion->vdo, __func__); in flush_block_map_updates()
355 vdo_log_info("Flushing block map changes"); in flush_block_map_updates()
358 vdo_drain_block_map(completion->vdo->block_map, VDO_ADMIN_STATE_RECOVERING, in flush_block_map_updates()
366 * handle_page_load_error() - Handle an error loading a page.
371 struct repair_completion *repair = completion->parent; in handle_page_load_error()
373 repair->outstanding--; in handle_page_load_error()
374 vdo_set_completion_result(&repair->completion, completion->result); in handle_page_load_error()
380 * unmap_entry() - Unmap an invalid entry and indicate that its page must be written out.
388 page->entries[slot] = UNMAPPED_BLOCK_MAP_ENTRY; in unmap_entry()
393 * remove_out_of_bounds_entries() - Unmap entries which outside the logical space.
405 struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); in remove_out_of_bounds_entries()
413 * process_slot() - Update the reference counts for a single entry.
423 struct slab_depot *depot = completion->vdo->depot; in process_slot()
425 struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); in process_slot()
463 * rebuild_reference_counts_from_page() - Rebuild reference counts from a block map page.
476 vdo_set_completion_result(&repair->completion, result); in rebuild_reference_counts_from_page()
480 if (!page->header.initialized) in rebuild_reference_counts_from_page()
484 if (vdo_get_block_map_page_pbn(page) == repair->last_slot.pbn) { in rebuild_reference_counts_from_page()
485 last_slot = repair->last_slot.slot; in rebuild_reference_counts_from_page()
494 repair->logical_blocks_used++; in rebuild_reference_counts_from_page()
499 * page_loaded() - Process a page which has just been loaded.
506 struct repair_completion *repair = completion->parent; in page_loaded()
508 repair->outstanding--; in page_loaded()
512 /* Advance progress to the next page, and fetch the next page we haven't yet requested. */ in page_loaded()
521 if (repair->completion.result != VDO_SUCCESS) in get_pbn_to_fetch()
524 while ((pbn == VDO_ZERO_BLOCK) && (repair->page_to_fetch < repair->leaf_pages)) in get_pbn_to_fetch()
525 pbn = vdo_find_block_map_page_pbn(block_map, repair->page_to_fetch++); in get_pbn_to_fetch()
527 if (vdo_is_physical_data_block(repair->completion.vdo->depot, pbn)) in get_pbn_to_fetch()
530 vdo_set_completion_result(&repair->completion, VDO_BAD_MAPPING); in get_pbn_to_fetch()
535 * fetch_page() - Fetch a page from the block map.
545 struct block_map *block_map = repair->completion.vdo->block_map; in fetch_page()
549 repair->outstanding++; in fetch_page()
554 vdo_get_page(page_completion, &block_map->zones[0], pbn, true, repair, in fetch_page()
558 if (repair->outstanding > 0) in fetch_page()
566 * rebuild_from_leaves() - Rebuild reference counts from the leaf block map pages.
569 * Rebuilds reference counts from the leaf block map pages now that reference counts have been
577 struct block_map *map = completion->vdo->block_map; in rebuild_from_leaves()
579 repair->logical_blocks_used = 0; in rebuild_from_leaves()
585 repair->leaf_pages = vdo_compute_block_map_page_count(map->entry_count); in rebuild_from_leaves()
586 repair->last_slot = (struct block_map_slot) { in rebuild_from_leaves()
587 .slot = map->entry_count % VDO_BLOCK_MAP_ENTRIES_PER_PAGE, in rebuild_from_leaves()
588 .pbn = vdo_find_block_map_page_pbn(map, repair->leaf_pages - 1), in rebuild_from_leaves()
590 if (repair->last_slot.slot == 0) in rebuild_from_leaves()
591 repair->last_slot.slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; in rebuild_from_leaves()
593 for (i = 0; i < repair->page_count; i++) { in rebuild_from_leaves()
594 if (fetch_page(repair, &repair->page_completions[i].completion)) { in rebuild_from_leaves()
605 * process_entry() - Process a single entry from the block map tree.
606 * @pbn: A pbn which holds a block map tree page.
616 struct slab_depot *depot = completion->vdo->depot; in process_entry()
629 "Could not adjust reference count for block map tree PBN %llu", in process_entry()
633 repair->block_map_data_blocks++; in process_entry()
640 struct vdo *vdo = completion->vdo; in rebuild_reference_counts()
641 struct vdo_page_cache *cache = &vdo->block_map->zones[0].page_cache; in rebuild_reference_counts()
644 if (abort_on_error(vdo_allocate_reference_counters(vdo->depot), repair)) in rebuild_reference_counts()
655 vdo_traverse_forest(vdo->block_map, process_entry, completion); in rebuild_reference_counts()
660 if (++point->entry_count < RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) in increment_recovery_point()
663 point->entry_count = 0; in increment_recovery_point()
664 if (point->sector_count < (VDO_SECTORS_PER_BLOCK - 1)) { in increment_recovery_point()
665 point->sector_count++; in increment_recovery_point()
669 point->sequence_number++; in increment_recovery_point()
670 point->sector_count = 1; in increment_recovery_point()
674 * advance_points() - Advance the current recovery and journal points.
676 * @entries_per_block: The number of entries in a recovery journal block.
681 if (!repair->next_recovery_point.increment_applied) { in advance_points()
682 repair->next_recovery_point.increment_applied = true; in advance_points()
686 increment_recovery_point(&repair->next_recovery_point); in advance_points()
687 vdo_advance_journal_point(&repair->next_journal_point, entries_per_block); in advance_points()
688 repair->next_recovery_point.increment_applied = false; in advance_points()
692 * before_recovery_point() - Check whether the first point precedes the second point.
701 if (first->sequence_number < second->sequence_number) in before_recovery_point()
704 if (first->sequence_number > second->sequence_number) in before_recovery_point()
707 if (first->sector_count < second->sector_count) in before_recovery_point()
710 return ((first->sector_count == second->sector_count) && in before_recovery_point()
711 (first->entry_count < second->entry_count)); in before_recovery_point()
727 * get_entry() - Unpack the recovery journal entry associated with the given recovery point.
738 sector = get_sector(repair->completion.vdo->recovery_journal, in get_entry()
739 repair->journal_data, point->sequence_number, in get_entry()
740 point->sector_count); in get_entry()
741 return vdo_unpack_recovery_journal_entry(§or->entries[point->entry_count]); in get_entry()
745 * validate_recovery_journal_entry() - Validate a recovery journal entry.
754 if ((entry->slot.pbn >= vdo->states.vdo.config.physical_blocks) || in validate_recovery_journal_entry()
755 (entry->slot.slot >= VDO_BLOCK_MAP_ENTRIES_PER_PAGE) || in validate_recovery_journal_entry()
756 !vdo_is_valid_location(&entry->mapping) || in validate_recovery_journal_entry()
757 !vdo_is_valid_location(&entry->unmapping) || in validate_recovery_journal_entry()
758 !vdo_is_physical_data_block(vdo->depot, entry->mapping.pbn) || in validate_recovery_journal_entry()
759 !vdo_is_physical_data_block(vdo->depot, entry->unmapping.pbn)) { in validate_recovery_journal_entry()
762 vdo_get_journal_operation_name(entry->operation), in validate_recovery_journal_entry()
763 (unsigned long long) entry->slot.pbn, in validate_recovery_journal_entry()
764 entry->slot.slot, in validate_recovery_journal_entry()
765 (unsigned long long) entry->unmapping.pbn, in validate_recovery_journal_entry()
766 (unsigned long long) entry->mapping.pbn); in validate_recovery_journal_entry()
769 if ((entry->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) && in validate_recovery_journal_entry()
770 (vdo_is_state_compressed(entry->mapping.state) || in validate_recovery_journal_entry()
771 (entry->mapping.pbn == VDO_ZERO_BLOCK) || in validate_recovery_journal_entry()
772 (entry->unmapping.state != VDO_MAPPING_STATE_UNMAPPED) || in validate_recovery_journal_entry()
773 (entry->unmapping.pbn != VDO_ZERO_BLOCK))) { in validate_recovery_journal_entry()
776 vdo_get_journal_operation_name(entry->operation), in validate_recovery_journal_entry()
777 (unsigned long long) entry->slot.pbn, in validate_recovery_journal_entry()
778 entry->slot.slot, in validate_recovery_journal_entry()
779 (unsigned long long) entry->unmapping.pbn, in validate_recovery_journal_entry()
780 (unsigned long long) entry->mapping.pbn); in validate_recovery_journal_entry()
787 * add_slab_journal_entries() - Replay recovery journal entries into the slab journals of the
796 struct repair_completion *repair = completion->parent; in add_slab_journal_entries()
797 struct vdo *vdo = completion->vdo; in add_slab_journal_entries()
798 struct recovery_journal *journal = vdo->recovery_journal; in add_slab_journal_entries()
804 completion->callback_thread_id, repair); in add_slab_journal_entries()
805 for (recovery_point = &repair->next_recovery_point; in add_slab_journal_entries()
806 before_recovery_point(recovery_point, &repair->tail_recovery_point); in add_slab_journal_entries()
807 advance_points(repair, journal->entries_per_block)) { in add_slab_journal_entries()
812 bool increment = !repair->next_recovery_point.increment_applied; in add_slab_journal_entries()
830 slab = vdo_get_slab(vdo->depot, pbn); in add_slab_journal_entries()
831 if (slab->allocator != allocator) in add_slab_journal_entries()
835 &repair->next_journal_point, in add_slab_journal_entries()
839 repair->entries_added_to_slab_journals++; in add_slab_journal_entries()
846 * vdo_replay_into_slab_journals() - Replay recovery journal entries in the slab journals of slabs
853 struct vdo_completion *completion = &allocator->completion; in vdo_replay_into_slab_journals()
855 struct vdo *vdo = completion->vdo; in vdo_replay_into_slab_journals()
857 vdo_assert_on_physical_zone_thread(vdo, allocator->zone_number, __func__); in vdo_replay_into_slab_journals()
858 if (repair->entry_count == 0) { in vdo_replay_into_slab_journals()
860 repair->logical_blocks_used = vdo->recovery_journal->logical_blocks_used; in vdo_replay_into_slab_journals()
861 repair->block_map_data_blocks = vdo->recovery_journal->block_map_data_blocks; in vdo_replay_into_slab_journals()
866 repair->next_recovery_point = (struct recovery_point) { in vdo_replay_into_slab_journals()
867 .sequence_number = repair->slab_journal_head, in vdo_replay_into_slab_journals()
872 repair->next_journal_point = (struct journal_point) { in vdo_replay_into_slab_journals()
873 .sequence_number = repair->slab_journal_head, in vdo_replay_into_slab_journals()
878 allocator->zone_number); in vdo_replay_into_slab_journals()
879 completion->parent = repair; in vdo_replay_into_slab_journals()
888 vdo_assert_on_admin_thread(completion->vdo, __func__); in load_slab_depot()
890 if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) { in load_slab_depot()
899 vdo_load_slab_depot(completion->vdo->depot, operation, completion, repair); in load_slab_depot()
907 vdo_assert_on_admin_thread(completion->vdo, __func__); in flush_block_map()
909 vdo_log_info("Flushing block map changes"); in flush_block_map()
911 operation = (vdo_state_requires_read_only_rebuild(completion->vdo->load_state) ? in flush_block_map()
914 vdo_drain_block_map(completion->vdo->block_map, operation, completion); in flush_block_map()
920 if (repair->launching || (repair->outstanding > 0)) in finish_if_done()
923 if (repair->completion.result != VDO_SUCCESS) { in finish_if_done()
926 for (i = 0; i < repair->page_count; i++) { in finish_if_done()
928 &repair->page_completions[i]; in finish_if_done()
930 if (page_completion->ready) in finish_if_done()
931 vdo_release_page_completion(&page_completion->completion); in finish_if_done()
934 vdo_launch_completion(&repair->completion); in finish_if_done()
938 if (repair->current_entry >= repair->entries) in finish_if_done()
947 vdo_set_completion_result(&repair->completion, result); in abort_block_map_recovery()
952 * find_entry_starting_next_page() - Find the first journal entry after a given entry which is not
953 * on the same block map page.
958 * Return: Pointer to the first later journal entry on a different block map page, or a pointer to
959 * just before the journal entries if no subsequent entry is on a different block map page.
968 if (current_entry < repair->entries) in find_entry_starting_next_page()
971 current_page = current_entry->block_map_slot.pbn; in find_entry_starting_next_page()
974 while ((current_entry >= repair->entries) && in find_entry_starting_next_page()
975 (current_entry->block_map_slot.pbn == current_page)) { in find_entry_starting_next_page()
983 current_entry--; in find_entry_starting_next_page()
991 * entries to a block map page.
1000 page->entries[current_entry->block_map_slot.slot] = current_entry->block_map_entry; in apply_journal_entries_to_page()
1001 current_entry--; in apply_journal_entries_to_page()
1010 struct repair_completion *repair = as_repair_completion(completion->parent); in block_map_page_loaded()
1012 repair->outstanding--; in block_map_page_loaded()
1013 if (!repair->launching) in block_map_page_loaded()
1019 struct repair_completion *repair = as_repair_completion(completion->parent); in handle_block_map_page_load_error()
1021 repair->outstanding--; in handle_block_map_page_load_error()
1022 abort_block_map_recovery(repair, completion->result); in handle_block_map_page_load_error()
1030 if (repair->current_unfetched_entry < repair->entries) in fetch_block_map_page()
1031 /* Nothing left to fetch. */ in fetch_block_map_page()
1034 /* Fetch the next page we haven't yet requested. */ in fetch_block_map_page()
1035 pbn = repair->current_unfetched_entry->block_map_slot.pbn; in fetch_block_map_page()
1036 repair->current_unfetched_entry = in fetch_block_map_page()
1037 find_entry_starting_next_page(repair, repair->current_unfetched_entry, in fetch_block_map_page()
1039 repair->outstanding++; in fetch_block_map_page()
1041 &repair->completion.vdo->block_map->zones[0], pbn, true, in fetch_block_map_page()
1042 &repair->completion, block_map_page_loaded, in fetch_block_map_page()
1050 if (completion == (&repair->page_completions[repair->page_count])) in get_next_page_completion()
1051 completion = &repair->page_completions[0]; in get_next_page_completion()
1063 if (repair->pbn != page_completion->pbn) in recover_ready_pages()
1066 while (page_completion->ready) { in recover_ready_pages()
1078 find_entry_starting_next_page(repair, repair->current_entry, in recover_ready_pages()
1080 apply_journal_entries_to_page(page, repair->current_entry, in recover_ready_pages()
1082 repair->current_entry = start_of_next_page; in recover_ready_pages()
1089 repair->pbn = repair->current_entry->block_map_slot.pbn; in recover_ready_pages()
1092 completion = &page_completion->completion; in recover_ready_pages()
1099 struct vdo *vdo = completion->vdo; in recover_block_map()
1105 /* Suppress block map errors. */ in recover_block_map()
1106 vdo->block_map->zones[0].page_cache.rebuilding = in recover_block_map()
1107 vdo_state_requires_read_only_rebuild(vdo->load_state); in recover_block_map()
1109 if (repair->block_map_entry_count == 0) { in recover_block_map()
1110 vdo_log_info("Replaying 0 recovery entries into block map"); in recover_block_map()
1111 vdo_free(vdo_forget(repair->journal_data)); in recover_block_map()
1120 repair->replay_heap = (struct replay_heap) { in recover_block_map()
1121 .data = repair->entries, in recover_block_map()
1122 .nr = repair->block_map_entry_count, in recover_block_map()
1123 .size = repair->block_map_entry_count, in recover_block_map()
1125 min_heapify_all(&repair->replay_heap, &repair_min_heap, NULL); in recover_block_map()
1127 vdo_log_info("Replaying %zu recovery entries into block map", in recover_block_map()
1128 repair->block_map_entry_count); in recover_block_map()
1130 repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; in recover_block_map()
1132 VDO_ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, in recover_block_map()
1136 repair->launching = true; in recover_block_map()
1137 repair->pbn = repair->current_entry->block_map_slot.pbn; in recover_block_map()
1138 repair->current_unfetched_entry = repair->current_entry; in recover_block_map()
1139 for (i = 0; i < repair->page_count; i++) { in recover_block_map()
1140 if (repair->current_unfetched_entry < repair->entries) in recover_block_map()
1143 fetch_block_map_page(repair, &repair->page_completions[i].completion); in recover_block_map()
1145 repair->launching = false; in recover_block_map()
1148 recover_ready_pages(repair, &repair->page_completions[0].completion); in recover_block_map()
1152 * get_recovery_journal_block_header() - Get the block header for a block at a position in the
1172 * is_valid_recovery_journal_block() - Determine whether the given header describes a valid block
1175 * @header: The unpacked block header to check.
1178 * A block is not valid if it is unformatted, or if it is older than the last successful recovery
1187 if ((header->nonce != journal->nonce) || in is_valid_recovery_journal_block()
1188 (header->recovery_count != journal->recovery_count)) in is_valid_recovery_journal_block()
1191 if (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL_2) in is_valid_recovery_journal_block()
1192 return (header->entry_count <= journal->entries_per_block); in is_valid_recovery_journal_block()
1195 (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL) && in is_valid_recovery_journal_block()
1196 (header->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK)); in is_valid_recovery_journal_block()
1200 * is_exact_recovery_journal_block() - Determine whether the given header describes the exact block
1203 * @header: The unpacked block header to check.
1206 * Return: True if the block matches.
1212 return ((header->sequence_number == sequence) && in is_exact_recovery_journal_block()
1217 * find_recovery_journal_head_and_tail() - Find the tail and head of the journal.
1224 struct recovery_journal *journal = repair->completion.vdo->recovery_journal; in find_recovery_journal_head_and_tail()
1230 * block must be a lower bound. Not doing so can result in extra data loss by setting the in find_recovery_journal_head_and_tail()
1233 repair->highest_tail = journal->tail; in find_recovery_journal_head_and_tail()
1234 for (i = 0; i < journal->size; i++) { in find_recovery_journal_head_and_tail()
1236 get_recovery_journal_block_header(journal, repair->journal_data, i); in find_recovery_journal_head_and_tail()
1239 /* This block is old or incorrectly formatted */ in find_recovery_journal_head_and_tail()
1244 /* This block is in the wrong location */ in find_recovery_journal_head_and_tail()
1248 if (header.sequence_number >= repair->highest_tail) { in find_recovery_journal_head_and_tail()
1250 repair->highest_tail = header.sequence_number; in find_recovery_journal_head_and_tail()
1256 if (header.block_map_head > repair->block_map_head) in find_recovery_journal_head_and_tail()
1257 repair->block_map_head = header.block_map_head; in find_recovery_journal_head_and_tail()
1259 if (header.slab_journal_head > repair->slab_journal_head) in find_recovery_journal_head_and_tail()
1260 repair->slab_journal_head = header.slab_journal_head; in find_recovery_journal_head_and_tail()
1267 * unpack_entry() - Unpack a recovery journal entry in either format.
1289 if (packed_entry->operation == VDO_JOURNAL_DATA_INCREMENT) in unpack_entry()
1290 entry->operation = VDO_JOURNAL_DATA_REMAPPING; in unpack_entry()
1291 else if (packed_entry->operation == VDO_JOURNAL_BLOCK_MAP_INCREMENT) in unpack_entry()
1292 entry->operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING; in unpack_entry()
1296 low32 = __le32_to_cpu(packed_entry->pbn_low_word); in unpack_entry()
1297 high4 = packed_entry->pbn_high_nibble; in unpack_entry()
1298 entry->slot = (struct block_map_slot) { in unpack_entry()
1300 .slot = (packed_entry->slot_low | (packed_entry->slot_high << 6)), in unpack_entry()
1302 entry->mapping = vdo_unpack_block_map_entry(&packed_entry->block_map_entry); in unpack_entry()
1303 entry->unmapping = (struct data_location) { in unpack_entry()
1313 * append_sector_entries() - Append an array of recovery journal entries from a journal block
1326 struct vdo *vdo = repair->completion.vdo; in append_sector_entries()
1335 /* When recovering from read-only mode, ignore damaged entries. */ in append_sector_entries()
1338 repair->entries[repair->block_map_entry_count] = in append_sector_entries()
1343 .number = repair->block_map_entry_count, in append_sector_entries()
1345 repair->block_map_entry_count++; in append_sector_entries()
1355 return ((sector_number == (VDO_SECTORS_PER_BLOCK - 1)) in entries_per_sector()
1368 get_recovery_journal_block_header(journal, repair->journal_data, in extract_entries_from_block()
1373 /* This block is invalid, so skip it. */ in extract_entries_from_block()
1380 get_sector(journal, repair->journal_data, sequence, i); in extract_entries_from_block()
1385 /* Only extract as many as the block header calls for. */ in extract_entries_from_block()
1386 append_sector_entries(repair, (char *) sector->entries, format, in extract_entries_from_block()
1388 sector->entry_count, in extract_entries_from_block()
1394 * entry count the block header claims. in extract_entries_from_block()
1396 entries -= sector_entries; in extract_entries_from_block()
1406 struct vdo *vdo = repair->completion.vdo; in parse_journal_for_rebuild()
1407 struct recovery_journal *journal = vdo->recovery_journal; in parse_journal_for_rebuild()
1408 journal_entry_count_t entries_per_block = journal->entries_per_block; in parse_journal_for_rebuild()
1410 format = get_recovery_journal_block_header(journal, repair->journal_data, in parse_journal_for_rebuild()
1411 repair->highest_tail).metadata_type; in parse_journal_for_rebuild()
1417 * packed_recovery_journal_entry from every valid journal block. in parse_journal_for_rebuild()
1419 count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block); in parse_journal_for_rebuild()
1421 &repair->entries); in parse_journal_for_rebuild()
1425 for (i = repair->block_map_head; i <= repair->highest_tail; i++) in parse_journal_for_rebuild()
1434 if ((repair->block_map_head <= repair->tail) && in validate_heads()
1435 (repair->slab_journal_head <= repair->tail)) in validate_heads()
1440 "Journal tail too early. block map head: %llu, slab journal head: %llu, tail: %llu", in validate_heads()
1441 (unsigned long long) repair->block_map_head, in validate_heads()
1442 (unsigned long long) repair->slab_journal_head, in validate_heads()
1443 (unsigned long long) repair->tail); in validate_heads()
1447 * extract_new_mappings() - Find all valid new mappings to be applied to the block map.
1451 * mappings to be applied to a given block map page can be done in a single page fetch.
1456 struct vdo *vdo = repair->completion.vdo; in extract_new_mappings()
1458 .sequence_number = repair->block_map_head, in extract_new_mappings()
1465 * every packed_recovery_journal_entry from every valid journal block. in extract_new_mappings()
1467 result = vdo_allocate(repair->entry_count, struct numbered_block_mapping, in extract_new_mappings()
1468 __func__, &repair->entries); in extract_new_mappings()
1472 for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); in extract_new_mappings()
1482 repair->entries[repair->block_map_entry_count] = in extract_new_mappings()
1487 .number = repair->block_map_entry_count, in extract_new_mappings()
1489 repair->block_map_entry_count++; in extract_new_mappings()
1492 result = VDO_ASSERT((repair->block_map_entry_count <= repair->entry_count), in extract_new_mappings()
1501 * compute_usages() - Compute the lbns in use and block map data blocks counts from the tail of
1512 .sequence_number = repair->tail, in compute_usages()
1517 struct vdo *vdo = repair->completion.vdo; in compute_usages()
1518 struct recovery_journal *journal = vdo->recovery_journal; in compute_usages()
1520 get_recovery_journal_block_header(journal, repair->journal_data, in compute_usages()
1521 repair->tail); in compute_usages()
1523 repair->logical_blocks_used = header.logical_blocks_used; in compute_usages()
1524 repair->block_map_data_blocks = header.block_map_data_blocks; in compute_usages()
1526 for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); in compute_usages()
1538 repair->block_map_data_blocks++; in compute_usages()
1543 repair->logical_blocks_used++; in compute_usages()
1546 repair->logical_blocks_used--; in compute_usages()
1557 struct recovery_journal *journal = repair->completion.vdo->recovery_journal; in parse_journal_for_recovery()
1561 head = min(repair->block_map_head, repair->slab_journal_head); in parse_journal_for_recovery()
1562 header = get_recovery_journal_block_header(journal, repair->journal_data, head); in parse_journal_for_recovery()
1564 for (i = head; i <= repair->highest_tail; i++) { in parse_journal_for_recovery()
1568 repair->tail = i; in parse_journal_for_recovery()
1569 repair->tail_recovery_point = (struct recovery_point) { in parse_journal_for_recovery()
1575 header = get_recovery_journal_block_header(journal, repair->journal_data, i); in parse_journal_for_recovery()
1577 /* A bad block header was found so this must be the end of the journal. */ in parse_journal_for_recovery()
1582 "Recovery journal is in an invalid format, a read-only rebuild is required."); in parse_journal_for_recovery()
1583 vdo_enter_read_only_mode(repair->completion.vdo, VDO_CORRUPT_JOURNAL); in parse_journal_for_recovery()
1592 get_sector(journal, repair->journal_data, i, j); in parse_journal_for_recovery()
1594 min_t(journal_entry_count_t, sector->entry_count, in parse_journal_for_recovery()
1597 /* A bad sector means that this block was torn. */ in parse_journal_for_recovery()
1603 repair->tail_recovery_point.sector_count++; in parse_journal_for_recovery()
1604 repair->tail_recovery_point.entry_count = sector_entries; in parse_journal_for_recovery()
1605 block_entries -= sector_entries; in parse_journal_for_recovery()
1606 repair->entry_count += sector_entries; in parse_journal_for_recovery()
1615 /* If this block was not filled, or if it tore, no later block can matter. */ in parse_journal_for_recovery()
1616 if ((header.entry_count != journal->entries_per_block) || (block_entries > 0)) in parse_journal_for_recovery()
1629 /* Set the tail to the last valid tail block, if there is one. */ in parse_journal_for_recovery()
1630 if (repair->tail_recovery_point.sector_count == 0) in parse_journal_for_recovery()
1631 repair->tail--; in parse_journal_for_recovery()
1637 …vdo_log_info("Highest-numbered recovery journal block has sequence number %llu, and the highest-nu… in parse_journal_for_recovery()
1638 (unsigned long long) repair->highest_tail, in parse_journal_for_recovery()
1639 (unsigned long long) repair->tail); in parse_journal_for_recovery()
1653 return (vdo_state_requires_read_only_rebuild(repair->completion.vdo->load_state) ? in parse_journal()
1660 struct repair_completion *repair = completion->parent; in finish_journal_load()
1662 if (++repair->vios_complete != repair->vio_count) in finish_journal_load()
1668 vdo_continue_completion(&repair->completion, parse_journal(repair)); in finish_journal_load()
1673 struct repair_completion *repair = completion->parent; in handle_journal_load_error()
1676 vdo_set_completion_result(&repair->completion, completion->result); in handle_journal_load_error()
1678 completion->callback(completion); in handle_journal_load_error()
1683 struct vio *vio = bio->bi_private; in read_journal_endio()
1684 struct vdo *vdo = vio->completion.vdo; in read_journal_endio()
1686 continue_vio_after_io(vio, finish_journal_load, vdo->thread_config.admin_thread); in read_journal_endio()
1690 * vdo_repair() - Load the recovery journal and then recover or rebuild a vdo.
1698 struct vdo *vdo = parent->vdo; in vdo_repair()
1699 struct recovery_journal *journal = vdo->recovery_journal; in vdo_repair()
1700 physical_block_number_t pbn = journal->origin; in vdo_repair()
1701 block_count_t remaining = journal->size; in vdo_repair()
1704 vdo->device_config->cache_size >> 1, in vdo_repair()
1709 if (vdo->load_state == VDO_FORCE_REBUILD) { in vdo_repair()
1710 vdo_log_warning("Rebuilding reference counts to clear read-only mode"); in vdo_repair()
1711 vdo->states.vdo.read_only_recoveries++; in vdo_repair()
1712 } else if (vdo->load_state == VDO_REBUILD_FOR_UPGRADE) { in vdo_repair()
1726 vdo_initialize_completion(&repair->completion, vdo, VDO_REPAIR_COMPLETION); in vdo_repair()
1727 repair->completion.error_handler = abort_repair; in vdo_repair()
1728 repair->completion.parent = parent; in vdo_repair()
1730 repair->page_count = page_count; in vdo_repair()
1733 &repair->journal_data); in vdo_repair()
1737 result = vdo_allocate(vio_count, struct vio, __func__, &repair->vios); in vdo_repair()
1741 ptr = repair->journal_data; in vdo_repair()
1742 for (repair->vio_count = 0; repair->vio_count < vio_count; repair->vio_count++) { in vdo_repair()
1749 &repair->vios[repair->vio_count]); in vdo_repair()
1754 remaining -= blocks; in vdo_repair()
1757 for (vio_count = 0; vio_count < repair->vio_count; in vdo_repair()
1759 vdo_submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, in vdo_repair()