15 #include "block-map.h"
31  * block write from overwriting a block which appears to still be a valid head block of the
129  *                                                  block.
131  * @sequence_number: The journal sequence number of the referenced block.
195  * pop_free_list() - Get a block from the end of the free list.
198  * Return: The block or NULL if the list is empty.
202 	struct recovery_journal_block *block;  in pop_free_list()  local
207 	block = list_last_entry(&journal->free_tail_blocks,  in pop_free_list()
209 	list_del_init(&block->list_node);  in pop_free_list()
210 	return block;  in pop_free_list()
214  * is_block_dirty() - Check whether a recovery block is dirty.
215  * @block: The block to check.
220  * Return: True if the block has any uncommitted entries.
222 static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block)  in is_block_dirty()  argument
224 	return (block->uncommitted_entry_count > 0);  in is_block_dirty()
228  * is_block_empty() - Check whether a journal block is empty.
229  * @block: The block to check.
231  * Return: True if the block has no entries.
233 static inline bool __must_check is_block_empty(const struct recovery_journal_block *block)  in is_block_empty()  argument
235 	return (block->entry_count == 0);  in is_block_empty()
239  * is_block_full() - Check whether a journal block is full.
240  * @block: The block to check.
242  * Return: True if the block is full.
244 static inline bool __must_check is_block_full(const struct recovery_journal_block *block)  in is_block_full()  argument
246 	return ((block == NULL) || (block->journal->entries_per_block == block->entry_count));  in is_block_full()
278  * Return: True if any block has a waiter.
282 	struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks);  in has_block_waiters()  local
285 	 * Either the first active tail block (if it exists) has waiters, or no active tail block  in has_block_waiters()
288 	return ((block != NULL) &&  in has_block_waiters()
289 		(vdo_waitq_has_waiters(&block->entry_waiters) ||  in has_block_waiters()
290 		 vdo_waitq_has_waiters(&block->commit_waiters)));  in has_block_waiters()
294 static void recycle_journal_block(struct recovery_journal_block *block);
360 					    "journal being saved has clean active block");  in check_for_drain_complete()
406  * Exposed only so the block map can be initialized therefrom.
408  * Return: The sequence number of the tail block.
419  * The head is the lowest sequence number of the block map head and the slab journal head.
441  *                                         so, force the oldest slab journal tail block to commit.
516  *                              current active block.
517  * @journal: The journal to be reset based on its active block.
555  * Attempts to reap the journal now that all the locks on some journal block have been released.
651  * initialize_recovery_block() - Initialize a journal block.
653  * @journal: The journal to which the block will belong.
654  * @block: The block to initialize.
659 				     struct recovery_journal_block *block)  in initialize_recovery_block()  argument
665 	 * Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries.  in initialize_recovery_block()
672 	 * Allocate a full block for the journal block even though not all of the space is used  in initialize_recovery_block()
673 	 * since the VIO needs to write a full disk block.  in initialize_recovery_block()
680 					 VIO_PRIORITY_HIGH, block, 1, data, &block->vio);  in initialize_recovery_block()
686 	list_add_tail(&block->list_node, &journal->free_tail_blocks);  in initialize_recovery_block()
687 	block->journal = journal;  in initialize_recovery_block()
693  *                                 was decoded from the super block.
740 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_decode_recovery_journal()  local
742 		result = initialize_recovery_block(vdo, journal, block);  in vdo_decode_recovery_journal()
813 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_free_recovery_journal()  local
815 		vdo_free(vdo_forget(block->vio.data));  in vdo_free_recovery_journal()
816 		free_vio_components(&block->vio);  in vdo_free_recovery_journal()
826  * @tail: The new tail block sequence number.
828  * @block_map_data_blocks: The new number of block map data blocks.
844  * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from
848  * Return: The number of block map pages allocated from slabs.
870  * @block_map: The block map for this VDO.
882  *                                 block.
897 		 * If the journal is saved, we should start one past the active block (since the  in vdo_record_recovery_journal()
898 		 * active block is not guaranteed to be empty).  in vdo_record_recovery_journal()
904 		 * block that might have entries that need to be applied.  in vdo_record_recovery_journal()
913  * get_block_header() - Get a pointer to the packed journal block header in the block buffer.
914  * @block: The recovery block.
916  * Return: The block's header.
919 get_block_header(const struct recovery_journal_block *block)  in get_block_header()  argument
921 	return (struct packed_journal_header *) block->vio.data;  in get_block_header()
925  * set_active_sector() - Set the current sector of the current block and initialize it.
926  * @block: The block to update.
929 static void set_active_sector(struct recovery_journal_block *block, void *sector)  in set_active_sector()  argument
931 	block->sector = sector;  in set_active_sector()
932 	block->sector->check_byte = get_block_header(block)->check_byte;  in set_active_sector()
933 	block->sector->recovery_count = block->journal->recovery_count;  in set_active_sector()
934 	block->sector->entry_count = 0;  in set_active_sector()
947 	struct recovery_journal_block *block;  in advance_tail()  local
949 	block = journal->active_block = pop_free_list(journal);  in advance_tail()
950 	if (block == NULL)  in advance_tail()
953 	list_move_tail(&block->list_node, &journal->active_tail_blocks);  in advance_tail()
966 	header = get_block_header(block);  in advance_tail()
967 	memset(block->vio.data, 0x0, VDO_BLOCK_SIZE);  in advance_tail()
968 	block->sequence_number = journal->tail;  in advance_tail()
969 	block->entry_count = 0;  in advance_tail()
970 	block->uncommitted_entry_count = 0;  in advance_tail()
971 	block->block_number = vdo_get_recovery_journal_block_number(journal,  in advance_tail()
975 	set_active_sector(block, vdo_get_journal_block_sector(header, 1));  in advance_tail()
1001  * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check
1019 		/* Cannot use this block since the journal is full. */  in prepare_to_assign_entry()
1025 	 * Don't allow the new block to be reaped until all of its entries have been committed to  in prepare_to_assign_entry()
1026 	 * the block map and until the journal block has been fully committed as well. Because the  in prepare_to_assign_entry()
1027 	 * block map update is done only after any slab journal entries have been made, the  in prepare_to_assign_entry()
1028 	 * per-entry lock for the block map entry serves to protect those as well.  in prepare_to_assign_entry()
1037  * schedule_block_write() - Queue a block for writing.
1039  * @block: The block which is now ready to write.
1041  * The block is expected to be full. If the block is currently writing, this is a noop as the block
1042  * will be queued for writing when the write finishes. The block must not currently be queued for
1046 				 struct recovery_journal_block *block)  in schedule_block_write()  argument
1048 	if (!block->committing)  in schedule_block_write()
1049 		vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter);  in schedule_block_write()
1051 	 * At the end of adding entries, or discovering this partial block is now full and ready to  in schedule_block_write()
1057  * release_journal_block_reference() - Release a reference to a journal block.
1058  * @block: The journal block from which to release a reference.
1060 static void release_journal_block_reference(struct recovery_journal_block *block)  in release_journal_block_reference()  argument
1062 	vdo_release_recovery_journal_block_reference(block->journal,  in release_journal_block_reference()
1063 						     block->sequence_number,  in release_journal_block_reference()
1082  * assign_entry() - Assign an entry waiter to the active block.
1084  * @context: The recovery journal block.
1091 	struct recovery_journal_block *block = context;  in assign_entry()  local
1092 	struct recovery_journal *journal = block->journal;  in assign_entry()
1096 		.sequence_number = block->sequence_number,  in assign_entry()
1097 		.entry_count = block->entry_count,  in assign_entry()
1103 	if (!vdo_waitq_has_waiters(&block->entry_waiters))  in assign_entry()
1106 	vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);  in assign_entry()
1107 	block->entry_count++;  in assign_entry()
1108 	block->uncommitted_entry_count++;  in assign_entry()
1111 	if (is_block_full(block)) {  in assign_entry()
1113 		 * The block is full, so we can write it anytime henceforth. If it is already  in assign_entry()
1116 		schedule_block_write(journal, block);  in assign_entry()
1143  * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been
1145  * @block: The block to be recycled.
1147 static void recycle_journal_block(struct recovery_journal_block *block)  in recycle_journal_block()  argument
1149 	struct recovery_journal *journal = block->journal;  in recycle_journal_block()
1152 	list_move_tail(&block->list_node, &journal->free_tail_blocks);  in recycle_journal_block()
1155 	for (i = block->entry_count; i < journal->entries_per_block; i++)  in recycle_journal_block()
1156 		release_journal_block_reference(block);  in recycle_journal_block()
1159 	 * Release our own lock against reaping now that the block is completely committed, or  in recycle_journal_block()
1162 	if (block->entry_count > 0)  in recycle_journal_block()
1163 		release_journal_block_reference(block);  in recycle_journal_block()
1165 	if (block == journal->active_block)  in recycle_journal_block()
1217 	struct recovery_journal_block *block;  in notify_commit_waiters()  local
1219 	list_for_each_entry(block, &journal->active_tail_blocks, list_node) {  in notify_commit_waiters()
1220 		if (block->committing)  in notify_commit_waiters()
1223 		vdo_waitq_notify_all_waiters(&block->commit_waiters,  in notify_commit_waiters()
1226 			vdo_waitq_notify_all_waiters(&block->entry_waiters,  in notify_commit_waiters()
1229 		} else if (is_block_dirty(block) || !is_block_full(block)) {  in notify_commit_waiters()
1242 	struct recovery_journal_block *block, *tmp;  in recycle_journal_blocks()  local
1244 	list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) {  in recycle_journal_blocks()
1245 		if (block->committing) {  in recycle_journal_blocks()
1251 		    (is_block_dirty(block) || !is_block_full(block))) {  in recycle_journal_blocks()
1259 		recycle_journal_block(block);  in recycle_journal_blocks()
1265  * @completion: The completion of the VIO writing this block.
1267  * This is the callback registered by write_block(). If more entries accumulated in the block being
1272 	struct recovery_journal_block *block = completion->parent;  in complete_write()  local
1273 	struct recovery_journal *journal = block->journal;  in complete_write()
1280 	journal->events.entries.committed += block->entries_in_commit;  in complete_write()
1281 	block->uncommitted_entry_count -= block->entries_in_commit;  in complete_write()
1282 	block->entries_in_commit = 0;  in complete_write()
1283 	block->committing = false;  in complete_write()
1285 	/* If this block is the latest block to be acknowledged, record that fact. */  in complete_write()
1286 	if (block->sequence_number > journal->last_write_acknowledged)  in complete_write()
1287 		journal->last_write_acknowledged = block->sequence_number;  in complete_write()
1290 	VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number),  in complete_write()
1296 	 * Is this block now full? Reaping, and adding entries, might have already sent it off for  in complete_write()
1299 	if (is_block_dirty(block) && is_block_full(block))  in complete_write()
1300 		schedule_block_write(journal, block);  in complete_write()
1310 	struct recovery_journal_block *block = completion->parent;  in handle_write_error()  local
1311 	struct recovery_journal *journal = block->journal;  in handle_write_error()
1315 			       "cannot write recovery journal block %llu",  in handle_write_error()
1316 			       (unsigned long long) block->sequence_number);  in handle_write_error()
1324 	struct recovery_journal_block *block = vio->completion.parent;  in complete_write_endio()  local
1325 	struct recovery_journal *journal = block->journal;  in complete_write_endio()
1331  * add_queued_recovery_entries() - Actually add entries from the queue to the given block.
1332  * @block: The journal block.
1334 static void add_queued_recovery_entries(struct recovery_journal_block *block)  in add_queued_recovery_entries()  argument
1336 	while (vdo_waitq_has_waiters(&block->entry_waiters)) {  in add_queued_recovery_entries()
1338 			vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters));  in add_queued_recovery_entries()
1343 		if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR)  in add_queued_recovery_entries()
1344 			set_active_sector(block,  in add_queued_recovery_entries()
1345 					  (char *) block->sector + VDO_SECTOR_SIZE);  in add_queued_recovery_entries()
1348 		packed_entry = &block->sector->entries[block->sector->entry_count++];  in add_queued_recovery_entries()
1362 		data_vio->recovery_sequence_number = block->sequence_number;  in add_queued_recovery_entries()
1365 		vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);  in add_queued_recovery_entries()
1370  * write_block() - Issue a block for writing.
1371  * @waiter: The recovery journal block to write.
1378 	struct recovery_journal_block *block =  in write_block()  local
1380 	struct recovery_journal *journal = block->journal;  in write_block()
1381 	struct packed_journal_header *header = get_block_header(block);  in write_block()
1383 	if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) ||  in write_block()
1387 	block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters);  in write_block()
1388 	add_queued_recovery_entries(block);  in write_block()
1392 	journal->events.entries.written += block->entries_in_commit;  in write_block()
1396 	header->entry_count = __cpu_to_le16(block->entry_count);  in write_block()
1398 	block->committing = true;  in write_block()
1403 	 * block itself is stable before allowing overwrites of the lbn's previous data.  in write_block()
1405 	vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number,  in write_block()
1419 	 * We call this function after adding entries to the journal and after finishing a block  in write_blocks()
1424 	 * no outstanding writes and some unwritten entries, we must issue a block, even if it's  in write_blocks()
1425 	 * the active block and it isn't full.  in write_blocks()
1434 	 * Do we need to write the active block? Only if we have no outstanding writes, even after  in write_blocks()
1447  *	      sequence number of the journal block in which the entry was
1523 	 * block is referenced.  in reap_recovery_journal()
1548 	 * If the block map head will advance, we must flush any block map page modified by the  in reap_recovery_journal()
1557  * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block
1560  * @sequence_number: The journal sequence number of the referenced block.
1599  *                                    journal block.
1601  * @sequence_number: The journal sequence number of the referenced block.
1721  * dump_recovery_block() - Dump the contents of the recovery block to the log.
1722  * @block: The block to dump.
1724 static void dump_recovery_block(const struct recovery_journal_block *block)  in dump_recovery_block()  argument
1727 		     (unsigned long long) block->sequence_number, block->entry_count,  in dump_recovery_block()
1728 		     (block->committing ? "committing" : "waiting"),  in dump_recovery_block()
1729 		     vdo_waitq_num_waiters(&block->entry_waiters),  in dump_recovery_block()
1730 		     vdo_waitq_num_waiters(&block->commit_waiters));  in dump_recovery_block()
1740 	const struct recovery_journal_block *block;  in vdo_dump_recovery_journal_statistics()  local
1764 	list_for_each_entry(block, &journal->active_tail_blocks, list_node)  in vdo_dump_recovery_journal_statistics()
1765 		dump_recovery_block(block);  in vdo_dump_recovery_journal_statistics()