1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2023, Klara Inc.
15 */
16
17 #include <sys/zfs_context.h>
18 #include <sys/spa.h>
19 #include <sys/ddt.h>
20 #include <sys/dmu_tx.h>
21 #include <sys/dmu.h>
22 #include <sys/ddt_impl.h>
23 #include <sys/dnode.h>
24 #include <sys/dbuf.h>
25 #include <sys/zap.h>
26 #include <sys/zio_checksum.h>
27
28 /*
29 * No more than this many txgs before swapping logs.
30 */
31 uint_t zfs_dedup_log_txg_max = 8;
32
33 /*
34 * Max memory for the log AVL trees. If zfs_dedup_log_mem_max is zero at module
35 * load, it will be set to zfs_dedup_log_mem_max_percent% of total memory.
36 */
37 uint64_t zfs_dedup_log_mem_max = 0;
38 uint_t zfs_dedup_log_mem_max_percent = 1;
39
40
41 static kmem_cache_t *ddt_log_entry_flat_cache;
42 static kmem_cache_t *ddt_log_entry_trad_cache;
43
44 #define DDT_LOG_ENTRY_FLAT_SIZE \
45 (sizeof (ddt_log_entry_t) + DDT_FLAT_PHYS_SIZE)
46 #define DDT_LOG_ENTRY_TRAD_SIZE \
47 (sizeof (ddt_log_entry_t) + DDT_TRAD_PHYS_SIZE)
48
49 #define DDT_LOG_ENTRY_SIZE(ddt) \
50 _DDT_PHYS_SWITCH(ddt, DDT_LOG_ENTRY_FLAT_SIZE, DDT_LOG_ENTRY_TRAD_SIZE)
51
52 void
ddt_log_init(void)53 ddt_log_init(void)
54 {
55 ddt_log_entry_flat_cache = kmem_cache_create("ddt_log_entry_flat_cache",
56 DDT_LOG_ENTRY_FLAT_SIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);
57 ddt_log_entry_trad_cache = kmem_cache_create("ddt_log_entry_trad_cache",
58 DDT_LOG_ENTRY_TRAD_SIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);
59
60 /*
61 * Max memory for log AVL entries. At least 1M, because we need
62 * something (that's ~3800 entries per tree). They can say 100% if they
63 * want; it just means they're at the mercy of the the txg flush limit.
64 */
65 if (zfs_dedup_log_mem_max == 0) {
66 zfs_dedup_log_mem_max_percent =
67 MIN(zfs_dedup_log_mem_max_percent, 100);
68 zfs_dedup_log_mem_max = (physmem * PAGESIZE) *
69 zfs_dedup_log_mem_max_percent / 100;
70 }
71 zfs_dedup_log_mem_max = MAX(zfs_dedup_log_mem_max, 1*1024*1024);
72 }
73
74 void
ddt_log_fini(void)75 ddt_log_fini(void)
76 {
77 kmem_cache_destroy(ddt_log_entry_trad_cache);
78 kmem_cache_destroy(ddt_log_entry_flat_cache);
79 }
80
81 static void
ddt_log_name(ddt_t * ddt,char * name,uint_t n)82 ddt_log_name(ddt_t *ddt, char *name, uint_t n)
83 {
84 snprintf(name, DDT_NAMELEN, DMU_POOL_DDT_LOG,
85 zio_checksum_table[ddt->ddt_checksum].ci_name, n);
86 }
87
88 static void
ddt_log_update_header(ddt_t * ddt,ddt_log_t * ddl,dmu_tx_t * tx)89 ddt_log_update_header(ddt_t *ddt, ddt_log_t *ddl, dmu_tx_t *tx)
90 {
91 dmu_buf_t *db;
92 VERIFY0(dmu_bonus_hold(ddt->ddt_os, ddl->ddl_object, FTAG, &db));
93 dmu_buf_will_dirty(db, tx);
94
95 ddt_log_header_t *hdr = (ddt_log_header_t *)db->db_data;
96 DLH_SET_VERSION(hdr, 1);
97 DLH_SET_FLAGS(hdr, ddl->ddl_flags);
98 hdr->dlh_length = ddl->ddl_length;
99 hdr->dlh_first_txg = ddl->ddl_first_txg;
100 hdr->dlh_checkpoint = ddl->ddl_checkpoint;
101
102 dmu_buf_rele(db, FTAG);
103 }
104
105 static void
ddt_log_create_one(ddt_t * ddt,ddt_log_t * ddl,uint_t n,dmu_tx_t * tx)106 ddt_log_create_one(ddt_t *ddt, ddt_log_t *ddl, uint_t n, dmu_tx_t *tx)
107 {
108 ASSERT3U(ddt->ddt_dir_object, >, 0);
109 ASSERT0(ddl->ddl_object);
110
111 char name[DDT_NAMELEN];
112 ddt_log_name(ddt, name, n);
113
114 ddl->ddl_object = dmu_object_alloc(ddt->ddt_os,
115 DMU_OTN_UINT64_METADATA, SPA_OLD_MAXBLOCKSIZE,
116 DMU_OTN_UINT64_METADATA, sizeof (ddt_log_header_t), tx);
117 VERIFY0(zap_add(ddt->ddt_os, ddt->ddt_dir_object, name,
118 sizeof (uint64_t), 1, &ddl->ddl_object, tx));
119 ddl->ddl_length = 0;
120 ddl->ddl_first_txg = tx->tx_txg;
121 ddt_log_update_header(ddt, ddl, tx);
122 }
123
124 static void
ddt_log_create(ddt_t * ddt,dmu_tx_t * tx)125 ddt_log_create(ddt_t *ddt, dmu_tx_t *tx)
126 {
127 ddt_log_create_one(ddt, ddt->ddt_log_active, 0, tx);
128 ddt_log_create_one(ddt, ddt->ddt_log_flushing, 1, tx);
129 }
130
131 static void
ddt_log_destroy_one(ddt_t * ddt,ddt_log_t * ddl,uint_t n,dmu_tx_t * tx)132 ddt_log_destroy_one(ddt_t *ddt, ddt_log_t *ddl, uint_t n, dmu_tx_t *tx)
133 {
134 ASSERT3U(ddt->ddt_dir_object, >, 0);
135
136 if (ddl->ddl_object == 0)
137 return;
138
139 ASSERT0(ddl->ddl_length);
140
141 char name[DDT_NAMELEN];
142 ddt_log_name(ddt, name, n);
143
144 VERIFY0(zap_remove(ddt->ddt_os, ddt->ddt_dir_object, name, tx));
145 VERIFY0(dmu_object_free(ddt->ddt_os, ddl->ddl_object, tx));
146
147 ddl->ddl_object = 0;
148 }
149
150 void
ddt_log_destroy(ddt_t * ddt,dmu_tx_t * tx)151 ddt_log_destroy(ddt_t *ddt, dmu_tx_t *tx)
152 {
153 ddt_log_destroy_one(ddt, ddt->ddt_log_active, 0, tx);
154 ddt_log_destroy_one(ddt, ddt->ddt_log_flushing, 1, tx);
155 }
156
157 static void
ddt_log_update_stats(ddt_t * ddt)158 ddt_log_update_stats(ddt_t *ddt)
159 {
160 /*
161 * Log object stats. We count the number of live entries in the log
162 * tree, even if there are more than on disk, and even if the same
163 * entry is on both append and flush trees, because that's more what
164 * the user expects to see. This does mean the on-disk size is not
165 * really correlated with the number of entries, but I don't think
166 * that's reasonable to expect anyway.
167 */
168 dmu_object_info_t doi;
169 uint64_t nblocks = 0;
170 if (dmu_object_info(ddt->ddt_os, ddt->ddt_log_active->ddl_object,
171 &doi) == 0)
172 nblocks += doi.doi_physical_blocks_512;
173 if (dmu_object_info(ddt->ddt_os, ddt->ddt_log_flushing->ddl_object,
174 &doi) == 0)
175 nblocks += doi.doi_physical_blocks_512;
176
177 ddt_object_t *ddo = &ddt->ddt_log_stats;
178 ddo->ddo_count =
179 avl_numnodes(&ddt->ddt_log_active->ddl_tree) +
180 avl_numnodes(&ddt->ddt_log_flushing->ddl_tree);
181 ddo->ddo_mspace = ddo->ddo_count * DDT_LOG_ENTRY_SIZE(ddt);
182 ddo->ddo_dspace = nblocks << 9;
183 }
184
185 void
ddt_log_begin(ddt_t * ddt,size_t nentries,dmu_tx_t * tx,ddt_log_update_t * dlu)186 ddt_log_begin(ddt_t *ddt, size_t nentries, dmu_tx_t *tx, ddt_log_update_t *dlu)
187 {
188 ASSERT3U(nentries, >, 0);
189 ASSERT0P(dlu->dlu_dbp);
190
191 if (ddt->ddt_log_active->ddl_object == 0)
192 ddt_log_create(ddt, tx);
193
194 /*
195 * We want to store as many entries as we can in a block, but never
196 * split an entry across block boundaries.
197 */
198 size_t reclen = P2ALIGN_TYPED(
199 sizeof (ddt_log_record_t) + sizeof (ddt_log_record_entry_t) +
200 DDT_PHYS_SIZE(ddt), sizeof (uint64_t), size_t);
201 ASSERT3U(reclen, <=, UINT16_MAX);
202 dlu->dlu_reclen = reclen;
203
204 VERIFY0(dnode_hold(ddt->ddt_os, ddt->ddt_log_active->ddl_object, dlu,
205 &dlu->dlu_dn));
206 dnode_set_storage_type(dlu->dlu_dn, DMU_OT_DDT_ZAP);
207
208 uint64_t nblocks = howmany(nentries,
209 dlu->dlu_dn->dn_datablksz / dlu->dlu_reclen);
210 uint64_t offset = ddt->ddt_log_active->ddl_length;
211 uint64_t length = nblocks * dlu->dlu_dn->dn_datablksz;
212
213 VERIFY0(dmu_buf_hold_array_by_dnode(dlu->dlu_dn, offset, length,
214 B_FALSE, dlu, &dlu->dlu_ndbp, &dlu->dlu_dbp,
215 DMU_READ_NO_PREFETCH | DMU_UNCACHEDIO));
216
217 dlu->dlu_tx = tx;
218 dlu->dlu_block = dlu->dlu_offset = 0;
219 }
220
221 static ddt_log_entry_t *
ddt_log_alloc_entry(ddt_t * ddt)222 ddt_log_alloc_entry(ddt_t *ddt)
223 {
224 ddt_log_entry_t *ddle;
225
226 if (ddt->ddt_flags & DDT_FLAG_FLAT) {
227 ddle = kmem_cache_alloc(ddt_log_entry_flat_cache, KM_SLEEP);
228 memset(ddle, 0, DDT_LOG_ENTRY_FLAT_SIZE);
229 } else {
230 ddle = kmem_cache_alloc(ddt_log_entry_trad_cache, KM_SLEEP);
231 memset(ddle, 0, DDT_LOG_ENTRY_TRAD_SIZE);
232 }
233
234 return (ddle);
235 }
236
237 static void
ddt_log_free_entry(ddt_t * ddt,ddt_log_entry_t * ddle)238 ddt_log_free_entry(ddt_t *ddt, ddt_log_entry_t *ddle)
239 {
240 kmem_cache_free(ddt->ddt_flags & DDT_FLAG_FLAT ?
241 ddt_log_entry_flat_cache : ddt_log_entry_trad_cache, ddle);
242 }
243
244 static void
ddt_log_update_entry(ddt_t * ddt,ddt_log_t * ddl,ddt_lightweight_entry_t * ddlwe,boolean_t hist)245 ddt_log_update_entry(ddt_t *ddt, ddt_log_t *ddl, ddt_lightweight_entry_t *ddlwe,
246 boolean_t hist)
247 {
248 /* Create the log tree entry from a live or stored entry */
249 avl_index_t where;
250 ddt_log_entry_t *ddle =
251 avl_find(&ddl->ddl_tree, &ddlwe->ddlwe_key, &where);
252 if (ddle == NULL) {
253 ddle = ddt_log_alloc_entry(ddt);
254 ddle->ddle_key = ddlwe->ddlwe_key;
255 avl_insert(&ddl->ddl_tree, ddle, where);
256 } else if (hist) {
257 ddt_lightweight_entry_t oddlwe;
258 DDT_LOG_ENTRY_TO_LIGHTWEIGHT(ddt, ddle, &oddlwe);
259 ddt_histogram_sub_entry(ddt, &ddt->ddt_log_histogram, &oddlwe);
260 }
261 if (hist)
262 ddt_histogram_add_entry(ddt, &ddt->ddt_log_histogram, ddlwe);
263 ddle->ddle_type = ddlwe->ddlwe_type;
264 ddle->ddle_class = ddlwe->ddlwe_class;
265 memcpy(ddle->ddle_phys, &ddlwe->ddlwe_phys, DDT_PHYS_SIZE(ddt));
266 }
267
268 void
ddt_log_entry(ddt_t * ddt,ddt_lightweight_entry_t * ddlwe,ddt_log_update_t * dlu)269 ddt_log_entry(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, ddt_log_update_t *dlu)
270 {
271 ASSERT3P(dlu->dlu_dbp, !=, NULL);
272
273 ddt_log_update_entry(ddt, ddt->ddt_log_active, ddlwe, B_TRUE);
274
275 /* Get our block */
276 ASSERT3U(dlu->dlu_block, <, dlu->dlu_ndbp);
277 dmu_buf_t *db = dlu->dlu_dbp[dlu->dlu_block];
278
279 /*
280 * If this would take us past the end of the block, finish it and
281 * move to the next one.
282 */
283 if (db->db_size < (dlu->dlu_offset + dlu->dlu_reclen)) {
284 ASSERT3U(dlu->dlu_offset, >, 0);
285 dmu_buf_fill_done(db, dlu->dlu_tx, B_FALSE);
286 dlu->dlu_block++;
287 dlu->dlu_offset = 0;
288 ASSERT3U(dlu->dlu_block, <, dlu->dlu_ndbp);
289 db = dlu->dlu_dbp[dlu->dlu_block];
290 }
291
292 /*
293 * If this is the first time touching the block, inform the DMU that
294 * we will fill it, and zero it out.
295 */
296 if (dlu->dlu_offset == 0) {
297 dmu_buf_will_fill_flags(db, dlu->dlu_tx, B_FALSE,
298 DMU_UNCACHEDIO);
299 memset(db->db_data, 0, db->db_size);
300 }
301
302 /* Create the log record directly in the buffer */
303 ddt_log_record_t *dlr = (db->db_data + dlu->dlu_offset);
304 DLR_SET_TYPE(dlr, DLR_ENTRY);
305 DLR_SET_RECLEN(dlr, dlu->dlu_reclen);
306 DLR_SET_ENTRY_TYPE(dlr, ddlwe->ddlwe_type);
307 DLR_SET_ENTRY_CLASS(dlr, ddlwe->ddlwe_class);
308
309 ddt_log_record_entry_t *dlre =
310 (ddt_log_record_entry_t *)&dlr->dlr_payload;
311 dlre->dlre_key = ddlwe->ddlwe_key;
312 memcpy(dlre->dlre_phys, &ddlwe->ddlwe_phys, DDT_PHYS_SIZE(ddt));
313
314 /* Advance offset for next record. */
315 dlu->dlu_offset += dlu->dlu_reclen;
316 }
317
318 void
ddt_log_commit(ddt_t * ddt,ddt_log_update_t * dlu)319 ddt_log_commit(ddt_t *ddt, ddt_log_update_t *dlu)
320 {
321 ASSERT3P(dlu->dlu_dbp, !=, NULL);
322 ASSERT3U(dlu->dlu_block+1, ==, dlu->dlu_ndbp);
323 ASSERT3U(dlu->dlu_offset, >, 0);
324
325 /*
326 * Close out the last block. Whatever we haven't used will be zeroed,
327 * which matches DLR_INVALID, so we can detect this during load.
328 */
329 dmu_buf_fill_done(dlu->dlu_dbp[dlu->dlu_block], dlu->dlu_tx, B_FALSE);
330
331 dmu_buf_rele_array(dlu->dlu_dbp, dlu->dlu_ndbp, dlu);
332
333 ddt->ddt_log_active->ddl_length +=
334 dlu->dlu_ndbp * (uint64_t)dlu->dlu_dn->dn_datablksz;
335 dnode_rele(dlu->dlu_dn, dlu);
336
337 ddt_log_update_header(ddt, ddt->ddt_log_active, dlu->dlu_tx);
338
339 memset(dlu, 0, sizeof (ddt_log_update_t));
340
341 ddt_log_update_stats(ddt);
342 }
343
344 boolean_t
ddt_log_take_first(ddt_t * ddt,ddt_log_t * ddl,ddt_lightweight_entry_t * ddlwe)345 ddt_log_take_first(ddt_t *ddt, ddt_log_t *ddl, ddt_lightweight_entry_t *ddlwe)
346 {
347 ddt_log_entry_t *ddle = avl_first(&ddl->ddl_tree);
348 if (ddle == NULL)
349 return (B_FALSE);
350
351 DDT_LOG_ENTRY_TO_LIGHTWEIGHT(ddt, ddle, ddlwe);
352
353 ddt_histogram_sub_entry(ddt, &ddt->ddt_log_histogram, ddlwe);
354
355 avl_remove(&ddl->ddl_tree, ddle);
356 ddt_log_free_entry(ddt, ddle);
357
358 return (B_TRUE);
359 }
360
361 boolean_t
ddt_log_remove_key(ddt_t * ddt,ddt_log_t * ddl,const ddt_key_t * ddk)362 ddt_log_remove_key(ddt_t *ddt, ddt_log_t *ddl, const ddt_key_t *ddk)
363 {
364 ddt_log_entry_t *ddle = avl_find(&ddl->ddl_tree, ddk, NULL);
365 if (ddle == NULL)
366 return (B_FALSE);
367
368 ddt_lightweight_entry_t ddlwe;
369 DDT_LOG_ENTRY_TO_LIGHTWEIGHT(ddt, ddle, &ddlwe);
370 ddt_histogram_sub_entry(ddt, &ddt->ddt_log_histogram, &ddlwe);
371
372 avl_remove(&ddl->ddl_tree, ddle);
373 ddt_log_free_entry(ddt, ddle);
374
375 return (B_TRUE);
376 }
377
378 boolean_t
ddt_log_find_key(ddt_t * ddt,const ddt_key_t * ddk,ddt_lightweight_entry_t * ddlwe,boolean_t * from_flushing)379 ddt_log_find_key(ddt_t *ddt, const ddt_key_t *ddk,
380 ddt_lightweight_entry_t *ddlwe, boolean_t *from_flushing)
381 {
382 ddt_log_entry_t *ddle = avl_find(&ddt->ddt_log_active->ddl_tree,
383 ddk, NULL);
384 if (ddle) {
385 if (from_flushing)
386 *from_flushing = B_FALSE;
387 } else {
388 ddle = avl_find(&ddt->ddt_log_flushing->ddl_tree, ddk, NULL);
389 if (!ddle)
390 return (B_FALSE);
391 if (from_flushing)
392 *from_flushing = B_TRUE;
393 }
394 if (ddlwe)
395 DDT_LOG_ENTRY_TO_LIGHTWEIGHT(ddt, ddle, ddlwe);
396 return (B_TRUE);
397 }
398
399 void
ddt_log_checkpoint(ddt_t * ddt,ddt_lightweight_entry_t * ddlwe,dmu_tx_t * tx)400 ddt_log_checkpoint(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, dmu_tx_t *tx)
401 {
402 ddt_log_t *ddl = ddt->ddt_log_flushing;
403
404 ASSERT3U(ddl->ddl_object, !=, 0);
405
406 #ifdef ZFS_DEBUG
407 /*
408 * There should not be any entries on the log tree before the given
409 * checkpoint. Assert that this is the case.
410 */
411 ddt_log_entry_t *ddle = avl_first(&ddl->ddl_tree);
412 if (ddle != NULL)
413 VERIFY3U(ddt_key_compare(&ddle->ddle_key, &ddlwe->ddlwe_key),
414 >, 0);
415 #endif
416
417 ddl->ddl_flags |= DDL_FLAG_CHECKPOINT;
418 ddl->ddl_checkpoint = ddlwe->ddlwe_key;
419 ddt_log_update_header(ddt, ddl, tx);
420
421 ddt_log_update_stats(ddt);
422 }
423
424 void
ddt_log_truncate(ddt_t * ddt,dmu_tx_t * tx)425 ddt_log_truncate(ddt_t *ddt, dmu_tx_t *tx)
426 {
427 ddt_log_t *ddl = ddt->ddt_log_flushing;
428
429 if (ddl->ddl_object == 0)
430 return;
431
432 ASSERT(avl_is_empty(&ddl->ddl_tree));
433
434 /* Eject the entire object */
435 dmu_free_range(ddt->ddt_os, ddl->ddl_object, 0, DMU_OBJECT_END, tx);
436
437 ddl->ddl_length = 0;
438 ddl->ddl_flags &= ~DDL_FLAG_CHECKPOINT;
439 memset(&ddl->ddl_checkpoint, 0, sizeof (ddt_key_t));
440 ddt_log_update_header(ddt, ddl, tx);
441
442 ddt_log_update_stats(ddt);
443 }
444
445 boolean_t
ddt_log_swap(ddt_t * ddt,dmu_tx_t * tx)446 ddt_log_swap(ddt_t *ddt, dmu_tx_t *tx)
447 {
448 /* Swap the logs. The old flushing one must be empty */
449 VERIFY(avl_is_empty(&ddt->ddt_log_flushing->ddl_tree));
450
451 /*
452 * If there are still blocks on the flushing log, truncate it first.
453 * This can happen if there were entries on the flushing log that were
454 * removed in memory via ddt_lookup(); their vestigal remains are
455 * on disk.
456 */
457 if (ddt->ddt_log_flushing->ddl_length > 0)
458 ddt_log_truncate(ddt, tx);
459
460 /*
461 * Swap policy. We swap the logs (and so begin flushing) when the
462 * active tree grows too large, or when we haven't swapped it in
463 * some amount of time, or if something has requested the logs be
464 * flushed ASAP (see ddt_walk_init()).
465 */
466
467 /*
468 * The log tree is too large if the memory usage of its entries is over
469 * half of the memory limit. This effectively gives each log tree half
470 * the available memory.
471 */
472 const boolean_t too_large =
473 (avl_numnodes(&ddt->ddt_log_active->ddl_tree) *
474 DDT_LOG_ENTRY_SIZE(ddt)) >= (zfs_dedup_log_mem_max >> 1);
475
476 const boolean_t too_old =
477 tx->tx_txg >=
478 (ddt->ddt_log_active->ddl_first_txg +
479 MAX(1, zfs_dedup_log_txg_max));
480
481 const boolean_t force =
482 ddt->ddt_log_active->ddl_first_txg <= ddt->ddt_flush_force_txg;
483
484 if (!(too_large || too_old || force))
485 return (B_FALSE);
486
487 ddt_log_t *swap = ddt->ddt_log_active;
488 ddt->ddt_log_active = ddt->ddt_log_flushing;
489 ddt->ddt_log_flushing = swap;
490
491 ASSERT(ddt->ddt_log_active->ddl_flags & DDL_FLAG_FLUSHING);
492 ddt->ddt_log_active->ddl_flags &=
493 ~(DDL_FLAG_FLUSHING | DDL_FLAG_CHECKPOINT);
494
495 ASSERT(!(ddt->ddt_log_flushing->ddl_flags & DDL_FLAG_FLUSHING));
496 ddt->ddt_log_flushing->ddl_flags |= DDL_FLAG_FLUSHING;
497
498 ddt->ddt_log_active->ddl_first_txg = tx->tx_txg;
499
500 ddt_log_update_header(ddt, ddt->ddt_log_active, tx);
501 ddt_log_update_header(ddt, ddt->ddt_log_flushing, tx);
502
503 ddt_log_update_stats(ddt);
504
505 return (B_TRUE);
506 }
507
508 static inline void
ddt_log_load_entry(ddt_t * ddt,ddt_log_t * ddl,ddt_log_record_t * dlr,const ddt_key_t * checkpoint)509 ddt_log_load_entry(ddt_t *ddt, ddt_log_t *ddl, ddt_log_record_t *dlr,
510 const ddt_key_t *checkpoint)
511 {
512 ASSERT3U(DLR_GET_TYPE(dlr), ==, DLR_ENTRY);
513
514 ddt_log_record_entry_t *dlre =
515 (ddt_log_record_entry_t *)dlr->dlr_payload;
516 if (checkpoint != NULL &&
517 ddt_key_compare(&dlre->dlre_key, checkpoint) <= 0) {
518 /* Skip pre-checkpoint entries; they're already flushed. */
519 return;
520 }
521
522 ddt_lightweight_entry_t ddlwe;
523 ddlwe.ddlwe_type = DLR_GET_ENTRY_TYPE(dlr);
524 ddlwe.ddlwe_class = DLR_GET_ENTRY_CLASS(dlr);
525
526 ddlwe.ddlwe_key = dlre->dlre_key;
527 memcpy(&ddlwe.ddlwe_phys, dlre->dlre_phys, DDT_PHYS_SIZE(ddt));
528
529 ddt_log_update_entry(ddt, ddl, &ddlwe, B_FALSE);
530 }
531
532 static void
ddt_log_empty(ddt_t * ddt,ddt_log_t * ddl)533 ddt_log_empty(ddt_t *ddt, ddt_log_t *ddl)
534 {
535 void *cookie = NULL;
536 ddt_log_entry_t *ddle;
537 IMPLY(ddt->ddt_version == UINT64_MAX, avl_is_empty(&ddl->ddl_tree));
538 while ((ddle =
539 avl_destroy_nodes(&ddl->ddl_tree, &cookie)) != NULL) {
540 ddt_log_free_entry(ddt, ddle);
541 }
542 ASSERT(avl_is_empty(&ddl->ddl_tree));
543 }
544
545 static int
ddt_log_load_one(ddt_t * ddt,uint_t n)546 ddt_log_load_one(ddt_t *ddt, uint_t n)
547 {
548 ASSERT3U(n, <, 2);
549
550 ddt_log_t *ddl = &ddt->ddt_log[n];
551
552 char name[DDT_NAMELEN];
553 ddt_log_name(ddt, name, n);
554
555 uint64_t obj;
556 int err = zap_lookup(ddt->ddt_os, ddt->ddt_dir_object, name,
557 sizeof (uint64_t), 1, &obj);
558 if (err == ENOENT)
559 return (0);
560 if (err != 0)
561 return (err);
562
563 dnode_t *dn;
564 err = dnode_hold(ddt->ddt_os, obj, FTAG, &dn);
565 if (err != 0)
566 return (err);
567
568 ddt_log_header_t hdr;
569 dmu_buf_t *db;
570 err = dmu_bonus_hold_by_dnode(dn, FTAG, &db, DMU_READ_NO_PREFETCH);
571 if (err != 0) {
572 dnode_rele(dn, FTAG);
573 return (err);
574 }
575 memcpy(&hdr, db->db_data, sizeof (ddt_log_header_t));
576 dmu_buf_rele(db, FTAG);
577
578 if (DLH_GET_VERSION(&hdr) != 1) {
579 dnode_rele(dn, FTAG);
580 zfs_dbgmsg("ddt_log_load: spa=%s ddt_log=%s "
581 "unknown version=%llu", spa_name(ddt->ddt_spa), name,
582 (u_longlong_t)DLH_GET_VERSION(&hdr));
583 return (SET_ERROR(EINVAL));
584 }
585
586 ddt_key_t *checkpoint = NULL;
587 if (DLH_GET_FLAGS(&hdr) & DDL_FLAG_CHECKPOINT) {
588 /*
589 * If the log has a checkpoint, then we can ignore any entries
590 * that have already been flushed.
591 */
592 ASSERT(DLH_GET_FLAGS(&hdr) & DDL_FLAG_FLUSHING);
593 checkpoint = &hdr.dlh_checkpoint;
594 }
595
596 if (hdr.dlh_length > 0) {
597 dmu_prefetch_stream_by_dnode(dn, 0, hdr.dlh_length, B_FALSE);
598
599 for (uint64_t offset = 0; offset < hdr.dlh_length;
600 offset += dn->dn_datablksz) {
601 err = dmu_buf_hold_by_dnode(dn, offset, FTAG, &db,
602 DMU_READ_PREFETCH | DMU_UNCACHEDIO);
603 if (err != 0) {
604 dnode_rele(dn, FTAG);
605 ddt_log_empty(ddt, ddl);
606 return (err);
607 }
608
609 uint64_t boffset = 0;
610 while (boffset < db->db_size) {
611 ddt_log_record_t *dlr =
612 (ddt_log_record_t *)(db->db_data + boffset);
613
614 /* Partially-filled block, skip the rest */
615 if (DLR_GET_TYPE(dlr) == DLR_INVALID)
616 break;
617
618 switch (DLR_GET_TYPE(dlr)) {
619 case DLR_ENTRY:
620 ddt_log_load_entry(ddt, ddl, dlr,
621 checkpoint);
622 break;
623
624 default:
625 dmu_buf_rele(db, FTAG);
626 dnode_rele(dn, FTAG);
627 ddt_log_empty(ddt, ddl);
628 return (SET_ERROR(EINVAL));
629 }
630
631 boffset += DLR_GET_RECLEN(dlr);
632 }
633
634 dmu_buf_rele(db, FTAG);
635 }
636 }
637
638 dnode_rele(dn, FTAG);
639
640 ddl->ddl_object = obj;
641 ddl->ddl_flags = DLH_GET_FLAGS(&hdr);
642 ddl->ddl_length = hdr.dlh_length;
643 ddl->ddl_first_txg = hdr.dlh_first_txg;
644
645 if (ddl->ddl_flags & DDL_FLAG_FLUSHING)
646 ddt->ddt_log_flushing = ddl;
647 else
648 ddt->ddt_log_active = ddl;
649
650 return (0);
651 }
652
653 int
ddt_log_load(ddt_t * ddt)654 ddt_log_load(ddt_t *ddt)
655 {
656 int err;
657
658 if (spa_load_state(ddt->ddt_spa) == SPA_LOAD_TRYIMPORT) {
659 /*
660 * The DDT is going to be freed again in a moment, so there's
661 * no point loading the log; it'll just slow down import.
662 */
663 return (0);
664 }
665
666 ASSERT0(ddt->ddt_log[0].ddl_object);
667 ASSERT0(ddt->ddt_log[1].ddl_object);
668 if (ddt->ddt_dir_object == 0) {
669 /*
670 * If we're configured but the containing dir doesn't exist
671 * yet, then the log object can't possibly exist either.
672 */
673 ASSERT3U(ddt->ddt_version, !=, UINT64_MAX);
674 return (SET_ERROR(ENOENT));
675 }
676
677 if ((err = ddt_log_load_one(ddt, 0)) != 0)
678 return (err);
679 if ((err = ddt_log_load_one(ddt, 1)) != 0)
680 return (err);
681
682 VERIFY3P(ddt->ddt_log_active, !=, ddt->ddt_log_flushing);
683 VERIFY(!(ddt->ddt_log_active->ddl_flags & DDL_FLAG_FLUSHING));
684 VERIFY(!(ddt->ddt_log_active->ddl_flags & DDL_FLAG_CHECKPOINT));
685 VERIFY(ddt->ddt_log_flushing->ddl_flags & DDL_FLAG_FLUSHING);
686
687 /*
688 * We have two finalisation tasks:
689 *
690 * - rebuild the histogram. We do this at the end rather than while
691 * we're loading so we don't need to uncount and recount entries that
692 * appear multiple times in the log.
693 *
694 * - remove entries from the flushing tree that are on both trees. This
695 * happens when ddt_lookup() rehydrates an entry from the flushing
696 * tree, as ddt_log_take_key() removes the entry from the in-memory
697 * tree but doesn't remove it from disk.
698 */
699
700 /*
701 * We don't technically need a config lock here, since there shouldn't
702 * be pool config changes during DDT load. dva_get_dsize_sync() via
703 * ddt_stat_generate() is expecting it though, and it won't hurt
704 * anything, so we take it.
705 */
706 spa_config_enter(ddt->ddt_spa, SCL_STATE, FTAG, RW_READER);
707
708 avl_tree_t *al = &ddt->ddt_log_active->ddl_tree;
709 avl_tree_t *fl = &ddt->ddt_log_flushing->ddl_tree;
710 ddt_log_entry_t *ae = avl_first(al);
711 ddt_log_entry_t *fe = avl_first(fl);
712 while (ae != NULL || fe != NULL) {
713 ddt_log_entry_t *ddle;
714 if (ae == NULL) {
715 /* active exhausted, take flushing */
716 ddle = fe;
717 fe = AVL_NEXT(fl, fe);
718 } else if (fe == NULL) {
719 /* flushing exuhausted, take active */
720 ddle = ae;
721 ae = AVL_NEXT(al, ae);
722 } else {
723 /* compare active and flushing */
724 int c = ddt_key_compare(&ae->ddle_key, &fe->ddle_key);
725 if (c < 0) {
726 /* active behind, take and advance */
727 ddle = ae;
728 ae = AVL_NEXT(al, ae);
729 } else if (c > 0) {
730 /* flushing behind, take and advance */
731 ddle = fe;
732 fe = AVL_NEXT(fl, fe);
733 } else {
734 /* match. remove from flushing, take active */
735 ddle = fe;
736 fe = AVL_NEXT(fl, fe);
737 avl_remove(fl, ddle);
738 ddt_log_free_entry(ddt, ddle);
739 ddle = ae;
740 ae = AVL_NEXT(al, ae);
741 }
742 }
743
744 ddt_lightweight_entry_t ddlwe;
745 DDT_LOG_ENTRY_TO_LIGHTWEIGHT(ddt, ddle, &ddlwe);
746 ddt_histogram_add_entry(ddt, &ddt->ddt_log_histogram, &ddlwe);
747 }
748
749 spa_config_exit(ddt->ddt_spa, SCL_STATE, FTAG);
750
751 ddt_log_update_stats(ddt);
752
753 return (0);
754 }
755
756 void
ddt_log_alloc(ddt_t * ddt)757 ddt_log_alloc(ddt_t *ddt)
758 {
759 ASSERT0P(ddt->ddt_log_active);
760 ASSERT0P(ddt->ddt_log_flushing);
761
762 avl_create(&ddt->ddt_log[0].ddl_tree, ddt_key_compare,
763 sizeof (ddt_log_entry_t), offsetof(ddt_log_entry_t, ddle_node));
764 avl_create(&ddt->ddt_log[1].ddl_tree, ddt_key_compare,
765 sizeof (ddt_log_entry_t), offsetof(ddt_log_entry_t, ddle_node));
766 ddt->ddt_log_active = &ddt->ddt_log[0];
767 ddt->ddt_log_flushing = &ddt->ddt_log[1];
768 ddt->ddt_log_flushing->ddl_flags |= DDL_FLAG_FLUSHING;
769 }
770
771 void
ddt_log_free(ddt_t * ddt)772 ddt_log_free(ddt_t *ddt)
773 {
774 ddt_log_empty(ddt, &ddt->ddt_log[0]);
775 ddt_log_empty(ddt, &ddt->ddt_log[1]);
776 avl_destroy(&ddt->ddt_log[0].ddl_tree);
777 avl_destroy(&ddt->ddt_log[1].ddl_tree);
778 }
779
780 ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, log_txg_max, UINT, ZMOD_RW,
781 "Max transactions before starting to flush dedup logs");
782
783 ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, log_mem_max, U64, ZMOD_RD,
784 "Max memory for dedup logs");
785
786 ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, log_mem_max_percent, UINT, ZMOD_RD,
787 "Max memory for dedup logs, as % of total memory");
788