1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27 * Copyright 2017 Nexenta Systems, Inc.
28 * Copyright (c) 2024, Klara, Inc.
29 * Copyright (c) 2026, TrueNAS.
30 */
31
32 #include <sys/zfs_context.h>
33 #include <sys/dmu.h>
34 #include <sys/dnode.h>
35 #include <sys/dsl_dataset.h>
36 #include <sys/zap.h>
37 #include <sys/zap_impl.h>
38
39 static kmem_cache_t *zap_name_cache;
40 static kmem_cache_t *zap_attr_cache;
41 static kmem_cache_t *zap_name_long_cache;
42 static kmem_cache_t *zap_attr_long_cache;
43
44 /* Setup/teardown caches. Part of the public interface in zap.h. */
45 void
zap_init(void)46 zap_init(void)
47 {
48 zap_name_cache = kmem_cache_create("zap_name",
49 sizeof (zap_name_t) + ZAP_MAXNAMELEN, 0, NULL, NULL,
50 NULL, NULL, NULL, 0);
51
52 zap_attr_cache = kmem_cache_create("zap_attr_cache",
53 sizeof (zap_attribute_t) + ZAP_MAXNAMELEN, 0, NULL,
54 NULL, NULL, NULL, NULL, 0);
55
56 zap_name_long_cache = kmem_cache_create("zap_name_long",
57 sizeof (zap_name_t) + ZAP_MAXNAMELEN_NEW, 0, NULL, NULL,
58 NULL, NULL, NULL, 0);
59
60 zap_attr_long_cache = kmem_cache_create("zap_attr_long_cache",
61 sizeof (zap_attribute_t) + ZAP_MAXNAMELEN_NEW, 0, NULL,
62 NULL, NULL, NULL, NULL, 0);
63 }
64
65 void
zap_fini(void)66 zap_fini(void)
67 {
68 kmem_cache_destroy(zap_name_cache);
69 kmem_cache_destroy(zap_attr_cache);
70 kmem_cache_destroy(zap_name_long_cache);
71 kmem_cache_destroy(zap_attr_long_cache);
72 }
73
74 static int
zap_normalize(zap_t * zap,const char * name,char * namenorm,int normflags,size_t outlen)75 zap_normalize(zap_t *zap, const char *name, char *namenorm, int normflags,
76 size_t outlen)
77 {
78 ASSERT(!(zap_getflags(zap) & ZAP_FLAG_UINT64_KEY));
79
80 size_t inlen = strlen(name) + 1;
81
82 int err = 0;
83 (void) u8_textprep_str((char *)name, &inlen, namenorm, &outlen,
84 normflags | U8_TEXTPREP_IGNORE_NULL | U8_TEXTPREP_IGNORE_INVALID,
85 U8_UNICODE_LATEST, &err);
86
87 return (err);
88 }
89
90 zap_name_t *
zap_name_alloc(zap_t * zap,boolean_t longname)91 zap_name_alloc(zap_t *zap, boolean_t longname)
92 {
93 kmem_cache_t *cache = longname ? zap_name_long_cache : zap_name_cache;
94 zap_name_t *zn = kmem_cache_alloc(cache, KM_SLEEP);
95
96 zn->zn_zap = zap;
97 zn->zn_normbuf_len = longname ? ZAP_MAXNAMELEN_NEW : ZAP_MAXNAMELEN;
98 return (zn);
99 }
100
101 zap_name_t *
zap_name_alloc_str(zap_t * zap,const char * key,matchtype_t mt)102 zap_name_alloc_str(zap_t *zap, const char *key, matchtype_t mt)
103 {
104 size_t key_len = strlen(key) + 1;
105 zap_name_t *zn = zap_name_alloc(zap, (key_len > ZAP_MAXNAMELEN));
106 if (zap_name_init_str(zn, key, mt) != 0) {
107 zap_name_free(zn);
108 return (NULL);
109 }
110 return (zn);
111 }
112
113 zap_name_t *
zap_name_alloc_uint64(zap_t * zap,const uint64_t * key,int numints)114 zap_name_alloc_uint64(zap_t *zap, const uint64_t *key, int numints)
115 {
116 zap_name_t *zn = kmem_cache_alloc(zap_name_cache, KM_SLEEP);
117
118 ASSERT0(zap->zap_normflags);
119 zn->zn_zap = zap;
120 zn->zn_key_intlen = sizeof (*key);
121 zn->zn_key_orig = zn->zn_key_norm = key;
122 zn->zn_key_orig_numints = zn->zn_key_norm_numints = numints;
123 zn->zn_matchtype = 0;
124 zn->zn_normbuf_len = ZAP_MAXNAMELEN;
125
126 zn->zn_hash = zap_hash(zn);
127 return (zn);
128 }
129
130 void
zap_name_free(zap_name_t * zn)131 zap_name_free(zap_name_t *zn)
132 {
133 if (zn->zn_normbuf_len == ZAP_MAXNAMELEN) {
134 kmem_cache_free(zap_name_cache, zn);
135 } else {
136 ASSERT3U(zn->zn_normbuf_len, ==, ZAP_MAXNAMELEN_NEW);
137 kmem_cache_free(zap_name_long_cache, zn);
138 }
139 }
140
141 int
zap_name_init_str(zap_name_t * zn,const char * key,matchtype_t mt)142 zap_name_init_str(zap_name_t *zn, const char *key, matchtype_t mt)
143 {
144 zap_t *zap = zn->zn_zap;
145 size_t key_len = strlen(key) + 1;
146
147 /* Make sure zn is allocated for longname if key is long */
148 IMPLY(key_len > ZAP_MAXNAMELEN,
149 zn->zn_normbuf_len == ZAP_MAXNAMELEN_NEW);
150
151 zn->zn_key_intlen = sizeof (*key);
152 zn->zn_key_orig = key;
153 zn->zn_key_orig_numints = key_len;
154 zn->zn_matchtype = mt;
155 zn->zn_normflags = zap->zap_normflags;
156
157 /*
158 * If we're dealing with a case sensitive lookup on a mixed or
159 * insensitive fs, remove U8_TEXTPREP_TOUPPER or the lookup
160 * will fold case to all caps overriding the lookup request.
161 */
162 if (mt & MT_MATCH_CASE)
163 zn->zn_normflags &= ~U8_TEXTPREP_TOUPPER;
164
165 if (zap->zap_normflags) {
166 /*
167 * We *must* use zap_normflags because this normalization is
168 * what the hash is computed from.
169 */
170 if (zap_normalize(zap, key, zn->zn_normbuf,
171 zap->zap_normflags, zn->zn_normbuf_len) != 0)
172 return (SET_ERROR(ENOTSUP));
173 zn->zn_key_norm = zn->zn_normbuf;
174 zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
175 } else {
176 if (mt != 0)
177 return (SET_ERROR(ENOTSUP));
178 zn->zn_key_norm = zn->zn_key_orig;
179 zn->zn_key_norm_numints = zn->zn_key_orig_numints;
180 }
181
182 zn->zn_hash = zap_hash(zn);
183
184 if (zap->zap_normflags != zn->zn_normflags) {
185 /*
186 * We *must* use zn_normflags because this normalization is
187 * what the matching is based on. (Not the hash!)
188 */
189 if (zap_normalize(zap, key, zn->zn_normbuf,
190 zn->zn_normflags, zn->zn_normbuf_len) != 0)
191 return (SET_ERROR(ENOTSUP));
192 zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
193 }
194
195 return (0);
196 }
197
198 boolean_t
zap_match(zap_name_t * zn,const char * matchname)199 zap_match(zap_name_t *zn, const char *matchname)
200 {
201 boolean_t res = B_FALSE;
202 ASSERT(!(zap_getflags(zn->zn_zap) & ZAP_FLAG_UINT64_KEY));
203
204 if (zn->zn_matchtype & MT_NORMALIZE) {
205 size_t namelen = zn->zn_normbuf_len;
206 char normbuf[ZAP_MAXNAMELEN];
207 char *norm = normbuf;
208
209 /*
210 * Cannot allocate this on-stack as it exceed the stack-limit of
211 * 1024.
212 */
213 if (namelen > ZAP_MAXNAMELEN)
214 norm = kmem_alloc(namelen, KM_SLEEP);
215
216 if (zap_normalize(zn->zn_zap, matchname, norm,
217 zn->zn_normflags, namelen) != 0) {
218 res = B_FALSE;
219 } else {
220 res = (strcmp(zn->zn_key_norm, norm) == 0);
221 }
222 if (norm != normbuf)
223 kmem_free(norm, namelen);
224 } else {
225 res = (strcmp(zn->zn_key_orig, matchname) == 0);
226 }
227 return (res);
228 }
229
230 uint64_t
zap_hash(zap_name_t * zn)231 zap_hash(zap_name_t *zn)
232 {
233 zap_t *zap = zn->zn_zap;
234 uint64_t h = 0;
235
236 if (zap_getflags(zap) & ZAP_FLAG_PRE_HASHED_KEY) {
237 ASSERT(zap_getflags(zap) & ZAP_FLAG_UINT64_KEY);
238 h = *(uint64_t *)zn->zn_key_orig;
239 } else {
240 h = zap->zap_salt;
241 ASSERT(h != 0);
242 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
243
244 if (zap_getflags(zap) & ZAP_FLAG_UINT64_KEY) {
245 const uint64_t *wp = zn->zn_key_norm;
246
247 ASSERT(zn->zn_key_intlen == 8);
248 for (int i = 0; i < zn->zn_key_norm_numints;
249 wp++, i++) {
250 uint64_t word = *wp;
251
252 for (int j = 0; j < 8; j++) {
253 h = (h >> 8) ^
254 zfs_crc64_table[(h ^ word) & 0xFF];
255 word >>= NBBY;
256 }
257 }
258 } else {
259 const uint8_t *cp = zn->zn_key_norm;
260
261 /*
262 * We previously stored the terminating null on
263 * disk, but didn't hash it, so we need to
264 * continue to not hash it. (The
265 * zn_key_*_numints includes the terminating
266 * null for non-binary keys.)
267 */
268 int len = zn->zn_key_norm_numints - 1;
269
270 ASSERT(zn->zn_key_intlen == 1);
271 for (int i = 0; i < len; cp++, i++) {
272 h = (h >> 8) ^
273 zfs_crc64_table[(h ^ *cp) & 0xFF];
274 }
275 }
276 }
277 /*
278 * Don't use all 64 bits, since we need some in the cookie for
279 * the collision differentiator. We MUST use the high bits,
280 * since those are the ones that we first pay attention to when
281 * choosing the bucket.
282 */
283 h &= ~((1ULL << (64 - zap_hashbits(zap))) - 1);
284
285 return (h);
286 }
287
288 static int
zap_lock_impl(dnode_t * dn,dmu_buf_t * db,dmu_tx_t * tx,krw_t lti,boolean_t fatreader,boolean_t adding,zap_t ** zapp)289 zap_lock_impl(dnode_t *dn, dmu_buf_t *db, dmu_tx_t *tx,
290 krw_t lti, boolean_t fatreader, boolean_t adding, zap_t **zapp)
291 {
292 ASSERT0(db->db_offset);
293 objset_t *os = dmu_buf_get_objset(db);
294 uint64_t obj = db->db_object;
295
296 *zapp = NULL;
297
298 if (DMU_OT_BYTESWAP(dn->dn_type) != DMU_BSWAP_ZAP)
299 return (SET_ERROR(EINVAL));
300
301 zap_t *zap = dmu_buf_get_user(db);
302 if (zap == NULL) {
303 zap = mzap_open(db);
304 if (zap == NULL) {
305 /*
306 * mzap_open() didn't like what it saw on-disk.
307 * Check for corruption!
308 */
309 return (SET_ERROR(EIO));
310 }
311 }
312
313 /*
314 * We're checking zap_ismicro without the lock held, in order to
315 * tell what type of lock we want. Once we have some sort of
316 * lock, see if it really is the right type. In practice this
317 * can only be different if it was upgraded from micro to fat,
318 * and micro wanted WRITER but fat only needs READER.
319 */
320 krw_t lt = (!zap->zap_ismicro && fatreader) ? RW_READER : lti;
321 rw_enter(&zap->zap_rwlock, lt);
322 if (lt != ((!zap->zap_ismicro && fatreader) ? RW_READER : lti)) {
323 /* it was upgraded, now we only need reader */
324 ASSERT(lt == RW_WRITER);
325 ASSERT(RW_READER ==
326 ((!zap->zap_ismicro && fatreader) ? RW_READER : lti));
327 rw_downgrade(&zap->zap_rwlock);
328 lt = RW_READER;
329 }
330
331 zap->zap_objset = os;
332 zap->zap_dnode = dn;
333
334 if (lt == RW_WRITER)
335 dmu_buf_will_dirty(db, tx);
336
337 ASSERT3P(zap->zap_dbuf, ==, db);
338
339 ASSERT(!zap->zap_ismicro ||
340 zap->zap_m.zap_num_entries <= zap->zap_m.zap_num_chunks);
341 if (zap->zap_ismicro && tx && adding &&
342 zap->zap_m.zap_num_entries == zap->zap_m.zap_num_chunks) {
343 uint64_t newsz = db->db_size + SPA_MINBLOCKSIZE;
344 if (newsz > zap_get_micro_max_size(dmu_objset_spa(os))) {
345 dprintf("upgrading obj %llu: num_entries=%u\n",
346 (u_longlong_t)obj, zap->zap_m.zap_num_entries);
347 *zapp = zap;
348 int err = mzap_upgrade(zapp, tx, 0);
349 if (err != 0)
350 rw_exit(&zap->zap_rwlock);
351 return (err);
352 }
353 VERIFY0(dmu_object_set_blocksize(os, obj, newsz, 0, tx));
354 zap->zap_m.zap_num_chunks =
355 db->db_size / MZAP_ENT_LEN - 1;
356
357 if (newsz > SPA_OLD_MAXBLOCKSIZE) {
358 dsl_dataset_t *ds = dmu_objset_ds(os);
359 if (!dsl_dataset_feature_is_active(ds,
360 SPA_FEATURE_LARGE_MICROZAP)) {
361 /*
362 * A microzap just grew beyond the old limit
363 * for the first time, so we have to ensure the
364 * feature flag is activated.
365 * zap_get_micro_max_size() won't let us get
366 * here if the feature is not enabled, so we
367 * don't need any other checks beforehand.
368 *
369 * Since we're in open context, we can't
370 * activate the feature directly, so we instead
371 * flag it on the dataset for next sync.
372 */
373 dsl_dataset_dirty(ds, tx);
374 mutex_enter(&ds->ds_lock);
375 ds->ds_feature_activation
376 [SPA_FEATURE_LARGE_MICROZAP] =
377 (void *)B_TRUE;
378 mutex_exit(&ds->ds_lock);
379 }
380 }
381 }
382
383 *zapp = zap;
384 return (0);
385 }
386
387 int
zap_lock_by_dnode(dnode_t * dn,dmu_tx_t * tx,krw_t lti,boolean_t fatreader,boolean_t adding,const void * tag,zap_t ** zapp)388 zap_lock_by_dnode(dnode_t *dn, dmu_tx_t *tx,
389 krw_t lti, boolean_t fatreader, boolean_t adding, const void *tag,
390 zap_t **zapp)
391 {
392 dmu_buf_t *db;
393 int err;
394
395 err = dmu_buf_hold_by_dnode(dn, 0, tag, &db, DMU_READ_NO_PREFETCH);
396 if (err != 0)
397 return (err);
398 err = zap_lock_impl(dn, db, tx, lti, fatreader, adding, zapp);
399 if (err != 0)
400 dmu_buf_rele(db, tag);
401 else
402 VERIFY(dnode_add_ref(dn, tag));
403 return (err);
404 }
405
406 int
zap_lock(objset_t * os,uint64_t obj,dmu_tx_t * tx,krw_t lti,boolean_t fatreader,boolean_t adding,const void * tag,zap_t ** zapp)407 zap_lock(objset_t *os, uint64_t obj, dmu_tx_t *tx,
408 krw_t lti, boolean_t fatreader, boolean_t adding, const void *tag,
409 zap_t **zapp)
410 {
411 dnode_t *dn;
412 int err;
413
414 err = dnode_hold(os, obj, tag, &dn);
415 if (err != 0)
416 return (err);
417 err = zap_lock_by_dnode(dn, tx, lti, fatreader, adding, tag, zapp);
418 dnode_rele(dn, tag);
419 return (err);
420 }
421
422 void
zap_unlock(zap_t * zap,const void * tag)423 zap_unlock(zap_t *zap, const void *tag)
424 {
425 rw_exit(&zap->zap_rwlock);
426 dnode_rele(zap->zap_dnode, tag);
427 dmu_buf_rele(zap->zap_dbuf, tag);
428 }
429
430 int
zap_lock_try_upgrade(zap_t * zap,dmu_tx_t * tx)431 zap_lock_try_upgrade(zap_t *zap, dmu_tx_t *tx)
432 {
433 if (RW_WRITE_HELD(&zap->zap_rwlock))
434 /* Already have writer, nothing to do. */
435 return (1);
436
437 /* Try to upgrade the lock in-place. */
438 if (rw_tryupgrade(&zap->zap_rwlock)) {
439 /*
440 * Got it, mark buffer dirty, since we only do that in
441 * zap_lock_impl() for writer.
442 */
443 dmu_buf_will_dirty(zap->zap_dbuf, tx);
444 return (1);
445 }
446
447 return (0);
448 }
449
450 void
zap_lock_upgrade(zap_t * zap,dmu_tx_t * tx)451 zap_lock_upgrade(zap_t *zap, dmu_tx_t *tx)
452 {
453 if (zap_lock_try_upgrade(zap, tx))
454 return;
455
456 /*
457 * It's safe to drop the lock here because we still have a hold on
458 * zap_dbuf, which prevents the dbuf being evicted and the zap_t being
459 * deallocated.
460 */
461 rw_exit(&zap->zap_rwlock);
462
463 rw_enter(&zap->zap_rwlock, RW_WRITER);
464 dmu_buf_will_dirty(zap->zap_dbuf, tx);
465 }
466
467 void
zap_evict_sync(void * dbu)468 zap_evict_sync(void *dbu)
469 {
470 zap_t *zap = dbu;
471
472 rw_destroy(&zap->zap_rwlock);
473
474 if (zap->zap_ismicro)
475 mze_destroy(zap);
476 else
477 mutex_destroy(&zap->zap_f.zap_num_entries_mtx);
478
479 kmem_free(zap, sizeof (zap_t));
480 }
481
482 uint64_t
zap_getflags(zap_t * zap)483 zap_getflags(zap_t *zap)
484 {
485 if (zap->zap_ismicro)
486 return (0);
487 return (zap_f_phys(zap)->zap_flags);
488 }
489
490 int
zap_hashbits(zap_t * zap)491 zap_hashbits(zap_t *zap)
492 {
493 if (zap_getflags(zap) & ZAP_FLAG_HASH64)
494 return (48);
495 else
496 return (28);
497 }
498
499 uint32_t
zap_maxcd(zap_t * zap)500 zap_maxcd(zap_t *zap)
501 {
502 if (zap_getflags(zap) & ZAP_FLAG_HASH64)
503 return ((1<<16)-1);
504 else
505 return (-1U);
506 }
507
508 /* DNU byteswap callback for DMU_BSWAP_ZAP, see dmu_ot_byteswap. */
509 void
zap_byteswap(void * buf,size_t size)510 zap_byteswap(void *buf, size_t size)
511 {
512 uint64_t block_type = *(uint64_t *)buf;
513
514 if (block_type == ZBT_MICRO || block_type == BSWAP_64(ZBT_MICRO)) {
515 /* ASSERT(magic == ZAP_LEAF_MAGIC); */
516 mzap_byteswap(buf, size);
517 } else {
518 fzap_byteswap(buf, size);
519 }
520 }
521
522 /*
523 * Cursor attribute allocator/free. Part of the public interface in zap.h,
524 * in this file to get access to the kmem caches.
525 */
526 static zap_attribute_t *
zap_attribute_alloc_impl(boolean_t longname)527 zap_attribute_alloc_impl(boolean_t longname)
528 {
529 zap_attribute_t *za;
530
531 za = kmem_cache_alloc((longname)? zap_attr_long_cache : zap_attr_cache,
532 KM_SLEEP);
533 za->za_name_len = (longname)? ZAP_MAXNAMELEN_NEW : ZAP_MAXNAMELEN;
534 return (za);
535 }
536
537 zap_attribute_t *
zap_attribute_alloc(void)538 zap_attribute_alloc(void)
539 {
540 return (zap_attribute_alloc_impl(B_FALSE));
541 }
542
543 zap_attribute_t *
zap_attribute_long_alloc(void)544 zap_attribute_long_alloc(void)
545 {
546 return (zap_attribute_alloc_impl(B_TRUE));
547 }
548
549 void
zap_attribute_free(zap_attribute_t * za)550 zap_attribute_free(zap_attribute_t *za)
551 {
552 if (za->za_name_len == ZAP_MAXNAMELEN) {
553 kmem_cache_free(zap_attr_cache, za);
554 } else {
555 ASSERT3U(za->za_name_len, ==, ZAP_MAXNAMELEN_NEW);
556 kmem_cache_free(zap_attr_long_cache, za);
557 }
558 }
559