1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
24 */
25
26 #include <sys/zfs_context.h>
27 #include <sys/zfeature.h>
28 #include <sys/dmu.h>
29 #include <sys/nvpair.h>
30 #include <sys/zap.h>
31 #include <sys/dmu_tx.h>
32 #include "zfeature_common.h"
33 #include <sys/spa_impl.h>
34
35 /*
36 * ZFS Feature Flags
37 * -----------------
38 *
39 * ZFS feature flags are used to provide fine-grained versioning to the ZFS
40 * on-disk format. Once enabled on a pool feature flags replace the old
41 * spa_version() number.
42 *
43 * Each new on-disk format change will be given a uniquely identifying string
44 * GUID rather than a version number. This avoids the problem of different
45 * organizations creating new on-disk formats with the same version number. To
46 * keep feature GUIDs unique they should consist of the reverse dns name of the
47 * organization which implemented the feature and a short name for the feature,
48 * separated by a colon (e.g. com.delphix:async_destroy).
49 *
50 * Reference Counts
51 * ----------------
52 *
53 * Within each pool features can be in one of three states: disabled, enabled,
54 * or active. These states are differentiated by a reference count stored on
55 * disk for each feature:
56 *
57 * 1) If there is no reference count stored on disk the feature is disabled.
58 * 2) If the reference count is 0 a system administrator has enabled the
59 * feature, but the feature has not been used yet, so no on-disk
60 * format changes have been made.
61 * 3) If the reference count is greater than 0 the feature is active.
62 * The format changes required by the feature are currently on disk.
63 * Note that if the feature's format changes are reversed the feature
64 * may choose to set its reference count back to 0.
65 *
66 * Feature flags makes no differentiation between non-zero reference counts
67 * for an active feature (e.g. a reference count of 1 means the same thing as a
68 * reference count of 27834721), but feature implementations may choose to use
69 * the reference count to store meaningful information. For example, a new RAID
70 * implementation might set the reference count to the number of vdevs using
71 * it. If all those disks are removed from the pool the feature goes back to
72 * having a reference count of 0.
73 *
74 * It is the responsibility of the individual features to maintain a non-zero
75 * reference count as long as the feature's format changes are present on disk.
76 *
77 * Dependencies
78 * ------------
79 *
80 * Each feature may depend on other features. The only effect of this
81 * relationship is that when a feature is enabled all of its dependencies are
82 * automatically enabled as well. Any future work to support disabling of
83 * features would need to ensure that features cannot be disabled if other
84 * enabled features depend on them.
85 *
86 * On-disk Format
87 * --------------
88 *
89 * When feature flags are enabled spa_version() is set to SPA_VERSION_FEATURES
90 * (5000). In order for this to work the pool is automatically upgraded to
91 * SPA_VERSION_BEFORE_FEATURES (28) first, so all pre-feature flags on disk
92 * format changes will be in use.
93 *
94 * Information about features is stored in 3 ZAP objects in the pool's MOS.
95 * These objects are linked to by the following names in the pool directory
96 * object:
97 *
98 * 1) features_for_read: feature GUID -> reference count
99 * Features needed to open the pool for reading.
100 * 2) features_for_write: feature GUID -> reference count
101 * Features needed to open the pool for writing.
102 * 3) feature_descriptions: feature GUID -> descriptive string
103 * A human readable string.
104 *
105 * All enabled features appear in either features_for_read or
106 * features_for_write, but not both.
107 *
108 * To open a pool in read-only mode only the features listed in
109 * features_for_read need to be supported.
110 *
111 * To open the pool in read-write mode features in both features_for_read and
112 * features_for_write need to be supported.
113 *
114 * Some features may be required to read the ZAP objects containing feature
115 * information. To allow software to check for compatibility with these features
116 * before the pool is opened their names must be stored in the label in a
117 * new "features_for_read" entry (note that features that are only required
118 * to write to a pool never need to be stored in the label since the
119 * features_for_write ZAP object can be read before the pool is written to).
120 * To save space in the label features must be explicitly marked as needing to
121 * be written to the label. Also, reference counts are not stored in the label,
122 * instead any feature whose reference count drops to 0 is removed from the
123 * label.
124 *
125 * Adding New Features
126 * -------------------
127 *
128 * Features must be registered in zpool_feature_init() function in
129 * zfeature_common.c using the zfeature_register() function. This function
130 * has arguments to specify if the feature should be stored in the
131 * features_for_read or features_for_write ZAP object and if it needs to be
132 * written to the label when active.
133 *
134 * Once a feature is registered it will appear as a "feature@<feature name>"
135 * property which can be set by an administrator. Feature implementors should
136 * use the spa_feature_is_enabled() and spa_feature_is_active() functions to
137 * query the state of a feature and the spa_feature_incr() and
138 * spa_feature_decr() functions to change an enabled feature's reference count.
139 * Reference counts may only be updated in the syncing context.
140 *
141 * Features may not perform enable-time initialization. Instead, any such
142 * initialization should occur when the feature is first used. This design
143 * enforces that on-disk changes be made only when features are used. Code
144 * should only check if a feature is enabled using spa_feature_is_enabled(),
145 * not by relying on any feature specific metadata existing. If a feature is
146 * enabled, but the feature's metadata is not on disk yet then it should be
147 * created as needed.
148 *
149 * As an example, consider the com.delphix:async_destroy feature. This feature
150 * relies on the existence of a bptree in the MOS that store blocks for
151 * asynchronous freeing. This bptree is not created when async_destroy is
152 * enabled. Instead, when a dataset is destroyed spa_feature_is_enabled() is
153 * called to check if async_destroy is enabled. If it is and the bptree object
154 * does not exist yet, the bptree object is created as part of the dataset
155 * destroy and async_destroy's reference count is incremented to indicate it
156 * has made an on-disk format change. Later, after the destroyed dataset's
157 * blocks have all been asynchronously freed there is no longer any use for the
158 * bptree object, so it is destroyed and async_destroy's reference count is
159 * decremented back to 0 to indicate that it has undone its on-disk format
160 * changes.
161 */
162
163 typedef enum {
164 FEATURE_ACTION_INCR,
165 FEATURE_ACTION_DECR,
166 } feature_action_t;
167
168 /*
169 * Checks that the active features in the pool are supported by
170 * this software. Adds each unsupported feature (name -> description) to
171 * the supplied nvlist.
172 */
173 boolean_t
spa_features_check(spa_t * spa,boolean_t for_write,nvlist_t * unsup_feat,nvlist_t * enabled_feat)174 spa_features_check(spa_t *spa, boolean_t for_write,
175 nvlist_t *unsup_feat, nvlist_t *enabled_feat)
176 {
177 objset_t *os = spa->spa_meta_objset;
178 boolean_t supported;
179 zap_cursor_t *zc;
180 zap_attribute_t *za;
181 uint64_t obj = for_write ?
182 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
183 char *buf;
184
185 zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
186 za = zap_attribute_alloc();
187 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
188
189 supported = B_TRUE;
190 for (zap_cursor_init(zc, os, obj);
191 zap_cursor_retrieve(zc, za) == 0;
192 zap_cursor_advance(zc)) {
193 ASSERT(za->za_integer_length == sizeof (uint64_t) &&
194 za->za_num_integers == 1);
195
196 if (NULL != enabled_feat) {
197 fnvlist_add_uint64(enabled_feat, za->za_name,
198 za->za_first_integer);
199 }
200
201 if (za->za_first_integer != 0 &&
202 !zfeature_is_supported(za->za_name)) {
203 supported = B_FALSE;
204
205 if (NULL != unsup_feat) {
206 const char *desc = "";
207
208 if (zap_lookup(os, spa->spa_feat_desc_obj,
209 za->za_name, 1, MAXPATHLEN, buf) == 0)
210 desc = buf;
211
212 VERIFY(nvlist_add_string(unsup_feat,
213 za->za_name, desc) == 0);
214 }
215 }
216 }
217 zap_cursor_fini(zc);
218
219 kmem_free(buf, MAXPATHLEN);
220 zap_attribute_free(za);
221 kmem_free(zc, sizeof (zap_cursor_t));
222
223 return (supported);
224 }
225
226 /*
227 * Use an in-memory cache of feature refcounts for quick retrieval.
228 *
229 * Note: well-designed features will not need to use this; they should
230 * use spa_feature_is_enabled() and spa_feature_is_active() instead.
231 * However, this is non-static for zdb, zhack, and spa_add_feature_stats().
232 */
233 int
feature_get_refcount(spa_t * spa,zfeature_info_t * feature,uint64_t * res)234 feature_get_refcount(spa_t *spa, zfeature_info_t *feature, uint64_t *res)
235 {
236 ASSERT(VALID_FEATURE_FID(feature->fi_feature));
237 if (spa->spa_feat_refcount_cache[feature->fi_feature] ==
238 SPA_FEATURE_DISABLED) {
239 return (SET_ERROR(ENOTSUP));
240 }
241 *res = spa->spa_feat_refcount_cache[feature->fi_feature];
242 return (0);
243 }
244
245 /*
246 * Note: well-designed features will not need to use this; they should
247 * use spa_feature_is_enabled() and spa_feature_is_active() instead.
248 * However, this is non-static for zdb and zhack.
249 */
250 int
feature_get_refcount_from_disk(spa_t * spa,zfeature_info_t * feature,uint64_t * res)251 feature_get_refcount_from_disk(spa_t *spa, zfeature_info_t *feature,
252 uint64_t *res)
253 {
254 int err;
255 uint64_t refcount;
256 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
257 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
258
259 /*
260 * If the pool is currently being created, the feature objects may not
261 * have been allocated yet. Act as though all features are disabled.
262 */
263 if (zapobj == 0)
264 return (SET_ERROR(ENOTSUP));
265
266 err = zap_lookup(spa->spa_meta_objset, zapobj,
267 feature->fi_guid, sizeof (uint64_t), 1, &refcount);
268 if (err != 0) {
269 if (err == ENOENT)
270 return (SET_ERROR(ENOTSUP));
271 else
272 return (err);
273 }
274 *res = refcount;
275 return (0);
276 }
277
278
279 static int
feature_get_enabled_txg(spa_t * spa,zfeature_info_t * feature,uint64_t * res)280 feature_get_enabled_txg(spa_t *spa, zfeature_info_t *feature, uint64_t *res)
281 {
282 uint64_t enabled_txg_obj __maybe_unused = spa->spa_feat_enabled_txg_obj;
283
284 ASSERT(zfeature_depends_on(feature->fi_feature,
285 SPA_FEATURE_ENABLED_TXG));
286
287 if (!spa_feature_is_enabled(spa, feature->fi_feature)) {
288 return (SET_ERROR(ENOTSUP));
289 }
290
291 ASSERT(enabled_txg_obj != 0);
292
293 VERIFY0(zap_lookup(spa->spa_meta_objset, spa->spa_feat_enabled_txg_obj,
294 feature->fi_guid, sizeof (uint64_t), 1, res));
295
296 return (0);
297 }
298
299 /*
300 * This function is non-static for zhack; it should otherwise not be used
301 * outside this file.
302 */
303 void
feature_sync(spa_t * spa,zfeature_info_t * feature,uint64_t refcount,dmu_tx_t * tx)304 feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount,
305 dmu_tx_t *tx)
306 {
307 ASSERT(VALID_FEATURE_OR_NONE(feature->fi_feature));
308 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
309 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
310 VERIFY0(zap_update(spa->spa_meta_objset, zapobj, feature->fi_guid,
311 sizeof (uint64_t), 1, &refcount, tx));
312
313 /*
314 * feature_sync is called directly from zhack, allowing the
315 * creation of arbitrary features whose fi_feature field may
316 * be greater than SPA_FEATURES. When called from zhack, the
317 * zfeature_info_t object's fi_feature field will be set to
318 * SPA_FEATURE_NONE.
319 */
320 if (feature->fi_feature != SPA_FEATURE_NONE) {
321 uint64_t *refcount_cache =
322 &spa->spa_feat_refcount_cache[feature->fi_feature];
323 VERIFY3U(*refcount_cache, ==,
324 atomic_swap_64(refcount_cache, refcount));
325 }
326
327 if (refcount == 0)
328 spa_deactivate_mos_feature(spa, feature->fi_guid);
329 else if (feature->fi_flags & ZFEATURE_FLAG_MOS)
330 spa_activate_mos_feature(spa, feature->fi_guid, tx);
331 }
332
333 /*
334 * This function is non-static for zhack; it should otherwise not be used
335 * outside this file.
336 */
337 void
feature_enable_sync(spa_t * spa,zfeature_info_t * feature,dmu_tx_t * tx)338 feature_enable_sync(spa_t *spa, zfeature_info_t *feature, dmu_tx_t *tx)
339 {
340 uint64_t initial_refcount =
341 (feature->fi_flags & ZFEATURE_FLAG_ACTIVATE_ON_ENABLE) ? 1 : 0;
342 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
343 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
344
345 ASSERT(0 != zapobj);
346 ASSERT(zfeature_is_valid_guid(feature->fi_guid));
347 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES);
348
349 /*
350 * If the feature is already enabled, ignore the request.
351 */
352 if (zap_contains(spa->spa_meta_objset, zapobj, feature->fi_guid) == 0)
353 return;
354
355 for (int i = 0; feature->fi_depends[i] != SPA_FEATURE_NONE; i++)
356 spa_feature_enable(spa, feature->fi_depends[i], tx);
357
358 VERIFY0(zap_update(spa->spa_meta_objset, spa->spa_feat_desc_obj,
359 feature->fi_guid, 1, strlen(feature->fi_desc) + 1,
360 feature->fi_desc, tx));
361
362 feature_sync(spa, feature, initial_refcount, tx);
363
364 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENABLED_TXG)) {
365 uint64_t enabling_txg = dmu_tx_get_txg(tx);
366
367 if (spa->spa_feat_enabled_txg_obj == 0ULL) {
368 spa->spa_feat_enabled_txg_obj =
369 zap_create_link(spa->spa_meta_objset,
370 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
371 DMU_POOL_FEATURE_ENABLED_TXG, tx);
372 }
373 spa_feature_incr(spa, SPA_FEATURE_ENABLED_TXG, tx);
374
375 VERIFY0(zap_add(spa->spa_meta_objset,
376 spa->spa_feat_enabled_txg_obj, feature->fi_guid,
377 sizeof (uint64_t), 1, &enabling_txg, tx));
378 }
379
380 /*
381 * Errata #4 is mostly a problem with encrypted datasets, but it
382 * is also a problem where the old encryption feature did not
383 * depend on the bookmark_v2 feature. If the pool does not have
384 * any encrypted datasets we can resolve this issue simply by
385 * enabling this dependency.
386 */
387 if (spa->spa_errata == ZPOOL_ERRATA_ZOL_8308_ENCRYPTION &&
388 spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
389 !spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION) &&
390 feature->fi_feature == SPA_FEATURE_BOOKMARK_V2)
391 spa->spa_errata = 0;
392
393 /*
394 * Convert the old on-disk error log to the new format when activating
395 * the head_errlog feature.
396 */
397 if (feature->fi_feature == SPA_FEATURE_HEAD_ERRLOG)
398 spa_upgrade_errlog(spa, tx);
399 }
400
401 static void
feature_do_action(spa_t * spa,spa_feature_t fid,feature_action_t action,dmu_tx_t * tx)402 feature_do_action(spa_t *spa, spa_feature_t fid, feature_action_t action,
403 dmu_tx_t *tx)
404 {
405 uint64_t refcount = 0;
406 zfeature_info_t *feature = &spa_feature_table[fid];
407 uint64_t zapobj __maybe_unused =
408 (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
409 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
410
411 ASSERT(VALID_FEATURE_FID(fid));
412 ASSERT(0 != zapobj);
413 ASSERT(zfeature_is_valid_guid(feature->fi_guid));
414
415 ASSERT(dmu_tx_is_syncing(tx));
416 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES);
417
418 VERIFY3U(feature_get_refcount(spa, feature, &refcount), !=, ENOTSUP);
419
420 switch (action) {
421 case FEATURE_ACTION_INCR:
422 VERIFY3U(refcount, !=, UINT64_MAX);
423 refcount++;
424 break;
425 case FEATURE_ACTION_DECR:
426 VERIFY3U(refcount, !=, 0);
427 refcount--;
428 break;
429 default:
430 ASSERT(0);
431 break;
432 }
433
434 feature_sync(spa, feature, refcount, tx);
435 }
436
437 void
spa_feature_create_zap_objects(spa_t * spa,dmu_tx_t * tx)438 spa_feature_create_zap_objects(spa_t *spa, dmu_tx_t *tx)
439 {
440 /*
441 * We create feature flags ZAP objects in two instances: during pool
442 * creation and during pool upgrade.
443 */
444 ASSERT((!spa->spa_sync_on && tx->tx_txg == TXG_INITIAL) ||
445 dsl_pool_sync_context(spa_get_dsl(spa)));
446
447 spa->spa_feat_for_read_obj = zap_create_link(spa->spa_meta_objset,
448 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
449 DMU_POOL_FEATURES_FOR_READ, tx);
450 spa->spa_feat_for_write_obj = zap_create_link(spa->spa_meta_objset,
451 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
452 DMU_POOL_FEATURES_FOR_WRITE, tx);
453 spa->spa_feat_desc_obj = zap_create_link(spa->spa_meta_objset,
454 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT,
455 DMU_POOL_FEATURE_DESCRIPTIONS, tx);
456 }
457
458 /*
459 * Enable any required dependencies, then enable the requested feature.
460 */
461 void
spa_feature_enable(spa_t * spa,spa_feature_t fid,dmu_tx_t * tx)462 spa_feature_enable(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx)
463 {
464 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES);
465 ASSERT(VALID_FEATURE_FID(fid));
466 feature_enable_sync(spa, &spa_feature_table[fid], tx);
467 }
468
469 void
spa_feature_incr(spa_t * spa,spa_feature_t fid,dmu_tx_t * tx)470 spa_feature_incr(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx)
471 {
472 feature_do_action(spa, fid, FEATURE_ACTION_INCR, tx);
473 }
474
475 void
spa_feature_decr(spa_t * spa,spa_feature_t fid,dmu_tx_t * tx)476 spa_feature_decr(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx)
477 {
478 feature_do_action(spa, fid, FEATURE_ACTION_DECR, tx);
479 }
480
481 boolean_t
spa_feature_is_enabled(spa_t * spa,spa_feature_t fid)482 spa_feature_is_enabled(spa_t *spa, spa_feature_t fid)
483 {
484 int err;
485 uint64_t refcount = 0;
486
487 ASSERT(VALID_FEATURE_FID(fid));
488 if (spa_version(spa) < SPA_VERSION_FEATURES)
489 return (B_FALSE);
490
491 err = feature_get_refcount(spa, &spa_feature_table[fid], &refcount);
492 ASSERT(err == 0 || err == ENOTSUP);
493 return (err == 0);
494 }
495
496 boolean_t
spa_feature_is_active(spa_t * spa,spa_feature_t fid)497 spa_feature_is_active(spa_t *spa, spa_feature_t fid)
498 {
499 int err;
500 uint64_t refcount = 0;
501
502 ASSERT(VALID_FEATURE_FID(fid));
503 if (spa_version(spa) < SPA_VERSION_FEATURES)
504 return (B_FALSE);
505
506 err = feature_get_refcount(spa, &spa_feature_table[fid], &refcount);
507 ASSERT(err == 0 || err == ENOTSUP);
508 return (err == 0 && refcount > 0);
509 }
510
511 /*
512 * For the feature specified by fid (which must depend on
513 * SPA_FEATURE_ENABLED_TXG), return the TXG at which it was enabled in the
514 * OUT txg argument.
515 *
516 * Returns B_TRUE if the feature is enabled, in which case txg will be filled
517 * with the transaction group in which the specified feature was enabled.
518 * Returns B_FALSE otherwise (i.e. if the feature is not enabled).
519 */
520 boolean_t
spa_feature_enabled_txg(spa_t * spa,spa_feature_t fid,uint64_t * txg)521 spa_feature_enabled_txg(spa_t *spa, spa_feature_t fid, uint64_t *txg)
522 {
523 int err;
524
525 ASSERT(VALID_FEATURE_FID(fid));
526 if (spa_version(spa) < SPA_VERSION_FEATURES)
527 return (B_FALSE);
528
529 err = feature_get_enabled_txg(spa, &spa_feature_table[fid], txg);
530 ASSERT(err == 0 || err == ENOTSUP);
531
532 return (err == 0);
533 }
534