1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
26 * Copyright (c) 2017, Intel Corporation.
27 */
28
29 /*
30 * Virtual Device Labels
31 * ---------------------
32 *
33 * The vdev label serves several distinct purposes:
34 *
35 * 1. Uniquely identify this device as part of a ZFS pool and confirm its
36 * identity within the pool.
37 *
38 * 2. Verify that all the devices given in a configuration are present
39 * within the pool.
40 *
41 * 3. Determine the uberblock for the pool.
42 *
43 * 4. In case of an import operation, determine the configuration of the
44 * toplevel vdev of which it is a part.
45 *
46 * 5. If an import operation cannot find all the devices in the pool,
47 * provide enough information to the administrator to determine which
48 * devices are missing.
49 *
50 * It is important to note that while the kernel is responsible for writing the
51 * label, it only consumes the information in the first three cases. The
52 * latter information is only consumed in userland when determining the
53 * configuration to import a pool.
54 *
55 *
56 * Label Organization
57 * ------------------
58 *
59 * Before describing the contents of the label, it's important to understand how
60 * the labels are written and updated with respect to the uberblock.
61 *
62 * When the pool configuration is altered, either because it was newly created
63 * or a device was added, we want to update all the labels such that we can deal
64 * with fatal failure at any point. To this end, each disk has two labels which
65 * are updated before and after the uberblock is synced. Assuming we have
66 * labels and an uberblock with the following transaction groups:
67 *
68 * L1 UB L2
69 * +------+ +------+ +------+
70 * | | | | | |
71 * | t10 | | t10 | | t10 |
72 * | | | | | |
73 * +------+ +------+ +------+
74 *
75 * In this stable state, the labels and the uberblock were all updated within
76 * the same transaction group (10). Each label is mirrored and checksummed, so
77 * that we can detect when we fail partway through writing the label.
78 *
79 * In order to identify which labels are valid, the labels are written in the
80 * following manner:
81 *
82 * 1. For each vdev, update 'L1' to the new label
83 * 2. Update the uberblock
84 * 3. For each vdev, update 'L2' to the new label
85 *
86 * Given arbitrary failure, we can determine the correct label to use based on
87 * the transaction group. If we fail after updating L1 but before updating the
88 * UB, we will notice that L1's transaction group is greater than the uberblock,
89 * so L2 must be valid. If we fail after writing the uberblock but before
90 * writing L2, we will notice that L2's transaction group is less than L1, and
91 * therefore L1 is valid.
92 *
93 * Another added complexity is that not every label is updated when the config
94 * is synced. If we add a single device, we do not want to have to re-write
95 * every label for every device in the pool. This means that both L1 and L2 may
96 * be older than the pool uberblock, because the necessary information is stored
97 * on another vdev.
98 *
99 *
100 * On-disk Format
101 * --------------
102 *
103 * The vdev label consists of two distinct parts, and is wrapped within the
104 * vdev_label_t structure. The label includes 8k of padding to permit legacy
105 * VTOC disk labels, but is otherwise ignored.
106 *
107 * The first half of the label is a packed nvlist which contains pool wide
108 * properties, per-vdev properties, and configuration information. It is
109 * described in more detail below.
110 *
111 * The latter half of the label consists of a redundant array of uberblocks.
112 * These uberblocks are updated whenever a transaction group is committed,
113 * or when the configuration is updated. When a pool is loaded, we scan each
114 * vdev for the 'best' uberblock.
115 *
116 *
117 * Configuration Information
118 * -------------------------
119 *
120 * The nvlist describing the pool and vdev contains the following elements:
121 *
122 * version ZFS on-disk version
123 * name Pool name
124 * state Pool state
125 * txg Transaction group in which this label was written
126 * pool_guid Unique identifier for this pool
127 * vdev_tree An nvlist describing vdev tree.
128 * features_for_read
129 * An nvlist of the features necessary for reading the MOS.
130 *
131 * Each leaf device label also contains the following:
132 *
133 * top_guid Unique ID for top-level vdev in which this is contained
134 * guid Unique ID for the leaf vdev
135 *
136 * The 'vs' configuration follows the format described in 'spa_config.c'.
137 */
138
139 #include <sys/zfs_context.h>
140 #include <sys/spa.h>
141 #include <sys/spa_impl.h>
142 #include <sys/dmu.h>
143 #include <sys/zap.h>
144 #include <sys/vdev.h>
145 #include <sys/vdev_impl.h>
146 #include <sys/vdev_raidz.h>
147 #include <sys/vdev_draid.h>
148 #include <sys/uberblock_impl.h>
149 #include <sys/metaslab.h>
150 #include <sys/metaslab_impl.h>
151 #include <sys/zio.h>
152 #include <sys/dsl_scan.h>
153 #include <sys/abd.h>
154 #include <sys/fs/zfs.h>
155 #include <sys/byteorder.h>
156 #include <sys/zfs_bootenv.h>
157
158 /*
159 * Basic routines to read and write from a vdev label.
160 * Used throughout the rest of this file.
161 */
162 uint64_t
vdev_label_offset(uint64_t psize,int l,uint64_t offset)163 vdev_label_offset(uint64_t psize, int l, uint64_t offset)
164 {
165 ASSERT(offset < sizeof (vdev_label_t));
166 ASSERT(P2PHASE_TYPED(psize, sizeof (vdev_label_t), uint64_t) == 0);
167
168 return (offset + l * sizeof (vdev_label_t) + (l < VDEV_LABELS / 2 ?
169 0 : psize - VDEV_LABELS * sizeof (vdev_label_t)));
170 }
171
172 /*
173 * Returns back the vdev label associated with the passed in offset.
174 */
175 int
vdev_label_number(uint64_t psize,uint64_t offset)176 vdev_label_number(uint64_t psize, uint64_t offset)
177 {
178 int l;
179
180 if (offset >= psize - VDEV_LABEL_END_SIZE) {
181 offset -= psize - VDEV_LABEL_END_SIZE;
182 offset += (VDEV_LABELS / 2) * sizeof (vdev_label_t);
183 }
184 l = offset / sizeof (vdev_label_t);
185 return (l < VDEV_LABELS ? l : -1);
186 }
187
188 static void
vdev_label_read(zio_t * zio,vdev_t * vd,int l,abd_t * buf,uint64_t offset,uint64_t size,zio_done_func_t * done,void * private,int flags)189 vdev_label_read(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset,
190 uint64_t size, zio_done_func_t *done, void *private, int flags)
191 {
192 ASSERT(
193 spa_config_held(zio->io_spa, SCL_STATE, RW_READER) == SCL_STATE ||
194 spa_config_held(zio->io_spa, SCL_STATE, RW_WRITER) == SCL_STATE);
195 ASSERT(flags & ZIO_FLAG_CONFIG_WRITER);
196
197 zio_nowait(zio_read_phys(zio, vd,
198 vdev_label_offset(vd->vdev_psize, l, offset),
199 size, buf, ZIO_CHECKSUM_LABEL, done, private,
200 ZIO_PRIORITY_SYNC_READ, flags, B_TRUE));
201 }
202
203 void
vdev_label_write(zio_t * zio,vdev_t * vd,int l,abd_t * buf,uint64_t offset,uint64_t size,zio_done_func_t * done,void * private,int flags)204 vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset,
205 uint64_t size, zio_done_func_t *done, void *private, int flags)
206 {
207 ASSERT(
208 spa_config_held(zio->io_spa, SCL_STATE, RW_READER) == SCL_STATE ||
209 spa_config_held(zio->io_spa, SCL_STATE, RW_WRITER) == SCL_STATE);
210 ASSERT(flags & ZIO_FLAG_CONFIG_WRITER);
211
212 zio_nowait(zio_write_phys(zio, vd,
213 vdev_label_offset(vd->vdev_psize, l, offset),
214 size, buf, ZIO_CHECKSUM_LABEL, done, private,
215 ZIO_PRIORITY_SYNC_WRITE, flags, B_TRUE));
216 }
217
218 /*
219 * Generate the nvlist representing this vdev's stats
220 */
221 void
vdev_config_generate_stats(vdev_t * vd,nvlist_t * nv)222 vdev_config_generate_stats(vdev_t *vd, nvlist_t *nv)
223 {
224 nvlist_t *nvx;
225 vdev_stat_t *vs;
226 vdev_stat_ex_t *vsx;
227
228 vs = kmem_alloc(sizeof (*vs), KM_SLEEP);
229 vsx = kmem_alloc(sizeof (*vsx), KM_SLEEP);
230
231 vdev_get_stats_ex(vd, vs, vsx);
232 fnvlist_add_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
233 (uint64_t *)vs, sizeof (*vs) / sizeof (uint64_t));
234
235 /*
236 * Add extended stats into a special extended stats nvlist. This keeps
237 * all the extended stats nicely grouped together. The extended stats
238 * nvlist is then added to the main nvlist.
239 */
240 nvx = fnvlist_alloc();
241
242 /* ZIOs in flight to disk */
243 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
244 vsx->vsx_active_queue[ZIO_PRIORITY_SYNC_READ]);
245
246 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
247 vsx->vsx_active_queue[ZIO_PRIORITY_SYNC_WRITE]);
248
249 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
250 vsx->vsx_active_queue[ZIO_PRIORITY_ASYNC_READ]);
251
252 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
253 vsx->vsx_active_queue[ZIO_PRIORITY_ASYNC_WRITE]);
254
255 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
256 vsx->vsx_active_queue[ZIO_PRIORITY_SCRUB]);
257
258 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
259 vsx->vsx_active_queue[ZIO_PRIORITY_TRIM]);
260
261 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
262 vsx->vsx_active_queue[ZIO_PRIORITY_REBUILD]);
263
264 /* ZIOs pending */
265 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
266 vsx->vsx_pend_queue[ZIO_PRIORITY_SYNC_READ]);
267
268 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
269 vsx->vsx_pend_queue[ZIO_PRIORITY_SYNC_WRITE]);
270
271 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
272 vsx->vsx_pend_queue[ZIO_PRIORITY_ASYNC_READ]);
273
274 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
275 vsx->vsx_pend_queue[ZIO_PRIORITY_ASYNC_WRITE]);
276
277 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
278 vsx->vsx_pend_queue[ZIO_PRIORITY_SCRUB]);
279
280 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
281 vsx->vsx_pend_queue[ZIO_PRIORITY_TRIM]);
282
283 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
284 vsx->vsx_pend_queue[ZIO_PRIORITY_REBUILD]);
285
286 /* Histograms */
287 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
288 vsx->vsx_total_histo[ZIO_TYPE_READ],
289 ARRAY_SIZE(vsx->vsx_total_histo[ZIO_TYPE_READ]));
290
291 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
292 vsx->vsx_total_histo[ZIO_TYPE_WRITE],
293 ARRAY_SIZE(vsx->vsx_total_histo[ZIO_TYPE_WRITE]));
294
295 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
296 vsx->vsx_disk_histo[ZIO_TYPE_READ],
297 ARRAY_SIZE(vsx->vsx_disk_histo[ZIO_TYPE_READ]));
298
299 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
300 vsx->vsx_disk_histo[ZIO_TYPE_WRITE],
301 ARRAY_SIZE(vsx->vsx_disk_histo[ZIO_TYPE_WRITE]));
302
303 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
304 vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_READ],
305 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_READ]));
306
307 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
308 vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_WRITE],
309 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SYNC_WRITE]));
310
311 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
312 vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_READ],
313 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_READ]));
314
315 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
316 vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_WRITE],
317 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_ASYNC_WRITE]));
318
319 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
320 vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB],
321 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB]));
322
323 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
324 vsx->vsx_queue_histo[ZIO_PRIORITY_TRIM],
325 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_TRIM]));
326
327 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
328 vsx->vsx_queue_histo[ZIO_PRIORITY_REBUILD],
329 ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_REBUILD]));
330
331 /* Request sizes */
332 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
333 vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ],
334 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ]));
335
336 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
337 vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE],
338 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE]));
339
340 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
341 vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ],
342 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ]));
343
344 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
345 vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE],
346 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE]));
347
348 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
349 vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB],
350 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB]));
351
352 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
353 vsx->vsx_ind_histo[ZIO_PRIORITY_TRIM],
354 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_TRIM]));
355
356 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
357 vsx->vsx_ind_histo[ZIO_PRIORITY_REBUILD],
358 ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_REBUILD]));
359
360 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
361 vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ],
362 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ]));
363
364 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
365 vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE],
366 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE]));
367
368 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
369 vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ],
370 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ]));
371
372 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
373 vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE],
374 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE]));
375
376 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
377 vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB],
378 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB]));
379
380 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
381 vsx->vsx_agg_histo[ZIO_PRIORITY_TRIM],
382 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_TRIM]));
383
384 fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
385 vsx->vsx_agg_histo[ZIO_PRIORITY_REBUILD],
386 ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_REBUILD]));
387
388 /* IO delays */
389 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_SLOW_IOS, vs->vs_slow_ios);
390
391 /* Direct I/O write verify errors */
392 fnvlist_add_uint64(nvx, ZPOOL_CONFIG_VDEV_DIO_VERIFY_ERRORS,
393 vs->vs_dio_verify_errors);
394
395 /* Add extended stats nvlist to main nvlist */
396 fnvlist_add_nvlist(nv, ZPOOL_CONFIG_VDEV_STATS_EX, nvx);
397
398 fnvlist_free(nvx);
399 kmem_free(vs, sizeof (*vs));
400 kmem_free(vsx, sizeof (*vsx));
401 }
402
403 static void
root_vdev_actions_getprogress(vdev_t * vd,nvlist_t * nvl)404 root_vdev_actions_getprogress(vdev_t *vd, nvlist_t *nvl)
405 {
406 spa_t *spa = vd->vdev_spa;
407
408 if (vd != spa->spa_root_vdev)
409 return;
410
411 /* provide either current or previous scan information */
412 pool_scan_stat_t ps;
413 if (spa_scan_get_stats(spa, &ps) == 0) {
414 fnvlist_add_uint64_array(nvl,
415 ZPOOL_CONFIG_SCAN_STATS, (uint64_t *)&ps,
416 sizeof (pool_scan_stat_t) / sizeof (uint64_t));
417 }
418
419 pool_removal_stat_t prs;
420 if (spa_removal_get_stats(spa, &prs) == 0) {
421 fnvlist_add_uint64_array(nvl,
422 ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t *)&prs,
423 sizeof (prs) / sizeof (uint64_t));
424 }
425
426 pool_checkpoint_stat_t pcs;
427 if (spa_checkpoint_get_stats(spa, &pcs) == 0) {
428 fnvlist_add_uint64_array(nvl,
429 ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t *)&pcs,
430 sizeof (pcs) / sizeof (uint64_t));
431 }
432
433 pool_raidz_expand_stat_t pres;
434 if (spa_raidz_expand_get_stats(spa, &pres) == 0) {
435 fnvlist_add_uint64_array(nvl,
436 ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t *)&pres,
437 sizeof (pres) / sizeof (uint64_t));
438 }
439 }
440
441 static void
top_vdev_actions_getprogress(vdev_t * vd,nvlist_t * nvl)442 top_vdev_actions_getprogress(vdev_t *vd, nvlist_t *nvl)
443 {
444 if (vd == vd->vdev_top) {
445 vdev_rebuild_stat_t vrs;
446 if (vdev_rebuild_get_stats(vd, &vrs) == 0) {
447 fnvlist_add_uint64_array(nvl,
448 ZPOOL_CONFIG_REBUILD_STATS, (uint64_t *)&vrs,
449 sizeof (vrs) / sizeof (uint64_t));
450 }
451 }
452 }
453
454 /*
455 * Generate the nvlist representing this vdev's config.
456 */
457 nvlist_t *
vdev_config_generate(spa_t * spa,vdev_t * vd,boolean_t getstats,vdev_config_flag_t flags)458 vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats,
459 vdev_config_flag_t flags)
460 {
461 nvlist_t *nv = NULL;
462 vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
463
464 nv = fnvlist_alloc();
465
466 fnvlist_add_string(nv, ZPOOL_CONFIG_TYPE, vd->vdev_ops->vdev_op_type);
467 if (!(flags & (VDEV_CONFIG_SPARE | VDEV_CONFIG_L2CACHE)))
468 fnvlist_add_uint64(nv, ZPOOL_CONFIG_ID, vd->vdev_id);
469 fnvlist_add_uint64(nv, ZPOOL_CONFIG_GUID, vd->vdev_guid);
470
471 if (vd->vdev_path != NULL)
472 fnvlist_add_string(nv, ZPOOL_CONFIG_PATH, vd->vdev_path);
473
474 if (vd->vdev_devid != NULL)
475 fnvlist_add_string(nv, ZPOOL_CONFIG_DEVID, vd->vdev_devid);
476
477 if (vd->vdev_physpath != NULL)
478 fnvlist_add_string(nv, ZPOOL_CONFIG_PHYS_PATH,
479 vd->vdev_physpath);
480
481 if (vd->vdev_enc_sysfs_path != NULL)
482 fnvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
483 vd->vdev_enc_sysfs_path);
484
485 if (vd->vdev_fru != NULL)
486 fnvlist_add_string(nv, ZPOOL_CONFIG_FRU, vd->vdev_fru);
487
488 if (vd->vdev_ops->vdev_op_config_generate != NULL)
489 vd->vdev_ops->vdev_op_config_generate(vd, nv);
490
491 if (vd->vdev_wholedisk != -1ULL) {
492 fnvlist_add_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
493 vd->vdev_wholedisk);
494 }
495
496 if (vd->vdev_not_present && !(flags & VDEV_CONFIG_MISSING))
497 fnvlist_add_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT, 1);
498
499 if (vd->vdev_isspare)
500 fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_SPARE, 1);
501
502 if (flags & VDEV_CONFIG_L2CACHE)
503 fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);
504
505 if (!(flags & (VDEV_CONFIG_SPARE | VDEV_CONFIG_L2CACHE)) &&
506 vd == vd->vdev_top) {
507 fnvlist_add_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
508 vd->vdev_ms_array);
509 fnvlist_add_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
510 vd->vdev_ms_shift);
511 fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);
512 fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASIZE,
513 vd->vdev_asize);
514 fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_LOG, vd->vdev_islog);
515 if (vd->vdev_noalloc) {
516 fnvlist_add_uint64(nv, ZPOOL_CONFIG_NONALLOCATING,
517 vd->vdev_noalloc);
518 }
519
520 /*
521 * Slog devices are removed synchronously so don't
522 * persist the vdev_removing flag to the label.
523 */
524 if (vd->vdev_removing && !vd->vdev_islog) {
525 fnvlist_add_uint64(nv, ZPOOL_CONFIG_REMOVING,
526 vd->vdev_removing);
527 }
528
529 /* zpool command expects alloc class data */
530 if (getstats && vd->vdev_alloc_bias != VDEV_BIAS_NONE) {
531 const char *bias = NULL;
532
533 switch (vd->vdev_alloc_bias) {
534 case VDEV_BIAS_LOG:
535 bias = VDEV_ALLOC_BIAS_LOG;
536 break;
537 case VDEV_BIAS_SPECIAL:
538 bias = VDEV_ALLOC_BIAS_SPECIAL;
539 break;
540 case VDEV_BIAS_DEDUP:
541 bias = VDEV_ALLOC_BIAS_DEDUP;
542 break;
543 default:
544 ASSERT3U(vd->vdev_alloc_bias, ==,
545 VDEV_BIAS_NONE);
546 }
547 fnvlist_add_string(nv, ZPOOL_CONFIG_ALLOCATION_BIAS,
548 bias);
549 }
550 }
551
552 if (vd->vdev_dtl_sm != NULL) {
553 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DTL,
554 space_map_object(vd->vdev_dtl_sm));
555 }
556
557 if (vic->vic_mapping_object != 0) {
558 fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_OBJECT,
559 vic->vic_mapping_object);
560 }
561
562 if (vic->vic_births_object != 0) {
563 fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_BIRTHS,
564 vic->vic_births_object);
565 }
566
567 if (vic->vic_prev_indirect_vdev != UINT64_MAX) {
568 fnvlist_add_uint64(nv, ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
569 vic->vic_prev_indirect_vdev);
570 }
571
572 if (vd->vdev_crtxg)
573 fnvlist_add_uint64(nv, ZPOOL_CONFIG_CREATE_TXG, vd->vdev_crtxg);
574
575 if (vd->vdev_expansion_time)
576 fnvlist_add_uint64(nv, ZPOOL_CONFIG_EXPANSION_TIME,
577 vd->vdev_expansion_time);
578
579 if (flags & VDEV_CONFIG_MOS) {
580 if (vd->vdev_leaf_zap != 0) {
581 ASSERT(vd->vdev_ops->vdev_op_leaf);
582 fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_LEAF_ZAP,
583 vd->vdev_leaf_zap);
584 }
585
586 if (vd->vdev_top_zap != 0) {
587 ASSERT(vd == vd->vdev_top);
588 fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP,
589 vd->vdev_top_zap);
590 }
591
592 if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap != 0 &&
593 spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) {
594 fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP,
595 vd->vdev_root_zap);
596 }
597
598 if (vd->vdev_resilver_deferred) {
599 ASSERT(vd->vdev_ops->vdev_op_leaf);
600 ASSERT(spa->spa_resilver_deferred);
601 fnvlist_add_boolean(nv, ZPOOL_CONFIG_RESILVER_DEFER);
602 }
603 }
604
605 if (getstats) {
606 vdev_config_generate_stats(vd, nv);
607
608 root_vdev_actions_getprogress(vd, nv);
609 top_vdev_actions_getprogress(vd, nv);
610
611 /*
612 * Note: this can be called from open context
613 * (spa_get_stats()), so we need the rwlock to prevent
614 * the mapping from being changed by condensing.
615 */
616 rw_enter(&vd->vdev_indirect_rwlock, RW_READER);
617 if (vd->vdev_indirect_mapping != NULL) {
618 ASSERT(vd->vdev_indirect_births != NULL);
619 vdev_indirect_mapping_t *vim =
620 vd->vdev_indirect_mapping;
621 fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_SIZE,
622 vdev_indirect_mapping_size(vim));
623 }
624 rw_exit(&vd->vdev_indirect_rwlock);
625 if (vd->vdev_mg != NULL &&
626 vd->vdev_mg->mg_fragmentation != ZFS_FRAG_INVALID) {
627 /*
628 * Compute approximately how much memory would be used
629 * for the indirect mapping if this device were to
630 * be removed.
631 *
632 * Note: If the frag metric is invalid, then not
633 * enough metaslabs have been converted to have
634 * histograms.
635 */
636 uint64_t seg_count = 0;
637 uint64_t to_alloc = vd->vdev_stat.vs_alloc;
638
639 /*
640 * There are the same number of allocated segments
641 * as free segments, so we will have at least one
642 * entry per free segment. However, small free
643 * segments (smaller than vdev_removal_max_span)
644 * will be combined with adjacent allocated segments
645 * as a single mapping.
646 */
647 for (int i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE;
648 i++) {
649 if (i + 1 < highbit64(vdev_removal_max_span)
650 - 1) {
651 to_alloc +=
652 vd->vdev_mg->mg_histogram[i] <<
653 (i + 1);
654 } else {
655 seg_count +=
656 vd->vdev_mg->mg_histogram[i];
657 }
658 }
659
660 /*
661 * The maximum length of a mapping is
662 * zfs_remove_max_segment, so we need at least one entry
663 * per zfs_remove_max_segment of allocated data.
664 */
665 seg_count += to_alloc / spa_remove_max_segment(spa);
666
667 fnvlist_add_uint64(nv, ZPOOL_CONFIG_INDIRECT_SIZE,
668 seg_count *
669 sizeof (vdev_indirect_mapping_entry_phys_t));
670 }
671 }
672
673 if (!vd->vdev_ops->vdev_op_leaf) {
674 nvlist_t **child;
675 uint64_t c;
676
677 ASSERT(!vd->vdev_ishole);
678
679 child = kmem_alloc(vd->vdev_children * sizeof (nvlist_t *),
680 KM_SLEEP);
681
682 for (c = 0; c < vd->vdev_children; c++) {
683 child[c] = vdev_config_generate(spa, vd->vdev_child[c],
684 getstats, flags);
685 }
686
687 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
688 (const nvlist_t * const *)child, vd->vdev_children);
689
690 for (c = 0; c < vd->vdev_children; c++)
691 nvlist_free(child[c]);
692
693 kmem_free(child, vd->vdev_children * sizeof (nvlist_t *));
694
695 } else {
696 const char *aux = NULL;
697
698 if (vd->vdev_offline && !vd->vdev_tmpoffline)
699 fnvlist_add_uint64(nv, ZPOOL_CONFIG_OFFLINE, B_TRUE);
700 if (vd->vdev_resilver_txg != 0)
701 fnvlist_add_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG,
702 vd->vdev_resilver_txg);
703 if (vd->vdev_rebuild_txg != 0)
704 fnvlist_add_uint64(nv, ZPOOL_CONFIG_REBUILD_TXG,
705 vd->vdev_rebuild_txg);
706 if (vd->vdev_faulted)
707 fnvlist_add_uint64(nv, ZPOOL_CONFIG_FAULTED, B_TRUE);
708 if (vd->vdev_degraded)
709 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DEGRADED, B_TRUE);
710 if (vd->vdev_removed)
711 fnvlist_add_uint64(nv, ZPOOL_CONFIG_REMOVED, B_TRUE);
712 if (vd->vdev_unspare)
713 fnvlist_add_uint64(nv, ZPOOL_CONFIG_UNSPARE, B_TRUE);
714 if (vd->vdev_ishole)
715 fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_HOLE, B_TRUE);
716
717 /* Set the reason why we're FAULTED/DEGRADED. */
718 switch (vd->vdev_stat.vs_aux) {
719 case VDEV_AUX_ERR_EXCEEDED:
720 aux = "err_exceeded";
721 break;
722
723 case VDEV_AUX_EXTERNAL:
724 aux = "external";
725 break;
726 }
727
728 if (aux != NULL && !vd->vdev_tmpoffline) {
729 fnvlist_add_string(nv, ZPOOL_CONFIG_AUX_STATE, aux);
730 } else {
731 /*
732 * We're healthy - clear any previous AUX_STATE values.
733 */
734 if (nvlist_exists(nv, ZPOOL_CONFIG_AUX_STATE))
735 nvlist_remove_all(nv, ZPOOL_CONFIG_AUX_STATE);
736 }
737
738 if (vd->vdev_splitting && vd->vdev_orig_guid != 0LL) {
739 fnvlist_add_uint64(nv, ZPOOL_CONFIG_ORIG_GUID,
740 vd->vdev_orig_guid);
741 }
742 }
743
744 return (nv);
745 }
746
747 /*
748 * Generate a view of the top-level vdevs. If we currently have holes
749 * in the namespace, then generate an array which contains a list of holey
750 * vdevs. Additionally, add the number of top-level children that currently
751 * exist.
752 */
753 void
vdev_top_config_generate(spa_t * spa,nvlist_t * config)754 vdev_top_config_generate(spa_t *spa, nvlist_t *config)
755 {
756 vdev_t *rvd = spa->spa_root_vdev;
757 uint64_t *array;
758 uint_t c, idx;
759
760 array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_SLEEP);
761
762 for (c = 0, idx = 0; c < rvd->vdev_children; c++) {
763 vdev_t *tvd = rvd->vdev_child[c];
764
765 if (tvd->vdev_ishole) {
766 array[idx++] = c;
767 }
768 }
769
770 if (idx) {
771 VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_HOLE_ARRAY,
772 array, idx) == 0);
773 }
774
775 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
776 rvd->vdev_children) == 0);
777
778 kmem_free(array, rvd->vdev_children * sizeof (uint64_t));
779 }
780
781 /*
782 * Returns the configuration from the label of the given vdev. For vdevs
783 * which don't have a txg value stored on their label (i.e. spares/cache)
784 * or have not been completely initialized (txg = 0) just return
785 * the configuration from the first valid label we find. Otherwise,
786 * find the most up-to-date label that does not exceed the specified
787 * 'txg' value.
788 */
789 nvlist_t *
vdev_label_read_config(vdev_t * vd,uint64_t txg)790 vdev_label_read_config(vdev_t *vd, uint64_t txg)
791 {
792 spa_t *spa = vd->vdev_spa;
793 nvlist_t *config = NULL;
794 vdev_phys_t *vp[VDEV_LABELS];
795 abd_t *vp_abd[VDEV_LABELS];
796 zio_t *zio[VDEV_LABELS];
797 uint64_t best_txg = 0;
798 uint64_t label_txg = 0;
799 int error = 0;
800 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
801 ZIO_FLAG_SPECULATIVE;
802
803 ASSERT(vd->vdev_validate_thread == curthread ||
804 spa_config_held(spa, SCL_STATE_ALL, RW_WRITER) == SCL_STATE_ALL);
805
806 if (!vdev_readable(vd))
807 return (NULL);
808
809 /*
810 * The label for a dRAID distributed spare is not stored on disk.
811 * Instead it is generated when needed which allows us to bypass
812 * the pipeline when reading the config from the label.
813 */
814 if (vd->vdev_ops == &vdev_draid_spare_ops)
815 return (vdev_draid_read_config_spare(vd));
816
817 for (int l = 0; l < VDEV_LABELS; l++) {
818 vp_abd[l] = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
819 vp[l] = abd_to_buf(vp_abd[l]);
820 }
821
822 retry:
823 for (int l = 0; l < VDEV_LABELS; l++) {
824 zio[l] = zio_root(spa, NULL, NULL, flags);
825
826 vdev_label_read(zio[l], vd, l, vp_abd[l],
827 offsetof(vdev_label_t, vl_vdev_phys), sizeof (vdev_phys_t),
828 NULL, NULL, flags);
829 }
830 for (int l = 0; l < VDEV_LABELS; l++) {
831 nvlist_t *label = NULL;
832
833 if (zio_wait(zio[l]) == 0 &&
834 nvlist_unpack(vp[l]->vp_nvlist, sizeof (vp[l]->vp_nvlist),
835 &label, 0) == 0) {
836 /*
837 * Auxiliary vdevs won't have txg values in their
838 * labels and newly added vdevs may not have been
839 * completely initialized so just return the
840 * configuration from the first valid label we
841 * encounter.
842 */
843 error = nvlist_lookup_uint64(label,
844 ZPOOL_CONFIG_POOL_TXG, &label_txg);
845 if ((error || label_txg == 0) && !config) {
846 config = label;
847 for (l++; l < VDEV_LABELS; l++)
848 zio_wait(zio[l]);
849 break;
850 } else if (label_txg <= txg && label_txg > best_txg) {
851 best_txg = label_txg;
852 nvlist_free(config);
853 config = fnvlist_dup(label);
854 }
855 }
856
857 if (label != NULL) {
858 nvlist_free(label);
859 label = NULL;
860 }
861 }
862
863 if (config == NULL && !(flags & ZIO_FLAG_TRYHARD)) {
864 flags |= ZIO_FLAG_TRYHARD;
865 goto retry;
866 }
867
868 /*
869 * We found a valid label but it didn't pass txg restrictions.
870 */
871 if (config == NULL && label_txg != 0) {
872 vdev_dbgmsg(vd, "label discarded as txg is too large "
873 "(%llu > %llu)", (u_longlong_t)label_txg,
874 (u_longlong_t)txg);
875 }
876
877 for (int l = 0; l < VDEV_LABELS; l++) {
878 abd_free(vp_abd[l]);
879 }
880
881 return (config);
882 }
883
884 /*
885 * Determine if a device is in use. The 'spare_guid' parameter will be filled
886 * in with the device guid if this spare is active elsewhere on the system.
887 */
888 static boolean_t
vdev_inuse(vdev_t * vd,uint64_t crtxg,vdev_labeltype_t reason,uint64_t * spare_guid,uint64_t * l2cache_guid)889 vdev_inuse(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason,
890 uint64_t *spare_guid, uint64_t *l2cache_guid)
891 {
892 spa_t *spa = vd->vdev_spa;
893 uint64_t state, pool_guid, device_guid, txg, spare_pool;
894 uint64_t vdtxg = 0;
895 nvlist_t *label;
896
897 if (spare_guid)
898 *spare_guid = 0ULL;
899 if (l2cache_guid)
900 *l2cache_guid = 0ULL;
901
902 /*
903 * Read the label, if any, and perform some basic sanity checks.
904 */
905 if ((label = vdev_label_read_config(vd, -1ULL)) == NULL)
906 return (B_FALSE);
907
908 (void) nvlist_lookup_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
909 &vdtxg);
910
911 if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
912 &state) != 0 ||
913 nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID,
914 &device_guid) != 0) {
915 nvlist_free(label);
916 return (B_FALSE);
917 }
918
919 if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
920 (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID,
921 &pool_guid) != 0 ||
922 nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_TXG,
923 &txg) != 0)) {
924 nvlist_free(label);
925 return (B_FALSE);
926 }
927
928 nvlist_free(label);
929
930 /*
931 * Check to see if this device indeed belongs to the pool it claims to
932 * be a part of. The only way this is allowed is if the device is a hot
933 * spare (which we check for later on).
934 */
935 if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
936 !spa_guid_exists(pool_guid, device_guid) &&
937 !spa_spare_exists(device_guid, NULL, NULL) &&
938 !spa_l2cache_exists(device_guid, NULL))
939 return (B_FALSE);
940
941 /*
942 * If the transaction group is zero, then this an initialized (but
943 * unused) label. This is only an error if the create transaction
944 * on-disk is the same as the one we're using now, in which case the
945 * user has attempted to add the same vdev multiple times in the same
946 * transaction.
947 */
948 if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
949 txg == 0 && vdtxg == crtxg)
950 return (B_TRUE);
951
952 /*
953 * Check to see if this is a spare device. We do an explicit check for
954 * spa_has_spare() here because it may be on our pending list of spares
955 * to add.
956 */
957 if (spa_spare_exists(device_guid, &spare_pool, NULL) ||
958 spa_has_spare(spa, device_guid)) {
959 if (spare_guid)
960 *spare_guid = device_guid;
961
962 switch (reason) {
963 case VDEV_LABEL_CREATE:
964 return (B_TRUE);
965
966 case VDEV_LABEL_REPLACE:
967 return (!spa_has_spare(spa, device_guid) ||
968 spare_pool != 0ULL);
969
970 case VDEV_LABEL_SPARE:
971 return (spa_has_spare(spa, device_guid));
972 default:
973 break;
974 }
975 }
976
977 /*
978 * Check to see if this is an l2cache device.
979 */
980 if (spa_l2cache_exists(device_guid, NULL) ||
981 spa_has_l2cache(spa, device_guid)) {
982 if (l2cache_guid)
983 *l2cache_guid = device_guid;
984
985 switch (reason) {
986 case VDEV_LABEL_CREATE:
987 return (B_TRUE);
988
989 case VDEV_LABEL_REPLACE:
990 return (!spa_has_l2cache(spa, device_guid));
991
992 case VDEV_LABEL_L2CACHE:
993 return (spa_has_l2cache(spa, device_guid));
994 default:
995 break;
996 }
997 }
998
999 /*
1000 * We can't rely on a pool's state if it's been imported
1001 * read-only. Instead we look to see if the pools is marked
1002 * read-only in the namespace and set the state to active.
1003 */
1004 if (state != POOL_STATE_SPARE && state != POOL_STATE_L2CACHE &&
1005 (spa = spa_by_guid(pool_guid, device_guid)) != NULL &&
1006 spa_mode(spa) == SPA_MODE_READ)
1007 state = POOL_STATE_ACTIVE;
1008
1009 /*
1010 * If the device is marked ACTIVE, then this device is in use by another
1011 * pool on the system.
1012 */
1013 return (state == POOL_STATE_ACTIVE);
1014 }
1015
1016 static nvlist_t *
vdev_aux_label_generate(vdev_t * vd,boolean_t reason_spare)1017 vdev_aux_label_generate(vdev_t *vd, boolean_t reason_spare)
1018 {
1019 /*
1020 * For inactive hot spares and level 2 ARC devices, we generate
1021 * a special label that identifies as a mutually shared hot
1022 * spare or l2cache device. We write the label in case of
1023 * addition or removal of hot spare or l2cache vdev (in which
1024 * case we want to revert the labels).
1025 */
1026 nvlist_t *label = fnvlist_alloc();
1027 fnvlist_add_uint64(label, ZPOOL_CONFIG_VERSION,
1028 spa_version(vd->vdev_spa));
1029 fnvlist_add_uint64(label, ZPOOL_CONFIG_POOL_STATE, reason_spare ?
1030 POOL_STATE_SPARE : POOL_STATE_L2CACHE);
1031 fnvlist_add_uint64(label, ZPOOL_CONFIG_GUID, vd->vdev_guid);
1032
1033 /*
1034 * This is merely to facilitate reporting the ashift of the
1035 * cache device through zdb. The actual retrieval of the
1036 * ashift (in vdev_alloc()) uses the nvlist
1037 * spa->spa_l2cache->sav_config (populated in
1038 * spa_ld_open_aux_vdevs()).
1039 */
1040 if (!reason_spare)
1041 fnvlist_add_uint64(label, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);
1042
1043 /*
1044 * Add path information to help find it during pool import
1045 */
1046 if (vd->vdev_path != NULL)
1047 fnvlist_add_string(label, ZPOOL_CONFIG_PATH, vd->vdev_path);
1048 if (vd->vdev_devid != NULL)
1049 fnvlist_add_string(label, ZPOOL_CONFIG_DEVID, vd->vdev_devid);
1050 if (vd->vdev_physpath != NULL) {
1051 fnvlist_add_string(label, ZPOOL_CONFIG_PHYS_PATH,
1052 vd->vdev_physpath);
1053 }
1054 return (label);
1055 }
1056
1057 /*
1058 * Initialize a vdev label. We check to make sure each leaf device is not in
1059 * use, and writable. We put down an initial label which we will later
1060 * overwrite with a complete label. Note that it's important to do this
1061 * sequentially, not in parallel, so that we catch cases of multiple use of the
1062 * same leaf vdev in the vdev we're creating -- e.g. mirroring a disk with
1063 * itself.
1064 */
1065 int
vdev_label_init(vdev_t * vd,uint64_t crtxg,vdev_labeltype_t reason)1066 vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
1067 {
1068 spa_t *spa = vd->vdev_spa;
1069 nvlist_t *label;
1070 vdev_phys_t *vp;
1071 abd_t *vp_abd;
1072 abd_t *bootenv;
1073 uberblock_t *ub;
1074 abd_t *ub_abd;
1075 zio_t *zio;
1076 char *buf;
1077 size_t buflen;
1078 int error;
1079 uint64_t spare_guid = 0, l2cache_guid = 0;
1080 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
1081 boolean_t reason_spare = (reason == VDEV_LABEL_SPARE || (reason ==
1082 VDEV_LABEL_REMOVE && vd->vdev_isspare));
1083 boolean_t reason_l2cache = (reason == VDEV_LABEL_L2CACHE || (reason ==
1084 VDEV_LABEL_REMOVE && vd->vdev_isl2cache));
1085
1086 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1087
1088 for (int c = 0; c < vd->vdev_children; c++)
1089 if ((error = vdev_label_init(vd->vdev_child[c],
1090 crtxg, reason)) != 0)
1091 return (error);
1092
1093 /* Track the creation time for this vdev */
1094 vd->vdev_crtxg = crtxg;
1095
1096 if (!vd->vdev_ops->vdev_op_leaf || !spa_writeable(spa))
1097 return (0);
1098
1099 /*
1100 * Dead vdevs cannot be initialized.
1101 */
1102 if (vdev_is_dead(vd))
1103 return (SET_ERROR(EIO));
1104
1105 /*
1106 * Determine if the vdev is in use.
1107 */
1108 if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_SPLIT &&
1109 vdev_inuse(vd, crtxg, reason, &spare_guid, &l2cache_guid))
1110 return (SET_ERROR(EBUSY));
1111
1112 /*
1113 * If this is a request to add or replace a spare or l2cache device
1114 * that is in use elsewhere on the system, then we must update the
1115 * guid (which was initialized to a random value) to reflect the
1116 * actual GUID (which is shared between multiple pools).
1117 */
1118 if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_L2CACHE &&
1119 spare_guid != 0ULL) {
1120 uint64_t guid_delta = spare_guid - vd->vdev_guid;
1121
1122 vd->vdev_guid += guid_delta;
1123
1124 for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
1125 pvd->vdev_guid_sum += guid_delta;
1126
1127 /*
1128 * If this is a replacement, then we want to fallthrough to the
1129 * rest of the code. If we're adding a spare, then it's already
1130 * labeled appropriately and we can just return.
1131 */
1132 if (reason == VDEV_LABEL_SPARE)
1133 return (0);
1134 ASSERT(reason == VDEV_LABEL_REPLACE ||
1135 reason == VDEV_LABEL_SPLIT);
1136 }
1137
1138 if (reason != VDEV_LABEL_REMOVE && reason != VDEV_LABEL_SPARE &&
1139 l2cache_guid != 0ULL) {
1140 uint64_t guid_delta = l2cache_guid - vd->vdev_guid;
1141
1142 vd->vdev_guid += guid_delta;
1143
1144 for (vdev_t *pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
1145 pvd->vdev_guid_sum += guid_delta;
1146
1147 /*
1148 * If this is a replacement, then we want to fallthrough to the
1149 * rest of the code. If we're adding an l2cache, then it's
1150 * already labeled appropriately and we can just return.
1151 */
1152 if (reason == VDEV_LABEL_L2CACHE)
1153 return (0);
1154 ASSERT(reason == VDEV_LABEL_REPLACE);
1155 }
1156
1157 /*
1158 * Initialize its label.
1159 */
1160 vp_abd = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
1161 abd_zero(vp_abd, sizeof (vdev_phys_t));
1162 vp = abd_to_buf(vp_abd);
1163
1164 /*
1165 * Generate a label describing the pool and our top-level vdev.
1166 * We mark it as being from txg 0 to indicate that it's not
1167 * really part of an active pool just yet. The labels will
1168 * be written again with a meaningful txg by spa_sync().
1169 */
1170 if (reason_spare || reason_l2cache) {
1171 label = vdev_aux_label_generate(vd, reason_spare);
1172
1173 /*
1174 * When spare or l2cache (aux) vdev is added during pool
1175 * creation, spa->spa_uberblock is not written until this
1176 * point. Write it on next config sync.
1177 */
1178 if (uberblock_verify(&spa->spa_uberblock))
1179 spa->spa_aux_sync_uber = B_TRUE;
1180 } else {
1181 uint64_t txg = 0ULL;
1182
1183 if (reason == VDEV_LABEL_SPLIT)
1184 txg = spa->spa_uberblock.ub_txg;
1185 label = spa_config_generate(spa, vd, txg, B_FALSE);
1186
1187 /*
1188 * Add our creation time. This allows us to detect multiple
1189 * vdev uses as described above, and automatically expires if we
1190 * fail.
1191 */
1192 VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
1193 crtxg) == 0);
1194 }
1195
1196 buf = vp->vp_nvlist;
1197 buflen = sizeof (vp->vp_nvlist);
1198
1199 error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP);
1200 if (error != 0) {
1201 nvlist_free(label);
1202 abd_free(vp_abd);
1203 /* EFAULT means nvlist_pack ran out of room */
1204 return (SET_ERROR(error == EFAULT ? ENAMETOOLONG : EINVAL));
1205 }
1206
1207 /*
1208 * Initialize uberblock template.
1209 */
1210 ub_abd = abd_alloc_linear(VDEV_UBERBLOCK_RING, B_TRUE);
1211 abd_copy_from_buf(ub_abd, &spa->spa_uberblock, sizeof (uberblock_t));
1212 abd_zero_off(ub_abd, sizeof (uberblock_t),
1213 VDEV_UBERBLOCK_RING - sizeof (uberblock_t));
1214 ub = abd_to_buf(ub_abd);
1215 ub->ub_txg = 0;
1216
1217 /* Initialize the 2nd padding area. */
1218 bootenv = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
1219 abd_zero(bootenv, VDEV_PAD_SIZE);
1220
1221 /*
1222 * Write everything in parallel.
1223 */
1224 retry:
1225 zio = zio_root(spa, NULL, NULL, flags);
1226
1227 for (int l = 0; l < VDEV_LABELS; l++) {
1228
1229 vdev_label_write(zio, vd, l, vp_abd,
1230 offsetof(vdev_label_t, vl_vdev_phys),
1231 sizeof (vdev_phys_t), NULL, NULL, flags);
1232
1233 /*
1234 * Skip the 1st padding area.
1235 * Zero out the 2nd padding area where it might have
1236 * left over data from previous filesystem format.
1237 */
1238 vdev_label_write(zio, vd, l, bootenv,
1239 offsetof(vdev_label_t, vl_be),
1240 VDEV_PAD_SIZE, NULL, NULL, flags);
1241
1242 vdev_label_write(zio, vd, l, ub_abd,
1243 offsetof(vdev_label_t, vl_uberblock),
1244 VDEV_UBERBLOCK_RING, NULL, NULL, flags);
1245 }
1246
1247 error = zio_wait(zio);
1248
1249 if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {
1250 flags |= ZIO_FLAG_TRYHARD;
1251 goto retry;
1252 }
1253
1254 nvlist_free(label);
1255 abd_free(bootenv);
1256 abd_free(ub_abd);
1257 abd_free(vp_abd);
1258
1259 /*
1260 * If this vdev hasn't been previously identified as a spare, then we
1261 * mark it as such only if a) we are labeling it as a spare, or b) it
1262 * exists as a spare elsewhere in the system. Do the same for
1263 * level 2 ARC devices.
1264 */
1265 if (error == 0 && !vd->vdev_isspare &&
1266 (reason == VDEV_LABEL_SPARE ||
1267 spa_spare_exists(vd->vdev_guid, NULL, NULL)))
1268 spa_spare_add(vd);
1269
1270 if (error == 0 && !vd->vdev_isl2cache &&
1271 (reason == VDEV_LABEL_L2CACHE ||
1272 spa_l2cache_exists(vd->vdev_guid, NULL)))
1273 spa_l2cache_add(vd);
1274
1275 return (error);
1276 }
1277
1278 /*
1279 * Done callback for vdev_label_read_bootenv_impl. If this is the first
1280 * callback to finish, store our abd in the callback pointer. Otherwise, we
1281 * just free our abd and return.
1282 */
1283 static void
vdev_label_read_bootenv_done(zio_t * zio)1284 vdev_label_read_bootenv_done(zio_t *zio)
1285 {
1286 zio_t *rio = zio->io_private;
1287 abd_t **cbp = rio->io_private;
1288
1289 ASSERT3U(zio->io_size, ==, VDEV_PAD_SIZE);
1290
1291 if (zio->io_error == 0) {
1292 mutex_enter(&rio->io_lock);
1293 if (*cbp == NULL) {
1294 /* Will free this buffer in vdev_label_read_bootenv. */
1295 *cbp = zio->io_abd;
1296 } else {
1297 abd_free(zio->io_abd);
1298 }
1299 mutex_exit(&rio->io_lock);
1300 } else {
1301 abd_free(zio->io_abd);
1302 }
1303 }
1304
1305 static void
vdev_label_read_bootenv_impl(zio_t * zio,vdev_t * vd,int flags)1306 vdev_label_read_bootenv_impl(zio_t *zio, vdev_t *vd, int flags)
1307 {
1308 for (int c = 0; c < vd->vdev_children; c++)
1309 vdev_label_read_bootenv_impl(zio, vd->vdev_child[c], flags);
1310
1311 /*
1312 * We just use the first label that has a correct checksum; the
1313 * bootloader should have rewritten them all to be the same on boot,
1314 * and any changes we made since boot have been the same across all
1315 * labels.
1316 */
1317 if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) {
1318 for (int l = 0; l < VDEV_LABELS; l++) {
1319 vdev_label_read(zio, vd, l,
1320 abd_alloc_linear(VDEV_PAD_SIZE, B_FALSE),
1321 offsetof(vdev_label_t, vl_be), VDEV_PAD_SIZE,
1322 vdev_label_read_bootenv_done, zio, flags);
1323 }
1324 }
1325 }
1326
1327 int
vdev_label_read_bootenv(vdev_t * rvd,nvlist_t * bootenv)1328 vdev_label_read_bootenv(vdev_t *rvd, nvlist_t *bootenv)
1329 {
1330 nvlist_t *config;
1331 spa_t *spa = rvd->vdev_spa;
1332 abd_t *abd = NULL;
1333 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
1334 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_TRYHARD;
1335
1336 ASSERT(bootenv);
1337 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1338
1339 zio_t *zio = zio_root(spa, NULL, &abd, flags);
1340 vdev_label_read_bootenv_impl(zio, rvd, flags);
1341 int err = zio_wait(zio);
1342
1343 if (abd != NULL) {
1344 char *buf;
1345 vdev_boot_envblock_t *vbe = abd_to_buf(abd);
1346
1347 vbe->vbe_version = ntohll(vbe->vbe_version);
1348 switch (vbe->vbe_version) {
1349 case VB_RAW:
1350 /*
1351 * if we have textual data in vbe_bootenv, create nvlist
1352 * with key "envmap".
1353 */
1354 fnvlist_add_uint64(bootenv, BOOTENV_VERSION, VB_RAW);
1355 vbe->vbe_bootenv[sizeof (vbe->vbe_bootenv) - 1] = '\0';
1356 fnvlist_add_string(bootenv, GRUB_ENVMAP,
1357 vbe->vbe_bootenv);
1358 break;
1359
1360 case VB_NVLIST:
1361 err = nvlist_unpack(vbe->vbe_bootenv,
1362 sizeof (vbe->vbe_bootenv), &config, 0);
1363 if (err == 0) {
1364 fnvlist_merge(bootenv, config);
1365 nvlist_free(config);
1366 break;
1367 }
1368 zfs_fallthrough;
1369 default:
1370 /* Check for FreeBSD zfs bootonce command string */
1371 buf = abd_to_buf(abd);
1372 if (*buf == '\0') {
1373 fnvlist_add_uint64(bootenv, BOOTENV_VERSION,
1374 VB_NVLIST);
1375 break;
1376 }
1377 fnvlist_add_string(bootenv, FREEBSD_BOOTONCE, buf);
1378 }
1379
1380 /*
1381 * abd was allocated in vdev_label_read_bootenv_impl()
1382 */
1383 abd_free(abd);
1384 /*
1385 * If we managed to read any successfully,
1386 * return success.
1387 */
1388 return (0);
1389 }
1390 return (err);
1391 }
1392
1393 int
vdev_label_write_bootenv(vdev_t * vd,nvlist_t * env)1394 vdev_label_write_bootenv(vdev_t *vd, nvlist_t *env)
1395 {
1396 zio_t *zio;
1397 spa_t *spa = vd->vdev_spa;
1398 vdev_boot_envblock_t *bootenv;
1399 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
1400 int error;
1401 size_t nvsize;
1402 char *nvbuf;
1403 const char *tmp;
1404
1405 error = nvlist_size(env, &nvsize, NV_ENCODE_XDR);
1406 if (error != 0)
1407 return (SET_ERROR(error));
1408
1409 if (nvsize >= sizeof (bootenv->vbe_bootenv)) {
1410 return (SET_ERROR(E2BIG));
1411 }
1412
1413 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
1414
1415 error = ENXIO;
1416 for (int c = 0; c < vd->vdev_children; c++) {
1417 int child_err;
1418
1419 child_err = vdev_label_write_bootenv(vd->vdev_child[c], env);
1420 /*
1421 * As long as any of the disks managed to write all of their
1422 * labels successfully, return success.
1423 */
1424 if (child_err == 0)
1425 error = child_err;
1426 }
1427
1428 if (!vd->vdev_ops->vdev_op_leaf || vdev_is_dead(vd) ||
1429 !vdev_writeable(vd)) {
1430 return (error);
1431 }
1432 ASSERT3U(sizeof (*bootenv), ==, VDEV_PAD_SIZE);
1433 abd_t *abd = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
1434 abd_zero(abd, VDEV_PAD_SIZE);
1435
1436 bootenv = abd_borrow_buf_copy(abd, VDEV_PAD_SIZE);
1437 nvbuf = bootenv->vbe_bootenv;
1438 nvsize = sizeof (bootenv->vbe_bootenv);
1439
1440 bootenv->vbe_version = fnvlist_lookup_uint64(env, BOOTENV_VERSION);
1441 switch (bootenv->vbe_version) {
1442 case VB_RAW:
1443 if (nvlist_lookup_string(env, GRUB_ENVMAP, &tmp) == 0) {
1444 (void) strlcpy(bootenv->vbe_bootenv, tmp, nvsize);
1445 }
1446 error = 0;
1447 break;
1448
1449 case VB_NVLIST:
1450 error = nvlist_pack(env, &nvbuf, &nvsize, NV_ENCODE_XDR,
1451 KM_SLEEP);
1452 break;
1453
1454 default:
1455 error = EINVAL;
1456 break;
1457 }
1458
1459 if (error == 0) {
1460 bootenv->vbe_version = htonll(bootenv->vbe_version);
1461 abd_return_buf_copy(abd, bootenv, VDEV_PAD_SIZE);
1462 } else {
1463 abd_free(abd);
1464 return (SET_ERROR(error));
1465 }
1466
1467 retry:
1468 zio = zio_root(spa, NULL, NULL, flags);
1469 for (int l = 0; l < VDEV_LABELS; l++) {
1470 vdev_label_write(zio, vd, l, abd,
1471 offsetof(vdev_label_t, vl_be),
1472 VDEV_PAD_SIZE, NULL, NULL, flags);
1473 }
1474
1475 error = zio_wait(zio);
1476 if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {
1477 flags |= ZIO_FLAG_TRYHARD;
1478 goto retry;
1479 }
1480
1481 abd_free(abd);
1482 return (error);
1483 }
1484
1485 /*
1486 * ==========================================================================
1487 * uberblock load/sync
1488 * ==========================================================================
1489 */
1490
1491 /*
1492 * Consider the following situation: txg is safely synced to disk. We've
1493 * written the first uberblock for txg + 1, and then we lose power. When we
1494 * come back up, we fail to see the uberblock for txg + 1 because, say,
1495 * it was on a mirrored device and the replica to which we wrote txg + 1
1496 * is now offline. If we then make some changes and sync txg + 1, and then
1497 * the missing replica comes back, then for a few seconds we'll have two
1498 * conflicting uberblocks on disk with the same txg. The solution is simple:
1499 * among uberblocks with equal txg, choose the one with the latest timestamp.
1500 */
1501 static int
vdev_uberblock_compare(const uberblock_t * ub1,const uberblock_t * ub2)1502 vdev_uberblock_compare(const uberblock_t *ub1, const uberblock_t *ub2)
1503 {
1504 int cmp = TREE_CMP(ub1->ub_txg, ub2->ub_txg);
1505
1506 if (likely(cmp))
1507 return (cmp);
1508
1509 cmp = TREE_CMP(ub1->ub_timestamp, ub2->ub_timestamp);
1510 if (likely(cmp))
1511 return (cmp);
1512
1513 /*
1514 * If MMP_VALID(ub) && MMP_SEQ_VALID(ub) then the host has an MMP-aware
1515 * ZFS, e.g. OpenZFS >= 0.7.
1516 *
1517 * If one ub has MMP and the other does not, they were written by
1518 * different hosts, which matters for MMP. So we treat no MMP/no SEQ as
1519 * a 0 value.
1520 *
1521 * Since timestamp and txg are the same if we get this far, either is
1522 * acceptable for importing the pool.
1523 */
1524 unsigned int seq1 = 0;
1525 unsigned int seq2 = 0;
1526
1527 if (MMP_VALID(ub1) && MMP_SEQ_VALID(ub1))
1528 seq1 = MMP_SEQ(ub1);
1529
1530 if (MMP_VALID(ub2) && MMP_SEQ_VALID(ub2))
1531 seq2 = MMP_SEQ(ub2);
1532
1533 return (TREE_CMP(seq1, seq2));
1534 }
1535
1536 struct ubl_cbdata {
1537 uberblock_t ubl_latest; /* Most recent uberblock */
1538 uberblock_t *ubl_ubbest; /* Best uberblock (w/r/t max_txg) */
1539 vdev_t *ubl_vd; /* vdev associated with the above */
1540 };
1541
1542 static void
vdev_uberblock_load_done(zio_t * zio)1543 vdev_uberblock_load_done(zio_t *zio)
1544 {
1545 vdev_t *vd = zio->io_vd;
1546 spa_t *spa = zio->io_spa;
1547 zio_t *rio = zio->io_private;
1548 uberblock_t *ub = abd_to_buf(zio->io_abd);
1549 struct ubl_cbdata *cbp = rio->io_private;
1550
1551 ASSERT3U(zio->io_size, ==, VDEV_UBERBLOCK_SIZE(vd));
1552
1553 if (zio->io_error == 0 && uberblock_verify(ub) == 0) {
1554 mutex_enter(&rio->io_lock);
1555 if (vdev_uberblock_compare(ub, &cbp->ubl_latest) > 0) {
1556 cbp->ubl_latest = *ub;
1557 }
1558 if (ub->ub_txg <= spa->spa_load_max_txg &&
1559 vdev_uberblock_compare(ub, cbp->ubl_ubbest) > 0) {
1560 /*
1561 * Keep track of the vdev in which this uberblock
1562 * was found. We will use this information later
1563 * to obtain the config nvlist associated with
1564 * this uberblock.
1565 */
1566 *cbp->ubl_ubbest = *ub;
1567 cbp->ubl_vd = vd;
1568 }
1569 mutex_exit(&rio->io_lock);
1570 }
1571
1572 abd_free(zio->io_abd);
1573 }
1574
1575 static void
vdev_uberblock_load_impl(zio_t * zio,vdev_t * vd,int flags,struct ubl_cbdata * cbp)1576 vdev_uberblock_load_impl(zio_t *zio, vdev_t *vd, int flags,
1577 struct ubl_cbdata *cbp)
1578 {
1579 for (int c = 0; c < vd->vdev_children; c++)
1580 vdev_uberblock_load_impl(zio, vd->vdev_child[c], flags, cbp);
1581
1582 if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd) &&
1583 vd->vdev_ops != &vdev_draid_spare_ops) {
1584 for (int l = 0; l < VDEV_LABELS; l++) {
1585 for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
1586 vdev_label_read(zio, vd, l,
1587 abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd),
1588 B_TRUE), VDEV_UBERBLOCK_OFFSET(vd, n),
1589 VDEV_UBERBLOCK_SIZE(vd),
1590 vdev_uberblock_load_done, zio, flags);
1591 }
1592 }
1593 }
1594 }
1595
1596 /*
1597 * Reads the 'best' uberblock from disk along with its associated
1598 * configuration. First, we read the uberblock array of each label of each
1599 * vdev, keeping track of the uberblock with the highest txg in each array.
1600 * Then, we read the configuration from the same vdev as the best uberblock.
1601 */
1602 void
vdev_uberblock_load(vdev_t * rvd,uberblock_t * ub,nvlist_t ** config)1603 vdev_uberblock_load(vdev_t *rvd, uberblock_t *ub, nvlist_t **config)
1604 {
1605 zio_t *zio;
1606 spa_t *spa = rvd->vdev_spa;
1607 struct ubl_cbdata cb;
1608 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
1609 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_TRYHARD;
1610
1611 ASSERT(ub);
1612 ASSERT(config);
1613
1614 memset(ub, 0, sizeof (uberblock_t));
1615 memset(&cb, 0, sizeof (cb));
1616 *config = NULL;
1617
1618 cb.ubl_ubbest = ub;
1619
1620 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
1621 zio = zio_root(spa, NULL, &cb, flags);
1622 vdev_uberblock_load_impl(zio, rvd, flags, &cb);
1623 (void) zio_wait(zio);
1624
1625 /*
1626 * It's possible that the best uberblock was discovered on a label
1627 * that has a configuration which was written in a future txg.
1628 * Search all labels on this vdev to find the configuration that
1629 * matches the txg for our uberblock.
1630 */
1631 if (cb.ubl_vd != NULL) {
1632 vdev_dbgmsg(cb.ubl_vd, "best uberblock found for spa %s. "
1633 "txg %llu", spa->spa_name, (u_longlong_t)ub->ub_txg);
1634
1635 if (ub->ub_raidz_reflow_info !=
1636 cb.ubl_latest.ub_raidz_reflow_info) {
1637 vdev_dbgmsg(cb.ubl_vd,
1638 "spa=%s best uberblock (txg=%llu info=0x%llx) "
1639 "has different raidz_reflow_info than latest "
1640 "uberblock (txg=%llu info=0x%llx)",
1641 spa->spa_name,
1642 (u_longlong_t)ub->ub_txg,
1643 (u_longlong_t)ub->ub_raidz_reflow_info,
1644 (u_longlong_t)cb.ubl_latest.ub_txg,
1645 (u_longlong_t)cb.ubl_latest.ub_raidz_reflow_info);
1646 memset(ub, 0, sizeof (uberblock_t));
1647 spa_config_exit(spa, SCL_ALL, FTAG);
1648 return;
1649 }
1650
1651 *config = vdev_label_read_config(cb.ubl_vd, ub->ub_txg);
1652 if (*config == NULL && spa->spa_extreme_rewind) {
1653 vdev_dbgmsg(cb.ubl_vd, "failed to read label config. "
1654 "Trying again without txg restrictions.");
1655 *config = vdev_label_read_config(cb.ubl_vd, UINT64_MAX);
1656 }
1657 if (*config == NULL) {
1658 vdev_dbgmsg(cb.ubl_vd, "failed to read label config");
1659 }
1660 }
1661 spa_config_exit(spa, SCL_ALL, FTAG);
1662 }
1663
1664 /*
1665 * For use when a leaf vdev is expanded.
1666 * The location of labels 2 and 3 changed, and at the new location the
1667 * uberblock rings are either empty or contain garbage. The sync will write
1668 * new configs there because the vdev is dirty, but expansion also needs the
1669 * uberblock rings copied. Read them from label 0 which did not move.
1670 *
1671 * Since the point is to populate labels {2,3} with valid uberblocks,
1672 * we zero uberblocks we fail to read or which are not valid.
1673 */
1674
1675 static void
vdev_copy_uberblocks(vdev_t * vd)1676 vdev_copy_uberblocks(vdev_t *vd)
1677 {
1678 abd_t *ub_abd;
1679 zio_t *write_zio;
1680 int locks = (SCL_L2ARC | SCL_ZIO);
1681 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
1682 ZIO_FLAG_SPECULATIVE;
1683
1684 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_READER) ==
1685 SCL_STATE);
1686 ASSERT(vd->vdev_ops->vdev_op_leaf);
1687
1688 /*
1689 * No uberblocks are stored on distributed spares, they may be
1690 * safely skipped when expanding a leaf vdev.
1691 */
1692 if (vd->vdev_ops == &vdev_draid_spare_ops)
1693 return;
1694
1695 spa_config_enter(vd->vdev_spa, locks, FTAG, RW_READER);
1696
1697 ub_abd = abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
1698
1699 write_zio = zio_root(vd->vdev_spa, NULL, NULL, flags);
1700 for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
1701 const int src_label = 0;
1702 zio_t *zio;
1703
1704 zio = zio_root(vd->vdev_spa, NULL, NULL, flags);
1705 vdev_label_read(zio, vd, src_label, ub_abd,
1706 VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),
1707 NULL, NULL, flags);
1708
1709 if (zio_wait(zio) || uberblock_verify(abd_to_buf(ub_abd)))
1710 abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));
1711
1712 for (int l = 2; l < VDEV_LABELS; l++)
1713 vdev_label_write(write_zio, vd, l, ub_abd,
1714 VDEV_UBERBLOCK_OFFSET(vd, n),
1715 VDEV_UBERBLOCK_SIZE(vd), NULL, NULL,
1716 flags | ZIO_FLAG_DONT_PROPAGATE);
1717 }
1718 (void) zio_wait(write_zio);
1719
1720 spa_config_exit(vd->vdev_spa, locks, FTAG);
1721
1722 abd_free(ub_abd);
1723 }
1724
1725 /*
1726 * On success, increment root zio's count of good writes.
1727 * We only get credit for writes to known-visible vdevs; see spa_vdev_add().
1728 */
1729 static void
vdev_uberblock_sync_done(zio_t * zio)1730 vdev_uberblock_sync_done(zio_t *zio)
1731 {
1732 uint64_t *good_writes = zio->io_private;
1733
1734 if (zio->io_error == 0 && zio->io_vd->vdev_top->vdev_ms_array != 0)
1735 atomic_inc_64(good_writes);
1736 }
1737
1738 /*
1739 * Write the uberblock to all labels of all leaves of the specified vdev.
1740 */
1741 static void
vdev_uberblock_sync(zio_t * zio,uint64_t * good_writes,uberblock_t * ub,vdev_t * vd,int flags)1742 vdev_uberblock_sync(zio_t *zio, uint64_t *good_writes,
1743 uberblock_t *ub, vdev_t *vd, int flags)
1744 {
1745 for (uint64_t c = 0; c < vd->vdev_children; c++) {
1746 vdev_uberblock_sync(zio, good_writes,
1747 ub, vd->vdev_child[c], flags);
1748 }
1749
1750 if (!vd->vdev_ops->vdev_op_leaf)
1751 return;
1752
1753 if (!vdev_writeable(vd))
1754 return;
1755
1756 /*
1757 * There's no need to write uberblocks to a distributed spare, they
1758 * are already stored on all the leaves of the parent dRAID. For
1759 * this same reason vdev_uberblock_load_impl() skips distributed
1760 * spares when reading uberblocks.
1761 */
1762 if (vd->vdev_ops == &vdev_draid_spare_ops)
1763 return;
1764
1765 /* If the vdev was expanded, need to copy uberblock rings. */
1766 if (vd->vdev_state == VDEV_STATE_HEALTHY &&
1767 vd->vdev_copy_uberblocks == B_TRUE) {
1768 vdev_copy_uberblocks(vd);
1769 vd->vdev_copy_uberblocks = B_FALSE;
1770 }
1771
1772 /*
1773 * We chose a slot based on the txg. If this uberblock has a special
1774 * RAIDZ expansion state, then it is essentially an update of the
1775 * current uberblock (it has the same txg). However, the current
1776 * state is committed, so we want to write it to a different slot. If
1777 * we overwrote the same slot, and we lose power during the uberblock
1778 * write, and the disk does not do single-sector overwrites
1779 * atomically (even though it is required to - i.e. we should see
1780 * either the old or the new uberblock), then we could lose this
1781 * txg's uberblock. Rewinding to the previous txg's uberblock may not
1782 * be possible because RAIDZ expansion may have already overwritten
1783 * some of the data, so we need the progress indicator in the
1784 * uberblock.
1785 */
1786 int m = spa_multihost(vd->vdev_spa) ? MMP_BLOCKS_PER_LABEL : 0;
1787 int n = (ub->ub_txg - (RRSS_GET_STATE(ub) == RRSS_SCRATCH_VALID)) %
1788 (VDEV_UBERBLOCK_COUNT(vd) - m);
1789
1790 /* Copy the uberblock_t into the ABD */
1791 abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
1792 abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));
1793 abd_zero_off(ub_abd, sizeof (uberblock_t),
1794 VDEV_UBERBLOCK_SIZE(vd) - sizeof (uberblock_t));
1795
1796 for (int l = 0; l < VDEV_LABELS; l++)
1797 vdev_label_write(zio, vd, l, ub_abd,
1798 VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd),
1799 vdev_uberblock_sync_done, good_writes,
1800 flags | ZIO_FLAG_DONT_PROPAGATE);
1801
1802 abd_free(ub_abd);
1803 }
1804
1805 /* Sync the uberblocks to all vdevs in svd[] */
1806 int
vdev_uberblock_sync_list(vdev_t ** svd,int svdcount,uberblock_t * ub,int flags)1807 vdev_uberblock_sync_list(vdev_t **svd, int svdcount, uberblock_t *ub, int flags)
1808 {
1809 spa_t *spa = svd[0]->vdev_spa;
1810 zio_t *zio;
1811 uint64_t good_writes = 0;
1812
1813 zio = zio_root(spa, NULL, NULL, flags);
1814
1815 for (int v = 0; v < svdcount; v++)
1816 vdev_uberblock_sync(zio, &good_writes, ub, svd[v], flags);
1817
1818 if (spa->spa_aux_sync_uber) {
1819 for (int v = 0; v < spa->spa_spares.sav_count; v++) {
1820 vdev_uberblock_sync(zio, &good_writes, ub,
1821 spa->spa_spares.sav_vdevs[v], flags);
1822 }
1823 for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
1824 vdev_uberblock_sync(zio, &good_writes, ub,
1825 spa->spa_l2cache.sav_vdevs[v], flags);
1826 }
1827 }
1828 (void) zio_wait(zio);
1829
1830 /*
1831 * Flush the uberblocks to disk. This ensures that the odd labels
1832 * are no longer needed (because the new uberblocks and the even
1833 * labels are safely on disk), so it is safe to overwrite them.
1834 */
1835 zio = zio_root(spa, NULL, NULL, flags);
1836
1837 for (int v = 0; v < svdcount; v++) {
1838 if (vdev_writeable(svd[v])) {
1839 zio_flush(zio, svd[v]);
1840 }
1841 }
1842 if (spa->spa_aux_sync_uber) {
1843 spa->spa_aux_sync_uber = B_FALSE;
1844 for (int v = 0; v < spa->spa_spares.sav_count; v++) {
1845 if (vdev_writeable(spa->spa_spares.sav_vdevs[v])) {
1846 zio_flush(zio, spa->spa_spares.sav_vdevs[v]);
1847 }
1848 }
1849 for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
1850 if (vdev_writeable(spa->spa_l2cache.sav_vdevs[v])) {
1851 zio_flush(zio, spa->spa_l2cache.sav_vdevs[v]);
1852 }
1853 }
1854 }
1855
1856 (void) zio_wait(zio);
1857
1858 return (good_writes >= 1 ? 0 : EIO);
1859 }
1860
1861 /*
1862 * On success, increment the count of good writes for our top-level vdev.
1863 */
1864 static void
vdev_label_sync_done(zio_t * zio)1865 vdev_label_sync_done(zio_t *zio)
1866 {
1867 uint64_t *good_writes = zio->io_private;
1868
1869 if (zio->io_error == 0)
1870 atomic_inc_64(good_writes);
1871 }
1872
1873 /*
1874 * If there weren't enough good writes, indicate failure to the parent.
1875 */
1876 static void
vdev_label_sync_top_done(zio_t * zio)1877 vdev_label_sync_top_done(zio_t *zio)
1878 {
1879 uint64_t *good_writes = zio->io_private;
1880
1881 if (*good_writes == 0)
1882 zio->io_error = SET_ERROR(EIO);
1883
1884 kmem_free(good_writes, sizeof (uint64_t));
1885 }
1886
1887 /*
1888 * We ignore errors for log and cache devices, simply free the private data.
1889 */
1890 static void
vdev_label_sync_ignore_done(zio_t * zio)1891 vdev_label_sync_ignore_done(zio_t *zio)
1892 {
1893 kmem_free(zio->io_private, sizeof (uint64_t));
1894 }
1895
1896 /*
1897 * Write all even or odd labels to all leaves of the specified vdev.
1898 */
1899 static void
vdev_label_sync(zio_t * zio,uint64_t * good_writes,vdev_t * vd,int l,uint64_t txg,int flags)1900 vdev_label_sync(zio_t *zio, uint64_t *good_writes,
1901 vdev_t *vd, int l, uint64_t txg, int flags)
1902 {
1903 nvlist_t *label;
1904 vdev_phys_t *vp;
1905 abd_t *vp_abd;
1906 char *buf;
1907 size_t buflen;
1908 vdev_t *pvd = vd->vdev_parent;
1909 boolean_t spare_in_use = B_FALSE;
1910
1911 for (int c = 0; c < vd->vdev_children; c++) {
1912 vdev_label_sync(zio, good_writes,
1913 vd->vdev_child[c], l, txg, flags);
1914 }
1915
1916 if (!vd->vdev_ops->vdev_op_leaf)
1917 return;
1918
1919 if (!vdev_writeable(vd))
1920 return;
1921
1922 /*
1923 * The top-level config never needs to be written to a distributed
1924 * spare. When read vdev_dspare_label_read_config() will generate
1925 * the config for the vdev_label_read_config().
1926 */
1927 if (vd->vdev_ops == &vdev_draid_spare_ops)
1928 return;
1929
1930 if (pvd && pvd->vdev_ops == &vdev_spare_ops)
1931 spare_in_use = B_TRUE;
1932
1933 /*
1934 * Generate a label describing the top-level config to which we belong.
1935 */
1936 if ((vd->vdev_isspare && !spare_in_use) || vd->vdev_isl2cache) {
1937 label = vdev_aux_label_generate(vd, vd->vdev_isspare);
1938 } else {
1939 label = spa_config_generate(vd->vdev_spa, vd, txg, B_FALSE);
1940 }
1941
1942 vp_abd = abd_alloc_linear(sizeof (vdev_phys_t), B_TRUE);
1943 abd_zero(vp_abd, sizeof (vdev_phys_t));
1944 vp = abd_to_buf(vp_abd);
1945
1946 buf = vp->vp_nvlist;
1947 buflen = sizeof (vp->vp_nvlist);
1948
1949 if (!nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP)) {
1950 for (; l < VDEV_LABELS; l += 2) {
1951 vdev_label_write(zio, vd, l, vp_abd,
1952 offsetof(vdev_label_t, vl_vdev_phys),
1953 sizeof (vdev_phys_t),
1954 vdev_label_sync_done, good_writes,
1955 flags | ZIO_FLAG_DONT_PROPAGATE);
1956 }
1957 }
1958
1959 abd_free(vp_abd);
1960 nvlist_free(label);
1961 }
1962
1963 static int
vdev_label_sync_list(spa_t * spa,int l,uint64_t txg,int flags)1964 vdev_label_sync_list(spa_t *spa, int l, uint64_t txg, int flags)
1965 {
1966 list_t *dl = &spa->spa_config_dirty_list;
1967 vdev_t *vd;
1968 zio_t *zio;
1969 int error;
1970
1971 /*
1972 * Write the new labels to disk.
1973 */
1974 zio = zio_root(spa, NULL, NULL, flags);
1975
1976 for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd)) {
1977 uint64_t *good_writes;
1978
1979 ASSERT(!vd->vdev_ishole);
1980
1981 good_writes = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
1982 zio_t *vio = zio_null(zio, spa, NULL,
1983 (vd->vdev_islog || vd->vdev_aux != NULL) ?
1984 vdev_label_sync_ignore_done : vdev_label_sync_top_done,
1985 good_writes, flags);
1986 vdev_label_sync(vio, good_writes, vd, l, txg, flags);
1987 zio_nowait(vio);
1988 }
1989
1990 /*
1991 * AUX path may have changed during import
1992 */
1993 spa_aux_vdev_t *sav[2] = {&spa->spa_spares, &spa->spa_l2cache};
1994 for (int i = 0; i < 2; i++) {
1995 for (int v = 0; v < sav[i]->sav_count; v++) {
1996 uint64_t *good_writes;
1997 if (!sav[i]->sav_label_sync)
1998 continue;
1999 good_writes = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
2000 zio_t *vio = zio_null(zio, spa, NULL,
2001 vdev_label_sync_ignore_done, good_writes, flags);
2002 vdev_label_sync(vio, good_writes, sav[i]->sav_vdevs[v],
2003 l, txg, flags);
2004 zio_nowait(vio);
2005 }
2006 }
2007
2008 error = zio_wait(zio);
2009
2010 /*
2011 * Flush the new labels to disk.
2012 */
2013 zio = zio_root(spa, NULL, NULL, flags);
2014
2015 for (vd = list_head(dl); vd != NULL; vd = list_next(dl, vd))
2016 zio_flush(zio, vd);
2017
2018 for (int i = 0; i < 2; i++) {
2019 if (!sav[i]->sav_label_sync)
2020 continue;
2021 for (int v = 0; v < sav[i]->sav_count; v++)
2022 zio_flush(zio, sav[i]->sav_vdevs[v]);
2023 if (l == 1)
2024 sav[i]->sav_label_sync = B_FALSE;
2025 }
2026
2027 (void) zio_wait(zio);
2028
2029 return (error);
2030 }
2031
2032 /*
2033 * Sync the uberblock and any changes to the vdev configuration.
2034 *
2035 * The order of operations is carefully crafted to ensure that
2036 * if the system panics or loses power at any time, the state on disk
2037 * is still transactionally consistent. The in-line comments below
2038 * describe the failure semantics at each stage.
2039 *
2040 * Moreover, vdev_config_sync() is designed to be idempotent: if it fails
2041 * at any time, you can just call it again, and it will resume its work.
2042 */
2043 int
vdev_config_sync(vdev_t ** svd,int svdcount,uint64_t txg)2044 vdev_config_sync(vdev_t **svd, int svdcount, uint64_t txg)
2045 {
2046 spa_t *spa = svd[0]->vdev_spa;
2047 uberblock_t *ub = &spa->spa_uberblock;
2048 int error = 0;
2049 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
2050
2051 ASSERT(svdcount != 0);
2052 retry:
2053 /*
2054 * Normally, we don't want to try too hard to write every label and
2055 * uberblock. If there is a flaky disk, we don't want the rest of the
2056 * sync process to block while we retry. But if we can't write a
2057 * single label out, we should retry with ZIO_FLAG_TRYHARD before
2058 * bailing out and declaring the pool faulted.
2059 */
2060 if (error != 0) {
2061 if ((flags & ZIO_FLAG_TRYHARD) != 0)
2062 return (error);
2063 flags |= ZIO_FLAG_TRYHARD;
2064 }
2065
2066 ASSERT(ub->ub_txg <= txg);
2067
2068 /*
2069 * If this isn't a resync due to I/O errors,
2070 * and nothing changed in this transaction group,
2071 * and multihost protection isn't enabled,
2072 * and the vdev configuration hasn't changed,
2073 * then there's nothing to do.
2074 */
2075 if (ub->ub_txg < txg) {
2076 boolean_t changed = uberblock_update(ub, spa->spa_root_vdev,
2077 txg, spa->spa_mmp.mmp_delay);
2078
2079 if (!changed && list_is_empty(&spa->spa_config_dirty_list) &&
2080 !spa_multihost(spa))
2081 return (0);
2082 }
2083
2084 if (txg > spa_freeze_txg(spa))
2085 return (0);
2086
2087 ASSERT(txg <= spa->spa_final_txg);
2088
2089 /*
2090 * Flush the write cache of every disk that's been written to
2091 * in this transaction group. This ensures that all blocks
2092 * written in this txg will be committed to stable storage
2093 * before any uberblock that references them.
2094 */
2095 zio_t *zio = zio_root(spa, NULL, NULL, flags);
2096
2097 for (vdev_t *vd =
2098 txg_list_head(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)); vd != NULL;
2099 vd = txg_list_next(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg)))
2100 zio_flush(zio, vd);
2101
2102 (void) zio_wait(zio);
2103
2104 /*
2105 * Sync out the even labels (L0, L2) for every dirty vdev. If the
2106 * system dies in the middle of this process, that's OK: all of the
2107 * even labels that made it to disk will be newer than any uberblock,
2108 * and will therefore be considered invalid. The odd labels (L1, L3),
2109 * which have not yet been touched, will still be valid. We flush
2110 * the new labels to disk to ensure that all even-label updates
2111 * are committed to stable storage before the uberblock update.
2112 */
2113 if ((error = vdev_label_sync_list(spa, 0, txg, flags)) != 0) {
2114 if ((flags & ZIO_FLAG_TRYHARD) != 0) {
2115 zfs_dbgmsg("vdev_label_sync_list() returned error %d "
2116 "for pool '%s' when syncing out the even labels "
2117 "of dirty vdevs", error, spa_name(spa));
2118 }
2119 goto retry;
2120 }
2121
2122 /*
2123 * Sync the uberblocks to all vdevs in svd[].
2124 * If the system dies in the middle of this step, there are two cases
2125 * to consider, and the on-disk state is consistent either way:
2126 *
2127 * (1) If none of the new uberblocks made it to disk, then the
2128 * previous uberblock will be the newest, and the odd labels
2129 * (which had not yet been touched) will be valid with respect
2130 * to that uberblock.
2131 *
2132 * (2) If one or more new uberblocks made it to disk, then they
2133 * will be the newest, and the even labels (which had all
2134 * been successfully committed) will be valid with respect
2135 * to the new uberblocks.
2136 */
2137 if ((error = vdev_uberblock_sync_list(svd, svdcount, ub, flags)) != 0) {
2138 if ((flags & ZIO_FLAG_TRYHARD) != 0) {
2139 zfs_dbgmsg("vdev_uberblock_sync_list() returned error "
2140 "%d for pool '%s'", error, spa_name(spa));
2141 }
2142 goto retry;
2143 }
2144
2145 if (spa_multihost(spa))
2146 mmp_update_uberblock(spa, ub);
2147
2148 /*
2149 * Sync out odd labels for every dirty vdev. If the system dies
2150 * in the middle of this process, the even labels and the new
2151 * uberblocks will suffice to open the pool. The next time
2152 * the pool is opened, the first thing we'll do -- before any
2153 * user data is modified -- is mark every vdev dirty so that
2154 * all labels will be brought up to date. We flush the new labels
2155 * to disk to ensure that all odd-label updates are committed to
2156 * stable storage before the next transaction group begins.
2157 */
2158 if ((error = vdev_label_sync_list(spa, 1, txg, flags)) != 0) {
2159 if ((flags & ZIO_FLAG_TRYHARD) != 0) {
2160 zfs_dbgmsg("vdev_label_sync_list() returned error %d "
2161 "for pool '%s' when syncing out the odd labels of "
2162 "dirty vdevs", error, spa_name(spa));
2163 }
2164 goto retry;
2165 }
2166
2167 return (0);
2168 }
2169