1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (c) 2018 Intel Corporation. 14 * Copyright (c) 2020 by Lawrence Livermore National Security, LLC. 15 * Copyright (c) 2025, Klara, Inc. 16 * Copyright (c) 2026, Seagate Technology, LLC. 17 * Copyright (c) 2026, Wasabi Technologies, Inc. 18 */ 19 20 #include <sys/zfs_context.h> 21 #include <sys/spa.h> 22 #include <sys/spa_impl.h> 23 #include <sys/vdev_impl.h> 24 #include <sys/vdev_draid.h> 25 #include <sys/vdev_raidz.h> 26 #include <sys/vdev_rebuild.h> 27 #include <sys/abd.h> 28 #include <sys/zio.h> 29 #include <sys/nvpair.h> 30 #include <sys/zio_checksum.h> 31 #include <sys/fs/zfs.h> 32 #include <sys/fm/fs/zfs.h> 33 #include <zfs_fletcher.h> 34 35 #ifdef ZFS_DEBUG 36 #include <sys/vdev.h> /* For vdev_xlate() in vdev_draid_io_verify() */ 37 #endif 38 39 /* 40 * dRAID is a distributed spare implementation for ZFS. A dRAID vdev is 41 * comprised of multiple raidz redundancy groups which are spread over the 42 * dRAID children. To ensure an even distribution, and avoid hot spots, a 43 * permutation mapping is applied to the order of the dRAID children. 44 * This mixing effectively distributes the parity columns evenly over all 45 * of the disks in the dRAID. 46 * 47 * This is beneficial because it means when resilvering all of the disks 48 * can participate thereby increasing the available IOPs and bandwidth. 49 * Furthermore, by reserving a small fraction of each child's total capacity 50 * virtual distributed spare disks can be created. These spares similarly 51 * benefit from the performance gains of spanning all of the children. The 52 * consequence of which is that resilvering to a distributed spare can 53 * substantially reduce the time required to restore full parity to pool 54 * with a failed disks. 55 * 56 * === dRAID group layout === 57 * 58 * First, let's define a "row" in the configuration to be a 16M chunk from 59 * each physical drive at the same offset. This is the minimum allowable 60 * size since it must be possible to store a full 16M block when there is 61 * only a single data column. Next, we define a "group" to be a set of 62 * sequential disks containing both the parity and data columns. We allow 63 * groups to span multiple rows in order to align any group size to any 64 * number of physical drives. Finally, a "slice" is comprised of the rows 65 * which contain the target number of groups. The permutation mappings 66 * are applied in a round robin fashion to each slice. 67 * 68 * Given D+P drives in a group (including parity drives) and C-S physical 69 * drives (not including the spare drives), we can distribute the groups 70 * across R rows without remainder by selecting the least common multiple 71 * of D+P and C-S as the number of groups; i.e. ngroups = LCM(D+P, C-S). 72 * 73 * In the example below, there are C=14 physical drives in the configuration 74 * with S=2 drives worth of spare capacity. Each group has a width of 9 75 * which includes D=8 data and P=1 parity drive. There are 4 groups and 76 * 3 rows per slice. Each group has a size of 144M (16M * 9) and a slice 77 * size is 576M (144M * 4). When allocating from a dRAID each group is 78 * filled before moving on to the next as show in slice0 below. 79 * 80 * data disks (8 data + 1 parity) spares (2) 81 * +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 82 * ^ | 2 | 6 | 1 | 11| 4 | 0 | 7 | 10| 8 | 9 | 13| 5 | 12| 3 | device map 0 83 * | +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 84 * | | group 0 | group 1..| | 85 * | +-----------------------------------+-----------+-------| 86 * | | 0 1 2 3 4 5 6 7 8 | 36 37 38| | r 87 * | | 9 10 11 12 13 14 15 16 17| 45 46 47| | o 88 * | | 18 19 20 21 22 23 24 25 26| 54 55 56| | w 89 * | 27 28 29 30 31 32 33 34 35| 63 64 65| | 0 90 * s +-----------------------+-----------------------+-------+ 91 * l | ..group 1 | group 2.. | | 92 * i +-----------------------+-----------------------+-------+ 93 * c | 39 40 41 42 43 44| 72 73 74 75 76 77| | r 94 * e | 48 49 50 51 52 53| 81 82 83 84 85 86| | o 95 * 0 | 57 58 59 60 61 62| 90 91 92 93 94 95| | w 96 * | 66 67 68 69 70 71| 99 100 101 102 103 104| | 1 97 * | +-----------+-----------+-----------------------+-------+ 98 * | |..group 2 | group 3 | | 99 * | +-----------+-----------+-----------------------+-------+ 100 * | | 78 79 80|108 109 110 111 112 113 114 115 116| | r 101 * | | 87 88 89|117 118 119 120 121 122 123 124 125| | o 102 * | | 96 97 98|126 127 128 129 130 131 132 133 134| | w 103 * v |105 106 107|135 136 137 138 139 140 141 142 143| | 2 104 * +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 105 * | 9 | 11| 12| 2 | 4 | 1 | 3 | 0 | 10| 13| 8 | 5 | 6 | 7 | device map 1 106 * s +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 107 * l | group 4 | group 5..| | row 3 108 * i +-----------------------+-----------+-----------+-------| 109 * c | ..group 5 | group 6.. | | row 4 110 * e +-----------+-----------+-----------------------+-------+ 111 * 1 |..group 6 | group 7 | | row 5 112 * +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 113 * | 3 | 5 | 10| 8 | 6 | 11| 12| 0 | 2 | 4 | 7 | 1 | 9 | 13| device map 2 114 * s +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 115 * l | group 8 | group 9..| | row 6 116 * i +-----------------------------------------------+-------| 117 * c | ..group 9 | group 10.. | | row 7 118 * e +-----------------------+-----------------------+-------+ 119 * 2 |..group 10 | group 11 | | row 8 120 * +-----------+-----------------------------------+-------+ 121 * 122 * This layout has several advantages over requiring that each row contain 123 * a whole number of groups. 124 * 125 * 1. The group count is not a relevant parameter when defining a dRAID 126 * layout. Only the group width is needed, and *all* groups will have 127 * the desired size. 128 * 129 * 2. All possible group widths (<= physical disk count) can be supported. 130 * 131 * 3. The logic within vdev_draid.c is simplified when the group width is 132 * the same for all groups (although some of the logic around computing 133 * permutation numbers and drive offsets is more complicated). 134 * 135 * === dRAID failure domains === 136 * 137 * If we put several slices alongside in a row and configure each disk in 138 * slice to be from different failure domain (for example an enclosure), we 139 * can then tolerate the failure of the whole domain -- only one device 140 * will be failed in every slice in this case. The column of such slices 141 * we will call failure group, and the row with such slices alongside we 142 * will call "big width row", width being multiple of children (W = C*n). 143 * 144 * Here's an example of configuration with 7 failure domains and two 145 * failure groups: 146 * 147 * 7 C disks in each slice, 2 slices in big 14 W rows 148 * +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 149 * | 1 | 7 | 3 | 9 | 11| 5 | 13| 6 | 10| 4 | 8 | 0 | 12| 2 | device map 0 150 * s +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 151 * l | group 0 | gr1..| S | group 3 | gr4.. | S | row 0 152 * c +-------+-------+-------+---+-------+-------+-------+---+ 153 * 0,1 | ..gr1 | group 2 | S | ..gr4 | group 5 | S | row 1 154 * +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 155 * | 2 | 10| 12| 7 | 8 | 13| 11| 1 | 5 | 4 | 6 | 3 | 9 | 0 | device map 1 156 * s +===+===+===+===+===+===+===+===+===+===+===+===+===+===+ 157 * l | group 6 | gr7..| S | group 9 |gr10.. | S | row 2 158 * c +-------+-------+-------+---+---------------+-------+---+ 159 * 2,3 | ..gr7 | group 8 | S |..gr10 | group 11 | S | row 3 160 * +-------+---------------+---+-------+---------------+---+ 161 * failure group 0 failure group 1 162 * 163 * In practice, there might be much more failure groups. And in theory, the 164 * width of the big rows can be much larger than curent limit of 255 imposed 165 * for the number of children. But we kept the same limit for now for the 166 * sake of simplicity of implementation. 167 * 168 * In order to preserve fast sequential resilvering in case of a disk failure, 169 * all failure groups much share all disks between themselves, and this is 170 * achieved by shuffling the disks between the groups. But only i-th disks 171 * in each group are shuffled between themselves, i.e. the disks from the 172 * same failure domains (enclosures). After that, they are shuffled within 173 * each group. Thus, no more than one disk from any failure domain can appear 174 * in any failure group as a result of this shuffling. In the above example, 175 * you won't find any tuple of (0, 7) or (1, 8) or (2, 9) or ... (6, 13) 176 * mapped to the same slice. This is done in vdev_draid_shuffle_perms(). 177 * 178 * Spare disks are evenly distributed among failure groups, and they are 179 * shared by all groups. However, to support domain failure, we cannot have 180 * more than (nparity - 1) failed disks in any group, no matter if they are 181 * rebuilt to draid spares or not (the blocks of those spares can be mapped 182 * to the disks from the failed domain (enclosure), and we cannot tolerate 183 * more than nparity failures in any failure group). 184 * 185 * 186 * N.B. The following array describes all valid dRAID permutation maps. 187 * Each row is used to generate a permutation map for a different number 188 * of children from a unique seed. The seeds were generated and carefully 189 * evaluated by the 'draid' utility in order to provide balanced mappings. 190 * In addition to the seed a checksum of the in-memory mapping is stored 191 * for verification. 192 * 193 * The imbalance ratio of a given failure (e.g. 5 disks wide, child 3 failed, 194 * with a given permutation map) is the ratio of the amounts of I/O that will 195 * be sent to the least and most busy disks when resilvering. The average 196 * imbalance ratio (of a given number of disks and permutation map) is the 197 * average of the ratios of all possible single and double disk failures. 198 * 199 * In order to achieve a low imbalance ratio the number of permutations in 200 * the mapping must be significantly larger than the number of children. 201 * For dRAID the number of permutations has been limited to 512 to minimize 202 * the map size. This does result in a gradually increasing imbalance ratio 203 * as seen in the table below. Increasing the number of permutations for 204 * larger child counts would reduce the imbalance ratio. However, in practice 205 * when there are a large number of children each child is responsible for 206 * fewer total IOs so it's less of a concern. 207 * 208 * Note these values are hard coded and must never be changed. Existing 209 * pools depend on the same mapping always being generated in order to 210 * read and write from the correct locations. Any change would make 211 * existing pools completely inaccessible. 212 */ 213 static const draid_map_t draid_maps[VDEV_DRAID_MAX_MAPS] = { 214 { 2, 256, 0x89ef3dabbcc7de37, 0x00000000433d433d }, /* 1.000 */ 215 { 3, 256, 0x89a57f3de98121b4, 0x00000000bcd8b7b5 }, /* 1.000 */ 216 { 4, 256, 0xc9ea9ec82340c885, 0x00000001819d7c69 }, /* 1.000 */ 217 { 5, 256, 0xf46733b7f4d47dfd, 0x00000002a1648d74 }, /* 1.010 */ 218 { 6, 256, 0x88c3c62d8585b362, 0x00000003d3b0c2c4 }, /* 1.031 */ 219 { 7, 256, 0x3a65d809b4d1b9d5, 0x000000055c4183ee }, /* 1.043 */ 220 { 8, 256, 0xe98930e3c5d2e90a, 0x00000006edfb0329 }, /* 1.059 */ 221 { 9, 256, 0x5a5430036b982ccb, 0x00000008ceaf6934 }, /* 1.056 */ 222 { 10, 256, 0x92bf389e9eadac74, 0x0000000b26668c09 }, /* 1.072 */ 223 { 11, 256, 0x74ccebf1dcf3ae80, 0x0000000dd691358c }, /* 1.083 */ 224 { 12, 256, 0x8847e41a1a9f5671, 0x00000010a0c63c8e }, /* 1.097 */ 225 { 13, 256, 0x7481b56debf0e637, 0x0000001424121fe4 }, /* 1.100 */ 226 { 14, 256, 0x559b8c44065f8967, 0x00000016ab2ff079 }, /* 1.121 */ 227 { 15, 256, 0x34c49545a2ee7f01, 0x0000001a6028efd6 }, /* 1.103 */ 228 { 16, 256, 0xb85f4fa81a7698f7, 0x0000001e95ff5e66 }, /* 1.111 */ 229 { 17, 256, 0x6353e47b7e47aba0, 0x00000021a81fa0fe }, /* 1.133 */ 230 { 18, 256, 0xaa549746b1cbb81c, 0x00000026f02494c9 }, /* 1.131 */ 231 { 19, 256, 0x892e343f2f31d690, 0x00000029eb392835 }, /* 1.130 */ 232 { 20, 256, 0x76914824db98cc3f, 0x0000003004f31a7c }, /* 1.141 */ 233 { 21, 256, 0x4b3cbabf9cfb1d0f, 0x00000036363a2408 }, /* 1.139 */ 234 { 22, 256, 0xf45c77abb4f035d4, 0x00000038dd0f3e84 }, /* 1.150 */ 235 { 23, 256, 0x5e18bd7f3fd4baf4, 0x0000003f0660391f }, /* 1.174 */ 236 { 24, 256, 0xa7b3a4d285d6503b, 0x000000443dfc9ff6 }, /* 1.168 */ 237 { 25, 256, 0x56ac7dd967521f5a, 0x0000004b03a87eb7 }, /* 1.180 */ 238 { 26, 256, 0x3a42dfda4eb880f7, 0x000000522c719bba }, /* 1.226 */ 239 { 27, 256, 0xd200d2fc6b54bf60, 0x0000005760b4fdf5 }, /* 1.228 */ 240 { 28, 256, 0xc52605bbd486c546, 0x0000005e00d8f74c }, /* 1.217 */ 241 { 29, 256, 0xc761779e63cd762f, 0x00000067be3cd85c }, /* 1.239 */ 242 { 30, 256, 0xca577b1e07f85ca5, 0x0000006f5517f3e4 }, /* 1.238 */ 243 { 31, 256, 0xfd50a593c518b3d4, 0x0000007370e7778f }, /* 1.273 */ 244 { 32, 512, 0xc6c87ba5b042650b, 0x000000f7eb08a156 }, /* 1.191 */ 245 { 33, 512, 0xc3880d0c9d458304, 0x0000010734b5d160 }, /* 1.199 */ 246 { 34, 512, 0xe920927e4d8b2c97, 0x00000118c1edbce0 }, /* 1.195 */ 247 { 35, 512, 0x8da7fcda87bde316, 0x0000012a3e9f9110 }, /* 1.201 */ 248 { 36, 512, 0xcf09937491514a29, 0x0000013bd6a24bef }, /* 1.194 */ 249 { 37, 512, 0x9b5abbf345cbd7cc, 0x0000014b9d90fac3 }, /* 1.237 */ 250 { 38, 512, 0x506312a44668d6a9, 0x0000015e1b5f6148 }, /* 1.242 */ 251 { 39, 512, 0x71659ede62b4755f, 0x00000173ef029bcd }, /* 1.231 */ 252 { 40, 512, 0xa7fde73fb74cf2d7, 0x000001866fb72748 }, /* 1.233 */ 253 { 41, 512, 0x19e8b461a1dea1d3, 0x000001a046f76b23 }, /* 1.271 */ 254 { 42, 512, 0x031c9b868cc3e976, 0x000001afa64c49d3 }, /* 1.263 */ 255 { 43, 512, 0xbaa5125faa781854, 0x000001c76789e278 }, /* 1.270 */ 256 { 44, 512, 0x4ed55052550d721b, 0x000001d800ccd8eb }, /* 1.281 */ 257 { 45, 512, 0x0fd63ddbdff90677, 0x000001f08ad59ed2 }, /* 1.282 */ 258 { 46, 512, 0x36d66546de7fdd6f, 0x000002016f09574b }, /* 1.286 */ 259 { 47, 512, 0x99f997e7eafb69d7, 0x0000021e42e47cb6 }, /* 1.329 */ 260 { 48, 512, 0xbecd9c2571312c5d, 0x000002320fe2872b }, /* 1.286 */ 261 { 49, 512, 0xd97371329e488a32, 0x0000024cd73f2ca7 }, /* 1.322 */ 262 { 50, 512, 0x30e9b136670749ee, 0x000002681c83b0e0 }, /* 1.335 */ 263 { 51, 512, 0x11ad6bc8f47aaeb4, 0x0000027e9261b5d5 }, /* 1.305 */ 264 { 52, 512, 0x68e445300af432c1, 0x0000029aa0eb7dbf }, /* 1.330 */ 265 { 53, 512, 0x910fb561657ea98c, 0x000002b3dca04853 }, /* 1.365 */ 266 { 54, 512, 0xd619693d8ce5e7a5, 0x000002cc280e9c97 }, /* 1.334 */ 267 { 55, 512, 0x24e281f564dbb60a, 0x000002e9fa842713 }, /* 1.364 */ 268 { 56, 512, 0x947a7d3bdaab44c5, 0x000003046680f72e }, /* 1.374 */ 269 { 57, 512, 0x2d44fec9c093e0de, 0x00000324198ba810 }, /* 1.363 */ 270 { 58, 512, 0x87743c272d29bb4c, 0x0000033ec48c9ac9 }, /* 1.401 */ 271 { 59, 512, 0x96aa3b6f67f5d923, 0x0000034faead902c }, /* 1.392 */ 272 { 60, 512, 0x94a4f1faf520b0d3, 0x0000037d713ab005 }, /* 1.360 */ 273 { 61, 512, 0xb13ed3a272f711a2, 0x00000397368f3cbd }, /* 1.396 */ 274 { 62, 512, 0x3b1b11805fa4a64a, 0x000003b8a5e2840c }, /* 1.453 */ 275 { 63, 512, 0x4c74caad9172ba71, 0x000003d4be280290 }, /* 1.437 */ 276 { 64, 512, 0x035ff643923dd29e, 0x000003fad6c355e1 }, /* 1.402 */ 277 { 65, 512, 0x768e9171b11abd3c, 0x0000040eb07fed20 }, /* 1.459 */ 278 { 66, 512, 0x75880e6f78a13ddd, 0x000004433d6acf14 }, /* 1.423 */ 279 { 67, 512, 0x910b9714f698a877, 0x00000451ea65d5db }, /* 1.447 */ 280 { 68, 512, 0x87f5db6f9fdcf5c7, 0x000004732169e3f7 }, /* 1.450 */ 281 { 69, 512, 0x836d4968fbaa3706, 0x000004954068a380 }, /* 1.455 */ 282 { 70, 512, 0xc567d73a036421ab, 0x000004bd7cb7bd3d }, /* 1.463 */ 283 { 71, 512, 0x619df40f240b8fed, 0x000004e376c2e972 }, /* 1.463 */ 284 { 72, 512, 0x42763a680d5bed8e, 0x000005084275c680 }, /* 1.452 */ 285 { 73, 512, 0x5866f064b3230431, 0x0000052906f2c9ab }, /* 1.498 */ 286 { 74, 512, 0x9fa08548b1621a44, 0x0000054708019247 }, /* 1.526 */ 287 { 75, 512, 0xb6053078ce0fc303, 0x00000572cc5c72b0 }, /* 1.491 */ 288 { 76, 512, 0x4a7aad7bf3890923, 0x0000058e987bc8e9 }, /* 1.470 */ 289 { 77, 512, 0xe165613fd75b5a53, 0x000005c20473a211 }, /* 1.527 */ 290 { 78, 512, 0x3ff154ac878163a6, 0x000005d659194bf3 }, /* 1.509 */ 291 { 79, 512, 0x24b93ade0aa8a532, 0x0000060a201c4f8e }, /* 1.569 */ 292 { 80, 512, 0xc18e2d14cd9bb554, 0x0000062c55cfe48c }, /* 1.555 */ 293 { 81, 512, 0x98cc78302feb58b6, 0x0000066656a07194 }, /* 1.509 */ 294 { 82, 512, 0xc6c5fd5a2abc0543, 0x0000067cff94fbf8 }, /* 1.596 */ 295 { 83, 512, 0xa7962f514acbba21, 0x000006ab7b5afa2e }, /* 1.568 */ 296 { 84, 512, 0xba02545069ddc6dc, 0x000006d19861364f }, /* 1.541 */ 297 { 85, 512, 0x447c73192c35073e, 0x000006fce315ce35 }, /* 1.623 */ 298 { 86, 512, 0x48beef9e2d42b0c2, 0x00000720a8e38b6b }, /* 1.620 */ 299 { 87, 512, 0x4874cf98541a35e0, 0x00000758382a2273 }, /* 1.597 */ 300 { 88, 512, 0xad4cf8333a31127a, 0x00000781e1651b1b }, /* 1.575 */ 301 { 89, 512, 0x47ae4859d57888c1, 0x000007b27edbe5bc }, /* 1.627 */ 302 { 90, 512, 0x06f7723cfe5d1891, 0x000007dc2a96d8eb }, /* 1.596 */ 303 { 91, 512, 0xd4e44218d660576d, 0x0000080ac46f02d5 }, /* 1.622 */ 304 { 92, 512, 0x7066702b0d5be1f2, 0x00000832c96d154e }, /* 1.695 */ 305 { 93, 512, 0x011209b4f9e11fb9, 0x0000085eefda104c }, /* 1.605 */ 306 { 94, 512, 0x47ffba30a0b35708, 0x00000899badc32dc }, /* 1.625 */ 307 { 95, 512, 0x1a95a6ac4538aaa8, 0x000008b6b69a42b2 }, /* 1.687 */ 308 { 96, 512, 0xbda2b239bb2008eb, 0x000008f22d2de38a }, /* 1.621 */ 309 { 97, 512, 0x7ffa0bea90355c6c, 0x0000092e5b23b816 }, /* 1.699 */ 310 { 98, 512, 0x1d56ba34be426795, 0x0000094f482e5d1b }, /* 1.688 */ 311 { 99, 512, 0x0aa89d45c502e93d, 0x00000977d94a98ce }, /* 1.642 */ 312 { 100, 512, 0x54369449f6857774, 0x000009c06c9b34cc }, /* 1.683 */ 313 { 101, 512, 0xf7d4dd8445b46765, 0x000009e5dc542259 }, /* 1.755 */ 314 { 102, 512, 0xfa8866312f169469, 0x00000a16b54eae93 }, /* 1.692 */ 315 { 103, 512, 0xd8a5aea08aef3ff9, 0x00000a381d2cbfe7 }, /* 1.747 */ 316 { 104, 512, 0x66bcd2c3d5f9ef0e, 0x00000a8191817be7 }, /* 1.751 */ 317 { 105, 512, 0x3fb13a47a012ec81, 0x00000ab562b9a254 }, /* 1.751 */ 318 { 106, 512, 0x43100f01c9e5e3ca, 0x00000aeee84c185f }, /* 1.726 */ 319 { 107, 512, 0xca09c50ccee2d054, 0x00000b1c359c047d }, /* 1.788 */ 320 { 108, 512, 0xd7176732ac503f9b, 0x00000b578bc52a73 }, /* 1.740 */ 321 { 109, 512, 0xed206e51f8d9422d, 0x00000b8083e0d960 }, /* 1.780 */ 322 { 110, 512, 0x17ead5dc6ba0dcd6, 0x00000bcfb1a32ca8 }, /* 1.836 */ 323 { 111, 512, 0x5f1dc21e38a969eb, 0x00000c0171becdd6 }, /* 1.778 */ 324 { 112, 512, 0xddaa973de33ec528, 0x00000c3edaba4b95 }, /* 1.831 */ 325 { 113, 512, 0x2a5eccd7735a3630, 0x00000c630664e7df }, /* 1.825 */ 326 { 114, 512, 0xafcccee5c0b71446, 0x00000cb65392f6e4 }, /* 1.826 */ 327 { 115, 512, 0x8fa30c5e7b147e27, 0x00000cd4db391e55 }, /* 1.843 */ 328 { 116, 512, 0x5afe0711fdfafd82, 0x00000d08cb4ec35d }, /* 1.826 */ 329 { 117, 512, 0x533a6090238afd4c, 0x00000d336f115d1b }, /* 1.803 */ 330 { 118, 512, 0x90cf11b595e39a84, 0x00000d8e041c2048 }, /* 1.857 */ 331 { 119, 512, 0x0d61a3b809444009, 0x00000dcb798afe35 }, /* 1.877 */ 332 { 120, 512, 0x7f34da0f54b0d114, 0x00000df3922664e1 }, /* 1.849 */ 333 { 121, 512, 0xa52258d5b72f6551, 0x00000e4d37a9872d }, /* 1.867 */ 334 { 122, 512, 0xc1de54d7672878db, 0x00000e6583a94cf6 }, /* 1.978 */ 335 { 123, 512, 0x1d03354316a414ab, 0x00000ebffc50308d }, /* 1.947 */ 336 { 124, 512, 0xcebdcc377665412c, 0x00000edee1997cea }, /* 1.865 */ 337 { 125, 512, 0x4ddd4c04b1a12344, 0x00000f21d64b373f }, /* 1.881 */ 338 { 126, 512, 0x64fc8f94e3973658, 0x00000f8f87a8896b }, /* 1.882 */ 339 { 127, 512, 0x68765f78034a334e, 0x00000fb8fe62197e }, /* 1.867 */ 340 { 128, 512, 0xaf36b871a303e816, 0x00000fec6f3afb1e }, /* 1.972 */ 341 { 129, 512, 0x2a4cbf73866c3a28, 0x00001027febfe4e5 }, /* 1.896 */ 342 { 130, 512, 0x9cb128aacdcd3b2f, 0x0000106aa8ac569d }, /* 1.965 */ 343 { 131, 512, 0x5511d41c55869124, 0x000010bbd755ddf1 }, /* 1.963 */ 344 { 132, 512, 0x42f92461937f284a, 0x000010fb8bceb3b5 }, /* 1.925 */ 345 { 133, 512, 0xe2d89a1cf6f1f287, 0x0000114cf5331e34 }, /* 1.862 */ 346 { 134, 512, 0xdc631a038956200e, 0x0000116428d2adc5 }, /* 2.042 */ 347 { 135, 512, 0xb2e5ac222cd236be, 0x000011ca88e4d4d2 }, /* 1.935 */ 348 { 136, 512, 0xbc7d8236655d88e7, 0x000011e39cb94e66 }, /* 2.005 */ 349 { 137, 512, 0x073e02d88d2d8e75, 0x0000123136c7933c }, /* 2.041 */ 350 { 138, 512, 0x3ddb9c3873166be0, 0x00001280e4ec6d52 }, /* 1.997 */ 351 { 139, 512, 0x7d3b1a845420e1b5, 0x000012c2e7cd6a44 }, /* 1.996 */ 352 { 140, 512, 0x60102308aa7b2a6c, 0x000012fc490e6c7d }, /* 2.053 */ 353 { 141, 512, 0xdb22bb2f9eb894aa, 0x00001343f5a85a1a }, /* 1.971 */ 354 { 142, 512, 0xd853f879a13b1606, 0x000013bb7d5f9048 }, /* 2.018 */ 355 { 143, 512, 0x001620a03f804b1d, 0x000013e74cc794fd }, /* 1.961 */ 356 { 144, 512, 0xfdb52dda76fbf667, 0x00001442d2f22480 }, /* 2.046 */ 357 { 145, 512, 0xa9160110f66e24ff, 0x0000144b899f9dbb }, /* 1.968 */ 358 { 146, 512, 0x77306a30379ae03b, 0x000014cb98eb1f81 }, /* 2.143 */ 359 { 147, 512, 0x14f5985d2752319d, 0x000014feab821fc9 }, /* 2.064 */ 360 { 148, 512, 0xa4b8ff11de7863f8, 0x0000154a0e60b9c9 }, /* 2.023 */ 361 { 149, 512, 0x44b345426455c1b3, 0x000015999c3c569c }, /* 2.136 */ 362 { 150, 512, 0x272677826049b46c, 0x000015c9697f4b92 }, /* 2.063 */ 363 { 151, 512, 0x2f9216e2cd74fe40, 0x0000162b1f7bbd39 }, /* 1.974 */ 364 { 152, 512, 0x706ae3e763ad8771, 0x00001661371c55e1 }, /* 2.210 */ 365 { 153, 512, 0xf7fd345307c2480e, 0x000016e251f28b6a }, /* 2.006 */ 366 { 154, 512, 0x6e94e3d26b3139eb, 0x000016f2429bb8c6 }, /* 2.193 */ 367 { 155, 512, 0x5458bbfbb781fcba, 0x0000173efdeca1b9 }, /* 2.163 */ 368 { 156, 512, 0xa80e2afeccd93b33, 0x000017bfdcb78adc }, /* 2.046 */ 369 { 157, 512, 0x1e4ccbb22796cf9d, 0x00001826fdcc39c9 }, /* 2.084 */ 370 { 158, 512, 0x8fba4b676aaa3663, 0x00001841a1379480 }, /* 2.264 */ 371 { 159, 512, 0xf82b843814b315fa, 0x000018886e19b8a3 }, /* 2.074 */ 372 { 160, 512, 0x7f21e920ecf753a3, 0x0000191812ca0ea7 }, /* 2.282 */ 373 { 161, 512, 0x48bb8ea2c4caa620, 0x0000192f310faccf }, /* 2.148 */ 374 { 162, 512, 0x5cdb652b4952c91b, 0x0000199e1d7437c7 }, /* 2.355 */ 375 { 163, 512, 0x6ac1ba6f78c06cd4, 0x000019cd11f82c70 }, /* 2.164 */ 376 { 164, 512, 0x9faf5f9ca2669a56, 0x00001a18d5431f6a }, /* 2.393 */ 377 { 165, 512, 0xaa57e9383eb01194, 0x00001a9e7d253d85 }, /* 2.178 */ 378 { 166, 512, 0x896967bf495c34d2, 0x00001afb8319b9fc }, /* 2.334 */ 379 { 167, 512, 0xdfad5f05de225f1b, 0x00001b3a59c3093b }, /* 2.266 */ 380 { 168, 512, 0xfd299a99f9f2abdd, 0x00001bb6f1a10799 }, /* 2.304 */ 381 { 169, 512, 0xdda239e798fe9fd4, 0x00001bfae0c9692d }, /* 2.218 */ 382 { 170, 512, 0x5fca670414a32c3e, 0x00001c22129dbcff }, /* 2.377 */ 383 { 171, 512, 0x1bb8934314b087de, 0x00001c955db36cd0 }, /* 2.155 */ 384 { 172, 512, 0xd96394b4b082200d, 0x00001cfc8619b7e6 }, /* 2.404 */ 385 { 173, 512, 0xb612a7735b1c8cbc, 0x00001d303acdd585 }, /* 2.205 */ 386 { 174, 512, 0x28e7430fe5875fe1, 0x00001d7ed5b3697d }, /* 2.359 */ 387 { 175, 512, 0x5038e89efdd981b9, 0x00001dc40ec35c59 }, /* 2.158 */ 388 { 176, 512, 0x075fd78f1d14db7c, 0x00001e31c83b4a2b }, /* 2.614 */ 389 { 177, 512, 0xc50fafdb5021be15, 0x00001e7cdac82fbc }, /* 2.239 */ 390 { 178, 512, 0xe6dc7572ce7b91c7, 0x00001edd8bb454fc }, /* 2.493 */ 391 { 179, 512, 0x21f7843e7beda537, 0x00001f3a8e019d6c }, /* 2.327 */ 392 { 180, 512, 0xc83385e20b43ec82, 0x00001f70735ec137 }, /* 2.231 */ 393 { 181, 512, 0xca818217dddb21fd, 0x0000201ca44c5a3c }, /* 2.237 */ 394 { 182, 512, 0xe6035defea48f933, 0x00002038e3346658 }, /* 2.691 */ 395 { 183, 512, 0x47262a4f953dac5a, 0x000020c2e554314e }, /* 2.170 */ 396 { 184, 512, 0xe24c7246260873ea, 0x000021197e618d64 }, /* 2.600 */ 397 { 185, 512, 0xeef6b57c9b58e9e1, 0x0000217ea48ecddc }, /* 2.391 */ 398 { 186, 512, 0x2becd3346e386142, 0x000021c496d4a5f9 }, /* 2.677 */ 399 { 187, 512, 0x63c6207bdf3b40a3, 0x0000220e0f2eec0c }, /* 2.410 */ 400 { 188, 512, 0x3056ce8989767d4b, 0x0000228eb76cd137 }, /* 2.776 */ 401 { 189, 512, 0x91af61c307cee780, 0x000022e17e2ea501 }, /* 2.266 */ 402 { 190, 512, 0xda359da225f6d54f, 0x00002358a2debc19 }, /* 2.717 */ 403 { 191, 512, 0x0a5f7a2a55607ba0, 0x0000238a79dac18c }, /* 2.474 */ 404 { 192, 512, 0x27bb75bf5224638a, 0x00002403a58e2351 }, /* 2.673 */ 405 { 193, 512, 0x1ebfdb94630f5d0f, 0x00002492a10cb339 }, /* 2.420 */ 406 { 194, 512, 0x6eae5e51d9c5f6fb, 0x000024ce4bf98715 }, /* 2.898 */ 407 { 195, 512, 0x08d903b4daedc2e0, 0x0000250d1e15886c }, /* 2.363 */ 408 { 196, 512, 0xc722a2f7fa7cd686, 0x0000258a99ed0c9e }, /* 2.747 */ 409 { 197, 512, 0x8f71faf0e54e361d, 0x000025dee11976f5 }, /* 2.531 */ 410 { 198, 512, 0x87f64695c91a54e7, 0x0000264e00a43da0 }, /* 2.707 */ 411 { 199, 512, 0xc719cbac2c336b92, 0x000026d327277ac1 }, /* 2.315 */ 412 { 200, 512, 0xe7e647afaf771ade, 0x000027523a5c44bf }, /* 3.012 */ 413 { 201, 512, 0x12d4b5c38ce8c946, 0x0000273898432545 }, /* 2.378 */ 414 { 202, 512, 0xf2e0cd4067bdc94a, 0x000027e47bb2c935 }, /* 2.969 */ 415 { 203, 512, 0x21b79f14d6d947d3, 0x0000281e64977f0d }, /* 2.594 */ 416 { 204, 512, 0x515093f952f18cd6, 0x0000289691a473fd }, /* 2.763 */ 417 { 205, 512, 0xd47b160a1b1022c8, 0x00002903e8b52411 }, /* 2.457 */ 418 { 206, 512, 0xc02fc96684715a16, 0x0000297515608601 }, /* 3.057 */ 419 { 207, 512, 0xef51e68efba72ed0, 0x000029ef73604804 }, /* 2.590 */ 420 { 208, 512, 0x9e3be6e5448b4f33, 0x00002a2846ed074b }, /* 3.047 */ 421 { 209, 512, 0x81d446c6d5fec063, 0x00002a92ca693455 }, /* 2.676 */ 422 { 210, 512, 0xff215de8224e57d5, 0x00002b2271fe3729 }, /* 2.993 */ 423 { 211, 512, 0xe2524d9ba8f69796, 0x00002b64b99c3ba2 }, /* 2.457 */ 424 { 212, 512, 0xf6b28e26097b7e4b, 0x00002bd768b6e068 }, /* 3.182 */ 425 { 213, 512, 0x893a487f30ce1644, 0x00002c67f722b4b2 }, /* 2.563 */ 426 { 214, 512, 0x386566c3fc9871df, 0x00002cc1cf8b4037 }, /* 3.025 */ 427 { 215, 512, 0x1e0ed78edf1f558a, 0x00002d3948d36c7f }, /* 2.730 */ 428 { 216, 512, 0xe3bc20c31e61f113, 0x00002d6d6b12e025 }, /* 3.036 */ 429 { 217, 512, 0xd6c3ad2e23021882, 0x00002deff7572241 }, /* 2.722 */ 430 { 218, 512, 0xb4a9f95cf0f69c5a, 0x00002e67d537aa36 }, /* 3.356 */ 431 { 219, 512, 0x6e98ed6f6c38e82f, 0x00002e9720626789 }, /* 2.697 */ 432 { 220, 512, 0x2e01edba33fddac7, 0x00002f407c6b0198 }, /* 2.979 */ 433 { 221, 512, 0x559d02e1f5f57ccc, 0x00002fb6a5ab4f24 }, /* 2.858 */ 434 { 222, 512, 0xac18f5a916adcd8e, 0x0000304ae1c5c57e }, /* 3.258 */ 435 { 223, 512, 0x15789fbaddb86f4b, 0x0000306f6e019c78 }, /* 2.693 */ 436 { 224, 512, 0xf4a9c36d5bc4c408, 0x000030da40434213 }, /* 3.259 */ 437 { 225, 512, 0xf640f90fd2727f44, 0x00003189ed37b90c }, /* 2.733 */ 438 { 226, 512, 0xb5313d390d61884a, 0x000031e152616b37 }, /* 3.235 */ 439 { 227, 512, 0x4bae6b3ce9160939, 0x0000321f40aeac42 }, /* 2.983 */ 440 { 228, 512, 0x838c34480f1a66a1, 0x000032f389c0f78e }, /* 3.308 */ 441 { 229, 512, 0xb1c4a52c8e3d6060, 0x0000330062a40284 }, /* 2.715 */ 442 { 230, 512, 0xe0f1110c6d0ed822, 0x0000338be435644f }, /* 3.540 */ 443 { 231, 512, 0x9f1a8ccdcea68d4b, 0x000034045a4e97e1 }, /* 2.779 */ 444 { 232, 512, 0x3261ed62223f3099, 0x000034702cfc401c }, /* 3.084 */ 445 { 233, 512, 0xf2191e2311022d65, 0x00003509dd19c9fc }, /* 2.987 */ 446 { 234, 512, 0xf102a395c2033abc, 0x000035654dc96fae }, /* 3.341 */ 447 { 235, 512, 0x11fe378f027906b6, 0x000035b5193b0264 }, /* 2.793 */ 448 { 236, 512, 0xf777f2c026b337aa, 0x000036704f5d9297 }, /* 3.518 */ 449 { 237, 512, 0x1b04e9c2ee143f32, 0x000036dfbb7af218 }, /* 2.962 */ 450 { 238, 512, 0x2fcec95266f9352c, 0x00003785c8df24a9 }, /* 3.196 */ 451 { 239, 512, 0xfe2b0e47e427dd85, 0x000037cbdf5da729 }, /* 2.914 */ 452 { 240, 512, 0x72b49bf2225f6c6d, 0x0000382227c15855 }, /* 3.408 */ 453 { 241, 512, 0x50486b43df7df9c7, 0x0000389b88be6453 }, /* 2.903 */ 454 { 242, 512, 0x5192a3e53181c8ab, 0x000038ddf3d67263 }, /* 3.778 */ 455 { 243, 512, 0xe9f5d8365296fd5e, 0x0000399f1c6c9e9c }, /* 3.026 */ 456 { 244, 512, 0xc740263f0301efa8, 0x00003a147146512d }, /* 3.347 */ 457 { 245, 512, 0x23cd0f2b5671e67d, 0x00003ab10bcc0d9d }, /* 3.212 */ 458 { 246, 512, 0x002ccc7e5cd41390, 0x00003ad6cd14a6c0 }, /* 3.482 */ 459 { 247, 512, 0x9aafb3c02544b31b, 0x00003b8cb8779fb0 }, /* 3.146 */ 460 { 248, 512, 0x72ba07a78b121999, 0x00003c24142a5a3f }, /* 3.626 */ 461 { 249, 512, 0x3d784aa58edfc7b4, 0x00003cd084817d99 }, /* 2.952 */ 462 { 250, 512, 0xaab750424d8004af, 0x00003d506a8e098e }, /* 3.463 */ 463 { 251, 512, 0x84403fcf8e6b5ca2, 0x00003d4c54c2aec4 }, /* 3.131 */ 464 { 252, 512, 0x71eb7455ec98e207, 0x00003e655715cf2c }, /* 3.538 */ 465 { 253, 512, 0xd752b4f19301595b, 0x00003ecd7b2ca5ac }, /* 2.974 */ 466 { 254, 512, 0xc4674129750499de, 0x00003e99e86d3e95 }, /* 3.843 */ 467 { 255, 512, 0x9772baff5cd12ef5, 0x00003f895c019841 }, /* 3.088 */ 468 }; 469 470 /* 471 * Verify the map is valid. Each device index must appear exactly 472 * once in every row, and the permutation array checksum must match. 473 */ 474 static int 475 verify_perms(uint8_t *perms, uint64_t children, uint64_t nperms, 476 uint64_t checksum) 477 { 478 int countssz = sizeof (uint16_t) * children; 479 uint16_t *counts = kmem_zalloc(countssz, KM_SLEEP); 480 481 for (int i = 0; i < nperms; i++) { 482 for (int j = 0; j < children; j++) { 483 uint8_t val = perms[(i * children) + j]; 484 485 if (val >= children || counts[val] != i) { 486 kmem_free(counts, countssz); 487 return (EINVAL); 488 } 489 490 counts[val]++; 491 } 492 } 493 494 if (checksum != 0) { 495 int permssz = sizeof (uint8_t) * children * nperms; 496 zio_cksum_t cksum; 497 498 #if defined(_ZFS_BIG_ENDIAN) 499 fletcher_4_byteswap_varsize(perms, permssz, &cksum); 500 #else 501 fletcher_4_native_varsize(perms, permssz, &cksum); 502 #endif 503 504 if (checksum != cksum.zc_word[0]) { 505 kmem_free(counts, countssz); 506 return (ECKSUM); 507 } 508 } 509 510 kmem_free(counts, countssz); 511 512 return (0); 513 } 514 515 /* 516 * Generate the permutation array for the draid_map_t. These maps control 517 * the placement of all data in a dRAID. Therefore it's critical that the 518 * seed always generates the same mapping. We provide our own pseudo-random 519 * number generator for this purpose. 520 */ 521 int 522 vdev_draid_generate_perms(const draid_map_t *map, uint8_t **permsp) 523 { 524 VERIFY3U(map->dm_children, >=, VDEV_DRAID_MIN_CHILDREN); 525 VERIFY3U(map->dm_children, <=, VDEV_DRAID_MAX_CHILDREN); 526 VERIFY3U(map->dm_seed, !=, 0); 527 VERIFY3U(map->dm_nperms, !=, 0); 528 VERIFY0P(map->dm_perms); 529 530 #ifdef _KERNEL 531 /* 532 * The kernel code always provides both a map_seed and checksum. 533 * Only the tests/zfs-tests/cmd/draid/draid.c utility will provide 534 * a zero checksum when generating new candidate maps. 535 */ 536 VERIFY3U(map->dm_checksum, !=, 0); 537 #endif 538 uint64_t children = map->dm_children; 539 uint64_t nperms = map->dm_nperms; 540 int rowsz = sizeof (uint8_t) * children; 541 int permssz = rowsz * nperms; 542 uint8_t *perms; 543 544 /* Allocate the permutation array */ 545 perms = vmem_alloc(permssz, KM_SLEEP); 546 547 /* Setup an initial row with a known pattern */ 548 uint8_t *initial_row = kmem_alloc(rowsz, KM_SLEEP); 549 for (int i = 0; i < children; i++) 550 initial_row[i] = i; 551 552 uint64_t draid_seed[2] = { VDEV_DRAID_SEED, map->dm_seed }; 553 uint8_t *current_row, *previous_row = initial_row; 554 555 /* 556 * Perform a Fisher-Yates shuffle of each row using the previous 557 * row as the starting point. An initial_row with known pattern 558 * is used as the input for the first row. 559 */ 560 for (int i = 0; i < nperms; i++) { 561 current_row = &perms[i * children]; 562 memcpy(current_row, previous_row, rowsz); 563 564 for (int j = children - 1; j > 0; j--) { 565 uint64_t k = vdev_draid_rand(draid_seed) % (j + 1); 566 uint8_t val = current_row[j]; 567 current_row[j] = current_row[k]; 568 current_row[k] = val; 569 } 570 571 previous_row = current_row; 572 } 573 574 kmem_free(initial_row, rowsz); 575 576 int error = verify_perms(perms, children, nperms, map->dm_checksum); 577 if (error) { 578 vmem_free(perms, permssz); 579 return (error); 580 } 581 582 *permsp = perms; 583 584 return (0); 585 } 586 587 static void 588 vdev_draid_swap_perms(uint8_t *perms, uint64_t i, uint64_t j) 589 { 590 uint8_t val = perms[i]; 591 592 perms[i] = perms[j]; 593 perms[j] = val; 594 } 595 596 /* 597 * Shuffle every i-th disk in slices that lie alongside in the big width row, 598 * increasing disk indices in each next slice in the row accordingly. The 599 * input to this function is the array of ready permutations from 600 * vdev_draid_generate_perms(), so in order to correctly shuffle i-th disks, 601 * we need to locate their position first and build a map of their locations. 602 * 603 * Note: the same Fisher-Yates shuffle algorithm is used as in 604 * vdev_draid_generate_perms(). 605 */ 606 static void 607 vdev_draid_shuffle_perms(const draid_map_t *map, uint8_t *perms, uint64_t width) 608 { 609 uint64_t cn = map->dm_children; 610 uint64_t n = width / cn; 611 uint64_t nperms = map->dm_nperms / n * n; 612 613 if (width <= cn) 614 return; 615 616 VERIFY3U(width, >=, VDEV_DRAID_MIN_CHILDREN); 617 VERIFY3U(width, <=, VDEV_DRAID_MAX_CHILDREN); 618 ASSERT0(width % cn); 619 620 uint64_t draid_seed[2] = { VDEV_DRAID_SEED, map->dm_seed }; 621 622 uint8_t *cmap = kmem_alloc(n, KM_SLEEP); 623 624 for (int i = 0; i < nperms; i += n) { 625 for (int j = 0; j < cn; j++) { 626 627 /* locate position of the same child in other slices */ 628 for (int k = n - 1; k > 0; k--) 629 for (int l = 0; l < cn; l++) 630 if (perms[(i+k) * cn + l] == 631 perms[(i+0) * cn + j]) 632 cmap[k] = l; 633 cmap[0] = j; 634 635 /* increase index values for slices on the right */ 636 for (int k = n - 1; k > 0; k--) 637 perms[(i+k) * cn + cmap[k]] += k * cn; 638 639 /* shuffle */ 640 for (int k = n - 1; k > 0; k--) { 641 int l = vdev_draid_rand(draid_seed) % (k + 1); 642 if (k == l) 643 continue; 644 vdev_draid_swap_perms(perms, 645 (i+k) * cn + cmap[k], 646 (i+l) * cn + cmap[l]); 647 } 648 } 649 } 650 651 kmem_free(cmap, n); 652 } 653 654 /* 655 * Lookup the fixed draid_map_t for the requested number of children. 656 */ 657 int 658 vdev_draid_lookup_map(uint64_t children, const draid_map_t **mapp) 659 { 660 for (int i = 0; i < VDEV_DRAID_MAX_MAPS; i++) { 661 if (draid_maps[i].dm_children == children) { 662 *mapp = &draid_maps[i]; 663 return (0); 664 } 665 } 666 667 return (ENOENT); 668 } 669 670 /* 671 * Lookup the permutation array and iteration id for the provided offset. 672 */ 673 static void 674 vdev_draid_get_perm(vdev_draid_config_t *vdc, uint64_t pindex, 675 uint8_t **base, uint64_t *iter) 676 { 677 uint64_t n = vdc->vdc_width / vdc->vdc_children; 678 uint64_t ncols = vdc->vdc_children; 679 uint64_t nperms = (vdc->vdc_nperms / n) * n; 680 uint64_t poff = pindex % (nperms * ncols); 681 682 ASSERT3P(nperms, >=, ncols * n); 683 684 *base = vdc->vdc_perms + (poff / (ncols * n)) * (ncols * n); 685 *iter = (poff % ncols) + (pindex % n) * ncols; 686 } 687 688 static inline uint64_t 689 vdev_draid_permute_id(vdev_draid_config_t *vdc, 690 uint8_t *base, uint64_t iter, uint64_t index) 691 { 692 if (vdc->vdc_width > vdc->vdc_children) { 693 uint64_t off = (iter / vdc->vdc_children) * vdc->vdc_children; 694 return (base[(index + iter) % vdc->vdc_children + off]); 695 } 696 697 return ((base[index] + iter) % vdc->vdc_children); 698 } 699 700 /* 701 * Return the asize which is the psize rounded up to a full group width. 702 * i.e. vdev_draid_psize_to_asize(). 703 */ 704 static uint64_t 705 vdev_draid_psize_to_asize(vdev_t *vd, uint64_t psize, uint64_t txg) 706 { 707 (void) txg; 708 vdev_draid_config_t *vdc = vd->vdev_tsd; 709 uint64_t ashift = vd->vdev_ashift; 710 711 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 712 713 uint64_t rows = ((psize - 1) / (vdc->vdc_ndata << ashift)) + 1; 714 uint64_t asize = (rows * vdc->vdc_groupwidth) << ashift; 715 716 ASSERT3U(asize, !=, 0); 717 ASSERT0(asize % (vdc->vdc_groupwidth)); 718 719 return (asize); 720 } 721 722 /* 723 * Deflate the asize to the psize, this includes stripping parity. 724 */ 725 uint64_t 726 vdev_draid_asize_to_psize(vdev_t *vd, uint64_t asize, uint64_t txg) 727 { 728 (void) txg; 729 vdev_draid_config_t *vdc = vd->vdev_tsd; 730 731 ASSERT0(asize % vdc->vdc_groupwidth); 732 733 return ((asize / vdc->vdc_groupwidth) * vdc->vdc_ndata); 734 } 735 736 /* 737 * Convert a logical offset to the corresponding group number. 738 */ 739 static uint64_t 740 vdev_draid_offset_to_group(vdev_t *vd, uint64_t offset) 741 { 742 vdev_draid_config_t *vdc = vd->vdev_tsd; 743 744 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 745 746 return (offset / vdc->vdc_groupsz); 747 } 748 749 /* 750 * Convert a group number to the logical starting offset for that group. 751 */ 752 static uint64_t 753 vdev_draid_group_to_offset(vdev_t *vd, uint64_t group) 754 { 755 vdev_draid_config_t *vdc = vd->vdev_tsd; 756 757 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 758 759 return (group * vdc->vdc_groupsz); 760 } 761 762 /* 763 * Full stripe writes. When writing, all columns (D+P) are required. Parity 764 * is calculated over all the columns, including empty zero filled sectors, 765 * and each is written to disk. While only the data columns are needed for 766 * a normal read, all of the columns are required for reconstruction when 767 * performing a sequential resilver. 768 * 769 * For "big columns" it's sufficient to map the correct range of the zio ABD. 770 * Partial columns require allocating a gang ABD in order to zero fill the 771 * empty sectors. When the column is empty a zero filled sector must be 772 * mapped. In all cases the data ABDs must be the same size as the parity 773 * ABDs (e.g. rc->rc_size == parity_size). 774 */ 775 static void 776 vdev_draid_map_alloc_write(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) 777 { 778 uint64_t skip_size = 1ULL << zio->io_vd->vdev_top->vdev_ashift; 779 uint64_t parity_size = rr->rr_col[0].rc_size; 780 uint64_t abd_off = abd_offset; 781 782 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 783 ASSERT3U(parity_size, ==, abd_get_size(rr->rr_col[0].rc_abd)); 784 785 for (uint64_t c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { 786 raidz_col_t *rc = &rr->rr_col[c]; 787 788 if (rc->rc_size == 0) { 789 /* empty data column (small write), add a skip sector */ 790 ASSERT3U(skip_size, ==, parity_size); 791 rc->rc_abd = abd_get_zeros_struct(&rc->rc_abdstruct, 792 skip_size); 793 } else if (rc->rc_size == parity_size) { 794 /* this is a "big column" */ 795 rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct, 796 zio->io_abd, abd_off, rc->rc_size); 797 } else { 798 /* short data column, add a skip sector */ 799 ASSERT3U(rc->rc_size + skip_size, ==, parity_size); 800 rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); 801 abd_gang_add(rc->rc_abd, abd_get_offset_size( 802 zio->io_abd, abd_off, rc->rc_size), B_TRUE); 803 abd_gang_add(rc->rc_abd, abd_get_zeros(skip_size), 804 B_TRUE); 805 } 806 807 ASSERT3U(abd_get_size(rc->rc_abd), ==, parity_size); 808 809 abd_off += rc->rc_size; 810 rc->rc_size = parity_size; 811 } 812 813 IMPLY(abd_offset != 0, abd_off == zio->io_size); 814 } 815 816 /* 817 * Scrub/resilver reads. In order to store the contents of the skip sectors 818 * an additional ABD is allocated. The columns are handled in the same way 819 * as a full stripe write except instead of using the zero ABD the newly 820 * allocated skip ABD is used to back the skip sectors. In all cases the 821 * data ABD must be the same size as the parity ABDs. 822 */ 823 static void 824 vdev_draid_map_alloc_scrub(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) 825 { 826 uint64_t skip_size = 1ULL << zio->io_vd->vdev_top->vdev_ashift; 827 uint64_t parity_size = rr->rr_col[0].rc_size; 828 uint64_t abd_off = abd_offset; 829 uint64_t skip_off = 0; 830 831 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 832 ASSERT0P(rr->rr_abd_empty); 833 834 if (rr->rr_nempty > 0) { 835 rr->rr_abd_empty = abd_alloc_linear(rr->rr_nempty * skip_size, 836 B_FALSE); 837 } 838 839 for (uint64_t c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { 840 raidz_col_t *rc = &rr->rr_col[c]; 841 842 if (rc->rc_size == 0) { 843 /* empty data column (small read), add a skip sector */ 844 ASSERT3U(skip_size, ==, parity_size); 845 ASSERT3U(rr->rr_nempty, !=, 0); 846 rc->rc_abd = abd_get_offset_size(rr->rr_abd_empty, 847 skip_off, skip_size); 848 skip_off += skip_size; 849 } else if (rc->rc_size == parity_size) { 850 /* this is a "big column" */ 851 rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct, 852 zio->io_abd, abd_off, rc->rc_size); 853 } else { 854 /* short data column, add a skip sector */ 855 ASSERT3U(rc->rc_size + skip_size, ==, parity_size); 856 ASSERT3U(rr->rr_nempty, !=, 0); 857 rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); 858 abd_gang_add(rc->rc_abd, abd_get_offset_size( 859 zio->io_abd, abd_off, rc->rc_size), B_TRUE); 860 abd_gang_add(rc->rc_abd, abd_get_offset_size( 861 rr->rr_abd_empty, skip_off, skip_size), B_TRUE); 862 skip_off += skip_size; 863 } 864 865 uint64_t abd_size = abd_get_size(rc->rc_abd); 866 ASSERT3U(abd_size, ==, abd_get_size(rr->rr_col[0].rc_abd)); 867 868 /* 869 * Increase rc_size so the skip ABD is included in subsequent 870 * parity calculations. 871 */ 872 abd_off += rc->rc_size; 873 rc->rc_size = abd_size; 874 } 875 876 IMPLY(abd_offset != 0, abd_off == zio->io_size); 877 ASSERT3U(skip_off, ==, rr->rr_nempty * skip_size); 878 } 879 880 /* 881 * Normal reads. In this common case only the columns containing data 882 * are read in to the zio ABDs. Neither the parity columns or empty skip 883 * sectors are read unless the checksum fails verification. In which case 884 * vdev_raidz_read_all() will call vdev_draid_map_alloc_empty() to expand 885 * the raid map in order to allow reconstruction using the parity data and 886 * skip sectors. 887 */ 888 static void 889 vdev_draid_map_alloc_read(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) 890 { 891 uint64_t abd_off = abd_offset; 892 893 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 894 895 for (uint64_t c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { 896 raidz_col_t *rc = &rr->rr_col[c]; 897 898 if (rc->rc_size > 0) { 899 rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct, 900 zio->io_abd, abd_off, rc->rc_size); 901 abd_off += rc->rc_size; 902 } 903 } 904 905 IMPLY(abd_offset != 0, abd_off == zio->io_size); 906 } 907 908 /* 909 * Converts a normal "read" raidz_row_t to a "scrub" raidz_row_t. The key 910 * difference is that an ABD is allocated to back skip sectors so they may 911 * be read in to memory, verified, and repaired if needed. 912 */ 913 void 914 vdev_draid_map_alloc_empty(zio_t *zio, raidz_row_t *rr) 915 { 916 uint64_t skip_size = 1ULL << zio->io_vd->vdev_top->vdev_ashift; 917 uint64_t parity_size = rr->rr_col[0].rc_size; 918 uint64_t skip_off = 0; 919 920 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 921 ASSERT0P(rr->rr_abd_empty); 922 923 if (rr->rr_nempty > 0) { 924 rr->rr_abd_empty = abd_alloc_linear(rr->rr_nempty * skip_size, 925 B_FALSE); 926 } 927 928 for (uint64_t c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { 929 raidz_col_t *rc = &rr->rr_col[c]; 930 931 if (rc->rc_size == 0) { 932 /* empty data column (small read), add a skip sector */ 933 ASSERT3U(skip_size, ==, parity_size); 934 ASSERT3U(rr->rr_nempty, !=, 0); 935 ASSERT0P(rc->rc_abd); 936 rc->rc_abd = abd_get_offset_size(rr->rr_abd_empty, 937 skip_off, skip_size); 938 skip_off += skip_size; 939 } else if (rc->rc_size == parity_size) { 940 /* this is a "big column", nothing to add */ 941 ASSERT3P(rc->rc_abd, !=, NULL); 942 } else { 943 /* 944 * short data column, add a skip sector and clear 945 * rc_tried to force the entire column to be re-read 946 * thereby including the missing skip sector data 947 * which is needed for reconstruction. 948 */ 949 ASSERT3U(rc->rc_size + skip_size, ==, parity_size); 950 ASSERT3U(rr->rr_nempty, !=, 0); 951 ASSERT3P(rc->rc_abd, !=, NULL); 952 ASSERT(!abd_is_gang(rc->rc_abd)); 953 abd_t *read_abd = rc->rc_abd; 954 rc->rc_abd = abd_alloc_gang(); 955 abd_gang_add(rc->rc_abd, read_abd, B_TRUE); 956 abd_gang_add(rc->rc_abd, abd_get_offset_size( 957 rr->rr_abd_empty, skip_off, skip_size), B_TRUE); 958 skip_off += skip_size; 959 rc->rc_tried = 0; 960 } 961 962 /* 963 * Increase rc_size so the empty ABD is included in subsequent 964 * parity calculations. 965 */ 966 rc->rc_size = parity_size; 967 } 968 969 ASSERT3U(skip_off, ==, rr->rr_nempty * skip_size); 970 } 971 972 /* 973 * Verify that all empty sectors are zero filled before using them to 974 * calculate parity. Otherwise, silent corruption in an empty sector will 975 * result in bad parity being generated. That bad parity will then be 976 * considered authoritative and overwrite the good parity on disk. This 977 * is possible because the checksum is only calculated over the data, 978 * thus it cannot be used to detect damage in empty sectors. 979 */ 980 int 981 vdev_draid_map_verify_empty(zio_t *zio, raidz_row_t *rr) 982 { 983 uint64_t skip_size = 1ULL << zio->io_vd->vdev_top->vdev_ashift; 984 uint64_t parity_size = rr->rr_col[0].rc_size; 985 uint64_t skip_off = parity_size - skip_size; 986 uint64_t empty_off = 0; 987 int ret = 0; 988 989 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 990 ASSERT3P(rr->rr_abd_empty, !=, NULL); 991 ASSERT3U(rr->rr_bigcols, >, 0); 992 993 void *zero_buf = kmem_zalloc(skip_size, KM_SLEEP); 994 995 for (int c = rr->rr_bigcols; c < rr->rr_cols; c++) { 996 raidz_col_t *rc = &rr->rr_col[c]; 997 998 ASSERT3P(rc->rc_abd, !=, NULL); 999 ASSERT3U(rc->rc_size, ==, parity_size); 1000 1001 if (abd_cmp_buf_off(rc->rc_abd, zero_buf, skip_off, 1002 skip_size) != 0) { 1003 vdev_raidz_checksum_error(zio, rc, rc->rc_abd); 1004 abd_zero_off(rc->rc_abd, skip_off, skip_size); 1005 rc->rc_error = SET_ERROR(ECKSUM); 1006 ret++; 1007 } 1008 1009 empty_off += skip_size; 1010 } 1011 1012 ASSERT3U(empty_off, ==, abd_get_size(rr->rr_abd_empty)); 1013 1014 kmem_free(zero_buf, skip_size); 1015 1016 return (ret); 1017 } 1018 1019 /* 1020 * Given a logical address within a dRAID configuration, return the physical 1021 * address on the first drive in the group that this address maps to 1022 * (at position 'start' in permutation number 'perm'). 1023 */ 1024 static uint64_t 1025 vdev_draid_logical_to_physical(vdev_t *vd, uint64_t logical_offset, 1026 uint64_t *perm, uint64_t *start, uint64_t *ndisks) 1027 { 1028 vdev_draid_config_t *vdc = vd->vdev_tsd; 1029 1030 /* b is the dRAID (parent) sector offset. */ 1031 uint64_t ashift = vd->vdev_top->vdev_ashift; 1032 uint64_t b_offset = logical_offset >> ashift; 1033 1034 /* 1035 * The height of a row in units of the vdev's minimum sector size. 1036 * This is the amount of data written to each disk of each group 1037 * in a given permutation. 1038 */ 1039 uint64_t rowheight_sectors = VDEV_DRAID_ROWHEIGHT >> ashift; 1040 1041 /* 1042 * We cycle through a disk permutation every groupsz * ngroups chunk 1043 * of address space. Note that ngroups * groupsz must be a multiple 1044 * of the number of data drives (ndisks) in order to guarantee 1045 * alignment. So, for example, if our row height is 16MB, our group 1046 * size is 10, and there are 13 data drives in the draid, then ngroups 1047 * will be 13, we will change permutation every 2.08GB and each 1048 * disk will have 160MB of data per chunk. 1049 */ 1050 uint64_t groupwidth = vdc->vdc_groupwidth; 1051 uint64_t ngroups = vdc->vdc_ngroups; 1052 1053 uint64_t group = logical_offset / vdc->vdc_groupsz; 1054 uint64_t fgrps = vdc->vdc_width / vdc->vdc_children; 1055 1056 *perm = (group / ngroups) * fgrps; 1057 1058 /* 1059 * Failure groups starting from (vdc_nspares % fgrps) have one less 1060 * spare, so they have one more ndisks. 1061 */ 1062 uint64_t biggies = vdc->vdc_nspares % fgrps; 1063 1064 uint64_t poff = 0; 1065 group %= ngroups; 1066 uint64_t ngroups1 = ngroups / fgrps; 1067 if (!biggies || group < biggies * ngroups1) 1068 poff = group / ngroups1; 1069 else 1070 poff = biggies + 1071 (group - (biggies * ngroups1)) / (ngroups1 + 1); 1072 ASSERT3U(poff, <, fgrps); 1073 *perm += poff; 1074 1075 *ndisks = (vdc->vdc_ndisks / fgrps) + 1076 (biggies ? ((poff >= biggies) ? 1 : 0) : 0); 1077 1078 /* b_offset is the sector offset within a group chunk */ 1079 b_offset = b_offset % (rowheight_sectors * groupwidth); 1080 ASSERT0(b_offset % groupwidth); 1081 1082 /* 1083 * Find the starting byte offset on each child vdev: 1084 * - within a permutation there are ngroups groups spread over the 1085 * rows, where each row covers a slice portion of the disk 1086 * - each permutation has (groupwidth * ngroups) / ndisks rows 1087 * - so each permutation covers rows * slice portion of the disk 1088 * - so we need to find the row where this IO group target begins 1089 */ 1090 uint64_t perm_rows = (groupwidth * ngroups) / vdc->vdc_ndisks; 1091 1092 /* Adjust group for our failure group. */ 1093 if (!biggies || poff <= biggies) 1094 group -= poff * ngroups1; 1095 else 1096 group -= (biggies * ngroups1) + 1097 (poff - biggies) * (ngroups1 + 1); 1098 1099 IMPLY(poff < biggies, group < ngroups1); 1100 ASSERT3U(group, <=, ngroups1); 1101 1102 /* 1103 * groupstart is where the group this IO will land in "starts" in 1104 * the permutation array. 1105 */ 1106 uint64_t groupstart = (group * groupwidth) % *ndisks; 1107 ASSERT3U(groupstart + groupwidth, <=, *ndisks + groupstart); 1108 *start = groupstart; 1109 1110 /* Adjust ngroups for our failure group. */ 1111 ngroups = ngroups1 + ((biggies && poff >= biggies) ? 1 : 0); 1112 1113 ASSERT3U(group, <, ngroups); 1114 1115 uint64_t row = ((*perm / fgrps) * perm_rows) + 1116 (((group % ngroups) * groupwidth) / *ndisks); 1117 1118 return (((rowheight_sectors * row) + 1119 (b_offset / groupwidth)) << ashift); 1120 } 1121 1122 static uint64_t 1123 vdev_draid_map_alloc_row(zio_t *zio, raidz_row_t **rrp, uint64_t io_offset, 1124 uint64_t abd_offset, uint64_t abd_size) 1125 { 1126 vdev_t *vd = zio->io_vd; 1127 vdev_draid_config_t *vdc = vd->vdev_tsd; 1128 uint64_t ashift = vd->vdev_top->vdev_ashift; 1129 uint64_t io_size = abd_size; 1130 uint64_t io_asize = vdev_draid_psize_to_asize(vd, io_size, 0); 1131 uint64_t group = vdev_draid_offset_to_group(vd, io_offset); 1132 uint64_t start_offset = vdev_draid_group_to_offset(vd, group + 1); 1133 1134 /* 1135 * Limit the io_size to the space remaining in the group. A second 1136 * row in the raidz_map_t is created for the remainder. 1137 */ 1138 if (io_offset + io_asize > start_offset) { 1139 io_size = vdev_draid_asize_to_psize(vd, 1140 start_offset - io_offset, 0); 1141 } 1142 1143 /* 1144 * At most a block may span the logical end of one group and the start 1145 * of the next group. Therefore, at the end of a group the io_size must 1146 * span the group width evenly and the remainder must be aligned to the 1147 * start of the next group. 1148 */ 1149 IMPLY(abd_offset == 0 && io_size < zio->io_size, 1150 (io_asize >> ashift) % vdc->vdc_groupwidth == 0); 1151 IMPLY(abd_offset != 0, 1152 vdev_draid_group_to_offset(vd, group) == io_offset); 1153 1154 /* Lookup starting byte offset on each child vdev */ 1155 uint64_t groupstart, perm, ndisks; 1156 uint64_t physical_offset = vdev_draid_logical_to_physical(vd, 1157 io_offset, &perm, &groupstart, &ndisks); 1158 1159 /* 1160 * If there is less than groupwidth drives available after the group 1161 * start, the group is going to wrap onto the next row. 'wrap' is the 1162 * group disk number that starts on the next row. 1163 */ 1164 uint64_t groupwidth = vdc->vdc_groupwidth; 1165 uint64_t wrap = groupwidth; 1166 1167 if (groupstart + groupwidth > ndisks) 1168 wrap = ndisks - groupstart; 1169 1170 /* The io size in units of the vdev's minimum sector size. */ 1171 const uint64_t psize = io_size >> ashift; 1172 1173 /* 1174 * "Quotient": The number of data sectors for this stripe on all but 1175 * the "big column" child vdevs that also contain "remainder" data. 1176 */ 1177 uint64_t q = psize / vdc->vdc_ndata; 1178 1179 /* 1180 * "Remainder": The number of partial stripe data sectors in this I/O. 1181 * This will add a sector to some, but not all, child vdevs. 1182 */ 1183 uint64_t r = psize - q * vdc->vdc_ndata; 1184 1185 /* The number of "big columns" - those which contain remainder data. */ 1186 uint64_t bc = (r == 0 ? 0 : r + vdc->vdc_nparity); 1187 ASSERT3U(bc, <, groupwidth); 1188 1189 /* The total number of data and parity sectors for this I/O. */ 1190 uint64_t tot = psize + (vdc->vdc_nparity * (q + (r == 0 ? 0 : 1))); 1191 1192 ASSERT3U(vdc->vdc_nparity, >, 0); 1193 1194 raidz_row_t *rr = vdev_raidz_row_alloc(groupwidth, zio); 1195 rr->rr_bigcols = bc; 1196 rr->rr_firstdatacol = vdc->vdc_nparity; 1197 #ifdef ZFS_DEBUG 1198 rr->rr_offset = io_offset; 1199 rr->rr_size = io_size; 1200 #endif 1201 *rrp = rr; 1202 1203 uint8_t *base; 1204 uint64_t iter, asize = 0; 1205 vdev_draid_get_perm(vdc, perm, &base, &iter); 1206 for (uint64_t i = 0; i < groupwidth; i++) { 1207 raidz_col_t *rc = &rr->rr_col[i]; 1208 uint64_t c = (groupstart + i) % ndisks; 1209 1210 /* increment the offset if we wrap to the next row */ 1211 if (i == wrap) 1212 physical_offset += VDEV_DRAID_ROWHEIGHT; 1213 1214 rc->rc_devidx = vdev_draid_permute_id(vdc, base, iter, c); 1215 rc->rc_offset = physical_offset; 1216 1217 if (q == 0 && i >= bc) 1218 rc->rc_size = 0; 1219 else if (i < bc) 1220 rc->rc_size = (q + 1) << ashift; 1221 else 1222 rc->rc_size = q << ashift; 1223 1224 asize += rc->rc_size; 1225 } 1226 1227 ASSERT3U(asize, ==, tot << ashift); 1228 rr->rr_nempty = roundup(tot, groupwidth) - tot; 1229 IMPLY(bc > 0, rr->rr_nempty == groupwidth - bc); 1230 1231 /* Allocate buffers for the parity columns */ 1232 for (uint64_t c = 0; c < rr->rr_firstdatacol; c++) { 1233 raidz_col_t *rc = &rr->rr_col[c]; 1234 rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, 1235 rc->rc_size, B_FALSE); 1236 } 1237 1238 /* 1239 * Map buffers for data columns and allocate/map buffers for skip 1240 * sectors. There are three distinct cases for dRAID which are 1241 * required to support sequential rebuild. 1242 */ 1243 if (zio->io_type == ZIO_TYPE_WRITE) { 1244 vdev_draid_map_alloc_write(zio, abd_offset, rr); 1245 } else if ((rr->rr_nempty > 0) && 1246 (zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) { 1247 vdev_draid_map_alloc_scrub(zio, abd_offset, rr); 1248 } else { 1249 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 1250 vdev_draid_map_alloc_read(zio, abd_offset, rr); 1251 } 1252 1253 return (io_size); 1254 } 1255 1256 /* 1257 * Allocate the raidz mapping to be applied to the dRAID I/O. The parity 1258 * calculations for dRAID are identical to raidz however there are a few 1259 * differences in the layout. 1260 * 1261 * - dRAID always allocates a full stripe width. Any extra sectors due 1262 * this padding are zero filled and written to disk. They will be read 1263 * back during a scrub or repair operation since they are included in 1264 * the parity calculation. This property enables sequential resilvering. 1265 * 1266 * - When the block at the logical offset spans redundancy groups then two 1267 * rows are allocated in the raidz_map_t. One row resides at the end of 1268 * the first group and the other at the start of the following group. 1269 */ 1270 static raidz_map_t * 1271 vdev_draid_map_alloc(zio_t *zio) 1272 { 1273 raidz_row_t *rr[2]; 1274 uint64_t abd_offset = 0; 1275 uint64_t abd_size = zio->io_size; 1276 uint64_t io_offset = zio->io_offset; 1277 uint64_t size; 1278 int nrows = 1; 1279 1280 size = vdev_draid_map_alloc_row(zio, &rr[0], io_offset, 1281 abd_offset, abd_size); 1282 if (size < abd_size) { 1283 vdev_t *vd = zio->io_vd; 1284 1285 io_offset += vdev_draid_psize_to_asize(vd, size, 0); 1286 abd_offset += size; 1287 abd_size -= size; 1288 nrows++; 1289 1290 ASSERT3U(io_offset, ==, vdev_draid_group_to_offset( 1291 vd, vdev_draid_offset_to_group(vd, io_offset))); 1292 ASSERT3U(abd_offset, <, zio->io_size); 1293 ASSERT3U(abd_size, !=, 0); 1294 1295 size = vdev_draid_map_alloc_row(zio, &rr[1], 1296 io_offset, abd_offset, abd_size); 1297 VERIFY3U(size, ==, abd_size); 1298 } 1299 1300 raidz_map_t *rm; 1301 rm = kmem_zalloc(offsetof(raidz_map_t, rm_row[nrows]), KM_SLEEP); 1302 rm->rm_ops = vdev_raidz_math_get_ops(); 1303 rm->rm_nrows = nrows; 1304 rm->rm_row[0] = rr[0]; 1305 if (nrows == 2) 1306 rm->rm_row[1] = rr[1]; 1307 return (rm); 1308 } 1309 1310 /* 1311 * Given an offset into a dRAID return the next group width aligned offset 1312 * which can be used to start an allocation. 1313 */ 1314 static uint64_t 1315 vdev_draid_get_astart(vdev_t *vd, const uint64_t start) 1316 { 1317 vdev_draid_config_t *vdc = vd->vdev_tsd; 1318 1319 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1320 1321 return (roundup(start, vdc->vdc_groupwidth << vd->vdev_ashift)); 1322 } 1323 1324 /* 1325 * Allocatable space for dRAID is (children - nspares) * sizeof(smallest child) 1326 * rounded down to the last full slice. So each child must provide at least 1327 * 1 / (children - nspares) of its asize rounded up to VDEV_DRAID_ROWHEIGHT. 1328 */ 1329 static uint64_t 1330 vdev_draid_min_asize(vdev_t *vd) 1331 { 1332 vdev_draid_config_t *vdc = vd->vdev_tsd; 1333 1334 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1335 1336 return (VDEV_DRAID_REFLOW_RESERVE + 1337 DIV_ROUND_UP(DIV_ROUND_UP(vd->vdev_min_asize, vdc->vdc_ndisks), 1338 VDEV_DRAID_ROWHEIGHT) * VDEV_DRAID_ROWHEIGHT); 1339 } 1340 1341 /* 1342 * When using dRAID the minimum allocation size is determined by the number 1343 * of data disks in the redundancy group. Full stripes are always used. 1344 */ 1345 static uint64_t 1346 vdev_draid_min_alloc(vdev_t *vd) 1347 { 1348 vdev_draid_config_t *vdc = vd->vdev_tsd; 1349 1350 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1351 1352 return (vdc->vdc_ndata << vd->vdev_ashift); 1353 } 1354 1355 /* 1356 * Returns false if the txg range exists on any leaf vdev, true otherwise. 1357 * 1358 * A dRAID spare does not fit into the DTL model. While it has child vdevs 1359 * there is no redundancy among them, and the effective child vdev is 1360 * determined by offset. Essentially we do a vdev_dtl_reassess() on the 1361 * fly by replacing a dRAID spare with the child vdev under the offset. 1362 * Note that it is a recursive process because the child vdev can be 1363 * another dRAID spare and so on. 1364 */ 1365 boolean_t 1366 vdev_draid_missing(vdev_t *vd, uint64_t physical_offset, uint64_t txg, 1367 uint64_t size) 1368 { 1369 if (vd->vdev_ops == &vdev_spare_ops || 1370 vd->vdev_ops == &vdev_replacing_ops) { 1371 /* 1372 * Check all of the readable children, if any child 1373 * contains the txg range the data it is not missing. 1374 */ 1375 for (int c = 0; c < vd->vdev_children; c++) { 1376 vdev_t *cvd = vd->vdev_child[c]; 1377 1378 if (!vdev_readable(cvd)) 1379 continue; 1380 1381 if (!vdev_draid_missing(cvd, physical_offset, 1382 txg, size)) 1383 return (B_FALSE); 1384 } 1385 1386 return (B_TRUE); 1387 } 1388 1389 if (vd->vdev_ops == &vdev_draid_spare_ops) { 1390 /* 1391 * When sequentially resilvering we don't have a proper 1392 * txg range so instead we must presume all txgs are 1393 * missing on this vdev until the resilver completes. 1394 */ 1395 if (vd->vdev_rebuild_txg != 0) 1396 return (B_TRUE); 1397 1398 /* 1399 * DTL_MISSING is set for all prior txgs when a resilver 1400 * is started in spa_vdev_attach(). 1401 */ 1402 if (vdev_dtl_contains(vd, DTL_MISSING, txg, size)) 1403 return (B_TRUE); 1404 1405 /* 1406 * Consult the DTL on the relevant vdev. Either a vdev 1407 * leaf or spare/replace mirror child may be returned so 1408 * we must recursively call vdev_draid_missing_impl(). 1409 */ 1410 vd = vdev_draid_spare_get_child(vd, physical_offset); 1411 if (vd == NULL) 1412 return (B_TRUE); 1413 1414 return (vdev_draid_missing(vd, physical_offset, txg, size)); 1415 } 1416 1417 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size)); 1418 } 1419 1420 /* 1421 * Returns true if the txg is only partially replicated on the leaf vdevs. 1422 */ 1423 static boolean_t 1424 vdev_draid_partial(vdev_t *vd, uint64_t physical_offset, uint64_t txg, 1425 uint64_t size) 1426 { 1427 if (vd->vdev_ops == &vdev_spare_ops || 1428 vd->vdev_ops == &vdev_replacing_ops) { 1429 /* 1430 * Check all of the readable children, if any child is 1431 * missing the txg range then it is partially replicated. 1432 */ 1433 for (int c = 0; c < vd->vdev_children; c++) { 1434 vdev_t *cvd = vd->vdev_child[c]; 1435 1436 if (!vdev_readable(cvd)) 1437 continue; 1438 1439 if (vdev_draid_partial(cvd, physical_offset, txg, size)) 1440 return (B_TRUE); 1441 } 1442 1443 return (B_FALSE); 1444 } 1445 1446 if (vd->vdev_ops == &vdev_draid_spare_ops) { 1447 /* 1448 * When sequentially resilvering we don't have a proper 1449 * txg range so instead we must presume all txgs are 1450 * missing on this vdev until the resilver completes. 1451 */ 1452 if (vd->vdev_rebuild_txg != 0) 1453 return (B_TRUE); 1454 1455 /* 1456 * DTL_MISSING is set for all prior txgs when a resilver 1457 * is started in spa_vdev_attach(). 1458 */ 1459 if (vdev_dtl_contains(vd, DTL_MISSING, txg, size)) 1460 return (B_TRUE); 1461 1462 /* 1463 * Consult the DTL on the relevant vdev. Either a vdev 1464 * leaf or spare/replace mirror child may be returned so 1465 * we must recursively call vdev_draid_missing_impl(). 1466 */ 1467 vd = vdev_draid_spare_get_child(vd, physical_offset); 1468 if (vd == NULL) 1469 return (B_TRUE); 1470 1471 return (vdev_draid_partial(vd, physical_offset, txg, size)); 1472 } 1473 1474 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size)); 1475 } 1476 1477 /* 1478 * Determine if the vdev is readable at the given offset. 1479 */ 1480 boolean_t 1481 vdev_draid_readable(vdev_t *vd, uint64_t physical_offset) 1482 { 1483 if (vd->vdev_ops == &vdev_draid_spare_ops) { 1484 vd = vdev_draid_spare_get_child(vd, physical_offset); 1485 if (vd == NULL) 1486 return (B_FALSE); 1487 } 1488 1489 if (vd->vdev_ops == &vdev_spare_ops || 1490 vd->vdev_ops == &vdev_replacing_ops) { 1491 1492 for (int c = 0; c < vd->vdev_children; c++) { 1493 vdev_t *cvd = vd->vdev_child[c]; 1494 1495 if (!vdev_readable(cvd)) 1496 continue; 1497 1498 if (vdev_draid_readable(cvd, physical_offset)) 1499 return (B_TRUE); 1500 } 1501 1502 return (B_FALSE); 1503 } 1504 1505 return (vdev_readable(vd)); 1506 } 1507 1508 /* 1509 * Returns the first distributed spare found under the provided vdev tree. 1510 */ 1511 static vdev_t * 1512 vdev_draid_find_spare(vdev_t *vd) 1513 { 1514 if (vd->vdev_ops == &vdev_draid_spare_ops) 1515 return (vd); 1516 1517 for (int c = 0; c < vd->vdev_children; c++) { 1518 vdev_t *svd = vdev_draid_find_spare(vd->vdev_child[c]); 1519 if (svd != NULL) 1520 return (svd); 1521 } 1522 1523 return (NULL); 1524 } 1525 1526 /* 1527 * Returns B_TRUE if the passed in vdev is currently "faulted". 1528 * Faulted, in this context, means that the vdev represents a 1529 * replacing or sparing vdev tree. 1530 */ 1531 static boolean_t 1532 vdev_draid_faulted(vdev_t *vd, uint64_t physical_offset) 1533 { 1534 if (vd->vdev_ops == &vdev_draid_spare_ops) { 1535 vd = vdev_draid_spare_get_child(vd, physical_offset); 1536 if (vd == NULL) 1537 return (B_FALSE); 1538 1539 /* 1540 * After resolving the distributed spare to a leaf vdev 1541 * check the parent to determine if it's "faulted". 1542 */ 1543 vd = vd->vdev_parent; 1544 } 1545 1546 return (vd->vdev_ops == &vdev_replacing_ops || 1547 vd->vdev_ops == &vdev_spare_ops); 1548 } 1549 1550 /* 1551 * Determine if the dRAID block at the logical offset is degraded. 1552 * Used by sequential resilver. 1553 */ 1554 static boolean_t 1555 vdev_draid_group_degraded(vdev_t *vd, uint64_t offset) 1556 { 1557 vdev_draid_config_t *vdc = vd->vdev_tsd; 1558 1559 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1560 ASSERT3U(vdev_draid_get_astart(vd, offset), ==, offset); 1561 1562 uint64_t groupstart, perm, ndisks; 1563 uint64_t physical_offset = vdev_draid_logical_to_physical(vd, 1564 offset, &perm, &groupstart, &ndisks); 1565 1566 uint8_t *base; 1567 uint64_t iter; 1568 vdev_draid_get_perm(vdc, perm, &base, &iter); 1569 1570 for (uint64_t i = 0; i < vdc->vdc_groupwidth; i++) { 1571 uint64_t c = (groupstart + i) % ndisks; 1572 uint64_t cid = vdev_draid_permute_id(vdc, base, iter, c); 1573 vdev_t *cvd = vd->vdev_child[cid]; 1574 1575 /* Group contains a faulted vdev. */ 1576 if (vdev_draid_faulted(cvd, physical_offset)) 1577 return (B_TRUE); 1578 1579 /* 1580 * Always check groups with active distributed spares 1581 * because any vdev failure in the pool will affect them. 1582 */ 1583 if (vdev_draid_find_spare(cvd) != NULL) 1584 return (B_TRUE); 1585 } 1586 1587 return (B_FALSE); 1588 } 1589 1590 /* 1591 * Determine if the txg is missing. Used by healing resilver. 1592 */ 1593 static boolean_t 1594 vdev_draid_group_missing(vdev_t *vd, uint64_t offset, uint64_t txg, 1595 uint64_t size) 1596 { 1597 vdev_draid_config_t *vdc = vd->vdev_tsd; 1598 1599 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1600 ASSERT3U(vdev_draid_get_astart(vd, offset), ==, offset); 1601 1602 uint64_t groupstart, perm, ndisks; 1603 uint64_t physical_offset = vdev_draid_logical_to_physical(vd, 1604 offset, &perm, &groupstart, &ndisks); 1605 1606 uint8_t *base; 1607 uint64_t iter; 1608 vdev_draid_get_perm(vdc, perm, &base, &iter); 1609 1610 for (uint64_t i = 0; i < vdc->vdc_groupwidth; i++) { 1611 uint64_t c = (groupstart + i) % ndisks; 1612 uint64_t cid = vdev_draid_permute_id(vdc, base, iter, c); 1613 vdev_t *cvd = vd->vdev_child[cid]; 1614 1615 /* Transaction group is known to be partially replicated. */ 1616 if (vdev_draid_partial(cvd, physical_offset, txg, size)) 1617 return (B_TRUE); 1618 } 1619 1620 return (B_FALSE); 1621 } 1622 1623 /* 1624 * Find the smallest child asize and largest sector size to calculate the 1625 * available capacity. Distributed spares are ignored since their capacity 1626 * is also based of the minimum child size in the top-level dRAID. 1627 */ 1628 static void 1629 vdev_draid_calculate_asize(vdev_t *vd, uint64_t *asizep, uint64_t *max_asizep, 1630 uint64_t *logical_ashiftp, uint64_t *physical_ashiftp) 1631 { 1632 uint64_t logical_ashift = 0, physical_ashift = 0; 1633 uint64_t asize = 0, max_asize = 0; 1634 1635 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1636 1637 for (int c = 0; c < vd->vdev_children; c++) { 1638 vdev_t *cvd = vd->vdev_child[c]; 1639 1640 if (cvd->vdev_ops == &vdev_draid_spare_ops) 1641 continue; 1642 1643 asize = MIN(asize - 1, cvd->vdev_asize - 1) + 1; 1644 max_asize = MIN(max_asize - 1, cvd->vdev_max_asize - 1) + 1; 1645 logical_ashift = MAX(logical_ashift, cvd->vdev_ashift); 1646 } 1647 for (int c = 0; c < vd->vdev_children; c++) { 1648 vdev_t *cvd = vd->vdev_child[c]; 1649 1650 if (cvd->vdev_ops == &vdev_draid_spare_ops) 1651 continue; 1652 physical_ashift = vdev_best_ashift(logical_ashift, 1653 physical_ashift, cvd->vdev_physical_ashift); 1654 } 1655 1656 *asizep = asize; 1657 *max_asizep = max_asize; 1658 *logical_ashiftp = logical_ashift; 1659 *physical_ashiftp = physical_ashift; 1660 } 1661 1662 /* 1663 * Open spare vdevs. 1664 */ 1665 static boolean_t 1666 vdev_draid_open_spares(vdev_t *vd) 1667 { 1668 return (vd->vdev_ops == &vdev_draid_spare_ops || 1669 vd->vdev_ops == &vdev_replacing_ops || 1670 vd->vdev_ops == &vdev_spare_ops); 1671 } 1672 1673 /* 1674 * Open all children, excluding spares. 1675 */ 1676 static boolean_t 1677 vdev_draid_open_children(vdev_t *vd) 1678 { 1679 return (!vdev_draid_open_spares(vd)); 1680 } 1681 1682 /* 1683 * Open a top-level dRAID vdev. 1684 */ 1685 static int 1686 vdev_draid_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize, 1687 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr) 1688 { 1689 vdev_draid_config_t *vdc = vd->vdev_tsd; 1690 uint64_t nparity = vdc->vdc_nparity; 1691 int open_errors = 0; 1692 1693 if (nparity > VDEV_DRAID_MAXPARITY || 1694 vdc->vdc_children < nparity + 1) { 1695 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; 1696 return (SET_ERROR(EINVAL)); 1697 } 1698 1699 /* 1700 * First open the normal children then the distributed spares. This 1701 * ordering is important to ensure the distributed spares calculate 1702 * the correct psize in the event that the dRAID vdevs were expanded. 1703 */ 1704 vdev_open_children_subset(vd, cr, vdev_draid_open_children); 1705 vdev_open_children_subset(vd, cr, vdev_draid_open_spares); 1706 1707 /* 1708 * Verify enough of the children are available to continue. 1709 * If several disks got failed on i-th position in each slice in the 1710 * big width row (failure groups) - they are counted as one failure, 1711 * but only if the failures threshold is not reached in any group. 1712 */ 1713 boolean_t safe2skip = B_FALSE; 1714 if (vdc->vdc_width > vdc->vdc_children && 1715 vdev_draid_fail_domain_allowed(vd)) 1716 safe2skip = B_TRUE; 1717 for (int c = 0; c < vdc->vdc_children; c++) { 1718 for (int i = c; i < vdc->vdc_width; i += vdc->vdc_children) { 1719 if (vd->vdev_child[i]->vdev_open_error != 0) { 1720 if ((++open_errors) > nparity) { 1721 vd->vdev_stat.vs_aux = 1722 VDEV_AUX_NO_REPLICAS; 1723 return (SET_ERROR(ENXIO)); 1724 } 1725 if (safe2skip) 1726 break; 1727 } 1728 } 1729 } 1730 1731 /* 1732 * Allocatable capacity is the sum of the space on all children less 1733 * the number of distributed spares rounded down to last full row 1734 * and then to the last full group. An additional 32MB of scratch 1735 * space is reserved at the end of each child for use by the dRAID 1736 * expansion feature. 1737 */ 1738 uint64_t child_asize, child_max_asize; 1739 vdev_draid_calculate_asize(vd, &child_asize, &child_max_asize, 1740 logical_ashift, physical_ashift); 1741 1742 /* 1743 * Should be unreachable since the minimum child size is 64MB, but 1744 * we want to make sure an underflow absolutely cannot occur here. 1745 */ 1746 if (child_asize < VDEV_DRAID_REFLOW_RESERVE || 1747 child_max_asize < VDEV_DRAID_REFLOW_RESERVE) { 1748 return (SET_ERROR(ENXIO)); 1749 } 1750 1751 child_asize = ((child_asize - VDEV_DRAID_REFLOW_RESERVE) / 1752 VDEV_DRAID_ROWHEIGHT) * VDEV_DRAID_ROWHEIGHT; 1753 child_max_asize = ((child_max_asize - VDEV_DRAID_REFLOW_RESERVE) / 1754 VDEV_DRAID_ROWHEIGHT) * VDEV_DRAID_ROWHEIGHT; 1755 1756 *asize = (((child_asize * vdc->vdc_ndisks) / vdc->vdc_groupsz) * 1757 vdc->vdc_groupsz); 1758 *max_asize = (((child_max_asize * vdc->vdc_ndisks) / vdc->vdc_groupsz) * 1759 vdc->vdc_groupsz); 1760 1761 /* 1762 * For failure groups with multiple silices in the big width row, 1763 * round down to the big slice size. 1764 */ 1765 if (vdc->vdc_width > vdc->vdc_children) { 1766 uint64_t slicesz = vdc->vdc_devslicesz * vdc->vdc_ndisks; 1767 *asize = (*asize / slicesz) * slicesz; 1768 *max_asize = (*max_asize / slicesz) * slicesz; 1769 } 1770 1771 return (0); 1772 } 1773 1774 /* 1775 * Close a top-level dRAID vdev. 1776 */ 1777 static void 1778 vdev_draid_close(vdev_t *vd) 1779 { 1780 for (int c = 0; c < vd->vdev_children; c++) { 1781 if (vd->vdev_child[c] != NULL) 1782 vdev_close(vd->vdev_child[c]); 1783 } 1784 } 1785 1786 /* 1787 * Return the maximum asize for a rebuild zio in the provided range 1788 * given the following constraints. A dRAID chunks may not: 1789 * 1790 * - Exceed the maximum allowed block size (SPA_MAXBLOCKSIZE), or 1791 * - Span dRAID redundancy groups. 1792 */ 1793 static uint64_t 1794 vdev_draid_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize, 1795 uint64_t max_segment) 1796 { 1797 vdev_draid_config_t *vdc = vd->vdev_tsd; 1798 1799 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1800 1801 uint64_t ashift = vd->vdev_ashift; 1802 uint64_t ndata = vdc->vdc_ndata; 1803 uint64_t psize = MIN(P2ROUNDUP(max_segment * ndata, 1 << ashift), 1804 SPA_MAXBLOCKSIZE); 1805 1806 ASSERT3U(vdev_draid_get_astart(vd, start), ==, start); 1807 ASSERT0(asize % (vdc->vdc_groupwidth << ashift)); 1808 1809 /* Chunks must evenly span all data columns in the group. */ 1810 psize = (((psize >> ashift) / ndata) * ndata) << ashift; 1811 uint64_t chunk_size = MIN(asize, vdev_psize_to_asize(vd, psize)); 1812 1813 /* Reduce the chunk size to the group space remaining. */ 1814 uint64_t group = vdev_draid_offset_to_group(vd, start); 1815 uint64_t left = vdev_draid_group_to_offset(vd, group + 1) - start; 1816 chunk_size = MIN(chunk_size, left); 1817 1818 ASSERT0(chunk_size % (vdc->vdc_groupwidth << ashift)); 1819 ASSERT3U(vdev_draid_offset_to_group(vd, start), ==, 1820 vdev_draid_offset_to_group(vd, start + chunk_size - 1)); 1821 1822 return (chunk_size); 1823 } 1824 1825 /* 1826 * Align the start of the metaslab to the group width and slightly reduce 1827 * its size to a multiple of the group width. Since full stripe writes are 1828 * required by dRAID this space is unallocable. Furthermore, aligning the 1829 * metaslab start is important for vdev initialize and TRIM which both operate 1830 * on metaslab boundaries which vdev_xlate() expects to be aligned. 1831 */ 1832 static void 1833 vdev_draid_metaslab_init(vdev_t *vd, uint64_t *ms_start, uint64_t *ms_size) 1834 { 1835 vdev_draid_config_t *vdc = vd->vdev_tsd; 1836 1837 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 1838 1839 uint64_t sz = vdc->vdc_groupwidth << vd->vdev_ashift; 1840 uint64_t astart = vdev_draid_get_astart(vd, *ms_start); 1841 uint64_t asize = ((*ms_size - (astart - *ms_start)) / sz) * sz; 1842 1843 *ms_start = astart; 1844 *ms_size = asize; 1845 1846 ASSERT0(*ms_start % sz); 1847 ASSERT0(*ms_size % sz); 1848 } 1849 1850 /* 1851 * Add virtual dRAID spares to the list of valid spares. In order to accomplish 1852 * this the existing array must be freed and reallocated with the additional 1853 * entries. 1854 */ 1855 int 1856 vdev_draid_spare_create(nvlist_t *nvroot, vdev_t *vd, uint64_t *ndraidp, 1857 uint64_t *nfgroupp, uint64_t next_vdev_id) 1858 { 1859 uint64_t draid_nspares = 0; 1860 uint64_t ndraid = 0; 1861 uint64_t nfgroup = 0; 1862 int error; 1863 1864 for (uint64_t i = 0; i < vd->vdev_children; i++) { 1865 vdev_t *cvd = vd->vdev_child[i]; 1866 1867 if (cvd->vdev_ops == &vdev_draid_ops) { 1868 vdev_draid_config_t *vdc = cvd->vdev_tsd; 1869 draid_nspares += vdc->vdc_nspares; 1870 ndraid++; 1871 if (vdc->vdc_width > vdc->vdc_children) 1872 nfgroup++; 1873 } 1874 } 1875 1876 if (draid_nspares == 0) { 1877 *ndraidp = ndraid; 1878 *nfgroupp = nfgroup; 1879 return (0); 1880 } 1881 1882 nvlist_t **old_spares, **new_spares; 1883 uint_t old_nspares; 1884 error = nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 1885 &old_spares, &old_nspares); 1886 if (error) 1887 old_nspares = 0; 1888 1889 /* Allocate memory and copy of the existing spares. */ 1890 new_spares = kmem_alloc(sizeof (nvlist_t *) * 1891 (draid_nspares + old_nspares), KM_SLEEP); 1892 for (uint_t i = 0; i < old_nspares; i++) 1893 new_spares[i] = fnvlist_dup(old_spares[i]); 1894 1895 /* Add new distributed spares to ZPOOL_CONFIG_SPARES. */ 1896 uint64_t n = old_nspares; 1897 for (uint64_t vdev_id = 0; vdev_id < vd->vdev_children; vdev_id++) { 1898 vdev_t *cvd = vd->vdev_child[vdev_id]; 1899 char path[64]; 1900 1901 if (cvd->vdev_ops != &vdev_draid_ops) 1902 continue; 1903 1904 vdev_draid_config_t *vdc = cvd->vdev_tsd; 1905 uint64_t nspares = vdc->vdc_nspares; 1906 uint64_t nparity = vdc->vdc_nparity; 1907 1908 for (uint64_t spare_id = 0; spare_id < nspares; spare_id++) { 1909 memset(path, 0, sizeof (path)); 1910 (void) snprintf(path, sizeof (path) - 1, 1911 "%s%llu-%llu-%llu", VDEV_TYPE_DRAID, 1912 (u_longlong_t)nparity, 1913 (u_longlong_t)next_vdev_id + vdev_id, 1914 (u_longlong_t)spare_id); 1915 1916 nvlist_t *spare = fnvlist_alloc(); 1917 fnvlist_add_string(spare, ZPOOL_CONFIG_PATH, path); 1918 fnvlist_add_string(spare, ZPOOL_CONFIG_TYPE, 1919 VDEV_TYPE_DRAID_SPARE); 1920 fnvlist_add_uint64(spare, ZPOOL_CONFIG_TOP_GUID, 1921 cvd->vdev_guid); 1922 fnvlist_add_uint64(spare, ZPOOL_CONFIG_SPARE_ID, 1923 spare_id); 1924 fnvlist_add_uint64(spare, ZPOOL_CONFIG_IS_LOG, 0); 1925 fnvlist_add_uint64(spare, ZPOOL_CONFIG_IS_SPARE, 1); 1926 fnvlist_add_uint64(spare, ZPOOL_CONFIG_WHOLE_DISK, 1); 1927 fnvlist_add_uint64(spare, ZPOOL_CONFIG_ASHIFT, 1928 cvd->vdev_ashift); 1929 1930 new_spares[n] = spare; 1931 n++; 1932 } 1933 } 1934 1935 if (n > 0) { 1936 (void) nvlist_remove_all(nvroot, ZPOOL_CONFIG_SPARES); 1937 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 1938 (const nvlist_t **)new_spares, n); 1939 } 1940 1941 for (int i = 0; i < n; i++) 1942 nvlist_free(new_spares[i]); 1943 1944 kmem_free(new_spares, sizeof (*new_spares) * n); 1945 *ndraidp = ndraid; 1946 *nfgroupp = nfgroup; 1947 1948 return (0); 1949 } 1950 1951 /* 1952 * Determine if any portion of the provided block resides on a child vdev 1953 * with a dirty DTL and therefore needs to be resilvered. 1954 */ 1955 static boolean_t 1956 vdev_draid_need_resilver(vdev_t *vd, const dva_t *dva, size_t psize, 1957 uint64_t phys_birth) 1958 { 1959 uint64_t offset = DVA_GET_OFFSET(dva); 1960 uint64_t asize = vdev_draid_psize_to_asize(vd, psize, 0); 1961 1962 if (phys_birth == TXG_UNKNOWN) { 1963 /* 1964 * Sequential resilver. There is no meaningful phys_birth 1965 * for this block, we can only determine if block resides 1966 * in a degraded group in which case it must be resilvered. 1967 */ 1968 ASSERT3U(vdev_draid_offset_to_group(vd, offset), ==, 1969 vdev_draid_offset_to_group(vd, offset + asize - 1)); 1970 1971 return (vdev_draid_group_degraded(vd, offset)); 1972 } else { 1973 /* 1974 * Healing resilver. TXGs not in DTL_PARTIAL are intact, 1975 * as are blocks in non-degraded groups. 1976 */ 1977 if (!vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1)) 1978 return (B_FALSE); 1979 1980 if (vdev_draid_group_missing(vd, offset, phys_birth, 1)) 1981 return (B_TRUE); 1982 1983 /* The block may span groups in which case check both. */ 1984 if (vdev_draid_offset_to_group(vd, offset) != 1985 vdev_draid_offset_to_group(vd, offset + asize - 1)) { 1986 if (vdev_draid_group_missing(vd, 1987 offset + asize, phys_birth, 1)) 1988 return (B_TRUE); 1989 } 1990 1991 return (B_FALSE); 1992 } 1993 } 1994 1995 static boolean_t 1996 vdev_draid_rebuilding(vdev_t *vd) 1997 { 1998 if (vd->vdev_ops->vdev_op_leaf && vd->vdev_rebuild_txg) 1999 return (B_TRUE); 2000 2001 for (int i = 0; i < vd->vdev_children; i++) { 2002 if (vdev_draid_rebuilding(vd->vdev_child[i])) { 2003 return (B_TRUE); 2004 } 2005 } 2006 2007 return (B_FALSE); 2008 } 2009 2010 static void 2011 vdev_draid_io_verify(vdev_t *vd, raidz_row_t *rr, int col) 2012 { 2013 #ifdef ZFS_DEBUG 2014 zfs_range_seg64_t logical_rs, physical_rs, remain_rs; 2015 logical_rs.rs_start = rr->rr_offset; 2016 logical_rs.rs_end = logical_rs.rs_start + 2017 vdev_draid_psize_to_asize(vd, rr->rr_size, 0); 2018 2019 raidz_col_t *rc = &rr->rr_col[col]; 2020 vdev_t *cvd = vd->vdev_child[rc->rc_devidx]; 2021 2022 vdev_xlate(cvd, &logical_rs, &physical_rs, &remain_rs); 2023 ASSERT(vdev_xlate_is_empty(&remain_rs)); 2024 ASSERT3U(rc->rc_offset, ==, physical_rs.rs_start); 2025 ASSERT3U(rc->rc_offset, <, physical_rs.rs_end); 2026 ASSERT3U(rc->rc_offset + rc->rc_size, ==, physical_rs.rs_end); 2027 #endif 2028 } 2029 2030 /* 2031 * For write operations: 2032 * 1. Generate the parity data 2033 * 2. Create child zio write operations to each column's vdev, for both 2034 * data and parity. A gang ABD is allocated by vdev_draid_map_alloc() 2035 * if a skip sector needs to be added to a column. 2036 */ 2037 static void 2038 vdev_draid_io_start_write(zio_t *zio, raidz_row_t *rr) 2039 { 2040 vdev_t *vd = zio->io_vd; 2041 raidz_map_t *rm = zio->io_vsd; 2042 2043 vdev_raidz_generate_parity_row(rm, rr); 2044 2045 for (int c = 0; c < rr->rr_cols; c++) { 2046 raidz_col_t *rc = &rr->rr_col[c]; 2047 2048 /* 2049 * Empty columns are zero filled and included in the parity 2050 * calculation and therefore must be written. 2051 */ 2052 ASSERT3U(rc->rc_size, !=, 0); 2053 2054 /* Verify physical to logical translation */ 2055 vdev_draid_io_verify(vd, rr, c); 2056 2057 zio_nowait(zio_vdev_child_io(zio, NULL, 2058 vd->vdev_child[rc->rc_devidx], rc->rc_offset, 2059 rc->rc_abd, rc->rc_size, zio->io_type, zio->io_priority, 2060 0, vdev_raidz_child_done, rc)); 2061 } 2062 } 2063 2064 /* 2065 * For read operations: 2066 * 1. The vdev_draid_map_alloc() function will create a minimal raidz 2067 * mapping for the read based on the zio->io_flags. There are two 2068 * possible mappings either 1) a normal read, or 2) a scrub/resilver. 2069 * 2. Create the zio read operations. This will include all parity 2070 * columns and skip sectors for a scrub/resilver. 2071 */ 2072 static void 2073 vdev_draid_io_start_read(zio_t *zio, raidz_row_t *rr) 2074 { 2075 vdev_t *vd = zio->io_vd; 2076 2077 /* Sequential rebuild must do IO at redundancy group boundary. */ 2078 IMPLY(zio->io_priority == ZIO_PRIORITY_REBUILD, rr->rr_nempty == 0); 2079 2080 /* 2081 * Iterate over the columns in reverse order so that we hit the parity 2082 * last. Any errors along the way will force us to read the parity. 2083 * For scrub/resilver IOs which verify skip sectors, a gang ABD will 2084 * have been allocated to store them and rc->rc_size is increased. 2085 */ 2086 for (int c = rr->rr_cols - 1; c >= 0; c--) { 2087 raidz_col_t *rc = &rr->rr_col[c]; 2088 vdev_t *cvd = vd->vdev_child[rc->rc_devidx]; 2089 2090 if (!vdev_draid_readable(cvd, rc->rc_offset)) { 2091 if (c >= rr->rr_firstdatacol) 2092 rr->rr_missingdata++; 2093 else 2094 rr->rr_missingparity++; 2095 rc->rc_error = SET_ERROR(ENXIO); 2096 rc->rc_tried = 1; 2097 rc->rc_skipped = 1; 2098 continue; 2099 } 2100 2101 if (vdev_draid_missing(cvd, rc->rc_offset, zio->io_txg, 1)) { 2102 vdev_t *svd; 2103 2104 if (c >= rr->rr_firstdatacol) 2105 rr->rr_missingdata++; 2106 else 2107 rr->rr_missingparity++; 2108 rc->rc_error = SET_ERROR(ESTALE); 2109 rc->rc_skipped = 1; 2110 2111 /* 2112 * If this child has draid spare attached, and that 2113 * spare by rc_offset maps to another spare, the repair 2114 * would go to that spare, and we want all mirrored 2115 * children on it to be updated with the repaired data, 2116 * even when we cannot vouch for it during rebuilds 2117 * (which don't have checksums). Otherwise, we will have 2118 * a lot of checksum errors on that spares during scrub. 2119 * The worst thing that can happen in this case is that 2120 * we will update the reserved spare column on some 2121 * device with unverified data, which is harmless. 2122 */ 2123 if ((svd = vdev_draid_find_spare(cvd)) != NULL) { 2124 svd = vdev_draid_spare_get_child(svd, 2125 rc->rc_offset); 2126 if (svd && (svd->vdev_ops == &vdev_spare_ops || 2127 svd->vdev_ops == &vdev_replacing_ops)) 2128 rc->rc_tgt_is_dspare = 1; 2129 } 2130 continue; 2131 } 2132 2133 /* 2134 * Empty columns may be read during vdev_draid_io_done(). 2135 * Only skip them after the readable and missing checks 2136 * verify they are available. 2137 */ 2138 if (rc->rc_size == 0) { 2139 rc->rc_skipped = 1; 2140 continue; 2141 } 2142 2143 if (zio->io_flags & ZIO_FLAG_RESILVER) { 2144 vdev_t *svd; 2145 2146 /* 2147 * Repairs need to be constrained to the devices being 2148 * rebuilt since without a checksum we cannot verify the 2149 * data is actually correct and performing an incorrect 2150 * repair could result in locking in the damage and 2151 * making the data unrecoverable. 2152 */ 2153 if (zio->io_priority == ZIO_PRIORITY_REBUILD && 2154 !vdev_draid_rebuilding(cvd)) 2155 rc->rc_allow_repair = 0; 2156 2157 /* 2158 * If this child is a distributed spare then the 2159 * offset might reside on the vdev being replaced. 2160 * In which case this data must be written to the 2161 * new device. Failure to do so would result in 2162 * checksum errors when the old device is detached 2163 * and the pool is scrubbed. 2164 */ 2165 if ((svd = vdev_draid_find_spare(cvd)) != NULL) { 2166 svd = vdev_draid_spare_get_child(svd, 2167 rc->rc_offset); 2168 if (svd && (svd->vdev_ops == &vdev_spare_ops || 2169 svd->vdev_ops == &vdev_replacing_ops)) { 2170 rc->rc_force_repair = 1; 2171 2172 if (vdev_draid_rebuilding(svd)) 2173 rc->rc_allow_repair = 1; 2174 } 2175 } 2176 2177 /* 2178 * Always issue a repair IO to this child when its 2179 * a spare or replacing vdev with an active rebuild. 2180 */ 2181 if ((cvd->vdev_ops == &vdev_spare_ops || 2182 cvd->vdev_ops == &vdev_replacing_ops) && 2183 vdev_draid_rebuilding(cvd)) { 2184 rc->rc_force_repair = 1; 2185 rc->rc_allow_repair = 1; 2186 } 2187 } 2188 2189 if (vdev_sit_out_reads(cvd, zio->io_flags)) { 2190 rr->rr_outlier_cnt++; 2191 ASSERT0(rc->rc_latency_outlier); 2192 rc->rc_latency_outlier = 1; 2193 } 2194 } 2195 2196 /* 2197 * When the row contains a latency outlier and sufficient parity 2198 * exists to reconstruct the column data, then skip reading the 2199 * known slow child vdev as a performance optimization. 2200 */ 2201 if (rr->rr_outlier_cnt > 0 && 2202 (rr->rr_firstdatacol - rr->rr_missingparity) >= 2203 (rr->rr_missingdata + 1)) { 2204 2205 for (int c = rr->rr_cols - 1; c >= rr->rr_firstdatacol; c--) { 2206 raidz_col_t *rc = &rr->rr_col[c]; 2207 2208 if (rc->rc_error == 0 && rc->rc_latency_outlier) { 2209 rr->rr_missingdata++; 2210 rc->rc_error = SET_ERROR(EAGAIN); 2211 rc->rc_skipped = 1; 2212 break; 2213 } 2214 } 2215 } 2216 2217 /* 2218 * Either a parity or data column is missing this means a repair 2219 * may be attempted by vdev_draid_io_done(). Expand the raid map 2220 * to read in empty columns which are needed along with the parity 2221 * during reconstruction. 2222 */ 2223 if ((rr->rr_missingdata > 0 || rr->rr_missingparity > 0) && 2224 rr->rr_nempty > 0 && rr->rr_abd_empty == NULL) { 2225 vdev_draid_map_alloc_empty(zio, rr); 2226 } 2227 2228 for (int c = rr->rr_cols - 1; c >= 0; c--) { 2229 raidz_col_t *rc = &rr->rr_col[c]; 2230 vdev_t *cvd = vd->vdev_child[rc->rc_devidx]; 2231 2232 if (rc->rc_error || rc->rc_size == 0) 2233 continue; 2234 2235 if (c >= rr->rr_firstdatacol || rr->rr_missingdata > 0 || 2236 (zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) { 2237 zio_nowait(zio_vdev_child_io(zio, NULL, cvd, 2238 rc->rc_offset, rc->rc_abd, rc->rc_size, 2239 zio->io_type, zio->io_priority, 0, 2240 vdev_raidz_child_done, rc)); 2241 } 2242 } 2243 } 2244 2245 /* 2246 * Start an IO operation to a dRAID vdev. 2247 */ 2248 static void 2249 vdev_draid_io_start(zio_t *zio) 2250 { 2251 vdev_t *vd __maybe_unused = zio->io_vd; 2252 2253 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 2254 ASSERT3U(zio->io_offset, ==, vdev_draid_get_astart(vd, zio->io_offset)); 2255 2256 raidz_map_t *rm = vdev_draid_map_alloc(zio); 2257 zio->io_vsd = rm; 2258 zio->io_vsd_ops = &vdev_raidz_vsd_ops; 2259 zio_batch_create(zio); 2260 2261 if (zio->io_type == ZIO_TYPE_WRITE) { 2262 for (int i = 0; i < rm->rm_nrows; i++) { 2263 vdev_draid_io_start_write(zio, rm->rm_row[i]); 2264 } 2265 } else { 2266 ASSERT(zio->io_type == ZIO_TYPE_READ); 2267 2268 for (int i = 0; i < rm->rm_nrows; i++) { 2269 vdev_draid_io_start_read(zio, rm->rm_row[i]); 2270 } 2271 } 2272 2273 zio_execute(zio_batch_rele(zio)); 2274 } 2275 2276 /* 2277 * Complete an IO operation on a dRAID vdev. The raidz logic can be applied 2278 * to dRAID since the layout is fully described by the raidz_map_t. 2279 */ 2280 static void 2281 vdev_draid_io_done(zio_t *zio) 2282 { 2283 vdev_raidz_io_done(zio); 2284 } 2285 2286 static void 2287 vdev_draid_state_change(vdev_t *vd, int faulted, int degraded) 2288 { 2289 vdev_draid_config_t *vdc = vd->vdev_tsd; 2290 ASSERT(vd->vdev_ops == &vdev_draid_ops); 2291 2292 if (faulted > vdc->vdc_nparity * (vdc->vdc_width / vdc->vdc_children)) 2293 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN, 2294 VDEV_AUX_NO_REPLICAS); 2295 else if (degraded + faulted != 0) 2296 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE); 2297 else 2298 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE); 2299 } 2300 2301 static void 2302 vdev_draid_xlate(vdev_t *cvd, const zfs_range_seg64_t *logical_rs, 2303 zfs_range_seg64_t *physical_rs, zfs_range_seg64_t *remain_rs) 2304 { 2305 vdev_t *raidvd = cvd->vdev_parent; 2306 ASSERT(raidvd->vdev_ops == &vdev_draid_ops); 2307 2308 vdev_draid_config_t *vdc = raidvd->vdev_tsd; 2309 uint64_t ashift = raidvd->vdev_top->vdev_ashift; 2310 2311 /* Make sure the offsets are block-aligned */ 2312 ASSERT0(logical_rs->rs_start % (1 << ashift)); 2313 ASSERT0(logical_rs->rs_end % (1 << ashift)); 2314 2315 uint64_t logical_start = logical_rs->rs_start; 2316 uint64_t logical_end = logical_rs->rs_end; 2317 2318 /* 2319 * Unaligned ranges must be skipped. All metaslabs are correctly 2320 * aligned so this should not happen, but this case is handled in 2321 * case it's needed by future callers. 2322 */ 2323 uint64_t astart = vdev_draid_get_astart(raidvd, logical_start); 2324 if (astart != logical_start) { 2325 physical_rs->rs_start = logical_start; 2326 physical_rs->rs_end = logical_start; 2327 remain_rs->rs_start = MIN(astart, logical_end); 2328 remain_rs->rs_end = logical_end; 2329 return; 2330 } 2331 2332 /* 2333 * Unlike with mirrors and raidz a dRAID logical range can map 2334 * to multiple non-contiguous physical ranges. This is handled by 2335 * limiting the size of the logical range to a single group and 2336 * setting the remain argument such that it describes the remaining 2337 * unmapped logical range. This is stricter than absolutely 2338 * necessary but helps simplify the logic below. 2339 */ 2340 uint64_t group = vdev_draid_offset_to_group(raidvd, logical_start); 2341 uint64_t nextstart = vdev_draid_group_to_offset(raidvd, group + 1); 2342 if (logical_end > nextstart) 2343 logical_end = nextstart; 2344 2345 /* Find the starting offset for each vdev in the group */ 2346 uint64_t perm, groupstart, ndisks; 2347 uint64_t start = vdev_draid_logical_to_physical(raidvd, 2348 logical_start, &perm, &groupstart, &ndisks); 2349 uint64_t end = start; 2350 2351 uint8_t *base; 2352 uint64_t iter, id; 2353 vdev_draid_get_perm(vdc, perm, &base, &iter); 2354 2355 /* 2356 * Check if the passed child falls within the group. If it does 2357 * update the start and end to reflect the physical range. 2358 * Otherwise, leave them unmodified which will result in an empty 2359 * (zero-length) physical range being returned. 2360 */ 2361 for (uint64_t i = 0; i < vdc->vdc_groupwidth; i++) { 2362 uint64_t c = (groupstart + i) % ndisks; 2363 2364 if (c == 0 && i != 0) { 2365 /* the group wrapped, increment the start */ 2366 start += VDEV_DRAID_ROWHEIGHT; 2367 end = start; 2368 } 2369 2370 id = vdev_draid_permute_id(vdc, base, iter, c); 2371 if (id == cvd->vdev_id) { 2372 uint64_t b_size = (logical_end >> ashift) - 2373 (logical_start >> ashift); 2374 ASSERT3U(b_size, >, 0); 2375 end = start + ((((b_size - 1) / 2376 vdc->vdc_groupwidth) + 1) << ashift); 2377 break; 2378 } 2379 } 2380 physical_rs->rs_start = start; 2381 physical_rs->rs_end = end; 2382 2383 /* 2384 * Only top-level vdevs are allowed to set remain_rs because 2385 * when .vdev_op_xlate() is called for their children the full 2386 * logical range is not provided by vdev_xlate(). 2387 */ 2388 remain_rs->rs_start = logical_end; 2389 remain_rs->rs_end = logical_rs->rs_end; 2390 2391 ASSERT3U(physical_rs->rs_start, <=, logical_start); 2392 ASSERT3U(physical_rs->rs_end - physical_rs->rs_start, <=, 2393 logical_end - logical_start); 2394 } 2395 2396 /* 2397 * Add dRAID specific fields to the config nvlist. 2398 */ 2399 static void 2400 vdev_draid_config_generate(vdev_t *vd, nvlist_t *nv) 2401 { 2402 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 2403 vdev_draid_config_t *vdc = vd->vdev_tsd; 2404 2405 fnvlist_add_uint64(nv, ZPOOL_CONFIG_NPARITY, vdc->vdc_nparity); 2406 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NDATA, vdc->vdc_ndata); 2407 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NSPARES, vdc->vdc_nspares); 2408 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NGROUPS, vdc->vdc_ngroups); 2409 2410 if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_DRAID_FAIL_DOMAINS)) 2411 fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NCHILDREN, 2412 vdc->vdc_children); 2413 } 2414 2415 /* 2416 * Initialize private dRAID specific fields from the nvlist. 2417 */ 2418 static int 2419 vdev_draid_init(spa_t *spa, nvlist_t *nv, void **tsd) 2420 { 2421 (void) spa; 2422 uint64_t ndata, nparity, nspares, ngroups; 2423 int error; 2424 2425 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DRAID_NDATA, &ndata)) 2426 return (SET_ERROR(EINVAL)); 2427 2428 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY, &nparity) || 2429 nparity == 0 || nparity > VDEV_DRAID_MAXPARITY) { 2430 return (SET_ERROR(EINVAL)); 2431 } 2432 2433 uint_t width; 2434 uint64_t children; 2435 nvlist_t **child; 2436 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2437 &child, &width) != 0 || width == 0) { 2438 return (SET_ERROR(EINVAL)); 2439 } 2440 2441 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DRAID_NCHILDREN, &children)) { 2442 children = width; 2443 if (children > VDEV_DRAID_MAX_CHILDREN) 2444 return (SET_ERROR(EINVAL)); 2445 } 2446 2447 if (children == 0 || width % children != 0) 2448 return (SET_ERROR(EINVAL)); 2449 2450 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DRAID_NSPARES, &nspares) || 2451 nspares > 100) { 2452 return (SET_ERROR(EINVAL)); 2453 } 2454 2455 int fgrps = width / children; 2456 int nspare = nspares / fgrps; 2457 if (nspares % fgrps) 2458 nspare++; 2459 2460 /* 2461 * Validate the minimum number of children exist per group for the 2462 * specified parity level (draid1 >= 2, draid2 >= 3, draid3 >= 4). 2463 */ 2464 if (children < (ndata + nparity + nspare)) 2465 return (SET_ERROR(EINVAL)); 2466 2467 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DRAID_NGROUPS, &ngroups) || 2468 ngroups == 0 || ngroups > VDEV_DRAID_MAX_CHILDREN) { 2469 return (SET_ERROR(EINVAL)); 2470 } 2471 2472 /* 2473 * Create the dRAID configuration using the pool nvlist configuration 2474 * and the fixed mapping for the correct number of children. 2475 */ 2476 vdev_draid_config_t *vdc; 2477 const draid_map_t *map; 2478 2479 error = vdev_draid_lookup_map(children, &map); 2480 if (error) 2481 return (SET_ERROR(EINVAL)); 2482 2483 vdc = kmem_zalloc(sizeof (*vdc), KM_SLEEP); 2484 vdc->vdc_ndata = ndata; 2485 vdc->vdc_nparity = nparity; 2486 vdc->vdc_nspares = nspares; 2487 vdc->vdc_children = children; 2488 vdc->vdc_ngroups = ngroups; 2489 vdc->vdc_width = width; 2490 vdc->vdc_nperms = map->dm_nperms; 2491 2492 error = vdev_draid_generate_perms(map, &vdc->vdc_perms); 2493 if (error) { 2494 kmem_free(vdc, sizeof (*vdc)); 2495 return (SET_ERROR(EINVAL)); 2496 } 2497 2498 if (width > children) 2499 vdev_draid_shuffle_perms(map, vdc->vdc_perms, width); 2500 2501 /* 2502 * Derived constants. 2503 */ 2504 vdc->vdc_groupwidth = vdc->vdc_ndata + vdc->vdc_nparity; 2505 vdc->vdc_ndisks = vdc->vdc_width - vdc->vdc_nspares; 2506 vdc->vdc_groupsz = vdc->vdc_groupwidth * VDEV_DRAID_ROWHEIGHT; 2507 vdc->vdc_devslicesz = (vdc->vdc_groupsz * vdc->vdc_ngroups) / 2508 vdc->vdc_ndisks; 2509 2510 ASSERT3U(vdc->vdc_groupwidth, >=, 2); 2511 ASSERT3U(vdc->vdc_groupwidth, <=, vdc->vdc_ndisks); 2512 ASSERT3U(vdc->vdc_groupsz, >=, 2 * VDEV_DRAID_ROWHEIGHT); 2513 ASSERT3U(vdc->vdc_devslicesz, >=, VDEV_DRAID_ROWHEIGHT); 2514 ASSERT0(vdc->vdc_devslicesz % VDEV_DRAID_ROWHEIGHT); 2515 ASSERT3U((vdc->vdc_groupwidth * vdc->vdc_ngroups) % 2516 vdc->vdc_ndisks, ==, 0); 2517 2518 *tsd = vdc; 2519 2520 return (0); 2521 } 2522 2523 static void 2524 vdev_draid_fini(vdev_t *vd) 2525 { 2526 vdev_draid_config_t *vdc = vd->vdev_tsd; 2527 2528 vmem_free(vdc->vdc_perms, sizeof (uint8_t) * 2529 vdc->vdc_children * vdc->vdc_nperms); 2530 kmem_free(vdc, sizeof (*vdc)); 2531 } 2532 2533 static uint64_t 2534 vdev_draid_nparity(vdev_t *vd) 2535 { 2536 vdev_draid_config_t *vdc = vd->vdev_tsd; 2537 2538 return (vdc->vdc_nparity * (vdc->vdc_width / vdc->vdc_children)); 2539 } 2540 2541 static uint64_t 2542 vdev_draid_ndisks(vdev_t *vd) 2543 { 2544 vdev_draid_config_t *vdc = vd->vdev_tsd; 2545 2546 return (vdc->vdc_ndisks); 2547 } 2548 2549 vdev_ops_t vdev_draid_ops = { 2550 .vdev_op_init = vdev_draid_init, 2551 .vdev_op_fini = vdev_draid_fini, 2552 .vdev_op_open = vdev_draid_open, 2553 .vdev_op_close = vdev_draid_close, 2554 .vdev_op_psize_to_asize = vdev_draid_psize_to_asize, 2555 .vdev_op_asize_to_psize = vdev_draid_asize_to_psize, 2556 .vdev_op_min_asize = vdev_draid_min_asize, 2557 .vdev_op_min_alloc = vdev_draid_min_alloc, 2558 .vdev_op_io_start = vdev_draid_io_start, 2559 .vdev_op_io_done = vdev_draid_io_done, 2560 .vdev_op_state_change = vdev_draid_state_change, 2561 .vdev_op_need_resilver = vdev_draid_need_resilver, 2562 .vdev_op_hold = NULL, 2563 .vdev_op_rele = NULL, 2564 .vdev_op_remap = NULL, 2565 .vdev_op_xlate = vdev_draid_xlate, 2566 .vdev_op_rebuild_asize = vdev_draid_rebuild_asize, 2567 .vdev_op_metaslab_init = vdev_draid_metaslab_init, 2568 .vdev_op_config_generate = vdev_draid_config_generate, 2569 .vdev_op_nparity = vdev_draid_nparity, 2570 .vdev_op_ndisks = vdev_draid_ndisks, 2571 .vdev_op_type = VDEV_TYPE_DRAID, 2572 .vdev_op_leaf = B_FALSE, 2573 }; 2574 2575 2576 /* 2577 * A dRAID distributed spare is a virtual leaf vdev which is included in the 2578 * parent dRAID configuration. The last N columns of the dRAID permutation 2579 * table are used to determine on which dRAID children a specific offset 2580 * should be written. These spare leaf vdevs can only be used to replace 2581 * faulted children in the same dRAID configuration. 2582 */ 2583 2584 /* 2585 * Distributed spare state. All fields are set when the distributed spare is 2586 * first opened and are immutable. 2587 */ 2588 typedef struct { 2589 vdev_t *vds_draid_vdev; /* top-level parent dRAID vdev */ 2590 uint64_t vds_top_guid; /* top-level parent dRAID guid */ 2591 uint64_t vds_spare_id; /* spare id (0 - vdc->vdc_nspares-1) */ 2592 } vdev_draid_spare_t; 2593 2594 /* 2595 * Returns the parent dRAID vdev to which the distributed spare belongs. 2596 * This may be safely called even when the vdev is not open. 2597 */ 2598 vdev_t * 2599 vdev_draid_spare_get_parent(vdev_t *vd) 2600 { 2601 vdev_draid_spare_t *vds = vd->vdev_tsd; 2602 2603 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_spare_ops); 2604 2605 if (vds->vds_draid_vdev != NULL) 2606 return (vds->vds_draid_vdev); 2607 2608 return (vdev_lookup_by_guid(vd->vdev_spa->spa_root_vdev, 2609 vds->vds_top_guid)); 2610 } 2611 2612 /* 2613 * A dRAID space is active when it's the child of a vdev using the 2614 * vdev_spare_ops, vdev_replacing_ops or vdev_draid_ops. 2615 */ 2616 static boolean_t 2617 vdev_draid_spare_is_active(vdev_t *vd) 2618 { 2619 vdev_t *pvd = vd->vdev_parent; 2620 2621 if (pvd != NULL && (pvd->vdev_ops == &vdev_spare_ops || 2622 pvd->vdev_ops == &vdev_replacing_ops || 2623 pvd->vdev_ops == &vdev_draid_ops)) { 2624 return (B_TRUE); 2625 } else { 2626 return (B_FALSE); 2627 } 2628 } 2629 2630 /* 2631 * Given a dRAID distribute spare vdev, returns the physical child vdev 2632 * on which the provided offset resides. This may involve recursing through 2633 * multiple layers of distributed spares. Note that offset is relative to 2634 * this vdev. 2635 */ 2636 vdev_t * 2637 vdev_draid_spare_get_child(vdev_t *vd, uint64_t physical_offset) 2638 { 2639 vdev_draid_spare_t *vds = vd->vdev_tsd; 2640 2641 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_spare_ops); 2642 2643 /* The vdev is closed */ 2644 if (vds->vds_draid_vdev == NULL) 2645 return (NULL); 2646 2647 vdev_t *tvd = vds->vds_draid_vdev; 2648 vdev_draid_config_t *vdc = tvd->vdev_tsd; 2649 2650 uint64_t fgrps = vdc->vdc_width / vdc->vdc_children; 2651 2652 ASSERT3P(tvd->vdev_ops, ==, &vdev_draid_ops); 2653 ASSERT3U(vds->vds_spare_id, <, vdc->vdc_nspares); 2654 2655 uint8_t *base; 2656 uint64_t iter; 2657 uint64_t perm = (physical_offset / vdc->vdc_devslicesz) * fgrps; 2658 2659 /* 2660 * Adjust permutation so that it points to the correct slice in the 2661 * big width row. 2662 */ 2663 perm += vds->vds_spare_id % fgrps; 2664 2665 vdev_draid_get_perm(vdc, perm, &base, &iter); 2666 2667 uint64_t cid = vdev_draid_permute_id(vdc, base, iter, 2668 (vdc->vdc_children - 1) - (vds->vds_spare_id / fgrps)); 2669 vdev_t *cvd = tvd->vdev_child[cid]; 2670 2671 if (cvd->vdev_ops == &vdev_draid_spare_ops) 2672 return (vdev_draid_spare_get_child(cvd, physical_offset)); 2673 2674 return (cvd); 2675 } 2676 2677 /* 2678 * Returns true if no failure group reached failures threshold so that 2679 * enclosure failure cannot be tolerated anymore. Used spares are counted 2680 * as failures because in case of enclosure failure their blocks can belong 2681 * to the disks from that enclosure and can be lost. 2682 */ 2683 boolean_t 2684 vdev_draid_fail_domain_allowed(vdev_t *vd) 2685 { 2686 vdev_draid_config_t *vdc = vd->vdev_tsd; 2687 2688 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_ops); 2689 ASSERT3P(vdc->vdc_width, >, vdc->vdc_children); 2690 2691 int counter = 0; 2692 2693 for (int c = 0; c < vdc->vdc_width; c++) { 2694 vdev_t *cvd = vd->vdev_child[c]; 2695 2696 if ((c % vdc->vdc_children) == 0) 2697 counter = 0; 2698 2699 if (cvd->vdev_ops == &vdev_spare_ops || 2700 cvd->vdev_ops == &vdev_draid_spare_ops || 2701 !vdev_readable(cvd)) 2702 counter++; 2703 2704 if (counter > vdc->vdc_nparity) 2705 return (B_FALSE); 2706 } 2707 2708 return (B_TRUE); 2709 } 2710 2711 static void 2712 vdev_draid_spare_close(vdev_t *vd) 2713 { 2714 vdev_draid_spare_t *vds = vd->vdev_tsd; 2715 vds->vds_draid_vdev = NULL; 2716 } 2717 2718 /* 2719 * Opening a dRAID spare device is done by looking up the associated dRAID 2720 * top-level vdev guid from the spare configuration. 2721 */ 2722 static int 2723 vdev_draid_spare_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, 2724 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr) 2725 { 2726 (void) cr; 2727 vdev_draid_spare_t *vds = vd->vdev_tsd; 2728 vdev_t *rvd = vd->vdev_spa->spa_root_vdev; 2729 uint64_t asize, max_asize; 2730 2731 vdev_t *tvd = vdev_lookup_by_guid(rvd, vds->vds_top_guid); 2732 if (tvd == NULL) { 2733 /* 2734 * When spa_vdev_add() is labeling new spares the 2735 * associated dRAID is not attached to the root vdev 2736 * nor does this spare have a parent. Simulate a valid 2737 * device in order to allow the label to be initialized 2738 * and the distributed spare added to the configuration. 2739 */ 2740 if (vd->vdev_parent == NULL) { 2741 *psize = *max_psize = SPA_MINDEVSIZE; 2742 *logical_ashift = *physical_ashift = ASHIFT_MIN; 2743 return (0); 2744 } 2745 2746 return (SET_ERROR(EINVAL)); 2747 } 2748 2749 vdev_draid_config_t *vdc = tvd->vdev_tsd; 2750 if (tvd->vdev_ops != &vdev_draid_ops || vdc == NULL) 2751 return (SET_ERROR(EINVAL)); 2752 2753 if (vds->vds_spare_id >= vdc->vdc_nspares) 2754 return (SET_ERROR(EINVAL)); 2755 2756 /* 2757 * Neither tvd->vdev_asize or tvd->vdev_max_asize can be used here 2758 * because the caller may be vdev_draid_open() in which case the 2759 * values are stale as they haven't yet been updated by vdev_open(). 2760 * To avoid this always recalculate the dRAID asize and max_asize. 2761 */ 2762 vdev_draid_calculate_asize(tvd, &asize, &max_asize, 2763 logical_ashift, physical_ashift); 2764 2765 *psize = asize + VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE; 2766 *max_psize = max_asize + VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE; 2767 2768 vds->vds_draid_vdev = tvd; 2769 vd->vdev_nonrot = tvd->vdev_nonrot; 2770 2771 return (0); 2772 } 2773 2774 /* 2775 * Completed distributed spare IO. Store the result in the parent zio 2776 * as if it had performed the operation itself. Only the first error is 2777 * preserved if there are multiple errors. 2778 */ 2779 static void 2780 vdev_draid_spare_child_done(zio_t *zio) 2781 { 2782 zio_t *pio = zio->io_private; 2783 2784 /* 2785 * IOs are issued to non-writable vdevs in order to keep their 2786 * DTLs accurate. However, we don't want to propagate the 2787 * error in to the distributed spare's DTL. When resilvering 2788 * vdev_draid_need_resilver() will consult the relevant DTL 2789 * to determine if the data is missing and must be repaired. 2790 */ 2791 if (!vdev_writeable(zio->io_vd)) 2792 return; 2793 2794 if (pio->io_error == 0) 2795 pio->io_error = zio->io_error; 2796 } 2797 2798 /* 2799 * Returns a valid label nvlist for the distributed spare vdev. This is 2800 * used to bypass the IO pipeline to avoid the complexity of constructing 2801 * a complete label with valid checksum to return when read. 2802 */ 2803 nvlist_t * 2804 vdev_draid_read_config_spare(vdev_t *vd) 2805 { 2806 spa_t *spa = vd->vdev_spa; 2807 spa_aux_vdev_t *sav = &spa->spa_spares; 2808 uint64_t guid = vd->vdev_guid; 2809 2810 nvlist_t *nv = fnvlist_alloc(); 2811 fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_SPARE, 1); 2812 fnvlist_add_uint64(nv, ZPOOL_CONFIG_CREATE_TXG, vd->vdev_crtxg); 2813 fnvlist_add_uint64(nv, ZPOOL_CONFIG_VERSION, spa_version(spa)); 2814 fnvlist_add_string(nv, ZPOOL_CONFIG_POOL_NAME, spa_name(spa)); 2815 fnvlist_add_uint64(nv, ZPOOL_CONFIG_POOL_GUID, spa_guid(spa)); 2816 fnvlist_add_uint64(nv, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 2817 fnvlist_add_uint64(nv, ZPOOL_CONFIG_TOP_GUID, vd->vdev_top->vdev_guid); 2818 fnvlist_add_uint64(nv, ZPOOL_CONFIG_POOL_STATE, 2819 vdev_draid_spare_is_active(vd) ? 2820 POOL_STATE_ACTIVE : POOL_STATE_SPARE); 2821 2822 /* Set the vdev guid based on the vdev list in sav_count. */ 2823 for (int i = 0; i < sav->sav_count; i++) { 2824 if (sav->sav_vdevs[i]->vdev_ops == &vdev_draid_spare_ops && 2825 strcmp(sav->sav_vdevs[i]->vdev_path, vd->vdev_path) == 0) { 2826 guid = sav->sav_vdevs[i]->vdev_guid; 2827 break; 2828 } 2829 } 2830 2831 fnvlist_add_uint64(nv, ZPOOL_CONFIG_GUID, guid); 2832 2833 return (nv); 2834 } 2835 2836 /* 2837 * Handle any flush requested of the distributed spare. All children must be 2838 * flushed. 2839 */ 2840 static int 2841 vdev_draid_spare_flush(zio_t *zio) 2842 { 2843 vdev_t *vd = zio->io_vd; 2844 int error = 0; 2845 2846 for (int c = 0; c < vd->vdev_children; c++) { 2847 zio_nowait(zio_vdev_child_io(zio, NULL, 2848 vd->vdev_child[c], zio->io_offset, zio->io_abd, 2849 zio->io_size, zio->io_type, zio->io_priority, 0, 2850 vdev_draid_spare_child_done, zio)); 2851 } 2852 2853 return (error); 2854 } 2855 2856 /* 2857 * Initiate an IO to the distributed spare. For normal IOs this entails using 2858 * the zio->io_offset and permutation table to calculate which child dRAID vdev 2859 * is responsible for the data. Then passing along the zio to that child to 2860 * perform the actual IO. The label ranges are not stored on disk and require 2861 * some special handling which is described below. 2862 */ 2863 static void 2864 vdev_draid_spare_io_start(zio_t *zio) 2865 { 2866 vdev_t *cvd = NULL, *vd = zio->io_vd; 2867 vdev_draid_spare_t *vds = vd->vdev_tsd; 2868 uint64_t offset = zio->io_offset - VDEV_LABEL_START_SIZE; 2869 2870 /* 2871 * If the vdev is closed, it's likely in the REMOVED or FAULTED state. 2872 * Nothing to be done here but return failure. 2873 */ 2874 if (vds == NULL) { 2875 zio->io_error = ENXIO; 2876 zio_interrupt(zio); 2877 return; 2878 } 2879 2880 switch (zio->io_type) { 2881 case ZIO_TYPE_FLUSH: 2882 zio->io_error = vdev_draid_spare_flush(zio); 2883 break; 2884 2885 case ZIO_TYPE_WRITE: 2886 if (VDEV_OFFSET_IS_LABEL(vd, zio->io_offset)) { 2887 /* 2888 * Accept probe IOs and config writers to simulate the 2889 * existence of an on disk label. vdev_label_sync(), 2890 * vdev_uberblock_sync() and vdev_copy_uberblocks() 2891 * skip the distributed spares. This only leaves 2892 * vdev_label_init() which is allowed to succeed to 2893 * avoid adding special cases the function. 2894 */ 2895 if (zio->io_flags & ZIO_FLAG_PROBE || 2896 zio->io_flags & ZIO_FLAG_CONFIG_WRITER) { 2897 zio->io_error = 0; 2898 } else { 2899 zio->io_error = SET_ERROR(EIO); 2900 } 2901 } else { 2902 cvd = vdev_draid_spare_get_child(vd, offset); 2903 2904 if (cvd == NULL) { 2905 zio->io_error = SET_ERROR(ENXIO); 2906 } else { 2907 zio_nowait(zio_vdev_child_io(zio, NULL, cvd, 2908 offset, zio->io_abd, zio->io_size, 2909 zio->io_type, zio->io_priority, 0, 2910 vdev_draid_spare_child_done, zio)); 2911 } 2912 } 2913 break; 2914 2915 case ZIO_TYPE_READ: 2916 if (VDEV_OFFSET_IS_LABEL(vd, zio->io_offset)) { 2917 /* 2918 * Accept probe IOs to simulate the existence of a 2919 * label. vdev_label_read_config() bypasses the 2920 * pipeline to read the label configuration and 2921 * vdev_uberblock_load() skips distributed spares 2922 * when attempting to locate the best uberblock. 2923 */ 2924 if (zio->io_flags & ZIO_FLAG_PROBE) { 2925 zio->io_error = 0; 2926 } else { 2927 zio->io_error = SET_ERROR(EIO); 2928 } 2929 } else { 2930 cvd = vdev_draid_spare_get_child(vd, offset); 2931 2932 if (cvd == NULL || !vdev_readable(cvd)) { 2933 zio->io_error = SET_ERROR(ENXIO); 2934 } else { 2935 zio_nowait(zio_vdev_child_io(zio, NULL, cvd, 2936 offset, zio->io_abd, zio->io_size, 2937 zio->io_type, zio->io_priority, 0, 2938 vdev_draid_spare_child_done, zio)); 2939 } 2940 } 2941 break; 2942 2943 case ZIO_TYPE_TRIM: 2944 /* The vdev label ranges are never trimmed */ 2945 ASSERT0(VDEV_OFFSET_IS_LABEL(vd, zio->io_offset)); 2946 2947 cvd = vdev_draid_spare_get_child(vd, offset); 2948 2949 if (cvd == NULL || !cvd->vdev_has_trim) { 2950 zio->io_error = SET_ERROR(ENXIO); 2951 } else { 2952 zio_nowait(zio_vdev_child_io(zio, NULL, cvd, 2953 offset, zio->io_abd, zio->io_size, 2954 zio->io_type, zio->io_priority, 0, 2955 vdev_draid_spare_child_done, zio)); 2956 } 2957 break; 2958 2959 default: 2960 zio->io_error = SET_ERROR(ENOTSUP); 2961 break; 2962 } 2963 2964 zio_execute(zio); 2965 } 2966 2967 static void 2968 vdev_draid_spare_io_done(zio_t *zio) 2969 { 2970 (void) zio; 2971 } 2972 2973 /* 2974 * Lookup the full spare config in spa->spa_spares.sav_config and 2975 * return the top_guid and spare_id for the named spare. 2976 */ 2977 static int 2978 vdev_draid_spare_lookup(spa_t *spa, nvlist_t *nv, uint64_t *top_guidp, 2979 uint64_t *spare_idp) 2980 { 2981 nvlist_t **spares; 2982 uint_t nspares; 2983 int error; 2984 2985 if ((spa->spa_spares.sav_config == NULL) || 2986 (nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 2987 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0)) { 2988 return (SET_ERROR(ENOENT)); 2989 } 2990 2991 const char *spare_name; 2992 error = nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &spare_name); 2993 if (error != 0) 2994 return (SET_ERROR(EINVAL)); 2995 2996 for (int i = 0; i < nspares; i++) { 2997 nvlist_t *spare = spares[i]; 2998 uint64_t top_guid, spare_id; 2999 const char *type, *path; 3000 3001 /* Skip non-distributed spares */ 3002 error = nvlist_lookup_string(spare, ZPOOL_CONFIG_TYPE, &type); 3003 if (error != 0 || strcmp(type, VDEV_TYPE_DRAID_SPARE) != 0) 3004 continue; 3005 3006 /* Skip spares with the wrong name */ 3007 error = nvlist_lookup_string(spare, ZPOOL_CONFIG_PATH, &path); 3008 if (error != 0 || strcmp(path, spare_name) != 0) 3009 continue; 3010 3011 /* Found the matching spare */ 3012 error = nvlist_lookup_uint64(spare, 3013 ZPOOL_CONFIG_TOP_GUID, &top_guid); 3014 if (error == 0) { 3015 error = nvlist_lookup_uint64(spare, 3016 ZPOOL_CONFIG_SPARE_ID, &spare_id); 3017 } 3018 3019 if (error != 0) { 3020 return (SET_ERROR(EINVAL)); 3021 } else { 3022 *top_guidp = top_guid; 3023 *spare_idp = spare_id; 3024 return (0); 3025 } 3026 } 3027 3028 return (SET_ERROR(ENOENT)); 3029 } 3030 3031 /* 3032 * Initialize private dRAID spare specific fields from the nvlist. 3033 */ 3034 static int 3035 vdev_draid_spare_init(spa_t *spa, nvlist_t *nv, void **tsd) 3036 { 3037 vdev_draid_spare_t *vds; 3038 uint64_t top_guid = 0; 3039 uint64_t spare_id; 3040 3041 /* 3042 * In the normal case check the list of spares stored in the spa 3043 * to lookup the top_guid and spare_id for provided spare config. 3044 * When creating a new pool or adding vdevs the spare list is not 3045 * yet populated and the values are provided in the passed config. 3046 */ 3047 if (vdev_draid_spare_lookup(spa, nv, &top_guid, &spare_id) != 0) { 3048 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_TOP_GUID, 3049 &top_guid) != 0) 3050 return (SET_ERROR(EINVAL)); 3051 3052 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_SPARE_ID, 3053 &spare_id) != 0) 3054 return (SET_ERROR(EINVAL)); 3055 } 3056 3057 vds = kmem_alloc(sizeof (vdev_draid_spare_t), KM_SLEEP); 3058 vds->vds_draid_vdev = NULL; 3059 vds->vds_top_guid = top_guid; 3060 vds->vds_spare_id = spare_id; 3061 3062 *tsd = vds; 3063 3064 return (0); 3065 } 3066 3067 static void 3068 vdev_draid_spare_fini(vdev_t *vd) 3069 { 3070 kmem_free(vd->vdev_tsd, sizeof (vdev_draid_spare_t)); 3071 } 3072 3073 static void 3074 vdev_draid_spare_config_generate(vdev_t *vd, nvlist_t *nv) 3075 { 3076 vdev_draid_spare_t *vds = vd->vdev_tsd; 3077 3078 ASSERT3P(vd->vdev_ops, ==, &vdev_draid_spare_ops); 3079 3080 fnvlist_add_uint64(nv, ZPOOL_CONFIG_TOP_GUID, vds->vds_top_guid); 3081 fnvlist_add_uint64(nv, ZPOOL_CONFIG_SPARE_ID, vds->vds_spare_id); 3082 } 3083 3084 vdev_ops_t vdev_draid_spare_ops = { 3085 .vdev_op_init = vdev_draid_spare_init, 3086 .vdev_op_fini = vdev_draid_spare_fini, 3087 .vdev_op_open = vdev_draid_spare_open, 3088 .vdev_op_close = vdev_draid_spare_close, 3089 .vdev_op_psize_to_asize = vdev_default_asize, 3090 .vdev_op_asize_to_psize = vdev_default_psize, 3091 .vdev_op_min_asize = vdev_default_min_asize, 3092 .vdev_op_min_alloc = NULL, 3093 .vdev_op_io_start = vdev_draid_spare_io_start, 3094 .vdev_op_io_done = vdev_draid_spare_io_done, 3095 .vdev_op_state_change = NULL, 3096 .vdev_op_need_resilver = NULL, 3097 .vdev_op_hold = NULL, 3098 .vdev_op_rele = NULL, 3099 .vdev_op_remap = NULL, 3100 .vdev_op_xlate = vdev_default_xlate, 3101 .vdev_op_rebuild_asize = NULL, 3102 .vdev_op_metaslab_init = NULL, 3103 .vdev_op_config_generate = vdev_draid_spare_config_generate, 3104 .vdev_op_nparity = NULL, 3105 .vdev_op_ndisks = NULL, 3106 .vdev_op_type = VDEV_TYPE_DRAID_SPARE, 3107 .vdev_op_leaf = B_TRUE, 3108 }; 3109