1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * t10_pi.c - Functions for generating and verifying T10 Protection 4 * Information. 5 */ 6 7 #include <linux/t10-pi.h> 8 #include <linux/blk-integrity.h> 9 #include <linux/crc-t10dif.h> 10 #include <linux/crc64.h> 11 #include <net/checksum.h> 12 #include <asm/unaligned.h> 13 #include "blk.h" 14 15 struct blk_integrity_iter { 16 void *prot_buf; 17 void *data_buf; 18 sector_t seed; 19 unsigned int data_size; 20 unsigned short interval; 21 const char *disk_name; 22 }; 23 24 static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len, 25 unsigned char csum_type) 26 { 27 if (csum_type == BLK_INTEGRITY_CSUM_IP) 28 return (__force __be16)ip_compute_csum(data, len); 29 return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len)); 30 } 31 32 /* 33 * Type 1 and Type 2 protection use the same format: 16 bit guard tag, 34 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref 35 * tag. 36 */ 37 static void t10_pi_generate(struct blk_integrity_iter *iter, 38 struct blk_integrity *bi) 39 { 40 u8 offset = bi->pi_offset; 41 unsigned int i; 42 43 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 44 struct t10_pi_tuple *pi = iter->prot_buf + offset; 45 46 pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval, 47 bi->csum_type); 48 if (offset) 49 pi->guard_tag = t10_pi_csum(pi->guard_tag, 50 iter->prot_buf, offset, bi->csum_type); 51 pi->app_tag = 0; 52 53 if (bi->flags & BLK_INTEGRITY_REF_TAG) 54 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); 55 else 56 pi->ref_tag = 0; 57 58 iter->data_buf += iter->interval; 59 iter->prot_buf += bi->tuple_size; 60 iter->seed++; 61 } 62 } 63 64 static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter, 65 struct blk_integrity *bi) 66 { 67 u8 offset = bi->pi_offset; 68 unsigned int i; 69 70 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 71 struct t10_pi_tuple *pi = iter->prot_buf + offset; 72 __be16 csum; 73 74 if (bi->flags & BLK_INTEGRITY_REF_TAG) { 75 if (pi->app_tag == T10_PI_APP_ESCAPE) 76 goto next; 77 78 if (be32_to_cpu(pi->ref_tag) != 79 lower_32_bits(iter->seed)) { 80 pr_err("%s: ref tag error at location %llu " \ 81 "(rcvd %u)\n", iter->disk_name, 82 (unsigned long long) 83 iter->seed, be32_to_cpu(pi->ref_tag)); 84 return BLK_STS_PROTECTION; 85 } 86 } else { 87 if (pi->app_tag == T10_PI_APP_ESCAPE && 88 pi->ref_tag == T10_PI_REF_ESCAPE) 89 goto next; 90 } 91 92 csum = t10_pi_csum(0, iter->data_buf, iter->interval, 93 bi->csum_type); 94 if (offset) 95 csum = t10_pi_csum(csum, iter->prot_buf, offset, 96 bi->csum_type); 97 98 if (pi->guard_tag != csum) { 99 pr_err("%s: guard tag error at sector %llu " \ 100 "(rcvd %04x, want %04x)\n", iter->disk_name, 101 (unsigned long long)iter->seed, 102 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); 103 return BLK_STS_PROTECTION; 104 } 105 106 next: 107 iter->data_buf += iter->interval; 108 iter->prot_buf += bi->tuple_size; 109 iter->seed++; 110 } 111 112 return BLK_STS_OK; 113 } 114 115 /** 116 * t10_pi_type1_prepare - prepare PI prior submitting request to device 117 * @rq: request with PI that should be prepared 118 * 119 * For Type 1/Type 2, the virtual start sector is the one that was 120 * originally submitted by the block layer for the ref_tag usage. Due to 121 * partitioning, MD/DM cloning, etc. the actual physical start sector is 122 * likely to be different. Remap protection information to match the 123 * physical LBA. 124 */ 125 static void t10_pi_type1_prepare(struct request *rq) 126 { 127 struct blk_integrity *bi = &rq->q->limits.integrity; 128 const int tuple_sz = bi->tuple_size; 129 u32 ref_tag = t10_pi_ref_tag(rq); 130 u8 offset = bi->pi_offset; 131 struct bio *bio; 132 133 __rq_for_each_bio(bio, rq) { 134 struct bio_integrity_payload *bip = bio_integrity(bio); 135 u32 virt = bip_get_seed(bip) & 0xffffffff; 136 struct bio_vec iv; 137 struct bvec_iter iter; 138 139 /* Already remapped? */ 140 if (bip->bip_flags & BIP_MAPPED_INTEGRITY) 141 break; 142 143 bip_for_each_vec(iv, bip, iter) { 144 unsigned int j; 145 void *p; 146 147 p = bvec_kmap_local(&iv); 148 for (j = 0; j < iv.bv_len; j += tuple_sz) { 149 struct t10_pi_tuple *pi = p + offset; 150 151 if (be32_to_cpu(pi->ref_tag) == virt) 152 pi->ref_tag = cpu_to_be32(ref_tag); 153 virt++; 154 ref_tag++; 155 p += tuple_sz; 156 } 157 kunmap_local(p); 158 } 159 160 bip->bip_flags |= BIP_MAPPED_INTEGRITY; 161 } 162 } 163 164 /** 165 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer 166 * @rq: request with PI that should be prepared 167 * @nr_bytes: total bytes to prepare 168 * 169 * For Type 1/Type 2, the virtual start sector is the one that was 170 * originally submitted by the block layer for the ref_tag usage. Due to 171 * partitioning, MD/DM cloning, etc. the actual physical start sector is 172 * likely to be different. Since the physical start sector was submitted 173 * to the device, we should remap it back to virtual values expected by the 174 * block layer. 175 */ 176 static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes) 177 { 178 struct blk_integrity *bi = &rq->q->limits.integrity; 179 unsigned intervals = nr_bytes >> bi->interval_exp; 180 const int tuple_sz = bi->tuple_size; 181 u32 ref_tag = t10_pi_ref_tag(rq); 182 u8 offset = bi->pi_offset; 183 struct bio *bio; 184 185 __rq_for_each_bio(bio, rq) { 186 struct bio_integrity_payload *bip = bio_integrity(bio); 187 u32 virt = bip_get_seed(bip) & 0xffffffff; 188 struct bio_vec iv; 189 struct bvec_iter iter; 190 191 bip_for_each_vec(iv, bip, iter) { 192 unsigned int j; 193 void *p; 194 195 p = bvec_kmap_local(&iv); 196 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { 197 struct t10_pi_tuple *pi = p + offset; 198 199 if (be32_to_cpu(pi->ref_tag) == ref_tag) 200 pi->ref_tag = cpu_to_be32(virt); 201 virt++; 202 ref_tag++; 203 intervals--; 204 p += tuple_sz; 205 } 206 kunmap_local(p); 207 } 208 } 209 } 210 211 static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len) 212 { 213 return cpu_to_be64(crc64_rocksoft_update(crc, data, len)); 214 } 215 216 static void ext_pi_crc64_generate(struct blk_integrity_iter *iter, 217 struct blk_integrity *bi) 218 { 219 u8 offset = bi->pi_offset; 220 unsigned int i; 221 222 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 223 struct crc64_pi_tuple *pi = iter->prot_buf + offset; 224 225 pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval); 226 if (offset) 227 pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag), 228 iter->prot_buf, offset); 229 pi->app_tag = 0; 230 231 if (bi->flags & BLK_INTEGRITY_REF_TAG) 232 put_unaligned_be48(iter->seed, pi->ref_tag); 233 else 234 put_unaligned_be48(0ULL, pi->ref_tag); 235 236 iter->data_buf += iter->interval; 237 iter->prot_buf += bi->tuple_size; 238 iter->seed++; 239 } 240 } 241 242 static bool ext_pi_ref_escape(const u8 ref_tag[6]) 243 { 244 static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; 245 246 return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0; 247 } 248 249 static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter, 250 struct blk_integrity *bi) 251 { 252 u8 offset = bi->pi_offset; 253 unsigned int i; 254 255 for (i = 0; i < iter->data_size; i += iter->interval) { 256 struct crc64_pi_tuple *pi = iter->prot_buf + offset; 257 u64 ref, seed; 258 __be64 csum; 259 260 if (bi->flags & BLK_INTEGRITY_REF_TAG) { 261 if (pi->app_tag == T10_PI_APP_ESCAPE) 262 goto next; 263 264 ref = get_unaligned_be48(pi->ref_tag); 265 seed = lower_48_bits(iter->seed); 266 if (ref != seed) { 267 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n", 268 iter->disk_name, seed, ref); 269 return BLK_STS_PROTECTION; 270 } 271 } else { 272 if (pi->app_tag == T10_PI_APP_ESCAPE && 273 ext_pi_ref_escape(pi->ref_tag)) 274 goto next; 275 } 276 277 csum = ext_pi_crc64(0, iter->data_buf, iter->interval); 278 if (offset) 279 csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf, 280 offset); 281 282 if (pi->guard_tag != csum) { 283 pr_err("%s: guard tag error at sector %llu " \ 284 "(rcvd %016llx, want %016llx)\n", 285 iter->disk_name, (unsigned long long)iter->seed, 286 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum)); 287 return BLK_STS_PROTECTION; 288 } 289 290 next: 291 iter->data_buf += iter->interval; 292 iter->prot_buf += bi->tuple_size; 293 iter->seed++; 294 } 295 296 return BLK_STS_OK; 297 } 298 299 static void ext_pi_type1_prepare(struct request *rq) 300 { 301 struct blk_integrity *bi = &rq->q->limits.integrity; 302 const int tuple_sz = bi->tuple_size; 303 u64 ref_tag = ext_pi_ref_tag(rq); 304 u8 offset = bi->pi_offset; 305 struct bio *bio; 306 307 __rq_for_each_bio(bio, rq) { 308 struct bio_integrity_payload *bip = bio_integrity(bio); 309 u64 virt = lower_48_bits(bip_get_seed(bip)); 310 struct bio_vec iv; 311 struct bvec_iter iter; 312 313 /* Already remapped? */ 314 if (bip->bip_flags & BIP_MAPPED_INTEGRITY) 315 break; 316 317 bip_for_each_vec(iv, bip, iter) { 318 unsigned int j; 319 void *p; 320 321 p = bvec_kmap_local(&iv); 322 for (j = 0; j < iv.bv_len; j += tuple_sz) { 323 struct crc64_pi_tuple *pi = p + offset; 324 u64 ref = get_unaligned_be48(pi->ref_tag); 325 326 if (ref == virt) 327 put_unaligned_be48(ref_tag, pi->ref_tag); 328 virt++; 329 ref_tag++; 330 p += tuple_sz; 331 } 332 kunmap_local(p); 333 } 334 335 bip->bip_flags |= BIP_MAPPED_INTEGRITY; 336 } 337 } 338 339 static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes) 340 { 341 struct blk_integrity *bi = &rq->q->limits.integrity; 342 unsigned intervals = nr_bytes >> bi->interval_exp; 343 const int tuple_sz = bi->tuple_size; 344 u64 ref_tag = ext_pi_ref_tag(rq); 345 u8 offset = bi->pi_offset; 346 struct bio *bio; 347 348 __rq_for_each_bio(bio, rq) { 349 struct bio_integrity_payload *bip = bio_integrity(bio); 350 u64 virt = lower_48_bits(bip_get_seed(bip)); 351 struct bio_vec iv; 352 struct bvec_iter iter; 353 354 bip_for_each_vec(iv, bip, iter) { 355 unsigned int j; 356 void *p; 357 358 p = bvec_kmap_local(&iv); 359 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { 360 struct crc64_pi_tuple *pi = p + offset; 361 u64 ref = get_unaligned_be48(pi->ref_tag); 362 363 if (ref == ref_tag) 364 put_unaligned_be48(virt, pi->ref_tag); 365 virt++; 366 ref_tag++; 367 intervals--; 368 p += tuple_sz; 369 } 370 kunmap_local(p); 371 } 372 } 373 } 374 375 void blk_integrity_generate(struct bio *bio) 376 { 377 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 378 struct bio_integrity_payload *bip = bio_integrity(bio); 379 struct blk_integrity_iter iter; 380 struct bvec_iter bviter; 381 struct bio_vec bv; 382 383 iter.disk_name = bio->bi_bdev->bd_disk->disk_name; 384 iter.interval = 1 << bi->interval_exp; 385 iter.seed = bio->bi_iter.bi_sector; 386 iter.prot_buf = bvec_virt(bip->bip_vec); 387 bio_for_each_segment(bv, bio, bviter) { 388 void *kaddr = bvec_kmap_local(&bv); 389 390 iter.data_buf = kaddr; 391 iter.data_size = bv.bv_len; 392 switch (bi->csum_type) { 393 case BLK_INTEGRITY_CSUM_CRC64: 394 ext_pi_crc64_generate(&iter, bi); 395 break; 396 case BLK_INTEGRITY_CSUM_CRC: 397 case BLK_INTEGRITY_CSUM_IP: 398 t10_pi_generate(&iter, bi); 399 break; 400 default: 401 break; 402 } 403 kunmap_local(kaddr); 404 } 405 } 406 407 void blk_integrity_verify(struct bio *bio) 408 { 409 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 410 struct bio_integrity_payload *bip = bio_integrity(bio); 411 struct blk_integrity_iter iter; 412 struct bvec_iter bviter; 413 struct bio_vec bv; 414 415 /* 416 * At the moment verify is called bi_iter has been advanced during split 417 * and completion, so use the copy created during submission here. 418 */ 419 iter.disk_name = bio->bi_bdev->bd_disk->disk_name; 420 iter.interval = 1 << bi->interval_exp; 421 iter.seed = bip->bio_iter.bi_sector; 422 iter.prot_buf = bvec_virt(bip->bip_vec); 423 __bio_for_each_segment(bv, bio, bviter, bip->bio_iter) { 424 void *kaddr = bvec_kmap_local(&bv); 425 blk_status_t ret = BLK_STS_OK; 426 427 iter.data_buf = kaddr; 428 iter.data_size = bv.bv_len; 429 switch (bi->csum_type) { 430 case BLK_INTEGRITY_CSUM_CRC64: 431 ret = ext_pi_crc64_verify(&iter, bi); 432 break; 433 case BLK_INTEGRITY_CSUM_CRC: 434 case BLK_INTEGRITY_CSUM_IP: 435 ret = t10_pi_verify(&iter, bi); 436 break; 437 default: 438 break; 439 } 440 kunmap_local(kaddr); 441 442 if (ret) { 443 bio->bi_status = ret; 444 return; 445 } 446 } 447 } 448 449 void blk_integrity_prepare(struct request *rq) 450 { 451 struct blk_integrity *bi = &rq->q->limits.integrity; 452 453 if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) 454 return; 455 456 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) 457 ext_pi_type1_prepare(rq); 458 else 459 t10_pi_type1_prepare(rq); 460 } 461 462 void blk_integrity_complete(struct request *rq, unsigned int nr_bytes) 463 { 464 struct blk_integrity *bi = &rq->q->limits.integrity; 465 466 if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) 467 return; 468 469 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) 470 ext_pi_type1_complete(rq, nr_bytes); 471 else 472 t10_pi_type1_complete(rq, nr_bytes); 473 } 474