1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com> 4 * Copyright(c) 2009 Intel Corporation 5 */ 6 #include <linux/kernel.h> 7 #include <linux/interrupt.h> 8 #include <linux/module.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/raid/pq.h> 11 #include <linux/raid/pq_tables.h> 12 #include <linux/async_tx.h> 13 #include <linux/gfp.h> 14 15 /* 16 * struct pq_scribble_page - space to hold throwaway P or Q buffer for 17 * synchronous gen_syndrome 18 */ 19 static struct page *pq_scribble_page; 20 21 /* the struct page *blocks[] parameter passed to async_gen_syndrome() 22 * and async_syndrome_val() contains the 'P' destination address at 23 * blocks[disks-2] and the 'Q' destination address at blocks[disks-1] 24 * 25 * note: these are macros as they are used as lvalues 26 */ 27 #define P(b, d) (b[d-2]) 28 #define Q(b, d) (b[d-1]) 29 30 #define MAX_DISKS 255 31 32 /* 33 * do_async_gen_syndrome - asynchronously calculate P and/or Q 34 */ 35 static __async_inline struct dma_async_tx_descriptor * 36 do_async_gen_syndrome(struct dma_chan *chan, 37 const unsigned char *scfs, int disks, 38 struct dmaengine_unmap_data *unmap, 39 enum dma_ctrl_flags dma_flags, 40 struct async_submit_ctl *submit) 41 { 42 struct dma_async_tx_descriptor *tx = NULL; 43 struct dma_device *dma = chan->device; 44 enum async_tx_flags flags_orig = submit->flags; 45 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 46 dma_async_tx_callback cb_param_orig = submit->cb_param; 47 int src_cnt = disks - 2; 48 unsigned short pq_src_cnt; 49 dma_addr_t dma_dest[2]; 50 int src_off = 0; 51 52 while (src_cnt > 0) { 53 submit->flags = flags_orig; 54 pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); 55 /* if we are submitting additional pqs, leave the chain open, 56 * clear the callback parameters, and leave the destination 57 * buffers mapped 58 */ 59 if (src_cnt > pq_src_cnt) { 60 submit->flags &= ~ASYNC_TX_ACK; 61 submit->flags |= ASYNC_TX_FENCE; 62 submit->cb_fn = NULL; 63 submit->cb_param = NULL; 64 } else { 65 submit->cb_fn = cb_fn_orig; 66 submit->cb_param = cb_param_orig; 67 if (cb_fn_orig) 68 dma_flags |= DMA_PREP_INTERRUPT; 69 } 70 if (submit->flags & ASYNC_TX_FENCE) 71 dma_flags |= DMA_PREP_FENCE; 72 73 /* Drivers force forward progress in case they can not provide 74 * a descriptor 75 */ 76 for (;;) { 77 dma_dest[0] = unmap->addr[disks - 2]; 78 dma_dest[1] = unmap->addr[disks - 1]; 79 tx = dma->device_prep_dma_pq(chan, dma_dest, 80 &unmap->addr[src_off], 81 pq_src_cnt, 82 &scfs[src_off], unmap->len, 83 dma_flags); 84 if (likely(tx)) 85 break; 86 async_tx_quiesce(&submit->depend_tx); 87 dma_async_issue_pending(chan); 88 } 89 90 dma_set_unmap(tx, unmap); 91 async_tx_submit(chan, tx, submit); 92 submit->depend_tx = tx; 93 94 /* drop completed sources */ 95 src_cnt -= pq_src_cnt; 96 src_off += pq_src_cnt; 97 98 dma_flags |= DMA_PREP_CONTINUE; 99 } 100 101 return tx; 102 } 103 104 /* 105 * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome 106 */ 107 static void 108 do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, 109 size_t len, struct async_submit_ctl *submit) 110 { 111 void **srcs; 112 int i; 113 int start = -1, stop = disks - 3; 114 115 if (submit->scribble) 116 srcs = submit->scribble; 117 else 118 srcs = (void **) blocks; 119 120 for (i = 0; i < disks; i++) { 121 if (blocks[i] == NULL) { 122 BUG_ON(i > disks - 3); /* P or Q can't be zero */ 123 srcs[i] = page_address(ZERO_PAGE(0)); 124 } else { 125 srcs[i] = page_address(blocks[i]) + offsets[i]; 126 127 if (i < disks - 2) { 128 stop = i; 129 if (start == -1) 130 start = i; 131 } 132 } 133 } 134 if (submit->flags & ASYNC_TX_PQ_XOR_DST) { 135 BUG_ON(!raid6_can_xor_syndrome()); 136 if (start >= 0) 137 raid6_xor_syndrome(disks, start, stop, len, srcs); 138 } else 139 raid6_gen_syndrome(disks, len, srcs); 140 async_tx_sync_epilog(submit); 141 } 142 143 static inline bool 144 is_dma_pq_aligned_offs(struct dma_device *dev, unsigned int *offs, 145 int src_cnt, size_t len) 146 { 147 int i; 148 149 for (i = 0; i < src_cnt; i++) { 150 if (!is_dma_pq_aligned(dev, offs[i], 0, len)) 151 return false; 152 } 153 return true; 154 } 155 156 /** 157 * async_gen_syndrome - asynchronously calculate a raid6 syndrome 158 * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 159 * @offsets: offset array into each block (src and dest) to start transaction 160 * @disks: number of blocks (including missing P or Q, see below) 161 * @len: length of operation in bytes 162 * @submit: submission/completion modifiers 163 * 164 * General note: This routine assumes a field of GF(2^8) with a 165 * primitive polynomial of 0x11d and a generator of {02}. 166 * 167 * 'disks' note: callers can optionally omit either P or Q (but not 168 * both) from the calculation by setting blocks[disks-2] or 169 * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <= 170 * PAGE_SIZE as a temporary buffer of this size is used in the 171 * synchronous path. 'disks' always accounts for both destination 172 * buffers. If any source buffers (blocks[i] where i < disks - 2) are 173 * set to NULL those buffers will be replaced with the raid6_zero_page 174 * in the synchronous path and omitted in the hardware-asynchronous 175 * path. 176 */ 177 struct dma_async_tx_descriptor * 178 async_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, 179 size_t len, struct async_submit_ctl *submit) 180 { 181 int src_cnt = disks - 2; 182 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 183 &P(blocks, disks), 2, 184 blocks, src_cnt, len); 185 struct dma_device *device = chan ? chan->device : NULL; 186 struct dmaengine_unmap_data *unmap = NULL; 187 188 BUG_ON(disks > MAX_DISKS || !(P(blocks, disks) || Q(blocks, disks))); 189 190 if (device) 191 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); 192 193 /* XORing P/Q is only implemented in software */ 194 if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) && 195 (src_cnt <= dma_maxpq(device, 0) || 196 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && 197 is_dma_pq_aligned_offs(device, offsets, disks, len)) { 198 struct dma_async_tx_descriptor *tx; 199 enum dma_ctrl_flags dma_flags = 0; 200 unsigned char coefs[MAX_DISKS]; 201 int i, j; 202 203 /* run the p+q asynchronously */ 204 pr_debug("%s: (async) disks: %d len: %zu\n", 205 __func__, disks, len); 206 207 /* convert source addresses being careful to collapse 'empty' 208 * sources and update the coefficients accordingly 209 */ 210 unmap->len = len; 211 for (i = 0, j = 0; i < src_cnt; i++) { 212 if (blocks[i] == NULL) 213 continue; 214 unmap->addr[j] = dma_map_page(device->dev, blocks[i], 215 offsets[i], len, DMA_TO_DEVICE); 216 coefs[j] = raid6_gfexp[i]; 217 unmap->to_cnt++; 218 j++; 219 } 220 221 /* 222 * DMAs use destinations as sources, 223 * so use BIDIRECTIONAL mapping 224 */ 225 unmap->bidi_cnt++; 226 if (P(blocks, disks)) 227 unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks), 228 P(offsets, disks), 229 len, DMA_BIDIRECTIONAL); 230 else { 231 unmap->addr[j++] = 0; 232 dma_flags |= DMA_PREP_PQ_DISABLE_P; 233 } 234 235 unmap->bidi_cnt++; 236 if (Q(blocks, disks)) 237 unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks), 238 Q(offsets, disks), 239 len, DMA_BIDIRECTIONAL); 240 else { 241 unmap->addr[j++] = 0; 242 dma_flags |= DMA_PREP_PQ_DISABLE_Q; 243 } 244 245 tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit); 246 dmaengine_unmap_put(unmap); 247 return tx; 248 } 249 250 dmaengine_unmap_put(unmap); 251 252 /* run the pq synchronously */ 253 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); 254 255 /* wait for any prerequisite operations */ 256 async_tx_quiesce(&submit->depend_tx); 257 258 if (!P(blocks, disks)) { 259 P(blocks, disks) = pq_scribble_page; 260 P(offsets, disks) = 0; 261 } 262 if (!Q(blocks, disks)) { 263 Q(blocks, disks) = pq_scribble_page; 264 Q(offsets, disks) = 0; 265 } 266 do_sync_gen_syndrome(blocks, offsets, disks, len, submit); 267 268 return NULL; 269 } 270 EXPORT_SYMBOL_GPL(async_gen_syndrome); 271 272 static inline struct dma_chan * 273 pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len) 274 { 275 #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA 276 return NULL; 277 #endif 278 return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks, 279 disks, len); 280 } 281 282 /** 283 * async_syndrome_val - asynchronously validate a raid6 syndrome 284 * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 285 * @offsets: common offset into each block (src and dest) to start transaction 286 * @disks: number of blocks (including missing P or Q, see below) 287 * @len: length of operation in bytes 288 * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set 289 * @spare: temporary result buffer for the synchronous case 290 * @s_off: spare buffer page offset 291 * @submit: submission / completion modifiers 292 * 293 * The same notes from async_gen_syndrome apply to the 'blocks', 294 * and 'disks' parameters of this routine. The synchronous path 295 * requires a temporary result buffer and submit->scribble to be 296 * specified. 297 */ 298 struct dma_async_tx_descriptor * 299 async_syndrome_val(struct page **blocks, unsigned int *offsets, int disks, 300 size_t len, enum sum_check_flags *pqres, struct page *spare, 301 unsigned int s_off, struct async_submit_ctl *submit) 302 { 303 struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); 304 struct dma_device *device = chan ? chan->device : NULL; 305 struct dma_async_tx_descriptor *tx; 306 unsigned char coefs[MAX_DISKS]; 307 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; 308 struct dmaengine_unmap_data *unmap = NULL; 309 310 BUG_ON(disks < 4 || disks > MAX_DISKS); 311 312 if (device) 313 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); 314 315 if (unmap && disks <= dma_maxpq(device, 0) && 316 is_dma_pq_aligned_offs(device, offsets, disks, len)) { 317 struct device *dev = device->dev; 318 dma_addr_t pq[2]; 319 int i, j = 0, src_cnt = 0; 320 321 pr_debug("%s: (async) disks: %d len: %zu\n", 322 __func__, disks, len); 323 324 unmap->len = len; 325 for (i = 0; i < disks-2; i++) 326 if (likely(blocks[i])) { 327 unmap->addr[j] = dma_map_page(dev, blocks[i], 328 offsets[i], len, 329 DMA_TO_DEVICE); 330 coefs[j] = raid6_gfexp[i]; 331 unmap->to_cnt++; 332 src_cnt++; 333 j++; 334 } 335 336 if (!P(blocks, disks)) { 337 pq[0] = 0; 338 dma_flags |= DMA_PREP_PQ_DISABLE_P; 339 } else { 340 pq[0] = dma_map_page(dev, P(blocks, disks), 341 P(offsets, disks), len, 342 DMA_TO_DEVICE); 343 unmap->addr[j++] = pq[0]; 344 unmap->to_cnt++; 345 } 346 if (!Q(blocks, disks)) { 347 pq[1] = 0; 348 dma_flags |= DMA_PREP_PQ_DISABLE_Q; 349 } else { 350 pq[1] = dma_map_page(dev, Q(blocks, disks), 351 Q(offsets, disks), len, 352 DMA_TO_DEVICE); 353 unmap->addr[j++] = pq[1]; 354 unmap->to_cnt++; 355 } 356 357 if (submit->flags & ASYNC_TX_FENCE) 358 dma_flags |= DMA_PREP_FENCE; 359 for (;;) { 360 tx = device->device_prep_dma_pq_val(chan, pq, 361 unmap->addr, 362 src_cnt, 363 coefs, 364 len, pqres, 365 dma_flags); 366 if (likely(tx)) 367 break; 368 async_tx_quiesce(&submit->depend_tx); 369 dma_async_issue_pending(chan); 370 } 371 372 dma_set_unmap(tx, unmap); 373 async_tx_submit(chan, tx, submit); 374 } else { 375 struct page *p_src = P(blocks, disks); 376 unsigned int p_off = P(offsets, disks); 377 struct page *q_src = Q(blocks, disks); 378 unsigned int q_off = Q(offsets, disks); 379 enum async_tx_flags flags_orig = submit->flags; 380 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 381 void *scribble = submit->scribble; 382 void *cb_param_orig = submit->cb_param; 383 void *p, *q, *s; 384 385 pr_debug("%s: (sync) disks: %d len: %zu\n", 386 __func__, disks, len); 387 388 /* caller must provide a temporary result buffer and 389 * allow the input parameters to be preserved 390 */ 391 BUG_ON(!spare || !scribble); 392 393 /* wait for any prerequisite operations */ 394 async_tx_quiesce(&submit->depend_tx); 395 396 /* recompute p and/or q into the temporary buffer and then 397 * check to see the result matches the current value 398 */ 399 tx = NULL; 400 *pqres = 0; 401 if (p_src) { 402 init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, 403 NULL, NULL, scribble); 404 tx = async_xor_offs(spare, s_off, 405 blocks, offsets, disks-2, len, submit); 406 async_tx_quiesce(&tx); 407 p = page_address(p_src) + p_off; 408 s = page_address(spare) + s_off; 409 *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; 410 } 411 412 if (q_src) { 413 P(blocks, disks) = NULL; 414 Q(blocks, disks) = spare; 415 Q(offsets, disks) = s_off; 416 init_async_submit(submit, 0, NULL, NULL, NULL, scribble); 417 tx = async_gen_syndrome(blocks, offsets, disks, 418 len, submit); 419 async_tx_quiesce(&tx); 420 q = page_address(q_src) + q_off; 421 s = page_address(spare) + s_off; 422 *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; 423 } 424 425 /* restore P, Q and submit */ 426 P(blocks, disks) = p_src; 427 P(offsets, disks) = p_off; 428 Q(blocks, disks) = q_src; 429 Q(offsets, disks) = q_off; 430 431 submit->cb_fn = cb_fn_orig; 432 submit->cb_param = cb_param_orig; 433 submit->flags = flags_orig; 434 async_tx_sync_epilog(submit); 435 tx = NULL; 436 } 437 dmaengine_unmap_put(unmap); 438 439 return tx; 440 } 441 EXPORT_SYMBOL_GPL(async_syndrome_val); 442 443 static int __init async_pq_init(void) 444 { 445 pq_scribble_page = alloc_page(GFP_KERNEL); 446 447 if (pq_scribble_page) 448 return 0; 449 450 pr_err("%s: failed to allocate required spare page\n", __func__); 451 452 return -ENOMEM; 453 } 454 455 static void __exit async_pq_exit(void) 456 { 457 __free_page(pq_scribble_page); 458 } 459 460 module_init(async_pq_init); 461 module_exit(async_pq_exit); 462 463 MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation"); 464 MODULE_LICENSE("GPL"); 465