1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Gary R Hook <gary.hook@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/kthread.h> 17 #include <linux/debugfs.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/interrupt.h> 20 #include <linux/compiler.h> 21 #include <linux/ccp.h> 22 23 #include "ccp-dev.h" 24 25 /* Allocate the requested number of contiguous LSB slots 26 * from the LSB bitmap. Look in the private range for this 27 * queue first; failing that, check the public area. 28 * If no space is available, wait around. 29 * Return: first slot number 30 */ 31 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) 32 { 33 struct ccp_device *ccp; 34 int start; 35 36 /* First look at the map for the queue */ 37 if (cmd_q->lsb >= 0) { 38 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, 39 LSB_SIZE, 40 0, count, 0); 41 if (start < LSB_SIZE) { 42 bitmap_set(cmd_q->lsbmap, start, count); 43 return start + cmd_q->lsb * LSB_SIZE; 44 } 45 } 46 47 /* No joy; try to get an entry from the shared blocks */ 48 ccp = cmd_q->ccp; 49 for (;;) { 50 mutex_lock(&ccp->sb_mutex); 51 52 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, 53 MAX_LSB_CNT * LSB_SIZE, 54 0, 55 count, 0); 56 if (start <= MAX_LSB_CNT * LSB_SIZE) { 57 bitmap_set(ccp->lsbmap, start, count); 58 59 mutex_unlock(&ccp->sb_mutex); 60 return start; 61 } 62 63 ccp->sb_avail = 0; 64 65 mutex_unlock(&ccp->sb_mutex); 66 67 /* Wait for KSB entries to become available */ 68 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 69 return 0; 70 } 71 } 72 73 /* Free a number of LSB slots from the bitmap, starting at 74 * the indicated starting slot number. 75 */ 76 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, 77 unsigned int count) 78 { 79 if (!start) 80 return; 81 82 if (cmd_q->lsb == start) { 83 /* An entry from the private LSB */ 84 bitmap_clear(cmd_q->lsbmap, start, count); 85 } else { 86 /* From the shared LSBs */ 87 struct ccp_device *ccp = cmd_q->ccp; 88 89 mutex_lock(&ccp->sb_mutex); 90 bitmap_clear(ccp->lsbmap, start, count); 91 ccp->sb_avail = 1; 92 mutex_unlock(&ccp->sb_mutex); 93 wake_up_interruptible_all(&ccp->sb_queue); 94 } 95 } 96 97 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 98 union ccp_function { 99 struct { 100 u16 size:7; 101 u16 encrypt:1; 102 u16 mode:5; 103 u16 type:2; 104 } aes; 105 struct { 106 u16 size:7; 107 u16 encrypt:1; 108 u16 rsvd:5; 109 u16 type:2; 110 } aes_xts; 111 struct { 112 u16 size:7; 113 u16 encrypt:1; 114 u16 mode:5; 115 u16 type:2; 116 } des3; 117 struct { 118 u16 rsvd1:10; 119 u16 type:4; 120 u16 rsvd2:1; 121 } sha; 122 struct { 123 u16 mode:3; 124 u16 size:12; 125 } rsa; 126 struct { 127 u16 byteswap:2; 128 u16 bitwise:3; 129 u16 reflect:2; 130 u16 rsvd:8; 131 } pt; 132 struct { 133 u16 rsvd:13; 134 } zlib; 135 struct { 136 u16 size:10; 137 u16 type:2; 138 u16 mode:3; 139 } ecc; 140 u16 raw; 141 }; 142 143 #define CCP_AES_SIZE(p) ((p)->aes.size) 144 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) 145 #define CCP_AES_MODE(p) ((p)->aes.mode) 146 #define CCP_AES_TYPE(p) ((p)->aes.type) 147 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) 148 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) 149 #define CCP_DES3_SIZE(p) ((p)->des3.size) 150 #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) 151 #define CCP_DES3_MODE(p) ((p)->des3.mode) 152 #define CCP_DES3_TYPE(p) ((p)->des3.type) 153 #define CCP_SHA_TYPE(p) ((p)->sha.type) 154 #define CCP_RSA_SIZE(p) ((p)->rsa.size) 155 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) 156 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) 157 #define CCP_ECC_MODE(p) ((p)->ecc.mode) 158 #define CCP_ECC_AFFINE(p) ((p)->ecc.one) 159 160 /* Word 0 */ 161 #define CCP5_CMD_DW0(p) ((p)->dw0) 162 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) 163 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) 164 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) 165 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) 166 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) 167 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) 168 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) 169 170 /* Word 1 */ 171 #define CCP5_CMD_DW1(p) ((p)->length) 172 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) 173 174 /* Word 2 */ 175 #define CCP5_CMD_DW2(p) ((p)->src_lo) 176 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) 177 178 /* Word 3 */ 179 #define CCP5_CMD_DW3(p) ((p)->dw3) 180 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 181 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) 182 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 183 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) 184 185 /* Words 4/5 */ 186 #define CCP5_CMD_DW4(p) ((p)->dw4) 187 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) 188 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 189 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) 190 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 191 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 192 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 193 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 194 195 /* Word 6/7 */ 196 #define CCP5_CMD_DW6(p) ((p)->key_lo) 197 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) 198 #define CCP5_CMD_DW7(p) ((p)->dw7) 199 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) 200 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 201 202 static inline u32 low_address(unsigned long addr) 203 { 204 return (u64)addr & 0x0ffffffff; 205 } 206 207 static inline u32 high_address(unsigned long addr) 208 { 209 return ((u64)addr >> 32) & 0x00000ffff; 210 } 211 212 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) 213 { 214 unsigned int head_idx, n; 215 u32 head_lo, queue_start; 216 217 queue_start = low_address(cmd_q->qdma_tail); 218 head_lo = ioread32(cmd_q->reg_head_lo); 219 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); 220 221 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; 222 223 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ 224 } 225 226 static int ccp5_do_cmd(struct ccp5_desc *desc, 227 struct ccp_cmd_queue *cmd_q) 228 { 229 u32 *mP; 230 __le32 *dP; 231 u32 tail; 232 int i; 233 int ret = 0; 234 235 cmd_q->total_ops++; 236 237 if (CCP5_CMD_SOC(desc)) { 238 CCP5_CMD_IOC(desc) = 1; 239 CCP5_CMD_SOC(desc) = 0; 240 } 241 mutex_lock(&cmd_q->q_mutex); 242 243 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx]; 244 dP = (__le32 *) desc; 245 for (i = 0; i < 8; i++) 246 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ 247 248 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 249 250 /* The data used by this command must be flushed to memory */ 251 wmb(); 252 253 /* Write the new tail address back to the queue register */ 254 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 255 iowrite32(tail, cmd_q->reg_tail_lo); 256 257 /* Turn the queue back on using our cached control register */ 258 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); 259 mutex_unlock(&cmd_q->q_mutex); 260 261 if (CCP5_CMD_IOC(desc)) { 262 /* Wait for the job to complete */ 263 ret = wait_event_interruptible(cmd_q->int_queue, 264 cmd_q->int_rcvd); 265 if (ret || cmd_q->cmd_error) { 266 /* Log the error and flush the queue by 267 * moving the head pointer 268 */ 269 if (cmd_q->cmd_error) 270 ccp_log_error(cmd_q->ccp, 271 cmd_q->cmd_error); 272 iowrite32(tail, cmd_q->reg_head_lo); 273 if (!ret) 274 ret = -EIO; 275 } 276 cmd_q->int_rcvd = 0; 277 } 278 279 return ret; 280 } 281 282 static int ccp5_perform_aes(struct ccp_op *op) 283 { 284 struct ccp5_desc desc; 285 union ccp_function function; 286 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 287 288 op->cmd_q->total_aes_ops++; 289 290 /* Zero out all the fields of the command desc */ 291 memset(&desc, 0, Q_DESC_SIZE); 292 293 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; 294 295 CCP5_CMD_SOC(&desc) = op->soc; 296 CCP5_CMD_IOC(&desc) = 1; 297 CCP5_CMD_INIT(&desc) = op->init; 298 CCP5_CMD_EOM(&desc) = op->eom; 299 CCP5_CMD_PROT(&desc) = 0; 300 301 function.raw = 0; 302 CCP_AES_ENCRYPT(&function) = op->u.aes.action; 303 CCP_AES_MODE(&function) = op->u.aes.mode; 304 CCP_AES_TYPE(&function) = op->u.aes.type; 305 CCP_AES_SIZE(&function) = op->u.aes.size; 306 307 CCP5_CMD_FUNCTION(&desc) = function.raw; 308 309 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 310 311 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 312 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 313 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 314 315 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 316 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 317 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 318 319 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 320 CCP5_CMD_KEY_HI(&desc) = 0; 321 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 322 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 323 324 return ccp5_do_cmd(&desc, op->cmd_q); 325 } 326 327 static int ccp5_perform_xts_aes(struct ccp_op *op) 328 { 329 struct ccp5_desc desc; 330 union ccp_function function; 331 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 332 333 op->cmd_q->total_xts_aes_ops++; 334 335 /* Zero out all the fields of the command desc */ 336 memset(&desc, 0, Q_DESC_SIZE); 337 338 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; 339 340 CCP5_CMD_SOC(&desc) = op->soc; 341 CCP5_CMD_IOC(&desc) = 1; 342 CCP5_CMD_INIT(&desc) = op->init; 343 CCP5_CMD_EOM(&desc) = op->eom; 344 CCP5_CMD_PROT(&desc) = 0; 345 346 function.raw = 0; 347 CCP_XTS_ENCRYPT(&function) = op->u.xts.action; 348 CCP_XTS_SIZE(&function) = op->u.xts.unit_size; 349 CCP5_CMD_FUNCTION(&desc) = function.raw; 350 351 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 352 353 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 354 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 355 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 356 357 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 358 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 359 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 360 361 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 362 CCP5_CMD_KEY_HI(&desc) = 0; 363 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 364 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 365 366 return ccp5_do_cmd(&desc, op->cmd_q); 367 } 368 369 static int ccp5_perform_sha(struct ccp_op *op) 370 { 371 struct ccp5_desc desc; 372 union ccp_function function; 373 374 op->cmd_q->total_sha_ops++; 375 376 /* Zero out all the fields of the command desc */ 377 memset(&desc, 0, Q_DESC_SIZE); 378 379 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; 380 381 CCP5_CMD_SOC(&desc) = op->soc; 382 CCP5_CMD_IOC(&desc) = 1; 383 CCP5_CMD_INIT(&desc) = 1; 384 CCP5_CMD_EOM(&desc) = op->eom; 385 CCP5_CMD_PROT(&desc) = 0; 386 387 function.raw = 0; 388 CCP_SHA_TYPE(&function) = op->u.sha.type; 389 CCP5_CMD_FUNCTION(&desc) = function.raw; 390 391 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 392 393 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 394 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 395 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 396 397 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 398 399 if (op->eom) { 400 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); 401 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); 402 } else { 403 CCP5_CMD_SHA_LO(&desc) = 0; 404 CCP5_CMD_SHA_HI(&desc) = 0; 405 } 406 407 return ccp5_do_cmd(&desc, op->cmd_q); 408 } 409 410 static int ccp5_perform_des3(struct ccp_op *op) 411 { 412 struct ccp5_desc desc; 413 union ccp_function function; 414 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 415 416 op->cmd_q->total_3des_ops++; 417 418 /* Zero out all the fields of the command desc */ 419 memset(&desc, 0, sizeof(struct ccp5_desc)); 420 421 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; 422 423 CCP5_CMD_SOC(&desc) = op->soc; 424 CCP5_CMD_IOC(&desc) = 1; 425 CCP5_CMD_INIT(&desc) = op->init; 426 CCP5_CMD_EOM(&desc) = op->eom; 427 CCP5_CMD_PROT(&desc) = 0; 428 429 function.raw = 0; 430 CCP_DES3_ENCRYPT(&function) = op->u.des3.action; 431 CCP_DES3_MODE(&function) = op->u.des3.mode; 432 CCP_DES3_TYPE(&function) = op->u.des3.type; 433 CCP5_CMD_FUNCTION(&desc) = function.raw; 434 435 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 436 437 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 438 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 439 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 440 441 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 442 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 443 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 444 445 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 446 CCP5_CMD_KEY_HI(&desc) = 0; 447 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 448 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 449 450 return ccp5_do_cmd(&desc, op->cmd_q); 451 } 452 453 static int ccp5_perform_rsa(struct ccp_op *op) 454 { 455 struct ccp5_desc desc; 456 union ccp_function function; 457 458 op->cmd_q->total_rsa_ops++; 459 460 /* Zero out all the fields of the command desc */ 461 memset(&desc, 0, Q_DESC_SIZE); 462 463 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; 464 465 CCP5_CMD_SOC(&desc) = op->soc; 466 CCP5_CMD_IOC(&desc) = 1; 467 CCP5_CMD_INIT(&desc) = 0; 468 CCP5_CMD_EOM(&desc) = 1; 469 CCP5_CMD_PROT(&desc) = 0; 470 471 function.raw = 0; 472 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3; 473 CCP5_CMD_FUNCTION(&desc) = function.raw; 474 475 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; 476 477 /* Source is from external memory */ 478 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 479 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 480 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 481 482 /* Destination is in external memory */ 483 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 484 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 485 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 486 487 /* Exponent is in LSB memory */ 488 CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE; 489 CCP5_CMD_KEY_HI(&desc) = 0; 490 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 491 492 return ccp5_do_cmd(&desc, op->cmd_q); 493 } 494 495 static int ccp5_perform_passthru(struct ccp_op *op) 496 { 497 struct ccp5_desc desc; 498 union ccp_function function; 499 struct ccp_dma_info *saddr = &op->src.u.dma; 500 struct ccp_dma_info *daddr = &op->dst.u.dma; 501 502 503 op->cmd_q->total_pt_ops++; 504 505 memset(&desc, 0, Q_DESC_SIZE); 506 507 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; 508 509 CCP5_CMD_SOC(&desc) = 0; 510 CCP5_CMD_IOC(&desc) = 1; 511 CCP5_CMD_INIT(&desc) = 0; 512 CCP5_CMD_EOM(&desc) = op->eom; 513 CCP5_CMD_PROT(&desc) = 0; 514 515 function.raw = 0; 516 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; 517 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; 518 CCP5_CMD_FUNCTION(&desc) = function.raw; 519 520 /* Length of source data is always 256 bytes */ 521 if (op->src.type == CCP_MEMTYPE_SYSTEM) 522 CCP5_CMD_LEN(&desc) = saddr->length; 523 else 524 CCP5_CMD_LEN(&desc) = daddr->length; 525 526 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 527 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 528 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 529 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 530 531 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 532 CCP5_CMD_LSB_ID(&desc) = op->sb_key; 533 } else { 534 u32 key_addr = op->src.u.sb * CCP_SB_BYTES; 535 536 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); 537 CCP5_CMD_SRC_HI(&desc) = 0; 538 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; 539 } 540 541 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 542 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 543 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 544 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 545 } else { 546 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; 547 548 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); 549 CCP5_CMD_DST_HI(&desc) = 0; 550 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; 551 } 552 553 return ccp5_do_cmd(&desc, op->cmd_q); 554 } 555 556 static int ccp5_perform_ecc(struct ccp_op *op) 557 { 558 struct ccp5_desc desc; 559 union ccp_function function; 560 561 op->cmd_q->total_ecc_ops++; 562 563 /* Zero out all the fields of the command desc */ 564 memset(&desc, 0, Q_DESC_SIZE); 565 566 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; 567 568 CCP5_CMD_SOC(&desc) = 0; 569 CCP5_CMD_IOC(&desc) = 1; 570 CCP5_CMD_INIT(&desc) = 0; 571 CCP5_CMD_EOM(&desc) = 1; 572 CCP5_CMD_PROT(&desc) = 0; 573 574 function.raw = 0; 575 function.ecc.mode = op->u.ecc.function; 576 CCP5_CMD_FUNCTION(&desc) = function.raw; 577 578 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 579 580 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 581 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 582 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 583 584 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 585 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 586 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 587 588 return ccp5_do_cmd(&desc, op->cmd_q); 589 } 590 591 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) 592 { 593 int q_mask = 1 << cmd_q->id; 594 int queues = 0; 595 int j; 596 597 /* Build a bit mask to know which LSBs this queue has access to. 598 * Don't bother with segment 0 as it has special privileges. 599 */ 600 for (j = 1; j < MAX_LSB_CNT; j++) { 601 if (status & q_mask) 602 bitmap_set(cmd_q->lsbmask, j, 1); 603 status >>= LSB_REGION_WIDTH; 604 } 605 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 606 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", 607 cmd_q->id, queues); 608 609 return queues ? 0 : -EINVAL; 610 } 611 612 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, 613 int lsb_cnt, int n_lsbs, 614 unsigned long *lsb_pub) 615 { 616 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 617 int bitno; 618 int qlsb_wgt; 619 int i; 620 621 /* For each queue: 622 * If the count of potential LSBs available to a queue matches the 623 * ordinal given to us in lsb_cnt: 624 * Copy the mask of possible LSBs for this queue into "qlsb"; 625 * For each bit in qlsb, see if the corresponding bit in the 626 * aggregation mask is set; if so, we have a match. 627 * If we have a match, clear the bit in the aggregation to 628 * mark it as no longer available. 629 * If there is no match, clear the bit in qlsb and keep looking. 630 */ 631 for (i = 0; i < ccp->cmd_q_count; i++) { 632 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 633 634 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 635 636 if (qlsb_wgt == lsb_cnt) { 637 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); 638 639 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 640 while (bitno < MAX_LSB_CNT) { 641 if (test_bit(bitno, lsb_pub)) { 642 /* We found an available LSB 643 * that this queue can access 644 */ 645 cmd_q->lsb = bitno; 646 bitmap_clear(lsb_pub, bitno, 1); 647 dev_dbg(ccp->dev, 648 "Queue %d gets LSB %d\n", 649 i, bitno); 650 break; 651 } 652 bitmap_clear(qlsb, bitno, 1); 653 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 654 } 655 if (bitno >= MAX_LSB_CNT) 656 return -EINVAL; 657 n_lsbs--; 658 } 659 } 660 return n_lsbs; 661 } 662 663 /* For each queue, from the most- to least-constrained: 664 * find an LSB that can be assigned to the queue. If there are N queues that 665 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a 666 * dedicated LSB. Remaining LSB regions become a shared resource. 667 * If we have fewer LSBs than queues, all LSB regions become shared resources. 668 */ 669 static int ccp_assign_lsbs(struct ccp_device *ccp) 670 { 671 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); 672 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 673 int n_lsbs = 0; 674 int bitno; 675 int i, lsb_cnt; 676 int rc = 0; 677 678 bitmap_zero(lsb_pub, MAX_LSB_CNT); 679 680 /* Create an aggregate bitmap to get a total count of available LSBs */ 681 for (i = 0; i < ccp->cmd_q_count; i++) 682 bitmap_or(lsb_pub, 683 lsb_pub, ccp->cmd_q[i].lsbmask, 684 MAX_LSB_CNT); 685 686 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); 687 688 if (n_lsbs >= ccp->cmd_q_count) { 689 /* We have enough LSBS to give every queue a private LSB. 690 * Brute force search to start with the queues that are more 691 * constrained in LSB choice. When an LSB is privately 692 * assigned, it is removed from the public mask. 693 * This is an ugly N squared algorithm with some optimization. 694 */ 695 for (lsb_cnt = 1; 696 n_lsbs && (lsb_cnt <= MAX_LSB_CNT); 697 lsb_cnt++) { 698 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, 699 lsb_pub); 700 if (rc < 0) 701 return -EINVAL; 702 n_lsbs = rc; 703 } 704 } 705 706 rc = 0; 707 /* What's left of the LSBs, according to the public mask, now become 708 * shared. Any zero bits in the lsb_pub mask represent an LSB region 709 * that can't be used as a shared resource, so mark the LSB slots for 710 * them as "in use". 711 */ 712 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); 713 714 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 715 while (bitno < MAX_LSB_CNT) { 716 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); 717 bitmap_set(qlsb, bitno, 1); 718 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 719 } 720 721 return rc; 722 } 723 724 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) 725 { 726 unsigned int i; 727 728 for (i = 0; i < ccp->cmd_q_count; i++) 729 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); 730 } 731 732 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) 733 { 734 unsigned int i; 735 736 for (i = 0; i < ccp->cmd_q_count; i++) 737 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); 738 } 739 740 static void ccp5_irq_bh(unsigned long data) 741 { 742 struct ccp_device *ccp = (struct ccp_device *)data; 743 u32 status; 744 unsigned int i; 745 746 for (i = 0; i < ccp->cmd_q_count; i++) { 747 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 748 749 status = ioread32(cmd_q->reg_interrupt_status); 750 751 if (status) { 752 cmd_q->int_status = status; 753 cmd_q->q_status = ioread32(cmd_q->reg_status); 754 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 755 756 /* On error, only save the first error value */ 757 if ((status & INT_ERROR) && !cmd_q->cmd_error) 758 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 759 760 cmd_q->int_rcvd = 1; 761 762 /* Acknowledge the interrupt and wake the kthread */ 763 iowrite32(status, cmd_q->reg_interrupt_status); 764 wake_up_interruptible(&cmd_q->int_queue); 765 } 766 } 767 ccp5_enable_queue_interrupts(ccp); 768 } 769 770 static irqreturn_t ccp5_irq_handler(int irq, void *data) 771 { 772 struct device *dev = data; 773 struct ccp_device *ccp = dev_get_drvdata(dev); 774 775 ccp5_disable_queue_interrupts(ccp); 776 ccp->total_interrupts++; 777 if (ccp->use_tasklet) 778 tasklet_schedule(&ccp->irq_tasklet); 779 else 780 ccp5_irq_bh((unsigned long)ccp); 781 return IRQ_HANDLED; 782 } 783 784 static int ccp5_init(struct ccp_device *ccp) 785 { 786 struct device *dev = ccp->dev; 787 struct ccp_cmd_queue *cmd_q; 788 struct dma_pool *dma_pool; 789 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 790 unsigned int qmr, qim, i; 791 u64 status; 792 u32 status_lo, status_hi; 793 int ret; 794 795 /* Find available queues */ 796 qim = 0; 797 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 798 for (i = 0; i < MAX_HW_QUEUES; i++) { 799 800 if (!(qmr & (1 << i))) 801 continue; 802 803 /* Allocate a dma pool for this queue */ 804 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 805 ccp->name, i); 806 dma_pool = dma_pool_create(dma_pool_name, dev, 807 CCP_DMAPOOL_MAX_SIZE, 808 CCP_DMAPOOL_ALIGN, 0); 809 if (!dma_pool) { 810 dev_err(dev, "unable to allocate dma pool\n"); 811 ret = -ENOMEM; 812 } 813 814 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 815 ccp->cmd_q_count++; 816 817 cmd_q->ccp = ccp; 818 cmd_q->id = i; 819 cmd_q->dma_pool = dma_pool; 820 mutex_init(&cmd_q->q_mutex); 821 822 /* Page alignment satisfies our needs for N <= 128 */ 823 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); 824 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 825 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize, 826 &cmd_q->qbase_dma, 827 GFP_KERNEL); 828 if (!cmd_q->qbase) { 829 dev_err(dev, "unable to allocate command queue\n"); 830 ret = -ENOMEM; 831 goto e_pool; 832 } 833 834 cmd_q->qidx = 0; 835 /* Preset some register values and masks that are queue 836 * number dependent 837 */ 838 cmd_q->reg_control = ccp->io_regs + 839 CMD5_Q_STATUS_INCR * (i + 1); 840 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; 841 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; 842 cmd_q->reg_int_enable = cmd_q->reg_control + 843 CMD5_Q_INT_ENABLE_BASE; 844 cmd_q->reg_interrupt_status = cmd_q->reg_control + 845 CMD5_Q_INTERRUPT_STATUS_BASE; 846 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; 847 cmd_q->reg_int_status = cmd_q->reg_control + 848 CMD5_Q_INT_STATUS_BASE; 849 cmd_q->reg_dma_status = cmd_q->reg_control + 850 CMD5_Q_DMA_STATUS_BASE; 851 cmd_q->reg_dma_read_status = cmd_q->reg_control + 852 CMD5_Q_DMA_READ_STATUS_BASE; 853 cmd_q->reg_dma_write_status = cmd_q->reg_control + 854 CMD5_Q_DMA_WRITE_STATUS_BASE; 855 856 init_waitqueue_head(&cmd_q->int_queue); 857 858 dev_dbg(dev, "queue #%u available\n", i); 859 } 860 861 if (ccp->cmd_q_count == 0) { 862 dev_notice(dev, "no command queues available\n"); 863 ret = -EIO; 864 goto e_pool; 865 } 866 867 /* Turn off the queues and disable interrupts until ready */ 868 ccp5_disable_queue_interrupts(ccp); 869 for (i = 0; i < ccp->cmd_q_count; i++) { 870 cmd_q = &ccp->cmd_q[i]; 871 872 cmd_q->qcontrol = 0; /* Start with nothing */ 873 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 874 875 ioread32(cmd_q->reg_int_status); 876 ioread32(cmd_q->reg_status); 877 878 /* Clear the interrupt status */ 879 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 880 } 881 882 dev_dbg(dev, "Requesting an IRQ...\n"); 883 /* Request an irq */ 884 ret = ccp->get_irq(ccp); 885 if (ret) { 886 dev_err(dev, "unable to allocate an IRQ\n"); 887 goto e_pool; 888 } 889 /* Initialize the ISR tasklet */ 890 if (ccp->use_tasklet) 891 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, 892 (unsigned long)ccp); 893 894 dev_dbg(dev, "Loading LSB map...\n"); 895 /* Copy the private LSB mask to the public registers */ 896 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 897 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 898 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); 899 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); 900 status = ((u64)status_hi<<30) | (u64)status_lo; 901 902 dev_dbg(dev, "Configuring virtual queues...\n"); 903 /* Configure size of each virtual queue accessible to host */ 904 for (i = 0; i < ccp->cmd_q_count; i++) { 905 u32 dma_addr_lo; 906 u32 dma_addr_hi; 907 908 cmd_q = &ccp->cmd_q[i]; 909 910 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); 911 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; 912 913 cmd_q->qdma_tail = cmd_q->qbase_dma; 914 dma_addr_lo = low_address(cmd_q->qdma_tail); 915 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); 916 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); 917 918 dma_addr_hi = high_address(cmd_q->qdma_tail); 919 cmd_q->qcontrol |= (dma_addr_hi << 16); 920 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 921 922 /* Find the LSB regions accessible to the queue */ 923 ccp_find_lsb_regions(cmd_q, status); 924 cmd_q->lsb = -1; /* Unassigned value */ 925 } 926 927 dev_dbg(dev, "Assigning LSBs...\n"); 928 ret = ccp_assign_lsbs(ccp); 929 if (ret) { 930 dev_err(dev, "Unable to assign LSBs (%d)\n", ret); 931 goto e_irq; 932 } 933 934 /* Optimization: pre-allocate LSB slots for each queue */ 935 for (i = 0; i < ccp->cmd_q_count; i++) { 936 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 937 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 938 } 939 940 dev_dbg(dev, "Starting threads...\n"); 941 /* Create a kthread for each queue */ 942 for (i = 0; i < ccp->cmd_q_count; i++) { 943 struct task_struct *kthread; 944 945 cmd_q = &ccp->cmd_q[i]; 946 947 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 948 "%s-q%u", ccp->name, cmd_q->id); 949 if (IS_ERR(kthread)) { 950 dev_err(dev, "error creating queue thread (%ld)\n", 951 PTR_ERR(kthread)); 952 ret = PTR_ERR(kthread); 953 goto e_kthread; 954 } 955 956 cmd_q->kthread = kthread; 957 wake_up_process(kthread); 958 } 959 960 dev_dbg(dev, "Enabling interrupts...\n"); 961 ccp5_enable_queue_interrupts(ccp); 962 963 dev_dbg(dev, "Registering device...\n"); 964 /* Put this on the unit list to make it available */ 965 ccp_add_device(ccp); 966 967 ret = ccp_register_rng(ccp); 968 if (ret) 969 goto e_kthread; 970 971 /* Register the DMA engine support */ 972 ret = ccp_dmaengine_register(ccp); 973 if (ret) 974 goto e_hwrng; 975 976 /* Set up debugfs entries */ 977 ccp5_debugfs_setup(ccp); 978 979 return 0; 980 981 e_hwrng: 982 ccp_unregister_rng(ccp); 983 984 e_kthread: 985 for (i = 0; i < ccp->cmd_q_count; i++) 986 if (ccp->cmd_q[i].kthread) 987 kthread_stop(ccp->cmd_q[i].kthread); 988 989 e_irq: 990 ccp->free_irq(ccp); 991 992 e_pool: 993 for (i = 0; i < ccp->cmd_q_count; i++) 994 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 995 996 return ret; 997 } 998 999 static void ccp5_destroy(struct ccp_device *ccp) 1000 { 1001 struct device *dev = ccp->dev; 1002 struct ccp_cmd_queue *cmd_q; 1003 struct ccp_cmd *cmd; 1004 unsigned int i; 1005 1006 /* Unregister the DMA engine */ 1007 ccp_dmaengine_unregister(ccp); 1008 1009 /* Unregister the RNG */ 1010 ccp_unregister_rng(ccp); 1011 1012 /* Remove this device from the list of available units first */ 1013 ccp_del_device(ccp); 1014 1015 /* We're in the process of tearing down the entire driver; 1016 * when all the devices are gone clean up debugfs 1017 */ 1018 if (ccp_present()) 1019 ccp5_debugfs_destroy(); 1020 1021 /* Disable and clear interrupts */ 1022 ccp5_disable_queue_interrupts(ccp); 1023 for (i = 0; i < ccp->cmd_q_count; i++) { 1024 cmd_q = &ccp->cmd_q[i]; 1025 1026 /* Turn off the run bit */ 1027 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); 1028 1029 /* Clear the interrupt status */ 1030 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 1031 ioread32(cmd_q->reg_int_status); 1032 ioread32(cmd_q->reg_status); 1033 } 1034 1035 /* Stop the queue kthreads */ 1036 for (i = 0; i < ccp->cmd_q_count; i++) 1037 if (ccp->cmd_q[i].kthread) 1038 kthread_stop(ccp->cmd_q[i].kthread); 1039 1040 ccp->free_irq(ccp); 1041 1042 for (i = 0; i < ccp->cmd_q_count; i++) { 1043 cmd_q = &ccp->cmd_q[i]; 1044 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, 1045 cmd_q->qbase_dma); 1046 } 1047 1048 /* Flush the cmd and backlog queue */ 1049 while (!list_empty(&ccp->cmd)) { 1050 /* Invoke the callback directly with an error code */ 1051 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 1052 list_del(&cmd->entry); 1053 cmd->callback(cmd->data, -ENODEV); 1054 } 1055 while (!list_empty(&ccp->backlog)) { 1056 /* Invoke the callback directly with an error code */ 1057 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 1058 list_del(&cmd->entry); 1059 cmd->callback(cmd->data, -ENODEV); 1060 } 1061 } 1062 1063 static void ccp5_config(struct ccp_device *ccp) 1064 { 1065 /* Public side */ 1066 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); 1067 } 1068 1069 static void ccp5other_config(struct ccp_device *ccp) 1070 { 1071 int i; 1072 u32 rnd; 1073 1074 /* We own all of the queues on the NTB CCP */ 1075 1076 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); 1077 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); 1078 for (i = 0; i < 12; i++) { 1079 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); 1080 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); 1081 } 1082 1083 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); 1084 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); 1085 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); 1086 1087 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 1088 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 1089 1090 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); 1091 1092 ccp5_config(ccp); 1093 } 1094 1095 /* Version 5 adds some function, but is essentially the same as v5 */ 1096 static const struct ccp_actions ccp5_actions = { 1097 .aes = ccp5_perform_aes, 1098 .xts_aes = ccp5_perform_xts_aes, 1099 .sha = ccp5_perform_sha, 1100 .des3 = ccp5_perform_des3, 1101 .rsa = ccp5_perform_rsa, 1102 .passthru = ccp5_perform_passthru, 1103 .ecc = ccp5_perform_ecc, 1104 .sballoc = ccp_lsb_alloc, 1105 .sbfree = ccp_lsb_free, 1106 .init = ccp5_init, 1107 .destroy = ccp5_destroy, 1108 .get_free_slots = ccp5_get_free_slots, 1109 .irqhandler = ccp5_irq_handler, 1110 }; 1111 1112 const struct ccp_vdata ccpv5a = { 1113 .version = CCP_VERSION(5, 0), 1114 .setup = ccp5_config, 1115 .perform = &ccp5_actions, 1116 .bar = 2, 1117 .offset = 0x0, 1118 }; 1119 1120 const struct ccp_vdata ccpv5b = { 1121 .version = CCP_VERSION(5, 0), 1122 .dma_chan_attr = DMA_PRIVATE, 1123 .setup = ccp5other_config, 1124 .perform = &ccp5_actions, 1125 .bar = 2, 1126 .offset = 0x0, 1127 }; 1128