1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 4 * All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/moduleparam.h> 10 #include <linux/mod_devicetable.h> 11 #include <linux/interrupt.h> 12 #include <linux/pci.h> 13 #include <linux/slab.h> 14 #include <linux/delay.h> 15 #include <linux/mm.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/scatterlist.h> 18 #include <linux/highmem.h> 19 #include <linux/crypto.h> 20 #include <linux/hw_random.h> 21 #include <linux/ktime.h> 22 23 #include <crypto/algapi.h> 24 #include <crypto/internal/des.h> 25 #include <crypto/internal/skcipher.h> 26 27 static char hifn_pll_ref[sizeof("extNNN")] = "ext"; 28 module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); 29 MODULE_PARM_DESC(hifn_pll_ref, 30 "PLL reference clock (pci[freq] or ext[freq], default ext)"); 31 32 static atomic_t hifn_dev_number; 33 34 #define ACRYPTO_OP_DECRYPT 0 35 #define ACRYPTO_OP_ENCRYPT 1 36 #define ACRYPTO_OP_HMAC 2 37 #define ACRYPTO_OP_RNG 3 38 39 #define ACRYPTO_MODE_ECB 0 40 #define ACRYPTO_MODE_CBC 1 41 #define ACRYPTO_MODE_CFB 2 42 #define ACRYPTO_MODE_OFB 3 43 44 #define ACRYPTO_TYPE_AES_128 0 45 #define ACRYPTO_TYPE_AES_192 1 46 #define ACRYPTO_TYPE_AES_256 2 47 #define ACRYPTO_TYPE_3DES 3 48 #define ACRYPTO_TYPE_DES 4 49 50 #define PCI_VENDOR_ID_HIFN 0x13A3 51 #define PCI_DEVICE_ID_HIFN_7955 0x0020 52 #define PCI_DEVICE_ID_HIFN_7956 0x001d 53 54 /* I/O region sizes */ 55 56 #define HIFN_BAR0_SIZE 0x1000 57 #define HIFN_BAR1_SIZE 0x2000 58 #define HIFN_BAR2_SIZE 0x8000 59 60 /* DMA registres */ 61 62 #define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ 63 #define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ 64 #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ 65 #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ 66 #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ 67 #define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ 68 #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ 69 #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ 70 #define HIFN_CHIP_ID 0x98 /* Chip ID */ 71 72 /* 73 * Processing Unit Registers (offset from BASEREG0) 74 */ 75 #define HIFN_0_PUDATA 0x00 /* Processing Unit Data */ 76 #define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */ 77 #define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */ 78 #define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */ 79 #define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */ 80 #define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */ 81 #define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */ 82 #define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */ 83 #define HIFN_0_SPACESIZE 0x20 /* Register space size */ 84 85 /* Processing Unit Control Register (HIFN_0_PUCTRL) */ 86 #define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */ 87 #define HIFN_PUCTRL_STOP 0x0008 /* stop pu */ 88 #define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */ 89 #define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */ 90 #define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */ 91 92 /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */ 93 #define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */ 94 #define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */ 95 #define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 96 #define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 97 #define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */ 98 #define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */ 99 #define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */ 100 #define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */ 101 #define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */ 102 #define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */ 103 104 /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */ 105 #define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */ 106 #define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */ 107 #define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */ 108 #define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */ 109 #define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */ 110 #define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */ 111 #define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */ 112 #define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */ 113 #define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */ 114 #define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */ 115 #define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */ 116 #define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */ 117 #define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */ 118 #define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */ 119 #define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */ 120 #define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */ 121 #define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */ 122 #define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */ 123 #define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */ 124 #define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */ 125 #define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */ 126 #define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */ 127 #define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */ 128 129 /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */ 130 #define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */ 131 #define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */ 132 #define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 133 #define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 134 #define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */ 135 #define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */ 136 #define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */ 137 #define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */ 138 #define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */ 139 #define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */ 140 141 /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */ 142 #define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */ 143 #define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */ 144 #define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 145 #define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 146 #define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */ 147 #define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */ 148 #define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */ 149 #define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */ 150 #define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */ 151 #define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */ 152 #define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */ 153 #define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */ 154 #define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */ 155 #define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */ 156 #define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */ 157 #define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */ 158 #define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */ 159 160 /* FIFO Status Register (HIFN_0_FIFOSTAT) */ 161 #define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */ 162 #define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */ 163 164 /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */ 165 #define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */ 166 167 /* 168 * DMA Interface Registers (offset from BASEREG1) 169 */ 170 #define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */ 171 #define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */ 172 #define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */ 173 #define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */ 174 #define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */ 175 #define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */ 176 #define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */ 177 #define HIFN_1_PLL 0x4c /* 795x: PLL config */ 178 #define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */ 179 #define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */ 180 #define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */ 181 #define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */ 182 #define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */ 183 #define HIFN_1_REVID 0x98 /* Revision ID */ 184 #define HIFN_1_UNLOCK_SECRET1 0xf4 185 #define HIFN_1_UNLOCK_SECRET2 0xfc 186 #define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */ 187 #define HIFN_1_PUB_BASE 0x300 /* Public Base Address */ 188 #define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */ 189 #define HIFN_1_PUB_OP 0x308 /* Public Operand */ 190 #define HIFN_1_PUB_STATUS 0x30c /* Public Status */ 191 #define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */ 192 #define HIFN_1_RNG_CONFIG 0x314 /* RNG config */ 193 #define HIFN_1_RNG_DATA 0x318 /* RNG data */ 194 #define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */ 195 #define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */ 196 197 /* DMA Status and Control Register (HIFN_1_DMA_CSR) */ 198 #define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */ 199 #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */ 200 #define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */ 201 #define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */ 202 #define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */ 203 #define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */ 204 #define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */ 205 #define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */ 206 #define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */ 207 #define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */ 208 #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */ 209 #define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */ 210 #define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */ 211 #define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 212 #define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */ 213 #define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */ 214 #define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */ 215 #define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */ 216 #define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */ 217 #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */ 218 #define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */ 219 #define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */ 220 #define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 221 #define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */ 222 #define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */ 223 #define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */ 224 #define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */ 225 #define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */ 226 #define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */ 227 #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */ 228 #define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */ 229 #define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */ 230 #define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 231 #define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */ 232 #define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */ 233 #define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */ 234 #define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */ 235 #define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */ 236 237 /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */ 238 #define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */ 239 #define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */ 240 #define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */ 241 #define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */ 242 #define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */ 243 #define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 244 #define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */ 245 #define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */ 246 #define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */ 247 #define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */ 248 #define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 249 #define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */ 250 #define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */ 251 #define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */ 252 #define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */ 253 #define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */ 254 #define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 255 #define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */ 256 #define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */ 257 #define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */ 258 #define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */ 259 #define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */ 260 261 /* DMA Configuration Register (HIFN_1_DMA_CNFG) */ 262 #define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */ 263 #define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */ 264 #define HIFN_DMACNFG_UNLOCK 0x00000800 265 #define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */ 266 #define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */ 267 #define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */ 268 #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ 269 #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ 270 271 /* PLL configuration register */ 272 #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ 273 #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ 274 #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ 275 #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ 276 #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ 277 #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ 278 #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ 279 #define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ 280 #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ 281 #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ 282 #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ 283 #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ 284 #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ 285 #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ 286 #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ 287 #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ 288 #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ 289 290 #define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ 291 292 /* Public key reset register (HIFN_1_PUB_RESET) */ 293 #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ 294 295 /* Public base address register (HIFN_1_PUB_BASE) */ 296 #define HIFN_PUBBASE_ADDR 0x00003fff /* base address */ 297 298 /* Public operand length register (HIFN_1_PUB_OPLEN) */ 299 #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ 300 #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ 301 #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ 302 #define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ 303 #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ 304 #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ 305 306 /* Public operation register (HIFN_1_PUB_OP) */ 307 #define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */ 308 #define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */ 309 #define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */ 310 #define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */ 311 #define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */ 312 #define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */ 313 #define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */ 314 #define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */ 315 #define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */ 316 #define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */ 317 #define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */ 318 #define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */ 319 #define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */ 320 #define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */ 321 #define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */ 322 #define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */ 323 #define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */ 324 #define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */ 325 #define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */ 326 #define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */ 327 328 /* Public status register (HIFN_1_PUB_STATUS) */ 329 #define HIFN_PUBSTS_DONE 0x00000001 /* operation done */ 330 #define HIFN_PUBSTS_CARRY 0x00000002 /* carry */ 331 332 /* Public interrupt enable register (HIFN_1_PUB_IEN) */ 333 #define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */ 334 335 /* Random number generator config register (HIFN_1_RNG_CONFIG) */ 336 #define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */ 337 338 #define HIFN_NAMESIZE 32 339 #define HIFN_MAX_RESULT_ORDER 5 340 341 #define HIFN_D_CMD_RSIZE (24 * 1) 342 #define HIFN_D_SRC_RSIZE (80 * 1) 343 #define HIFN_D_DST_RSIZE (80 * 1) 344 #define HIFN_D_RES_RSIZE (24 * 1) 345 346 #define HIFN_D_DST_DALIGN 4 347 348 #define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) 349 350 #define AES_MIN_KEY_SIZE 16 351 #define AES_MAX_KEY_SIZE 32 352 353 #define HIFN_DES_KEY_LENGTH 8 354 #define HIFN_3DES_KEY_LENGTH 24 355 #define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE 356 #define HIFN_IV_LENGTH 8 357 #define HIFN_AES_IV_LENGTH 16 358 #define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH 359 360 #define HIFN_MAC_KEY_LENGTH 64 361 #define HIFN_MD5_LENGTH 16 362 #define HIFN_SHA1_LENGTH 20 363 #define HIFN_MAC_TRUNC_LENGTH 12 364 365 #define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260) 366 #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) 367 #define HIFN_USED_RESULT 12 368 369 struct hifn_desc { 370 volatile __le32 l; 371 volatile __le32 p; 372 }; 373 374 struct hifn_dma { 375 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; 376 struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; 377 struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; 378 struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; 379 380 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; 381 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; 382 383 /* 384 * Our current positions for insertion and removal from the descriptor 385 * rings. 386 */ 387 volatile int cmdi, srci, dsti, resi; 388 volatile int cmdu, srcu, dstu, resu; 389 int cmdk, srck, dstk, resk; 390 }; 391 392 #define HIFN_FLAG_CMD_BUSY (1 << 0) 393 #define HIFN_FLAG_SRC_BUSY (1 << 1) 394 #define HIFN_FLAG_DST_BUSY (1 << 2) 395 #define HIFN_FLAG_RES_BUSY (1 << 3) 396 #define HIFN_FLAG_OLD_KEY (1 << 4) 397 398 #define HIFN_DEFAULT_ACTIVE_NUM 5 399 400 struct hifn_device { 401 char name[HIFN_NAMESIZE]; 402 403 int irq; 404 405 struct pci_dev *pdev; 406 void __iomem *bar[3]; 407 408 void *desc_virt; 409 dma_addr_t desc_dma; 410 411 u32 dmareg; 412 413 void *sa[HIFN_D_RES_RSIZE]; 414 415 spinlock_t lock; 416 417 u32 flags; 418 int active, started; 419 struct delayed_work work; 420 unsigned long reset; 421 unsigned long success; 422 unsigned long prev_success; 423 424 u8 snum; 425 426 struct tasklet_struct tasklet; 427 428 struct crypto_queue queue; 429 struct list_head alg_list; 430 431 unsigned int pk_clk_freq; 432 433 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 434 unsigned int rng_wait_time; 435 ktime_t rngtime; 436 struct hwrng rng; 437 #endif 438 }; 439 440 #define HIFN_D_LENGTH 0x0000ffff 441 #define HIFN_D_NOINVALID 0x01000000 442 #define HIFN_D_MASKDONEIRQ 0x02000000 443 #define HIFN_D_DESTOVER 0x04000000 444 #define HIFN_D_OVER 0x08000000 445 #define HIFN_D_LAST 0x20000000 446 #define HIFN_D_JUMP 0x40000000 447 #define HIFN_D_VALID 0x80000000 448 449 struct hifn_base_command { 450 volatile __le16 masks; 451 volatile __le16 session_num; 452 volatile __le16 total_source_count; 453 volatile __le16 total_dest_count; 454 }; 455 456 #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ 457 #define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */ 458 #define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */ 459 #define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */ 460 #define HIFN_BASE_CMD_DECODE 0x2000 461 #define HIFN_BASE_CMD_SRCLEN_M 0xc000 462 #define HIFN_BASE_CMD_SRCLEN_S 14 463 #define HIFN_BASE_CMD_DSTLEN_M 0x3000 464 #define HIFN_BASE_CMD_DSTLEN_S 12 465 #define HIFN_BASE_CMD_LENMASK_HI 0x30000 466 #define HIFN_BASE_CMD_LENMASK_LO 0x0ffff 467 468 /* 469 * Structure to help build up the command data structure. 470 */ 471 struct hifn_crypt_command { 472 volatile __le16 masks; 473 volatile __le16 header_skip; 474 volatile __le16 source_count; 475 volatile __le16 reserved; 476 }; 477 478 #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ 479 #define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */ 480 #define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */ 481 #define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */ 482 #define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */ 483 #define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */ 484 #define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */ 485 #define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */ 486 #define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */ 487 #define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */ 488 #define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */ 489 #define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */ 490 #define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */ 491 #define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */ 492 #define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */ 493 #define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */ 494 #define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */ 495 #define HIFN_CRYPT_CMD_SRCLEN_M 0xc000 496 #define HIFN_CRYPT_CMD_SRCLEN_S 14 497 498 #define HIFN_MAC_CMD_ALG_MASK 0x0001 499 #define HIFN_MAC_CMD_ALG_SHA1 0x0000 500 #define HIFN_MAC_CMD_ALG_MD5 0x0001 501 #define HIFN_MAC_CMD_MODE_MASK 0x000c 502 #define HIFN_MAC_CMD_MODE_HMAC 0x0000 503 #define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004 504 #define HIFN_MAC_CMD_MODE_HASH 0x0008 505 #define HIFN_MAC_CMD_MODE_FULL 0x0004 506 #define HIFN_MAC_CMD_TRUNC 0x0010 507 #define HIFN_MAC_CMD_RESULT 0x0020 508 #define HIFN_MAC_CMD_APPEND 0x0040 509 #define HIFN_MAC_CMD_SRCLEN_M 0xc000 510 #define HIFN_MAC_CMD_SRCLEN_S 14 511 512 /* 513 * MAC POS IPsec initiates authentication after encryption on encodes 514 * and before decryption on decodes. 515 */ 516 #define HIFN_MAC_CMD_POS_IPSEC 0x0200 517 #define HIFN_MAC_CMD_NEW_KEY 0x0800 518 519 #define HIFN_COMP_CMD_SRCLEN_M 0xc000 520 #define HIFN_COMP_CMD_SRCLEN_S 14 521 #define HIFN_COMP_CMD_ONE 0x0100 /* must be one */ 522 #define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */ 523 #define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */ 524 #define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */ 525 #define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */ 526 #define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */ 527 #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ 528 #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ 529 530 struct hifn_base_result { 531 volatile __le16 flags; 532 volatile __le16 session; 533 volatile __le16 src_cnt; /* 15:0 of source count */ 534 volatile __le16 dst_cnt; /* 15:0 of dest count */ 535 }; 536 537 #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ 538 #define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */ 539 #define HIFN_BASE_RES_SRCLEN_S 14 540 #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ 541 #define HIFN_BASE_RES_DSTLEN_S 12 542 543 struct hifn_comp_result { 544 volatile __le16 flags; 545 volatile __le16 crc; 546 }; 547 548 #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ 549 #define HIFN_COMP_RES_LCB_S 8 550 #define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */ 551 #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ 552 #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ 553 554 struct hifn_mac_result { 555 volatile __le16 flags; 556 volatile __le16 reserved; 557 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ 558 }; 559 560 #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ 561 #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ 562 563 struct hifn_crypt_result { 564 volatile __le16 flags; 565 volatile __le16 reserved; 566 }; 567 568 #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ 569 570 #ifndef HIFN_POLL_FREQUENCY 571 #define HIFN_POLL_FREQUENCY 0x1 572 #endif 573 574 #ifndef HIFN_POLL_SCALAR 575 #define HIFN_POLL_SCALAR 0x0 576 #endif 577 578 #define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ 579 #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ 580 581 struct hifn_crypto_alg { 582 struct list_head entry; 583 struct skcipher_alg alg; 584 struct hifn_device *dev; 585 }; 586 587 #define ASYNC_SCATTERLIST_CACHE 16 588 589 #define ASYNC_FLAGS_MISALIGNED (1 << 0) 590 591 struct hifn_cipher_walk { 592 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; 593 u32 flags; 594 int num; 595 }; 596 597 struct hifn_context { 598 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; 599 struct hifn_device *dev; 600 unsigned int keysize; 601 }; 602 603 struct hifn_request_context { 604 u8 *iv; 605 unsigned int ivsize; 606 u8 op, type, mode, unused; 607 struct hifn_cipher_walk walk; 608 }; 609 610 #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) 611 612 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) 613 { 614 return readl(dev->bar[0] + reg); 615 } 616 617 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) 618 { 619 return readl(dev->bar[1] + reg); 620 } 621 622 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) 623 { 624 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); 625 } 626 627 static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) 628 { 629 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); 630 } 631 632 static void hifn_wait_puc(struct hifn_device *dev) 633 { 634 int i; 635 u32 ret; 636 637 for (i = 10000; i > 0; --i) { 638 ret = hifn_read_0(dev, HIFN_0_PUCTRL); 639 if (!(ret & HIFN_PUCTRL_RESET)) 640 break; 641 642 udelay(1); 643 } 644 645 if (!i) 646 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); 647 } 648 649 static void hifn_reset_puc(struct hifn_device *dev) 650 { 651 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 652 hifn_wait_puc(dev); 653 } 654 655 static void hifn_stop_device(struct hifn_device *dev) 656 { 657 hifn_write_1(dev, HIFN_1_DMA_CSR, 658 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 659 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS); 660 hifn_write_0(dev, HIFN_0_PUIER, 0); 661 hifn_write_1(dev, HIFN_1_DMA_IER, 0); 662 } 663 664 static void hifn_reset_dma(struct hifn_device *dev, int full) 665 { 666 hifn_stop_device(dev); 667 668 /* 669 * Setting poll frequency and others to 0. 670 */ 671 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 672 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 673 mdelay(1); 674 675 /* 676 * Reset DMA. 677 */ 678 if (full) { 679 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 680 mdelay(1); 681 } else { 682 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE | 683 HIFN_DMACNFG_MSTRESET); 684 hifn_reset_puc(dev); 685 } 686 687 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 688 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 689 690 hifn_reset_puc(dev); 691 } 692 693 static u32 hifn_next_signature(u32 a, u_int cnt) 694 { 695 int i; 696 u32 v; 697 698 for (i = 0; i < cnt; i++) { 699 /* get the parity */ 700 v = a & 0x80080125; 701 v ^= v >> 16; 702 v ^= v >> 8; 703 v ^= v >> 4; 704 v ^= v >> 2; 705 v ^= v >> 1; 706 707 a = (v & 1) ^ (a << 1); 708 } 709 710 return a; 711 } 712 713 static struct pci2id { 714 u_short pci_vendor; 715 u_short pci_prod; 716 char card_id[13]; 717 } pci2id[] = { 718 { 719 PCI_VENDOR_ID_HIFN, 720 PCI_DEVICE_ID_HIFN_7955, 721 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 722 0x00, 0x00, 0x00, 0x00, 0x00 } 723 }, 724 { 725 PCI_VENDOR_ID_HIFN, 726 PCI_DEVICE_ID_HIFN_7956, 727 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 728 0x00, 0x00, 0x00, 0x00, 0x00 } 729 } 730 }; 731 732 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 733 static int hifn_rng_data_present(struct hwrng *rng, int wait) 734 { 735 struct hifn_device *dev = (struct hifn_device *)rng->priv; 736 s64 nsec; 737 738 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); 739 nsec -= dev->rng_wait_time; 740 if (nsec <= 0) 741 return 1; 742 if (!wait) 743 return 0; 744 ndelay(nsec); 745 return 1; 746 } 747 748 static int hifn_rng_data_read(struct hwrng *rng, u32 *data) 749 { 750 struct hifn_device *dev = (struct hifn_device *)rng->priv; 751 752 *data = hifn_read_1(dev, HIFN_1_RNG_DATA); 753 dev->rngtime = ktime_get(); 754 return 4; 755 } 756 757 static int hifn_register_rng(struct hifn_device *dev) 758 { 759 /* 760 * We must wait at least 256 Pk_clk cycles between two reads of the rng. 761 */ 762 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, 763 dev->pk_clk_freq) * 256; 764 765 dev->rng.name = dev->name; 766 dev->rng.data_present = hifn_rng_data_present; 767 dev->rng.data_read = hifn_rng_data_read; 768 dev->rng.priv = (unsigned long)dev; 769 770 return hwrng_register(&dev->rng); 771 } 772 773 static void hifn_unregister_rng(struct hifn_device *dev) 774 { 775 hwrng_unregister(&dev->rng); 776 } 777 #else 778 #define hifn_register_rng(dev) 0 779 #define hifn_unregister_rng(dev) 780 #endif 781 782 static int hifn_init_pubrng(struct hifn_device *dev) 783 { 784 int i; 785 786 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | 787 HIFN_PUBRST_RESET); 788 789 for (i = 100; i > 0; --i) { 790 mdelay(1); 791 792 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) 793 break; 794 } 795 796 if (!i) { 797 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); 798 } else { 799 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 800 dev->dmareg |= HIFN_DMAIER_PUBDONE; 801 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 802 803 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); 804 } 805 806 /* Enable RNG engine. */ 807 808 hifn_write_1(dev, HIFN_1_RNG_CONFIG, 809 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); 810 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); 811 812 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 813 /* First value must be discarded */ 814 hifn_read_1(dev, HIFN_1_RNG_DATA); 815 dev->rngtime = ktime_get(); 816 #endif 817 return 0; 818 } 819 820 static int hifn_enable_crypto(struct hifn_device *dev) 821 { 822 u32 dmacfg, addr; 823 char *offtbl = NULL; 824 int i; 825 826 for (i = 0; i < ARRAY_SIZE(pci2id); i++) { 827 if (pci2id[i].pci_vendor == dev->pdev->vendor && 828 pci2id[i].pci_prod == dev->pdev->device) { 829 offtbl = pci2id[i].card_id; 830 break; 831 } 832 } 833 834 if (!offtbl) { 835 dev_err(&dev->pdev->dev, "Unknown card!\n"); 836 return -ENODEV; 837 } 838 839 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG); 840 841 hifn_write_1(dev, HIFN_1_DMA_CNFG, 842 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET | 843 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 844 mdelay(1); 845 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1); 846 mdelay(1); 847 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); 848 mdelay(1); 849 850 for (i = 0; i < 12; ++i) { 851 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 852 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); 853 854 mdelay(1); 855 } 856 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); 857 858 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); 859 860 return 0; 861 } 862 863 static void hifn_init_dma(struct hifn_device *dev) 864 { 865 struct hifn_dma *dma = dev->desc_virt; 866 u32 dptr = dev->desc_dma; 867 int i; 868 869 for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) 870 dma->cmdr[i].p = __cpu_to_le32(dptr + 871 offsetof(struct hifn_dma, command_bufs[i][0])); 872 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 873 dma->resr[i].p = __cpu_to_le32(dptr + 874 offsetof(struct hifn_dma, result_bufs[i][0])); 875 876 /* Setup LAST descriptors. */ 877 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + 878 offsetof(struct hifn_dma, cmdr[0])); 879 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + 880 offsetof(struct hifn_dma, srcr[0])); 881 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + 882 offsetof(struct hifn_dma, dstr[0])); 883 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + 884 offsetof(struct hifn_dma, resr[0])); 885 886 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 887 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 888 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 889 } 890 891 /* 892 * Initialize the PLL. We need to know the frequency of the reference clock 893 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that 894 * allows us to operate without the risk of overclocking the chip. If it 895 * actually uses 33MHz, the chip will operate at half the speed, this can be 896 * overridden by specifying the frequency as module parameter (pci33). 897 * 898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a 899 * stable clock and the PCI clock frequency may vary, so the default is the 900 * external clock. There is no way to find out its frequency, we default to 901 * 66MHz since according to Mike Ham of HiFn, almost every board in existence 902 * has an external crystal populated at 66MHz. 903 */ 904 static void hifn_init_pll(struct hifn_device *dev) 905 { 906 unsigned int freq, m; 907 u32 pllcfg; 908 909 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; 910 911 if (strncmp(hifn_pll_ref, "ext", 3) == 0) 912 pllcfg |= HIFN_PLL_REF_CLK_PLL; 913 else 914 pllcfg |= HIFN_PLL_REF_CLK_HBI; 915 916 if (hifn_pll_ref[3] != '\0') 917 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 918 else { 919 freq = 66; 920 dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", 921 freq, hifn_pll_ref); 922 } 923 924 m = HIFN_PLL_FCK_MAX / freq; 925 926 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; 927 if (m <= 8) 928 pllcfg |= HIFN_PLL_IS_1_8; 929 else 930 pllcfg |= HIFN_PLL_IS_9_12; 931 932 /* Select clock source and enable clock bypass */ 933 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 934 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); 935 936 /* Let the chip lock to the input clock */ 937 mdelay(10); 938 939 /* Disable clock bypass */ 940 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 941 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); 942 943 /* Switch the engines to the PLL */ 944 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 945 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); 946 947 /* 948 * The Fpk_clk runs at half the total speed. Its frequency is needed to 949 * calculate the minimum time between two reads of the rng. Since 33MHz 950 * is actually 33.333... we overestimate the frequency here, resulting 951 * in slightly larger intervals. 952 */ 953 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; 954 } 955 956 static void hifn_init_registers(struct hifn_device *dev) 957 { 958 u32 dptr = dev->desc_dma; 959 960 /* Initialization magic... */ 961 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 962 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 963 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 964 965 /* write all 4 ring address registers */ 966 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + 967 offsetof(struct hifn_dma, cmdr[0])); 968 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + 969 offsetof(struct hifn_dma, srcr[0])); 970 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + 971 offsetof(struct hifn_dma, dstr[0])); 972 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + 973 offsetof(struct hifn_dma, resr[0])); 974 975 mdelay(2); 976 #if 0 977 hifn_write_1(dev, HIFN_1_DMA_CSR, 978 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 979 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 980 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 981 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 982 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 983 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 984 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 985 HIFN_DMACSR_S_WAIT | 986 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 987 HIFN_DMACSR_C_WAIT | 988 HIFN_DMACSR_ENGINE | 989 HIFN_DMACSR_PUBDONE); 990 #else 991 hifn_write_1(dev, HIFN_1_DMA_CSR, 992 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 993 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA | 994 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 995 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 996 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 997 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 998 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 999 HIFN_DMACSR_S_WAIT | 1000 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1001 HIFN_DMACSR_C_WAIT | 1002 HIFN_DMACSR_ENGINE | 1003 HIFN_DMACSR_PUBDONE); 1004 #endif 1005 hifn_read_1(dev, HIFN_1_DMA_CSR); 1006 1007 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1008 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1009 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1010 HIFN_DMAIER_ENGINE; 1011 dev->dmareg &= ~HIFN_DMAIER_C_WAIT; 1012 1013 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1014 hifn_read_1(dev, HIFN_1_DMA_IER); 1015 #if 0 1016 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG | 1017 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1018 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1019 HIFN_PUCNFG_DRAM); 1020 #else 1021 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); 1022 #endif 1023 hifn_init_pll(dev); 1024 1025 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1026 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1027 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1028 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1029 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1030 } 1031 1032 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, 1033 unsigned dlen, unsigned slen, u16 mask, u8 snum) 1034 { 1035 struct hifn_base_command *base_cmd; 1036 u8 *buf_pos = buf; 1037 1038 base_cmd = (struct hifn_base_command *)buf_pos; 1039 base_cmd->masks = __cpu_to_le16(mask); 1040 base_cmd->total_source_count = 1041 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO); 1042 base_cmd->total_dest_count = 1043 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1044 1045 dlen >>= 16; 1046 slen >>= 16; 1047 base_cmd->session_num = __cpu_to_le16(snum | 1048 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1049 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1050 1051 return sizeof(struct hifn_base_command); 1052 } 1053 1054 static int hifn_setup_crypto_command(struct hifn_device *dev, 1055 u8 *buf, unsigned dlen, unsigned slen, 1056 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) 1057 { 1058 struct hifn_dma *dma = dev->desc_virt; 1059 struct hifn_crypt_command *cry_cmd; 1060 u8 *buf_pos = buf; 1061 u16 cmd_len; 1062 1063 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1064 1065 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); 1066 dlen >>= 16; 1067 cry_cmd->masks = __cpu_to_le16(mode | 1068 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & 1069 HIFN_CRYPT_CMD_SRCLEN_M)); 1070 cry_cmd->header_skip = 0; 1071 cry_cmd->reserved = 0; 1072 1073 buf_pos += sizeof(struct hifn_crypt_command); 1074 1075 dma->cmdu++; 1076 if (dma->cmdu > 1) { 1077 dev->dmareg |= HIFN_DMAIER_C_WAIT; 1078 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1079 } 1080 1081 if (keylen) { 1082 memcpy(buf_pos, key, keylen); 1083 buf_pos += keylen; 1084 } 1085 if (ivsize) { 1086 memcpy(buf_pos, iv, ivsize); 1087 buf_pos += ivsize; 1088 } 1089 1090 cmd_len = buf_pos - buf; 1091 1092 return cmd_len; 1093 } 1094 1095 static int hifn_setup_cmd_desc(struct hifn_device *dev, 1096 struct hifn_context *ctx, struct hifn_request_context *rctx, 1097 void *priv, unsigned int nbytes) 1098 { 1099 struct hifn_dma *dma = dev->desc_virt; 1100 int cmd_len, sa_idx; 1101 u8 *buf, *buf_pos; 1102 u16 mask; 1103 1104 sa_idx = dma->cmdi; 1105 buf_pos = buf = dma->command_bufs[dma->cmdi]; 1106 1107 mask = 0; 1108 switch (rctx->op) { 1109 case ACRYPTO_OP_DECRYPT: 1110 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; 1111 break; 1112 case ACRYPTO_OP_ENCRYPT: 1113 mask = HIFN_BASE_CMD_CRYPT; 1114 break; 1115 case ACRYPTO_OP_HMAC: 1116 mask = HIFN_BASE_CMD_MAC; 1117 break; 1118 default: 1119 goto err_out; 1120 } 1121 1122 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, 1123 nbytes, mask, dev->snum); 1124 1125 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { 1126 u16 md = 0; 1127 1128 if (ctx->keysize) 1129 md |= HIFN_CRYPT_CMD_NEW_KEY; 1130 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) 1131 md |= HIFN_CRYPT_CMD_NEW_IV; 1132 1133 switch (rctx->mode) { 1134 case ACRYPTO_MODE_ECB: 1135 md |= HIFN_CRYPT_CMD_MODE_ECB; 1136 break; 1137 case ACRYPTO_MODE_CBC: 1138 md |= HIFN_CRYPT_CMD_MODE_CBC; 1139 break; 1140 case ACRYPTO_MODE_CFB: 1141 md |= HIFN_CRYPT_CMD_MODE_CFB; 1142 break; 1143 case ACRYPTO_MODE_OFB: 1144 md |= HIFN_CRYPT_CMD_MODE_OFB; 1145 break; 1146 default: 1147 goto err_out; 1148 } 1149 1150 switch (rctx->type) { 1151 case ACRYPTO_TYPE_AES_128: 1152 if (ctx->keysize != 16) 1153 goto err_out; 1154 md |= HIFN_CRYPT_CMD_KSZ_128 | 1155 HIFN_CRYPT_CMD_ALG_AES; 1156 break; 1157 case ACRYPTO_TYPE_AES_192: 1158 if (ctx->keysize != 24) 1159 goto err_out; 1160 md |= HIFN_CRYPT_CMD_KSZ_192 | 1161 HIFN_CRYPT_CMD_ALG_AES; 1162 break; 1163 case ACRYPTO_TYPE_AES_256: 1164 if (ctx->keysize != 32) 1165 goto err_out; 1166 md |= HIFN_CRYPT_CMD_KSZ_256 | 1167 HIFN_CRYPT_CMD_ALG_AES; 1168 break; 1169 case ACRYPTO_TYPE_3DES: 1170 if (ctx->keysize != 24) 1171 goto err_out; 1172 md |= HIFN_CRYPT_CMD_ALG_3DES; 1173 break; 1174 case ACRYPTO_TYPE_DES: 1175 if (ctx->keysize != 8) 1176 goto err_out; 1177 md |= HIFN_CRYPT_CMD_ALG_DES; 1178 break; 1179 default: 1180 goto err_out; 1181 } 1182 1183 buf_pos += hifn_setup_crypto_command(dev, buf_pos, 1184 nbytes, nbytes, ctx->key, ctx->keysize, 1185 rctx->iv, rctx->ivsize, md); 1186 } 1187 1188 dev->sa[sa_idx] = priv; 1189 dev->started++; 1190 1191 cmd_len = buf_pos - buf; 1192 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | 1193 HIFN_D_LAST | HIFN_D_MASKDONEIRQ); 1194 1195 if (++dma->cmdi == HIFN_D_CMD_RSIZE) { 1196 dma->cmdr[dma->cmdi].l = __cpu_to_le32( 1197 HIFN_D_VALID | HIFN_D_LAST | 1198 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); 1199 dma->cmdi = 0; 1200 } else { 1201 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); 1202 } 1203 1204 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { 1205 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1206 dev->flags |= HIFN_FLAG_CMD_BUSY; 1207 } 1208 return 0; 1209 1210 err_out: 1211 return -EINVAL; 1212 } 1213 1214 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, 1215 unsigned int offset, unsigned int size, int last) 1216 { 1217 struct hifn_dma *dma = dev->desc_virt; 1218 int idx; 1219 dma_addr_t addr; 1220 1221 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1222 DMA_TO_DEVICE); 1223 1224 idx = dma->srci; 1225 1226 dma->srcr[idx].p = __cpu_to_le32(addr); 1227 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1228 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1229 1230 if (++idx == HIFN_D_SRC_RSIZE) { 1231 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1232 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1233 (last ? HIFN_D_LAST : 0)); 1234 idx = 0; 1235 } 1236 1237 dma->srci = idx; 1238 dma->srcu++; 1239 1240 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { 1241 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1242 dev->flags |= HIFN_FLAG_SRC_BUSY; 1243 } 1244 1245 return size; 1246 } 1247 1248 static void hifn_setup_res_desc(struct hifn_device *dev) 1249 { 1250 struct hifn_dma *dma = dev->desc_virt; 1251 1252 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | 1253 HIFN_D_VALID | HIFN_D_LAST); 1254 /* 1255 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | 1256 * HIFN_D_LAST); 1257 */ 1258 1259 if (++dma->resi == HIFN_D_RES_RSIZE) { 1260 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | 1261 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); 1262 dma->resi = 0; 1263 } 1264 1265 dma->resu++; 1266 1267 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { 1268 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1269 dev->flags |= HIFN_FLAG_RES_BUSY; 1270 } 1271 } 1272 1273 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, 1274 unsigned offset, unsigned size, int last) 1275 { 1276 struct hifn_dma *dma = dev->desc_virt; 1277 int idx; 1278 dma_addr_t addr; 1279 1280 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1281 DMA_FROM_DEVICE); 1282 1283 idx = dma->dsti; 1284 dma->dstr[idx].p = __cpu_to_le32(addr); 1285 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1286 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1287 1288 if (++idx == HIFN_D_DST_RSIZE) { 1289 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1290 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1291 (last ? HIFN_D_LAST : 0)); 1292 idx = 0; 1293 } 1294 dma->dsti = idx; 1295 dma->dstu++; 1296 1297 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { 1298 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1299 dev->flags |= HIFN_FLAG_DST_BUSY; 1300 } 1301 } 1302 1303 static int hifn_setup_dma(struct hifn_device *dev, 1304 struct hifn_context *ctx, struct hifn_request_context *rctx, 1305 struct scatterlist *src, struct scatterlist *dst, 1306 unsigned int nbytes, void *priv) 1307 { 1308 struct scatterlist *t; 1309 struct page *spage, *dpage; 1310 unsigned int soff, doff; 1311 unsigned int n, len; 1312 1313 n = nbytes; 1314 while (n) { 1315 spage = sg_page(src); 1316 soff = src->offset; 1317 len = min(src->length, n); 1318 1319 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); 1320 1321 src++; 1322 n -= len; 1323 } 1324 1325 t = &rctx->walk.cache[0]; 1326 n = nbytes; 1327 while (n) { 1328 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1329 BUG_ON(!sg_page(t)); 1330 dpage = sg_page(t); 1331 doff = 0; 1332 len = t->length; 1333 } else { 1334 BUG_ON(!sg_page(dst)); 1335 dpage = sg_page(dst); 1336 doff = dst->offset; 1337 len = dst->length; 1338 } 1339 len = min(len, n); 1340 1341 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); 1342 1343 dst++; 1344 t++; 1345 n -= len; 1346 } 1347 1348 hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); 1349 hifn_setup_res_desc(dev); 1350 return 0; 1351 } 1352 1353 static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, 1354 int num, gfp_t gfp_flags) 1355 { 1356 int i; 1357 1358 num = min(ASYNC_SCATTERLIST_CACHE, num); 1359 sg_init_table(w->cache, num); 1360 1361 w->num = 0; 1362 for (i = 0; i < num; ++i) { 1363 struct page *page = alloc_page(gfp_flags); 1364 struct scatterlist *s; 1365 1366 if (!page) 1367 break; 1368 1369 s = &w->cache[i]; 1370 1371 sg_set_page(s, page, PAGE_SIZE, 0); 1372 w->num++; 1373 } 1374 1375 return i; 1376 } 1377 1378 static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) 1379 { 1380 int i; 1381 1382 for (i = 0; i < w->num; ++i) { 1383 struct scatterlist *s = &w->cache[i]; 1384 1385 __free_page(sg_page(s)); 1386 1387 s->length = 0; 1388 } 1389 1390 w->num = 0; 1391 } 1392 1393 static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, 1394 unsigned int size, unsigned int *nbytesp) 1395 { 1396 unsigned int copy, drest = *drestp, nbytes = *nbytesp; 1397 int idx = 0; 1398 1399 if (drest < size || size > nbytes) 1400 return -EINVAL; 1401 1402 while (size) { 1403 copy = min3(drest, size, dst->length); 1404 1405 size -= copy; 1406 drest -= copy; 1407 nbytes -= copy; 1408 1409 pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", 1410 __func__, copy, size, drest, nbytes); 1411 1412 dst++; 1413 idx++; 1414 } 1415 1416 *nbytesp = nbytes; 1417 *drestp = drest; 1418 1419 return idx; 1420 } 1421 1422 static int hifn_cipher_walk(struct skcipher_request *req, 1423 struct hifn_cipher_walk *w) 1424 { 1425 struct scatterlist *dst, *t; 1426 unsigned int nbytes = req->cryptlen, offset, copy, diff; 1427 int idx, tidx, err; 1428 1429 tidx = idx = 0; 1430 offset = 0; 1431 while (nbytes) { 1432 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) 1433 return -EINVAL; 1434 1435 dst = &req->dst[idx]; 1436 1437 pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", 1438 __func__, dst->length, dst->offset, offset, nbytes); 1439 1440 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1441 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || 1442 offset) { 1443 unsigned slen = min(dst->length - offset, nbytes); 1444 unsigned dlen = PAGE_SIZE; 1445 1446 t = &w->cache[idx]; 1447 1448 err = skcipher_add(&dlen, dst, slen, &nbytes); 1449 if (err < 0) 1450 return err; 1451 1452 idx += err; 1453 1454 copy = slen & ~(HIFN_D_DST_DALIGN - 1); 1455 diff = slen & (HIFN_D_DST_DALIGN - 1); 1456 1457 if (dlen < nbytes) { 1458 /* 1459 * Destination page does not have enough space 1460 * to put there additional blocksized chunk, 1461 * so we mark that page as containing only 1462 * blocksize aligned chunks: 1463 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); 1464 * and increase number of bytes to be processed 1465 * in next chunk: 1466 * nbytes += diff; 1467 */ 1468 nbytes += diff; 1469 1470 /* 1471 * Temporary of course... 1472 * Kick author if you will catch this one. 1473 */ 1474 pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", 1475 __func__, dlen, nbytes, slen, offset); 1476 pr_err("%s: please contact author to fix this " 1477 "issue, generally you should not catch " 1478 "this path under any condition but who " 1479 "knows how did you use crypto code.\n" 1480 "Thank you.\n", __func__); 1481 BUG(); 1482 } else { 1483 copy += diff + nbytes; 1484 1485 dst = &req->dst[idx]; 1486 1487 err = skcipher_add(&dlen, dst, nbytes, &nbytes); 1488 if (err < 0) 1489 return err; 1490 1491 idx += err; 1492 } 1493 1494 t->length = copy; 1495 t->offset = offset; 1496 } else { 1497 nbytes -= min(dst->length, nbytes); 1498 idx++; 1499 } 1500 1501 tidx++; 1502 } 1503 1504 return tidx; 1505 } 1506 1507 static int hifn_setup_session(struct skcipher_request *req) 1508 { 1509 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1510 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1511 struct hifn_device *dev = ctx->dev; 1512 unsigned long dlen, flags; 1513 unsigned int nbytes = req->cryptlen, idx = 0; 1514 int err = -EINVAL, sg_num; 1515 struct scatterlist *dst; 1516 1517 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) 1518 goto err_out_exit; 1519 1520 rctx->walk.flags = 0; 1521 1522 while (nbytes) { 1523 dst = &req->dst[idx]; 1524 dlen = min(dst->length, nbytes); 1525 1526 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1527 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) 1528 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; 1529 1530 nbytes -= dlen; 1531 idx++; 1532 } 1533 1534 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1535 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); 1536 if (err < 0) 1537 return err; 1538 } 1539 1540 sg_num = hifn_cipher_walk(req, &rctx->walk); 1541 if (sg_num < 0) { 1542 err = sg_num; 1543 goto err_out_exit; 1544 } 1545 1546 spin_lock_irqsave(&dev->lock, flags); 1547 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { 1548 err = -EAGAIN; 1549 goto err_out; 1550 } 1551 1552 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); 1553 if (err) 1554 goto err_out; 1555 1556 dev->snum++; 1557 1558 dev->active = HIFN_DEFAULT_ACTIVE_NUM; 1559 spin_unlock_irqrestore(&dev->lock, flags); 1560 1561 return 0; 1562 1563 err_out: 1564 spin_unlock_irqrestore(&dev->lock, flags); 1565 err_out_exit: 1566 if (err) { 1567 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " 1568 "type: %u, err: %d.\n", 1569 rctx->iv, rctx->ivsize, 1570 ctx->key, ctx->keysize, 1571 rctx->mode, rctx->op, rctx->type, err); 1572 } 1573 1574 return err; 1575 } 1576 1577 static int hifn_start_device(struct hifn_device *dev) 1578 { 1579 int err; 1580 1581 dev->started = dev->active = 0; 1582 hifn_reset_dma(dev, 1); 1583 1584 err = hifn_enable_crypto(dev); 1585 if (err) 1586 return err; 1587 1588 hifn_reset_puc(dev); 1589 1590 hifn_init_dma(dev); 1591 1592 hifn_init_registers(dev); 1593 1594 hifn_init_pubrng(dev); 1595 1596 return 0; 1597 } 1598 1599 static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, 1600 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) 1601 { 1602 unsigned int srest = *srestp, nbytes = *nbytesp, copy; 1603 void *daddr; 1604 int idx = 0; 1605 1606 if (srest < size || size > nbytes) 1607 return -EINVAL; 1608 1609 while (size) { 1610 copy = min3(srest, dst->length, size); 1611 1612 daddr = kmap_atomic(sg_page(dst)); 1613 memcpy(daddr + dst->offset + offset, saddr, copy); 1614 kunmap_atomic(daddr); 1615 1616 nbytes -= copy; 1617 size -= copy; 1618 srest -= copy; 1619 saddr += copy; 1620 offset = 0; 1621 1622 pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", 1623 __func__, copy, size, srest, nbytes); 1624 1625 dst++; 1626 idx++; 1627 } 1628 1629 *nbytesp = nbytes; 1630 *srestp = srest; 1631 1632 return idx; 1633 } 1634 1635 static inline void hifn_complete_sa(struct hifn_device *dev, int i) 1636 { 1637 unsigned long flags; 1638 1639 spin_lock_irqsave(&dev->lock, flags); 1640 dev->sa[i] = NULL; 1641 dev->started--; 1642 if (dev->started < 0) 1643 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, 1644 dev->started); 1645 spin_unlock_irqrestore(&dev->lock, flags); 1646 BUG_ON(dev->started < 0); 1647 } 1648 1649 static void hifn_process_ready(struct skcipher_request *req, int error) 1650 { 1651 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1652 1653 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1654 unsigned int nbytes = req->cryptlen; 1655 int idx = 0, err; 1656 struct scatterlist *dst, *t; 1657 void *saddr; 1658 1659 while (nbytes) { 1660 t = &rctx->walk.cache[idx]; 1661 dst = &req->dst[idx]; 1662 1663 pr_debug("\n%s: sg_page(t): %p, t->length: %u, " 1664 "sg_page(dst): %p, dst->length: %u, " 1665 "nbytes: %u.\n", 1666 __func__, sg_page(t), t->length, 1667 sg_page(dst), dst->length, nbytes); 1668 1669 if (!t->length) { 1670 nbytes -= min(dst->length, nbytes); 1671 idx++; 1672 continue; 1673 } 1674 1675 saddr = kmap_atomic(sg_page(t)); 1676 1677 err = skcipher_get(saddr, &t->length, t->offset, 1678 dst, nbytes, &nbytes); 1679 if (err < 0) { 1680 kunmap_atomic(saddr); 1681 break; 1682 } 1683 1684 idx += err; 1685 kunmap_atomic(saddr); 1686 } 1687 1688 hifn_cipher_walk_exit(&rctx->walk); 1689 } 1690 1691 skcipher_request_complete(req, error); 1692 } 1693 1694 static void hifn_clear_rings(struct hifn_device *dev, int error) 1695 { 1696 struct hifn_dma *dma = dev->desc_virt; 1697 int i, u; 1698 1699 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1700 "k: %d.%d.%d.%d.\n", 1701 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1702 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1703 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1704 1705 i = dma->resk; u = dma->resu; 1706 while (u != 0) { 1707 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1708 break; 1709 1710 if (dev->sa[i]) { 1711 dev->success++; 1712 dev->reset = 0; 1713 hifn_process_ready(dev->sa[i], error); 1714 hifn_complete_sa(dev, i); 1715 } 1716 1717 if (++i == HIFN_D_RES_RSIZE) 1718 i = 0; 1719 u--; 1720 } 1721 dma->resk = i; dma->resu = u; 1722 1723 i = dma->srck; u = dma->srcu; 1724 while (u != 0) { 1725 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1726 break; 1727 if (++i == HIFN_D_SRC_RSIZE) 1728 i = 0; 1729 u--; 1730 } 1731 dma->srck = i; dma->srcu = u; 1732 1733 i = dma->cmdk; u = dma->cmdu; 1734 while (u != 0) { 1735 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1736 break; 1737 if (++i == HIFN_D_CMD_RSIZE) 1738 i = 0; 1739 u--; 1740 } 1741 dma->cmdk = i; dma->cmdu = u; 1742 1743 i = dma->dstk; u = dma->dstu; 1744 while (u != 0) { 1745 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1746 break; 1747 if (++i == HIFN_D_DST_RSIZE) 1748 i = 0; 1749 u--; 1750 } 1751 dma->dstk = i; dma->dstu = u; 1752 1753 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1754 "k: %d.%d.%d.%d.\n", 1755 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1756 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1757 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1758 } 1759 1760 static void hifn_work(struct work_struct *work) 1761 { 1762 struct delayed_work *dw = to_delayed_work(work); 1763 struct hifn_device *dev = container_of(dw, struct hifn_device, work); 1764 unsigned long flags; 1765 int reset = 0; 1766 u32 r = 0; 1767 1768 spin_lock_irqsave(&dev->lock, flags); 1769 if (dev->active == 0) { 1770 struct hifn_dma *dma = dev->desc_virt; 1771 1772 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { 1773 dev->flags &= ~HIFN_FLAG_CMD_BUSY; 1774 r |= HIFN_DMACSR_C_CTRL_DIS; 1775 } 1776 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { 1777 dev->flags &= ~HIFN_FLAG_SRC_BUSY; 1778 r |= HIFN_DMACSR_S_CTRL_DIS; 1779 } 1780 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { 1781 dev->flags &= ~HIFN_FLAG_DST_BUSY; 1782 r |= HIFN_DMACSR_D_CTRL_DIS; 1783 } 1784 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { 1785 dev->flags &= ~HIFN_FLAG_RES_BUSY; 1786 r |= HIFN_DMACSR_R_CTRL_DIS; 1787 } 1788 if (r) 1789 hifn_write_1(dev, HIFN_1_DMA_CSR, r); 1790 } else 1791 dev->active--; 1792 1793 if ((dev->prev_success == dev->success) && dev->started) 1794 reset = 1; 1795 dev->prev_success = dev->success; 1796 spin_unlock_irqrestore(&dev->lock, flags); 1797 1798 if (reset) { 1799 if (++dev->reset >= 5) { 1800 int i; 1801 struct hifn_dma *dma = dev->desc_virt; 1802 1803 dev_info(&dev->pdev->dev, 1804 "r: %08x, active: %d, started: %d, " 1805 "success: %lu: qlen: %u/%u, reset: %d.\n", 1806 r, dev->active, dev->started, 1807 dev->success, dev->queue.qlen, dev->queue.max_qlen, 1808 reset); 1809 1810 dev_info(&dev->pdev->dev, "%s: res: ", __func__); 1811 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1812 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); 1813 if (dev->sa[i]) { 1814 hifn_process_ready(dev->sa[i], -ENODEV); 1815 hifn_complete_sa(dev, i); 1816 } 1817 } 1818 pr_info("\n"); 1819 1820 hifn_reset_dma(dev, 1); 1821 hifn_stop_device(dev); 1822 hifn_start_device(dev); 1823 dev->reset = 0; 1824 } 1825 1826 tasklet_schedule(&dev->tasklet); 1827 } 1828 1829 schedule_delayed_work(&dev->work, HZ); 1830 } 1831 1832 static irqreturn_t hifn_interrupt(int irq, void *data) 1833 { 1834 struct hifn_device *dev = data; 1835 struct hifn_dma *dma = dev->desc_virt; 1836 u32 dmacsr, restart; 1837 1838 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); 1839 1840 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " 1841 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", 1842 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, 1843 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1844 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1845 1846 if ((dmacsr & dev->dmareg) == 0) 1847 return IRQ_NONE; 1848 1849 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); 1850 1851 if (dmacsr & HIFN_DMACSR_ENGINE) 1852 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR)); 1853 if (dmacsr & HIFN_DMACSR_PUBDONE) 1854 hifn_write_1(dev, HIFN_1_PUB_STATUS, 1855 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1856 1857 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1858 if (restart) { 1859 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); 1860 1861 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", 1862 !!(dmacsr & HIFN_DMACSR_R_OVER), 1863 !!(dmacsr & HIFN_DMACSR_D_OVER), 1864 puisr, !!(puisr & HIFN_PUISR_DSTOVER)); 1865 if (!!(puisr & HIFN_PUISR_DSTOVER)) 1866 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1867 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | 1868 HIFN_DMACSR_D_OVER)); 1869 } 1870 1871 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1872 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1873 if (restart) { 1874 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", 1875 !!(dmacsr & HIFN_DMACSR_C_ABORT), 1876 !!(dmacsr & HIFN_DMACSR_S_ABORT), 1877 !!(dmacsr & HIFN_DMACSR_D_ABORT), 1878 !!(dmacsr & HIFN_DMACSR_R_ABORT)); 1879 hifn_reset_dma(dev, 1); 1880 hifn_init_dma(dev); 1881 hifn_init_registers(dev); 1882 } 1883 1884 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { 1885 dev_dbg(&dev->pdev->dev, "wait on command.\n"); 1886 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); 1887 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1888 } 1889 1890 tasklet_schedule(&dev->tasklet); 1891 1892 return IRQ_HANDLED; 1893 } 1894 1895 static void hifn_flush(struct hifn_device *dev) 1896 { 1897 unsigned long flags; 1898 struct crypto_async_request *async_req; 1899 struct skcipher_request *req; 1900 struct hifn_dma *dma = dev->desc_virt; 1901 int i; 1902 1903 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1904 struct hifn_desc *d = &dma->resr[i]; 1905 1906 if (dev->sa[i]) { 1907 hifn_process_ready(dev->sa[i], 1908 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); 1909 hifn_complete_sa(dev, i); 1910 } 1911 } 1912 1913 spin_lock_irqsave(&dev->lock, flags); 1914 while ((async_req = crypto_dequeue_request(&dev->queue))) { 1915 req = skcipher_request_cast(async_req); 1916 spin_unlock_irqrestore(&dev->lock, flags); 1917 1918 hifn_process_ready(req, -ENODEV); 1919 1920 spin_lock_irqsave(&dev->lock, flags); 1921 } 1922 spin_unlock_irqrestore(&dev->lock, flags); 1923 } 1924 1925 static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, 1926 unsigned int len) 1927 { 1928 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1929 struct hifn_device *dev = ctx->dev; 1930 int err; 1931 1932 err = verify_skcipher_des_key(cipher, key); 1933 if (err) 1934 return err; 1935 1936 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1937 1938 memcpy(ctx->key, key, len); 1939 ctx->keysize = len; 1940 1941 return 0; 1942 } 1943 1944 static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 1945 unsigned int len) 1946 { 1947 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1948 struct hifn_device *dev = ctx->dev; 1949 int err; 1950 1951 err = verify_skcipher_des3_key(cipher, key); 1952 if (err) 1953 return err; 1954 1955 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1956 1957 memcpy(ctx->key, key, len); 1958 ctx->keysize = len; 1959 1960 return 0; 1961 } 1962 1963 static int hifn_handle_req(struct skcipher_request *req) 1964 { 1965 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1966 struct hifn_device *dev = ctx->dev; 1967 int err = -EAGAIN; 1968 1969 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) 1970 err = hifn_setup_session(req); 1971 1972 if (err == -EAGAIN) { 1973 unsigned long flags; 1974 1975 spin_lock_irqsave(&dev->lock, flags); 1976 err = crypto_enqueue_request(&dev->queue, &req->base); 1977 spin_unlock_irqrestore(&dev->lock, flags); 1978 } 1979 1980 return err; 1981 } 1982 1983 static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, 1984 u8 type, u8 mode) 1985 { 1986 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1987 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1988 unsigned ivsize; 1989 1990 ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); 1991 1992 if (req->iv && mode != ACRYPTO_MODE_ECB) { 1993 if (type == ACRYPTO_TYPE_AES_128) 1994 ivsize = HIFN_AES_IV_LENGTH; 1995 else if (type == ACRYPTO_TYPE_DES) 1996 ivsize = HIFN_DES_KEY_LENGTH; 1997 else if (type == ACRYPTO_TYPE_3DES) 1998 ivsize = HIFN_3DES_KEY_LENGTH; 1999 } 2000 2001 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { 2002 if (ctx->keysize == 24) 2003 type = ACRYPTO_TYPE_AES_192; 2004 else if (ctx->keysize == 32) 2005 type = ACRYPTO_TYPE_AES_256; 2006 } 2007 2008 rctx->op = op; 2009 rctx->mode = mode; 2010 rctx->type = type; 2011 rctx->iv = req->iv; 2012 rctx->ivsize = ivsize; 2013 2014 /* 2015 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2016 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2017 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2018 */ 2019 2020 return hifn_handle_req(req); 2021 } 2022 2023 static int hifn_process_queue(struct hifn_device *dev) 2024 { 2025 struct crypto_async_request *async_req, *backlog; 2026 struct skcipher_request *req; 2027 unsigned long flags; 2028 int err = 0; 2029 2030 while (dev->started < HIFN_QUEUE_LENGTH) { 2031 spin_lock_irqsave(&dev->lock, flags); 2032 backlog = crypto_get_backlog(&dev->queue); 2033 async_req = crypto_dequeue_request(&dev->queue); 2034 spin_unlock_irqrestore(&dev->lock, flags); 2035 2036 if (!async_req) 2037 break; 2038 2039 if (backlog) 2040 crypto_request_complete(backlog, -EINPROGRESS); 2041 2042 req = skcipher_request_cast(async_req); 2043 2044 err = hifn_handle_req(req); 2045 if (err) 2046 break; 2047 } 2048 2049 return err; 2050 } 2051 2052 static int hifn_setup_crypto(struct skcipher_request *req, u8 op, 2053 u8 type, u8 mode) 2054 { 2055 int err; 2056 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2057 struct hifn_device *dev = ctx->dev; 2058 2059 err = hifn_setup_crypto_req(req, op, type, mode); 2060 if (err) 2061 return err; 2062 2063 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2064 hifn_process_queue(dev); 2065 2066 return -EINPROGRESS; 2067 } 2068 2069 /* 2070 * AES ecryption functions. 2071 */ 2072 static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) 2073 { 2074 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2075 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2076 } 2077 static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) 2078 { 2079 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2080 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2081 } 2082 2083 /* 2084 * AES decryption functions. 2085 */ 2086 static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) 2087 { 2088 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2089 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2090 } 2091 static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) 2092 { 2093 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2094 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2095 } 2096 2097 /* 2098 * DES ecryption functions. 2099 */ 2100 static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) 2101 { 2102 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2103 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2104 } 2105 static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) 2106 { 2107 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2108 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2109 } 2110 2111 /* 2112 * DES decryption functions. 2113 */ 2114 static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) 2115 { 2116 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2117 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2118 } 2119 static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) 2120 { 2121 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2122 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2123 } 2124 2125 /* 2126 * 3DES ecryption functions. 2127 */ 2128 static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) 2129 { 2130 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2131 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2132 } 2133 static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) 2134 { 2135 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2136 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2137 } 2138 2139 /* 3DES decryption functions. */ 2140 static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) 2141 { 2142 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2143 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2144 } 2145 static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) 2146 { 2147 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2148 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2149 } 2150 2151 struct hifn_alg_template { 2152 char name[CRYPTO_MAX_ALG_NAME]; 2153 char drv_name[CRYPTO_MAX_ALG_NAME]; 2154 unsigned int bsize; 2155 struct skcipher_alg skcipher; 2156 }; 2157 2158 static const struct hifn_alg_template hifn_alg_templates[] = { 2159 /* 2160 * 3DES ECB and CBC modes. 2161 */ 2162 { 2163 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, 2164 .skcipher = { 2165 .ivsize = HIFN_IV_LENGTH, 2166 .min_keysize = HIFN_3DES_KEY_LENGTH, 2167 .max_keysize = HIFN_3DES_KEY_LENGTH, 2168 .setkey = hifn_des3_setkey, 2169 .encrypt = hifn_encrypt_3des_cbc, 2170 .decrypt = hifn_decrypt_3des_cbc, 2171 }, 2172 }, 2173 { 2174 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, 2175 .skcipher = { 2176 .min_keysize = HIFN_3DES_KEY_LENGTH, 2177 .max_keysize = HIFN_3DES_KEY_LENGTH, 2178 .setkey = hifn_des3_setkey, 2179 .encrypt = hifn_encrypt_3des_ecb, 2180 .decrypt = hifn_decrypt_3des_ecb, 2181 }, 2182 }, 2183 2184 /* 2185 * DES ECB and CBC modes. 2186 */ 2187 { 2188 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, 2189 .skcipher = { 2190 .ivsize = HIFN_IV_LENGTH, 2191 .min_keysize = HIFN_DES_KEY_LENGTH, 2192 .max_keysize = HIFN_DES_KEY_LENGTH, 2193 .setkey = hifn_setkey, 2194 .encrypt = hifn_encrypt_des_cbc, 2195 .decrypt = hifn_decrypt_des_cbc, 2196 }, 2197 }, 2198 { 2199 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, 2200 .skcipher = { 2201 .min_keysize = HIFN_DES_KEY_LENGTH, 2202 .max_keysize = HIFN_DES_KEY_LENGTH, 2203 .setkey = hifn_setkey, 2204 .encrypt = hifn_encrypt_des_ecb, 2205 .decrypt = hifn_decrypt_des_ecb, 2206 }, 2207 }, 2208 2209 /* 2210 * AES ECB and CBC modes. 2211 */ 2212 { 2213 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, 2214 .skcipher = { 2215 .min_keysize = AES_MIN_KEY_SIZE, 2216 .max_keysize = AES_MAX_KEY_SIZE, 2217 .setkey = hifn_setkey, 2218 .encrypt = hifn_encrypt_aes_ecb, 2219 .decrypt = hifn_decrypt_aes_ecb, 2220 }, 2221 }, 2222 { 2223 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, 2224 .skcipher = { 2225 .ivsize = HIFN_AES_IV_LENGTH, 2226 .min_keysize = AES_MIN_KEY_SIZE, 2227 .max_keysize = AES_MAX_KEY_SIZE, 2228 .setkey = hifn_setkey, 2229 .encrypt = hifn_encrypt_aes_cbc, 2230 .decrypt = hifn_decrypt_aes_cbc, 2231 }, 2232 }, 2233 }; 2234 2235 static int hifn_init_tfm(struct crypto_skcipher *tfm) 2236 { 2237 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2238 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg); 2239 struct hifn_context *ctx = crypto_skcipher_ctx(tfm); 2240 2241 ctx->dev = ha->dev; 2242 crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context)); 2243 2244 return 0; 2245 } 2246 2247 static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) 2248 { 2249 struct hifn_crypto_alg *alg; 2250 int err; 2251 2252 alg = kzalloc(sizeof(*alg), GFP_KERNEL); 2253 if (!alg) 2254 return -ENOMEM; 2255 2256 alg->alg = t->skcipher; 2257 alg->alg.init = hifn_init_tfm; 2258 2259 err = -EINVAL; 2260 if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 2261 "%s", t->name) >= CRYPTO_MAX_ALG_NAME) 2262 goto out_free_alg; 2263 if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 2264 "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME) 2265 goto out_free_alg; 2266 2267 alg->alg.base.cra_priority = 300; 2268 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 2269 alg->alg.base.cra_blocksize = t->bsize; 2270 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); 2271 alg->alg.base.cra_alignmask = 0; 2272 alg->alg.base.cra_module = THIS_MODULE; 2273 2274 alg->dev = dev; 2275 2276 list_add_tail(&alg->entry, &dev->alg_list); 2277 2278 err = crypto_register_skcipher(&alg->alg); 2279 if (err) { 2280 list_del(&alg->entry); 2281 out_free_alg: 2282 kfree(alg); 2283 } 2284 2285 return err; 2286 } 2287 2288 static void hifn_unregister_alg(struct hifn_device *dev) 2289 { 2290 struct hifn_crypto_alg *a, *n; 2291 2292 list_for_each_entry_safe(a, n, &dev->alg_list, entry) { 2293 list_del(&a->entry); 2294 crypto_unregister_skcipher(&a->alg); 2295 kfree(a); 2296 } 2297 } 2298 2299 static int hifn_register_alg(struct hifn_device *dev) 2300 { 2301 int i, err; 2302 2303 for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { 2304 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); 2305 if (err) 2306 goto err_out_exit; 2307 } 2308 2309 return 0; 2310 2311 err_out_exit: 2312 hifn_unregister_alg(dev); 2313 return err; 2314 } 2315 2316 static void hifn_tasklet_callback(unsigned long data) 2317 { 2318 struct hifn_device *dev = (struct hifn_device *)data; 2319 2320 /* 2321 * This is ok to call this without lock being held, 2322 * althogh it modifies some parameters used in parallel, 2323 * (like dev->success), but they are used in process 2324 * context or update is atomic (like setting dev->sa[i] to NULL). 2325 */ 2326 hifn_clear_rings(dev, 0); 2327 2328 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2329 hifn_process_queue(dev); 2330 } 2331 2332 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2333 { 2334 int err, i; 2335 struct hifn_device *dev; 2336 char name[8]; 2337 2338 err = pci_enable_device(pdev); 2339 if (err) 2340 return err; 2341 pci_set_master(pdev); 2342 2343 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 2344 if (err) 2345 goto err_out_disable_pci_device; 2346 2347 snprintf(name, sizeof(name), "hifn%d", 2348 atomic_inc_return(&hifn_dev_number) - 1); 2349 2350 err = pci_request_regions(pdev, name); 2351 if (err) 2352 goto err_out_disable_pci_device; 2353 2354 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || 2355 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || 2356 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { 2357 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); 2358 err = -ENODEV; 2359 goto err_out_free_regions; 2360 } 2361 2362 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg), 2363 GFP_KERNEL); 2364 if (!dev) { 2365 err = -ENOMEM; 2366 goto err_out_free_regions; 2367 } 2368 2369 INIT_LIST_HEAD(&dev->alg_list); 2370 2371 snprintf(dev->name, sizeof(dev->name), "%s", name); 2372 spin_lock_init(&dev->lock); 2373 2374 for (i = 0; i < 3; ++i) { 2375 unsigned long addr, size; 2376 2377 addr = pci_resource_start(pdev, i); 2378 size = pci_resource_len(pdev, i); 2379 2380 dev->bar[i] = ioremap(addr, size); 2381 if (!dev->bar[i]) { 2382 err = -ENOMEM; 2383 goto err_out_unmap_bars; 2384 } 2385 } 2386 2387 dev->desc_virt = dma_alloc_coherent(&pdev->dev, 2388 sizeof(struct hifn_dma), 2389 &dev->desc_dma, GFP_KERNEL); 2390 if (!dev->desc_virt) { 2391 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); 2392 err = -ENOMEM; 2393 goto err_out_unmap_bars; 2394 } 2395 2396 dev->pdev = pdev; 2397 dev->irq = pdev->irq; 2398 2399 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 2400 dev->sa[i] = NULL; 2401 2402 pci_set_drvdata(pdev, dev); 2403 2404 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); 2405 2406 crypto_init_queue(&dev->queue, 1); 2407 2408 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); 2409 if (err) { 2410 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", 2411 dev->irq, err); 2412 dev->irq = 0; 2413 goto err_out_free_desc; 2414 } 2415 2416 err = hifn_start_device(dev); 2417 if (err) 2418 goto err_out_free_irq; 2419 2420 err = hifn_register_rng(dev); 2421 if (err) 2422 goto err_out_stop_device; 2423 2424 err = hifn_register_alg(dev); 2425 if (err) 2426 goto err_out_unregister_rng; 2427 2428 INIT_DELAYED_WORK(&dev->work, hifn_work); 2429 schedule_delayed_work(&dev->work, HZ); 2430 2431 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " 2432 "successfully registered as %s.\n", 2433 pci_name(pdev), dev->name); 2434 2435 return 0; 2436 2437 err_out_unregister_rng: 2438 hifn_unregister_rng(dev); 2439 err_out_stop_device: 2440 hifn_reset_dma(dev, 1); 2441 hifn_stop_device(dev); 2442 err_out_free_irq: 2443 free_irq(dev->irq, dev); 2444 tasklet_kill(&dev->tasklet); 2445 err_out_free_desc: 2446 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, 2447 dev->desc_dma); 2448 2449 err_out_unmap_bars: 2450 for (i = 0; i < 3; ++i) 2451 if (dev->bar[i]) 2452 iounmap(dev->bar[i]); 2453 kfree(dev); 2454 2455 err_out_free_regions: 2456 pci_release_regions(pdev); 2457 2458 err_out_disable_pci_device: 2459 pci_disable_device(pdev); 2460 2461 return err; 2462 } 2463 2464 static void hifn_remove(struct pci_dev *pdev) 2465 { 2466 int i; 2467 struct hifn_device *dev; 2468 2469 dev = pci_get_drvdata(pdev); 2470 2471 if (dev) { 2472 cancel_delayed_work_sync(&dev->work); 2473 2474 hifn_unregister_rng(dev); 2475 hifn_unregister_alg(dev); 2476 hifn_reset_dma(dev, 1); 2477 hifn_stop_device(dev); 2478 2479 free_irq(dev->irq, dev); 2480 tasklet_kill(&dev->tasklet); 2481 2482 hifn_flush(dev); 2483 2484 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), 2485 dev->desc_virt, dev->desc_dma); 2486 for (i = 0; i < 3; ++i) 2487 if (dev->bar[i]) 2488 iounmap(dev->bar[i]); 2489 2490 kfree(dev); 2491 } 2492 2493 pci_release_regions(pdev); 2494 pci_disable_device(pdev); 2495 } 2496 2497 static struct pci_device_id hifn_pci_tbl[] = { 2498 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) }, 2499 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) }, 2500 { 0 } 2501 }; 2502 MODULE_DEVICE_TABLE(pci, hifn_pci_tbl); 2503 2504 static struct pci_driver hifn_pci_driver = { 2505 .name = "hifn795x", 2506 .id_table = hifn_pci_tbl, 2507 .probe = hifn_probe, 2508 .remove = hifn_remove, 2509 }; 2510 2511 static int __init hifn_init(void) 2512 { 2513 unsigned int freq; 2514 int err; 2515 2516 if (strncmp(hifn_pll_ref, "ext", 3) && 2517 strncmp(hifn_pll_ref, "pci", 3)) { 2518 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); 2519 return -EINVAL; 2520 } 2521 2522 /* 2523 * For the 7955/7956 the reference clock frequency must be in the 2524 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, 2525 * but this chip is currently not supported. 2526 */ 2527 if (hifn_pll_ref[3] != '\0') { 2528 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 2529 if (freq < 20 || freq > 100) { 2530 pr_err("hifn795x: invalid hifn_pll_ref frequency, must" 2531 "be in the range of 20-100"); 2532 return -EINVAL; 2533 } 2534 } 2535 2536 err = pci_register_driver(&hifn_pci_driver); 2537 if (err < 0) { 2538 pr_err("Failed to register PCI driver for %s device.\n", 2539 hifn_pci_driver.name); 2540 return -ENODEV; 2541 } 2542 2543 pr_info("Driver for HIFN 795x crypto accelerator chip " 2544 "has been successfully registered.\n"); 2545 2546 return 0; 2547 } 2548 2549 static void __exit hifn_fini(void) 2550 { 2551 pci_unregister_driver(&hifn_pci_driver); 2552 2553 pr_info("Driver for HIFN 795x crypto accelerator chip " 2554 "has been successfully unregistered.\n"); 2555 } 2556 2557 module_init(hifn_init); 2558 module_exit(hifn_fini); 2559 2560 MODULE_LICENSE("GPL"); 2561 MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>"); 2562 MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip."); 2563