1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 4 * All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/moduleparam.h> 10 #include <linux/mod_devicetable.h> 11 #include <linux/interrupt.h> 12 #include <linux/pci.h> 13 #include <linux/slab.h> 14 #include <linux/delay.h> 15 #include <linux/mm.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/scatterlist.h> 18 #include <linux/highmem.h> 19 #include <linux/crypto.h> 20 #include <linux/hw_random.h> 21 #include <linux/ktime.h> 22 23 #include <crypto/algapi.h> 24 #include <crypto/internal/des.h> 25 #include <crypto/internal/skcipher.h> 26 27 static char hifn_pll_ref[sizeof("extNNN")] = "ext"; 28 module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); 29 MODULE_PARM_DESC(hifn_pll_ref, 30 "PLL reference clock (pci[freq] or ext[freq], default ext)"); 31 32 static atomic_t hifn_dev_number; 33 34 #define ACRYPTO_OP_DECRYPT 0 35 #define ACRYPTO_OP_ENCRYPT 1 36 #define ACRYPTO_OP_HMAC 2 37 #define ACRYPTO_OP_RNG 3 38 39 #define ACRYPTO_MODE_ECB 0 40 #define ACRYPTO_MODE_CBC 1 41 #define ACRYPTO_MODE_CFB 2 42 #define ACRYPTO_MODE_OFB 3 43 44 #define ACRYPTO_TYPE_AES_128 0 45 #define ACRYPTO_TYPE_AES_192 1 46 #define ACRYPTO_TYPE_AES_256 2 47 #define ACRYPTO_TYPE_3DES 3 48 #define ACRYPTO_TYPE_DES 4 49 50 #define PCI_VENDOR_ID_HIFN 0x13A3 51 #define PCI_DEVICE_ID_HIFN_7955 0x0020 52 #define PCI_DEVICE_ID_HIFN_7956 0x001d 53 54 /* I/O region sizes */ 55 56 #define HIFN_BAR0_SIZE 0x1000 57 #define HIFN_BAR1_SIZE 0x2000 58 #define HIFN_BAR2_SIZE 0x8000 59 60 /* DMA registres */ 61 62 #define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ 63 #define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ 64 #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ 65 #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ 66 #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ 67 #define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ 68 #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ 69 #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ 70 #define HIFN_CHIP_ID 0x98 /* Chip ID */ 71 72 /* 73 * Processing Unit Registers (offset from BASEREG0) 74 */ 75 #define HIFN_0_PUDATA 0x00 /* Processing Unit Data */ 76 #define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */ 77 #define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */ 78 #define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */ 79 #define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */ 80 #define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */ 81 #define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */ 82 #define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */ 83 #define HIFN_0_SPACESIZE 0x20 /* Register space size */ 84 85 /* Processing Unit Control Register (HIFN_0_PUCTRL) */ 86 #define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */ 87 #define HIFN_PUCTRL_STOP 0x0008 /* stop pu */ 88 #define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */ 89 #define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */ 90 #define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */ 91 92 /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */ 93 #define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */ 94 #define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */ 95 #define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 96 #define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 97 #define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */ 98 #define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */ 99 #define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */ 100 #define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */ 101 #define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */ 102 #define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */ 103 104 /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */ 105 #define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */ 106 #define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */ 107 #define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */ 108 #define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */ 109 #define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */ 110 #define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */ 111 #define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */ 112 #define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */ 113 #define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */ 114 #define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */ 115 #define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */ 116 #define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */ 117 #define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */ 118 #define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */ 119 #define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */ 120 #define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */ 121 #define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */ 122 #define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */ 123 #define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */ 124 #define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */ 125 #define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */ 126 #define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */ 127 #define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */ 128 129 /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */ 130 #define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */ 131 #define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */ 132 #define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 133 #define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 134 #define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */ 135 #define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */ 136 #define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */ 137 #define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */ 138 #define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */ 139 #define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */ 140 141 /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */ 142 #define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */ 143 #define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */ 144 #define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 145 #define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 146 #define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */ 147 #define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */ 148 #define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */ 149 #define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */ 150 #define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */ 151 #define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */ 152 #define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */ 153 #define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */ 154 #define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */ 155 #define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */ 156 #define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */ 157 #define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */ 158 #define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */ 159 160 /* FIFO Status Register (HIFN_0_FIFOSTAT) */ 161 #define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */ 162 #define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */ 163 164 /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */ 165 #define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */ 166 167 /* 168 * DMA Interface Registers (offset from BASEREG1) 169 */ 170 #define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */ 171 #define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */ 172 #define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */ 173 #define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */ 174 #define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */ 175 #define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */ 176 #define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */ 177 #define HIFN_1_PLL 0x4c /* 795x: PLL config */ 178 #define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */ 179 #define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */ 180 #define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */ 181 #define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */ 182 #define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */ 183 #define HIFN_1_REVID 0x98 /* Revision ID */ 184 #define HIFN_1_UNLOCK_SECRET1 0xf4 185 #define HIFN_1_UNLOCK_SECRET2 0xfc 186 #define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */ 187 #define HIFN_1_PUB_BASE 0x300 /* Public Base Address */ 188 #define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */ 189 #define HIFN_1_PUB_OP 0x308 /* Public Operand */ 190 #define HIFN_1_PUB_STATUS 0x30c /* Public Status */ 191 #define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */ 192 #define HIFN_1_RNG_CONFIG 0x314 /* RNG config */ 193 #define HIFN_1_RNG_DATA 0x318 /* RNG data */ 194 #define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */ 195 #define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */ 196 197 /* DMA Status and Control Register (HIFN_1_DMA_CSR) */ 198 #define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */ 199 #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */ 200 #define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */ 201 #define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */ 202 #define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */ 203 #define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */ 204 #define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */ 205 #define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */ 206 #define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */ 207 #define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */ 208 #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */ 209 #define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */ 210 #define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */ 211 #define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 212 #define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */ 213 #define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */ 214 #define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */ 215 #define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */ 216 #define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */ 217 #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */ 218 #define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */ 219 #define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */ 220 #define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 221 #define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */ 222 #define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */ 223 #define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */ 224 #define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */ 225 #define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */ 226 #define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */ 227 #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */ 228 #define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */ 229 #define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */ 230 #define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 231 #define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */ 232 #define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */ 233 #define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */ 234 #define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */ 235 #define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */ 236 237 /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */ 238 #define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */ 239 #define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */ 240 #define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */ 241 #define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */ 242 #define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */ 243 #define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 244 #define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */ 245 #define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */ 246 #define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */ 247 #define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */ 248 #define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 249 #define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */ 250 #define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */ 251 #define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */ 252 #define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */ 253 #define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */ 254 #define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 255 #define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */ 256 #define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */ 257 #define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */ 258 #define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */ 259 #define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */ 260 261 /* DMA Configuration Register (HIFN_1_DMA_CNFG) */ 262 #define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */ 263 #define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */ 264 #define HIFN_DMACNFG_UNLOCK 0x00000800 265 #define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */ 266 #define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */ 267 #define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */ 268 #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ 269 #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ 270 271 /* PLL configuration register */ 272 #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ 273 #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ 274 #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ 275 #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ 276 #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ 277 #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ 278 #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ 279 #define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ 280 #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ 281 #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ 282 #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ 283 #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ 284 #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ 285 #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ 286 #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ 287 #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ 288 #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ 289 290 #define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ 291 292 /* Public key reset register (HIFN_1_PUB_RESET) */ 293 #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ 294 295 /* Public base address register (HIFN_1_PUB_BASE) */ 296 #define HIFN_PUBBASE_ADDR 0x00003fff /* base address */ 297 298 /* Public operand length register (HIFN_1_PUB_OPLEN) */ 299 #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ 300 #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ 301 #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ 302 #define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ 303 #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ 304 #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ 305 306 /* Public operation register (HIFN_1_PUB_OP) */ 307 #define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */ 308 #define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */ 309 #define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */ 310 #define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */ 311 #define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */ 312 #define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */ 313 #define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */ 314 #define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */ 315 #define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */ 316 #define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */ 317 #define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */ 318 #define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */ 319 #define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */ 320 #define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */ 321 #define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */ 322 #define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */ 323 #define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */ 324 #define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */ 325 #define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */ 326 #define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */ 327 328 /* Public status register (HIFN_1_PUB_STATUS) */ 329 #define HIFN_PUBSTS_DONE 0x00000001 /* operation done */ 330 #define HIFN_PUBSTS_CARRY 0x00000002 /* carry */ 331 332 /* Public interrupt enable register (HIFN_1_PUB_IEN) */ 333 #define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */ 334 335 /* Random number generator config register (HIFN_1_RNG_CONFIG) */ 336 #define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */ 337 338 #define HIFN_NAMESIZE 32 339 #define HIFN_MAX_RESULT_ORDER 5 340 341 #define HIFN_D_CMD_RSIZE (24 * 1) 342 #define HIFN_D_SRC_RSIZE (80 * 1) 343 #define HIFN_D_DST_RSIZE (80 * 1) 344 #define HIFN_D_RES_RSIZE (24 * 1) 345 346 #define HIFN_D_DST_DALIGN 4 347 348 #define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) 349 350 #define AES_MIN_KEY_SIZE 16 351 #define AES_MAX_KEY_SIZE 32 352 353 #define HIFN_DES_KEY_LENGTH 8 354 #define HIFN_3DES_KEY_LENGTH 24 355 #define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE 356 #define HIFN_IV_LENGTH 8 357 #define HIFN_AES_IV_LENGTH 16 358 #define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH 359 360 #define HIFN_MAC_KEY_LENGTH 64 361 #define HIFN_MD5_LENGTH 16 362 #define HIFN_SHA1_LENGTH 20 363 #define HIFN_MAC_TRUNC_LENGTH 12 364 365 #define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260) 366 #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) 367 #define HIFN_USED_RESULT 12 368 369 struct hifn_desc { 370 volatile __le32 l; 371 volatile __le32 p; 372 }; 373 374 struct hifn_dma { 375 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; 376 struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; 377 struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; 378 struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; 379 380 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; 381 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; 382 383 /* 384 * Our current positions for insertion and removal from the descriptor 385 * rings. 386 */ 387 volatile int cmdi, srci, dsti, resi; 388 volatile int cmdu, srcu, dstu, resu; 389 int cmdk, srck, dstk, resk; 390 }; 391 392 #define HIFN_FLAG_CMD_BUSY (1 << 0) 393 #define HIFN_FLAG_SRC_BUSY (1 << 1) 394 #define HIFN_FLAG_DST_BUSY (1 << 2) 395 #define HIFN_FLAG_RES_BUSY (1 << 3) 396 #define HIFN_FLAG_OLD_KEY (1 << 4) 397 398 #define HIFN_DEFAULT_ACTIVE_NUM 5 399 400 struct hifn_device { 401 char name[HIFN_NAMESIZE]; 402 403 int irq; 404 405 struct pci_dev *pdev; 406 void __iomem *bar[3]; 407 408 void *desc_virt; 409 dma_addr_t desc_dma; 410 411 u32 dmareg; 412 413 void *sa[HIFN_D_RES_RSIZE]; 414 415 spinlock_t lock; 416 417 u32 flags; 418 int active, started; 419 struct delayed_work work; 420 unsigned long reset; 421 unsigned long success; 422 unsigned long prev_success; 423 424 u8 snum; 425 426 struct tasklet_struct tasklet; 427 428 struct crypto_queue queue; 429 struct list_head alg_list; 430 431 unsigned int pk_clk_freq; 432 433 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 434 unsigned int rng_wait_time; 435 ktime_t rngtime; 436 struct hwrng rng; 437 #endif 438 }; 439 440 #define HIFN_D_LENGTH 0x0000ffff 441 #define HIFN_D_NOINVALID 0x01000000 442 #define HIFN_D_MASKDONEIRQ 0x02000000 443 #define HIFN_D_DESTOVER 0x04000000 444 #define HIFN_D_OVER 0x08000000 445 #define HIFN_D_LAST 0x20000000 446 #define HIFN_D_JUMP 0x40000000 447 #define HIFN_D_VALID 0x80000000 448 449 struct hifn_base_command { 450 volatile __le16 masks; 451 volatile __le16 session_num; 452 volatile __le16 total_source_count; 453 volatile __le16 total_dest_count; 454 }; 455 456 #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ 457 #define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */ 458 #define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */ 459 #define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */ 460 #define HIFN_BASE_CMD_DECODE 0x2000 461 #define HIFN_BASE_CMD_SRCLEN_M 0xc000 462 #define HIFN_BASE_CMD_SRCLEN_S 14 463 #define HIFN_BASE_CMD_DSTLEN_M 0x3000 464 #define HIFN_BASE_CMD_DSTLEN_S 12 465 #define HIFN_BASE_CMD_LENMASK_HI 0x30000 466 #define HIFN_BASE_CMD_LENMASK_LO 0x0ffff 467 468 /* 469 * Structure to help build up the command data structure. 470 */ 471 struct hifn_crypt_command { 472 volatile __le16 masks; 473 volatile __le16 header_skip; 474 volatile __le16 source_count; 475 volatile __le16 reserved; 476 }; 477 478 #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ 479 #define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */ 480 #define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */ 481 #define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */ 482 #define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */ 483 #define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */ 484 #define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */ 485 #define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */ 486 #define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */ 487 #define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */ 488 #define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */ 489 #define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */ 490 #define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */ 491 #define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */ 492 #define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */ 493 #define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */ 494 #define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */ 495 #define HIFN_CRYPT_CMD_SRCLEN_M 0xc000 496 #define HIFN_CRYPT_CMD_SRCLEN_S 14 497 498 /* 499 * Structure to help build up the command data structure. 500 */ 501 struct hifn_mac_command { 502 volatile __le16 masks; 503 volatile __le16 header_skip; 504 volatile __le16 source_count; 505 volatile __le16 reserved; 506 }; 507 508 #define HIFN_MAC_CMD_ALG_MASK 0x0001 509 #define HIFN_MAC_CMD_ALG_SHA1 0x0000 510 #define HIFN_MAC_CMD_ALG_MD5 0x0001 511 #define HIFN_MAC_CMD_MODE_MASK 0x000c 512 #define HIFN_MAC_CMD_MODE_HMAC 0x0000 513 #define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004 514 #define HIFN_MAC_CMD_MODE_HASH 0x0008 515 #define HIFN_MAC_CMD_MODE_FULL 0x0004 516 #define HIFN_MAC_CMD_TRUNC 0x0010 517 #define HIFN_MAC_CMD_RESULT 0x0020 518 #define HIFN_MAC_CMD_APPEND 0x0040 519 #define HIFN_MAC_CMD_SRCLEN_M 0xc000 520 #define HIFN_MAC_CMD_SRCLEN_S 14 521 522 /* 523 * MAC POS IPsec initiates authentication after encryption on encodes 524 * and before decryption on decodes. 525 */ 526 #define HIFN_MAC_CMD_POS_IPSEC 0x0200 527 #define HIFN_MAC_CMD_NEW_KEY 0x0800 528 529 struct hifn_comp_command { 530 volatile __le16 masks; 531 volatile __le16 header_skip; 532 volatile __le16 source_count; 533 volatile __le16 reserved; 534 }; 535 536 #define HIFN_COMP_CMD_SRCLEN_M 0xc000 537 #define HIFN_COMP_CMD_SRCLEN_S 14 538 #define HIFN_COMP_CMD_ONE 0x0100 /* must be one */ 539 #define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */ 540 #define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */ 541 #define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */ 542 #define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */ 543 #define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */ 544 #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ 545 #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ 546 547 struct hifn_base_result { 548 volatile __le16 flags; 549 volatile __le16 session; 550 volatile __le16 src_cnt; /* 15:0 of source count */ 551 volatile __le16 dst_cnt; /* 15:0 of dest count */ 552 }; 553 554 #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ 555 #define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */ 556 #define HIFN_BASE_RES_SRCLEN_S 14 557 #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ 558 #define HIFN_BASE_RES_DSTLEN_S 12 559 560 struct hifn_comp_result { 561 volatile __le16 flags; 562 volatile __le16 crc; 563 }; 564 565 #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ 566 #define HIFN_COMP_RES_LCB_S 8 567 #define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */ 568 #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ 569 #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ 570 571 struct hifn_mac_result { 572 volatile __le16 flags; 573 volatile __le16 reserved; 574 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ 575 }; 576 577 #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ 578 #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ 579 580 struct hifn_crypt_result { 581 volatile __le16 flags; 582 volatile __le16 reserved; 583 }; 584 585 #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ 586 587 #ifndef HIFN_POLL_FREQUENCY 588 #define HIFN_POLL_FREQUENCY 0x1 589 #endif 590 591 #ifndef HIFN_POLL_SCALAR 592 #define HIFN_POLL_SCALAR 0x0 593 #endif 594 595 #define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ 596 #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ 597 598 struct hifn_crypto_alg { 599 struct list_head entry; 600 struct skcipher_alg alg; 601 struct hifn_device *dev; 602 }; 603 604 #define ASYNC_SCATTERLIST_CACHE 16 605 606 #define ASYNC_FLAGS_MISALIGNED (1 << 0) 607 608 struct hifn_cipher_walk { 609 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; 610 u32 flags; 611 int num; 612 }; 613 614 struct hifn_context { 615 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; 616 struct hifn_device *dev; 617 unsigned int keysize; 618 }; 619 620 struct hifn_request_context { 621 u8 *iv; 622 unsigned int ivsize; 623 u8 op, type, mode, unused; 624 struct hifn_cipher_walk walk; 625 }; 626 627 #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) 628 629 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) 630 { 631 return readl(dev->bar[0] + reg); 632 } 633 634 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) 635 { 636 return readl(dev->bar[1] + reg); 637 } 638 639 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) 640 { 641 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); 642 } 643 644 static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) 645 { 646 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); 647 } 648 649 static void hifn_wait_puc(struct hifn_device *dev) 650 { 651 int i; 652 u32 ret; 653 654 for (i = 10000; i > 0; --i) { 655 ret = hifn_read_0(dev, HIFN_0_PUCTRL); 656 if (!(ret & HIFN_PUCTRL_RESET)) 657 break; 658 659 udelay(1); 660 } 661 662 if (!i) 663 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); 664 } 665 666 static void hifn_reset_puc(struct hifn_device *dev) 667 { 668 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 669 hifn_wait_puc(dev); 670 } 671 672 static void hifn_stop_device(struct hifn_device *dev) 673 { 674 hifn_write_1(dev, HIFN_1_DMA_CSR, 675 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 676 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS); 677 hifn_write_0(dev, HIFN_0_PUIER, 0); 678 hifn_write_1(dev, HIFN_1_DMA_IER, 0); 679 } 680 681 static void hifn_reset_dma(struct hifn_device *dev, int full) 682 { 683 hifn_stop_device(dev); 684 685 /* 686 * Setting poll frequency and others to 0. 687 */ 688 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 689 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 690 mdelay(1); 691 692 /* 693 * Reset DMA. 694 */ 695 if (full) { 696 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 697 mdelay(1); 698 } else { 699 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE | 700 HIFN_DMACNFG_MSTRESET); 701 hifn_reset_puc(dev); 702 } 703 704 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 705 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 706 707 hifn_reset_puc(dev); 708 } 709 710 static u32 hifn_next_signature(u32 a, u_int cnt) 711 { 712 int i; 713 u32 v; 714 715 for (i = 0; i < cnt; i++) { 716 /* get the parity */ 717 v = a & 0x80080125; 718 v ^= v >> 16; 719 v ^= v >> 8; 720 v ^= v >> 4; 721 v ^= v >> 2; 722 v ^= v >> 1; 723 724 a = (v & 1) ^ (a << 1); 725 } 726 727 return a; 728 } 729 730 static struct pci2id { 731 u_short pci_vendor; 732 u_short pci_prod; 733 char card_id[13]; 734 } pci2id[] = { 735 { 736 PCI_VENDOR_ID_HIFN, 737 PCI_DEVICE_ID_HIFN_7955, 738 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 739 0x00, 0x00, 0x00, 0x00, 0x00 } 740 }, 741 { 742 PCI_VENDOR_ID_HIFN, 743 PCI_DEVICE_ID_HIFN_7956, 744 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 745 0x00, 0x00, 0x00, 0x00, 0x00 } 746 } 747 }; 748 749 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 750 static int hifn_rng_data_present(struct hwrng *rng, int wait) 751 { 752 struct hifn_device *dev = (struct hifn_device *)rng->priv; 753 s64 nsec; 754 755 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); 756 nsec -= dev->rng_wait_time; 757 if (nsec <= 0) 758 return 1; 759 if (!wait) 760 return 0; 761 ndelay(nsec); 762 return 1; 763 } 764 765 static int hifn_rng_data_read(struct hwrng *rng, u32 *data) 766 { 767 struct hifn_device *dev = (struct hifn_device *)rng->priv; 768 769 *data = hifn_read_1(dev, HIFN_1_RNG_DATA); 770 dev->rngtime = ktime_get(); 771 return 4; 772 } 773 774 static int hifn_register_rng(struct hifn_device *dev) 775 { 776 /* 777 * We must wait at least 256 Pk_clk cycles between two reads of the rng. 778 */ 779 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, 780 dev->pk_clk_freq) * 256; 781 782 dev->rng.name = dev->name; 783 dev->rng.data_present = hifn_rng_data_present; 784 dev->rng.data_read = hifn_rng_data_read; 785 dev->rng.priv = (unsigned long)dev; 786 787 return hwrng_register(&dev->rng); 788 } 789 790 static void hifn_unregister_rng(struct hifn_device *dev) 791 { 792 hwrng_unregister(&dev->rng); 793 } 794 #else 795 #define hifn_register_rng(dev) 0 796 #define hifn_unregister_rng(dev) 797 #endif 798 799 static int hifn_init_pubrng(struct hifn_device *dev) 800 { 801 int i; 802 803 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | 804 HIFN_PUBRST_RESET); 805 806 for (i = 100; i > 0; --i) { 807 mdelay(1); 808 809 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) 810 break; 811 } 812 813 if (!i) { 814 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); 815 } else { 816 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 817 dev->dmareg |= HIFN_DMAIER_PUBDONE; 818 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 819 820 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); 821 } 822 823 /* Enable RNG engine. */ 824 825 hifn_write_1(dev, HIFN_1_RNG_CONFIG, 826 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); 827 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); 828 829 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 830 /* First value must be discarded */ 831 hifn_read_1(dev, HIFN_1_RNG_DATA); 832 dev->rngtime = ktime_get(); 833 #endif 834 return 0; 835 } 836 837 static int hifn_enable_crypto(struct hifn_device *dev) 838 { 839 u32 dmacfg, addr; 840 char *offtbl = NULL; 841 int i; 842 843 for (i = 0; i < ARRAY_SIZE(pci2id); i++) { 844 if (pci2id[i].pci_vendor == dev->pdev->vendor && 845 pci2id[i].pci_prod == dev->pdev->device) { 846 offtbl = pci2id[i].card_id; 847 break; 848 } 849 } 850 851 if (!offtbl) { 852 dev_err(&dev->pdev->dev, "Unknown card!\n"); 853 return -ENODEV; 854 } 855 856 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG); 857 858 hifn_write_1(dev, HIFN_1_DMA_CNFG, 859 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET | 860 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 861 mdelay(1); 862 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1); 863 mdelay(1); 864 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); 865 mdelay(1); 866 867 for (i = 0; i < 12; ++i) { 868 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 869 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); 870 871 mdelay(1); 872 } 873 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); 874 875 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); 876 877 return 0; 878 } 879 880 static void hifn_init_dma(struct hifn_device *dev) 881 { 882 struct hifn_dma *dma = dev->desc_virt; 883 u32 dptr = dev->desc_dma; 884 int i; 885 886 for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) 887 dma->cmdr[i].p = __cpu_to_le32(dptr + 888 offsetof(struct hifn_dma, command_bufs[i][0])); 889 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 890 dma->resr[i].p = __cpu_to_le32(dptr + 891 offsetof(struct hifn_dma, result_bufs[i][0])); 892 893 /* Setup LAST descriptors. */ 894 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + 895 offsetof(struct hifn_dma, cmdr[0])); 896 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + 897 offsetof(struct hifn_dma, srcr[0])); 898 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + 899 offsetof(struct hifn_dma, dstr[0])); 900 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + 901 offsetof(struct hifn_dma, resr[0])); 902 903 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 904 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 905 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 906 } 907 908 /* 909 * Initialize the PLL. We need to know the frequency of the reference clock 910 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that 911 * allows us to operate without the risk of overclocking the chip. If it 912 * actually uses 33MHz, the chip will operate at half the speed, this can be 913 * overridden by specifying the frequency as module parameter (pci33). 914 * 915 * Unfortunately the PCI clock is not very suitable since the HIFN needs a 916 * stable clock and the PCI clock frequency may vary, so the default is the 917 * external clock. There is no way to find out its frequency, we default to 918 * 66MHz since according to Mike Ham of HiFn, almost every board in existence 919 * has an external crystal populated at 66MHz. 920 */ 921 static void hifn_init_pll(struct hifn_device *dev) 922 { 923 unsigned int freq, m; 924 u32 pllcfg; 925 926 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; 927 928 if (strncmp(hifn_pll_ref, "ext", 3) == 0) 929 pllcfg |= HIFN_PLL_REF_CLK_PLL; 930 else 931 pllcfg |= HIFN_PLL_REF_CLK_HBI; 932 933 if (hifn_pll_ref[3] != '\0') 934 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 935 else { 936 freq = 66; 937 dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", 938 freq, hifn_pll_ref); 939 } 940 941 m = HIFN_PLL_FCK_MAX / freq; 942 943 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; 944 if (m <= 8) 945 pllcfg |= HIFN_PLL_IS_1_8; 946 else 947 pllcfg |= HIFN_PLL_IS_9_12; 948 949 /* Select clock source and enable clock bypass */ 950 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 951 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); 952 953 /* Let the chip lock to the input clock */ 954 mdelay(10); 955 956 /* Disable clock bypass */ 957 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 958 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); 959 960 /* Switch the engines to the PLL */ 961 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 962 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); 963 964 /* 965 * The Fpk_clk runs at half the total speed. Its frequency is needed to 966 * calculate the minimum time between two reads of the rng. Since 33MHz 967 * is actually 33.333... we overestimate the frequency here, resulting 968 * in slightly larger intervals. 969 */ 970 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; 971 } 972 973 static void hifn_init_registers(struct hifn_device *dev) 974 { 975 u32 dptr = dev->desc_dma; 976 977 /* Initialization magic... */ 978 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 979 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 980 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 981 982 /* write all 4 ring address registers */ 983 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + 984 offsetof(struct hifn_dma, cmdr[0])); 985 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + 986 offsetof(struct hifn_dma, srcr[0])); 987 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + 988 offsetof(struct hifn_dma, dstr[0])); 989 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + 990 offsetof(struct hifn_dma, resr[0])); 991 992 mdelay(2); 993 #if 0 994 hifn_write_1(dev, HIFN_1_DMA_CSR, 995 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 996 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 997 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 998 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 999 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1000 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1001 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1002 HIFN_DMACSR_S_WAIT | 1003 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1004 HIFN_DMACSR_C_WAIT | 1005 HIFN_DMACSR_ENGINE | 1006 HIFN_DMACSR_PUBDONE); 1007 #else 1008 hifn_write_1(dev, HIFN_1_DMA_CSR, 1009 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1010 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA | 1011 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 1012 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 1013 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1014 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1015 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1016 HIFN_DMACSR_S_WAIT | 1017 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1018 HIFN_DMACSR_C_WAIT | 1019 HIFN_DMACSR_ENGINE | 1020 HIFN_DMACSR_PUBDONE); 1021 #endif 1022 hifn_read_1(dev, HIFN_1_DMA_CSR); 1023 1024 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1025 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1026 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1027 HIFN_DMAIER_ENGINE; 1028 dev->dmareg &= ~HIFN_DMAIER_C_WAIT; 1029 1030 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1031 hifn_read_1(dev, HIFN_1_DMA_IER); 1032 #if 0 1033 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG | 1034 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1035 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1036 HIFN_PUCNFG_DRAM); 1037 #else 1038 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); 1039 #endif 1040 hifn_init_pll(dev); 1041 1042 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1043 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1044 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1045 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1046 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1047 } 1048 1049 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, 1050 unsigned dlen, unsigned slen, u16 mask, u8 snum) 1051 { 1052 struct hifn_base_command *base_cmd; 1053 u8 *buf_pos = buf; 1054 1055 base_cmd = (struct hifn_base_command *)buf_pos; 1056 base_cmd->masks = __cpu_to_le16(mask); 1057 base_cmd->total_source_count = 1058 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO); 1059 base_cmd->total_dest_count = 1060 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1061 1062 dlen >>= 16; 1063 slen >>= 16; 1064 base_cmd->session_num = __cpu_to_le16(snum | 1065 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1066 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1067 1068 return sizeof(struct hifn_base_command); 1069 } 1070 1071 static int hifn_setup_crypto_command(struct hifn_device *dev, 1072 u8 *buf, unsigned dlen, unsigned slen, 1073 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) 1074 { 1075 struct hifn_dma *dma = dev->desc_virt; 1076 struct hifn_crypt_command *cry_cmd; 1077 u8 *buf_pos = buf; 1078 u16 cmd_len; 1079 1080 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1081 1082 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); 1083 dlen >>= 16; 1084 cry_cmd->masks = __cpu_to_le16(mode | 1085 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & 1086 HIFN_CRYPT_CMD_SRCLEN_M)); 1087 cry_cmd->header_skip = 0; 1088 cry_cmd->reserved = 0; 1089 1090 buf_pos += sizeof(struct hifn_crypt_command); 1091 1092 dma->cmdu++; 1093 if (dma->cmdu > 1) { 1094 dev->dmareg |= HIFN_DMAIER_C_WAIT; 1095 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1096 } 1097 1098 if (keylen) { 1099 memcpy(buf_pos, key, keylen); 1100 buf_pos += keylen; 1101 } 1102 if (ivsize) { 1103 memcpy(buf_pos, iv, ivsize); 1104 buf_pos += ivsize; 1105 } 1106 1107 cmd_len = buf_pos - buf; 1108 1109 return cmd_len; 1110 } 1111 1112 static int hifn_setup_cmd_desc(struct hifn_device *dev, 1113 struct hifn_context *ctx, struct hifn_request_context *rctx, 1114 void *priv, unsigned int nbytes) 1115 { 1116 struct hifn_dma *dma = dev->desc_virt; 1117 int cmd_len, sa_idx; 1118 u8 *buf, *buf_pos; 1119 u16 mask; 1120 1121 sa_idx = dma->cmdi; 1122 buf_pos = buf = dma->command_bufs[dma->cmdi]; 1123 1124 mask = 0; 1125 switch (rctx->op) { 1126 case ACRYPTO_OP_DECRYPT: 1127 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; 1128 break; 1129 case ACRYPTO_OP_ENCRYPT: 1130 mask = HIFN_BASE_CMD_CRYPT; 1131 break; 1132 case ACRYPTO_OP_HMAC: 1133 mask = HIFN_BASE_CMD_MAC; 1134 break; 1135 default: 1136 goto err_out; 1137 } 1138 1139 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, 1140 nbytes, mask, dev->snum); 1141 1142 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { 1143 u16 md = 0; 1144 1145 if (ctx->keysize) 1146 md |= HIFN_CRYPT_CMD_NEW_KEY; 1147 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) 1148 md |= HIFN_CRYPT_CMD_NEW_IV; 1149 1150 switch (rctx->mode) { 1151 case ACRYPTO_MODE_ECB: 1152 md |= HIFN_CRYPT_CMD_MODE_ECB; 1153 break; 1154 case ACRYPTO_MODE_CBC: 1155 md |= HIFN_CRYPT_CMD_MODE_CBC; 1156 break; 1157 case ACRYPTO_MODE_CFB: 1158 md |= HIFN_CRYPT_CMD_MODE_CFB; 1159 break; 1160 case ACRYPTO_MODE_OFB: 1161 md |= HIFN_CRYPT_CMD_MODE_OFB; 1162 break; 1163 default: 1164 goto err_out; 1165 } 1166 1167 switch (rctx->type) { 1168 case ACRYPTO_TYPE_AES_128: 1169 if (ctx->keysize != 16) 1170 goto err_out; 1171 md |= HIFN_CRYPT_CMD_KSZ_128 | 1172 HIFN_CRYPT_CMD_ALG_AES; 1173 break; 1174 case ACRYPTO_TYPE_AES_192: 1175 if (ctx->keysize != 24) 1176 goto err_out; 1177 md |= HIFN_CRYPT_CMD_KSZ_192 | 1178 HIFN_CRYPT_CMD_ALG_AES; 1179 break; 1180 case ACRYPTO_TYPE_AES_256: 1181 if (ctx->keysize != 32) 1182 goto err_out; 1183 md |= HIFN_CRYPT_CMD_KSZ_256 | 1184 HIFN_CRYPT_CMD_ALG_AES; 1185 break; 1186 case ACRYPTO_TYPE_3DES: 1187 if (ctx->keysize != 24) 1188 goto err_out; 1189 md |= HIFN_CRYPT_CMD_ALG_3DES; 1190 break; 1191 case ACRYPTO_TYPE_DES: 1192 if (ctx->keysize != 8) 1193 goto err_out; 1194 md |= HIFN_CRYPT_CMD_ALG_DES; 1195 break; 1196 default: 1197 goto err_out; 1198 } 1199 1200 buf_pos += hifn_setup_crypto_command(dev, buf_pos, 1201 nbytes, nbytes, ctx->key, ctx->keysize, 1202 rctx->iv, rctx->ivsize, md); 1203 } 1204 1205 dev->sa[sa_idx] = priv; 1206 dev->started++; 1207 1208 cmd_len = buf_pos - buf; 1209 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | 1210 HIFN_D_LAST | HIFN_D_MASKDONEIRQ); 1211 1212 if (++dma->cmdi == HIFN_D_CMD_RSIZE) { 1213 dma->cmdr[dma->cmdi].l = __cpu_to_le32( 1214 HIFN_D_VALID | HIFN_D_LAST | 1215 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); 1216 dma->cmdi = 0; 1217 } else { 1218 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); 1219 } 1220 1221 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { 1222 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1223 dev->flags |= HIFN_FLAG_CMD_BUSY; 1224 } 1225 return 0; 1226 1227 err_out: 1228 return -EINVAL; 1229 } 1230 1231 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, 1232 unsigned int offset, unsigned int size, int last) 1233 { 1234 struct hifn_dma *dma = dev->desc_virt; 1235 int idx; 1236 dma_addr_t addr; 1237 1238 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1239 DMA_TO_DEVICE); 1240 1241 idx = dma->srci; 1242 1243 dma->srcr[idx].p = __cpu_to_le32(addr); 1244 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1245 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1246 1247 if (++idx == HIFN_D_SRC_RSIZE) { 1248 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1249 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1250 (last ? HIFN_D_LAST : 0)); 1251 idx = 0; 1252 } 1253 1254 dma->srci = idx; 1255 dma->srcu++; 1256 1257 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { 1258 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1259 dev->flags |= HIFN_FLAG_SRC_BUSY; 1260 } 1261 1262 return size; 1263 } 1264 1265 static void hifn_setup_res_desc(struct hifn_device *dev) 1266 { 1267 struct hifn_dma *dma = dev->desc_virt; 1268 1269 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | 1270 HIFN_D_VALID | HIFN_D_LAST); 1271 /* 1272 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | 1273 * HIFN_D_LAST); 1274 */ 1275 1276 if (++dma->resi == HIFN_D_RES_RSIZE) { 1277 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | 1278 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); 1279 dma->resi = 0; 1280 } 1281 1282 dma->resu++; 1283 1284 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { 1285 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1286 dev->flags |= HIFN_FLAG_RES_BUSY; 1287 } 1288 } 1289 1290 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, 1291 unsigned offset, unsigned size, int last) 1292 { 1293 struct hifn_dma *dma = dev->desc_virt; 1294 int idx; 1295 dma_addr_t addr; 1296 1297 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1298 DMA_FROM_DEVICE); 1299 1300 idx = dma->dsti; 1301 dma->dstr[idx].p = __cpu_to_le32(addr); 1302 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1303 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1304 1305 if (++idx == HIFN_D_DST_RSIZE) { 1306 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1307 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1308 (last ? HIFN_D_LAST : 0)); 1309 idx = 0; 1310 } 1311 dma->dsti = idx; 1312 dma->dstu++; 1313 1314 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { 1315 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1316 dev->flags |= HIFN_FLAG_DST_BUSY; 1317 } 1318 } 1319 1320 static int hifn_setup_dma(struct hifn_device *dev, 1321 struct hifn_context *ctx, struct hifn_request_context *rctx, 1322 struct scatterlist *src, struct scatterlist *dst, 1323 unsigned int nbytes, void *priv) 1324 { 1325 struct scatterlist *t; 1326 struct page *spage, *dpage; 1327 unsigned int soff, doff; 1328 unsigned int n, len; 1329 1330 n = nbytes; 1331 while (n) { 1332 spage = sg_page(src); 1333 soff = src->offset; 1334 len = min(src->length, n); 1335 1336 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); 1337 1338 src++; 1339 n -= len; 1340 } 1341 1342 t = &rctx->walk.cache[0]; 1343 n = nbytes; 1344 while (n) { 1345 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1346 BUG_ON(!sg_page(t)); 1347 dpage = sg_page(t); 1348 doff = 0; 1349 len = t->length; 1350 } else { 1351 BUG_ON(!sg_page(dst)); 1352 dpage = sg_page(dst); 1353 doff = dst->offset; 1354 len = dst->length; 1355 } 1356 len = min(len, n); 1357 1358 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); 1359 1360 dst++; 1361 t++; 1362 n -= len; 1363 } 1364 1365 hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); 1366 hifn_setup_res_desc(dev); 1367 return 0; 1368 } 1369 1370 static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, 1371 int num, gfp_t gfp_flags) 1372 { 1373 int i; 1374 1375 num = min(ASYNC_SCATTERLIST_CACHE, num); 1376 sg_init_table(w->cache, num); 1377 1378 w->num = 0; 1379 for (i = 0; i < num; ++i) { 1380 struct page *page = alloc_page(gfp_flags); 1381 struct scatterlist *s; 1382 1383 if (!page) 1384 break; 1385 1386 s = &w->cache[i]; 1387 1388 sg_set_page(s, page, PAGE_SIZE, 0); 1389 w->num++; 1390 } 1391 1392 return i; 1393 } 1394 1395 static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) 1396 { 1397 int i; 1398 1399 for (i = 0; i < w->num; ++i) { 1400 struct scatterlist *s = &w->cache[i]; 1401 1402 __free_page(sg_page(s)); 1403 1404 s->length = 0; 1405 } 1406 1407 w->num = 0; 1408 } 1409 1410 static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, 1411 unsigned int size, unsigned int *nbytesp) 1412 { 1413 unsigned int copy, drest = *drestp, nbytes = *nbytesp; 1414 int idx = 0; 1415 1416 if (drest < size || size > nbytes) 1417 return -EINVAL; 1418 1419 while (size) { 1420 copy = min3(drest, size, dst->length); 1421 1422 size -= copy; 1423 drest -= copy; 1424 nbytes -= copy; 1425 1426 pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", 1427 __func__, copy, size, drest, nbytes); 1428 1429 dst++; 1430 idx++; 1431 } 1432 1433 *nbytesp = nbytes; 1434 *drestp = drest; 1435 1436 return idx; 1437 } 1438 1439 static int hifn_cipher_walk(struct skcipher_request *req, 1440 struct hifn_cipher_walk *w) 1441 { 1442 struct scatterlist *dst, *t; 1443 unsigned int nbytes = req->cryptlen, offset, copy, diff; 1444 int idx, tidx, err; 1445 1446 tidx = idx = 0; 1447 offset = 0; 1448 while (nbytes) { 1449 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) 1450 return -EINVAL; 1451 1452 dst = &req->dst[idx]; 1453 1454 pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", 1455 __func__, dst->length, dst->offset, offset, nbytes); 1456 1457 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1458 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || 1459 offset) { 1460 unsigned slen = min(dst->length - offset, nbytes); 1461 unsigned dlen = PAGE_SIZE; 1462 1463 t = &w->cache[idx]; 1464 1465 err = skcipher_add(&dlen, dst, slen, &nbytes); 1466 if (err < 0) 1467 return err; 1468 1469 idx += err; 1470 1471 copy = slen & ~(HIFN_D_DST_DALIGN - 1); 1472 diff = slen & (HIFN_D_DST_DALIGN - 1); 1473 1474 if (dlen < nbytes) { 1475 /* 1476 * Destination page does not have enough space 1477 * to put there additional blocksized chunk, 1478 * so we mark that page as containing only 1479 * blocksize aligned chunks: 1480 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); 1481 * and increase number of bytes to be processed 1482 * in next chunk: 1483 * nbytes += diff; 1484 */ 1485 nbytes += diff; 1486 1487 /* 1488 * Temporary of course... 1489 * Kick author if you will catch this one. 1490 */ 1491 pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", 1492 __func__, dlen, nbytes, slen, offset); 1493 pr_err("%s: please contact author to fix this " 1494 "issue, generally you should not catch " 1495 "this path under any condition but who " 1496 "knows how did you use crypto code.\n" 1497 "Thank you.\n", __func__); 1498 BUG(); 1499 } else { 1500 copy += diff + nbytes; 1501 1502 dst = &req->dst[idx]; 1503 1504 err = skcipher_add(&dlen, dst, nbytes, &nbytes); 1505 if (err < 0) 1506 return err; 1507 1508 idx += err; 1509 } 1510 1511 t->length = copy; 1512 t->offset = offset; 1513 } else { 1514 nbytes -= min(dst->length, nbytes); 1515 idx++; 1516 } 1517 1518 tidx++; 1519 } 1520 1521 return tidx; 1522 } 1523 1524 static int hifn_setup_session(struct skcipher_request *req) 1525 { 1526 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1527 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1528 struct hifn_device *dev = ctx->dev; 1529 unsigned long dlen, flags; 1530 unsigned int nbytes = req->cryptlen, idx = 0; 1531 int err = -EINVAL, sg_num; 1532 struct scatterlist *dst; 1533 1534 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) 1535 goto err_out_exit; 1536 1537 rctx->walk.flags = 0; 1538 1539 while (nbytes) { 1540 dst = &req->dst[idx]; 1541 dlen = min(dst->length, nbytes); 1542 1543 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1544 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) 1545 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; 1546 1547 nbytes -= dlen; 1548 idx++; 1549 } 1550 1551 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1552 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); 1553 if (err < 0) 1554 return err; 1555 } 1556 1557 sg_num = hifn_cipher_walk(req, &rctx->walk); 1558 if (sg_num < 0) { 1559 err = sg_num; 1560 goto err_out_exit; 1561 } 1562 1563 spin_lock_irqsave(&dev->lock, flags); 1564 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { 1565 err = -EAGAIN; 1566 goto err_out; 1567 } 1568 1569 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); 1570 if (err) 1571 goto err_out; 1572 1573 dev->snum++; 1574 1575 dev->active = HIFN_DEFAULT_ACTIVE_NUM; 1576 spin_unlock_irqrestore(&dev->lock, flags); 1577 1578 return 0; 1579 1580 err_out: 1581 spin_unlock_irqrestore(&dev->lock, flags); 1582 err_out_exit: 1583 if (err) { 1584 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " 1585 "type: %u, err: %d.\n", 1586 rctx->iv, rctx->ivsize, 1587 ctx->key, ctx->keysize, 1588 rctx->mode, rctx->op, rctx->type, err); 1589 } 1590 1591 return err; 1592 } 1593 1594 static int hifn_start_device(struct hifn_device *dev) 1595 { 1596 int err; 1597 1598 dev->started = dev->active = 0; 1599 hifn_reset_dma(dev, 1); 1600 1601 err = hifn_enable_crypto(dev); 1602 if (err) 1603 return err; 1604 1605 hifn_reset_puc(dev); 1606 1607 hifn_init_dma(dev); 1608 1609 hifn_init_registers(dev); 1610 1611 hifn_init_pubrng(dev); 1612 1613 return 0; 1614 } 1615 1616 static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, 1617 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) 1618 { 1619 unsigned int srest = *srestp, nbytes = *nbytesp, copy; 1620 void *daddr; 1621 int idx = 0; 1622 1623 if (srest < size || size > nbytes) 1624 return -EINVAL; 1625 1626 while (size) { 1627 copy = min3(srest, dst->length, size); 1628 1629 daddr = kmap_atomic(sg_page(dst)); 1630 memcpy(daddr + dst->offset + offset, saddr, copy); 1631 kunmap_atomic(daddr); 1632 1633 nbytes -= copy; 1634 size -= copy; 1635 srest -= copy; 1636 saddr += copy; 1637 offset = 0; 1638 1639 pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", 1640 __func__, copy, size, srest, nbytes); 1641 1642 dst++; 1643 idx++; 1644 } 1645 1646 *nbytesp = nbytes; 1647 *srestp = srest; 1648 1649 return idx; 1650 } 1651 1652 static inline void hifn_complete_sa(struct hifn_device *dev, int i) 1653 { 1654 unsigned long flags; 1655 1656 spin_lock_irqsave(&dev->lock, flags); 1657 dev->sa[i] = NULL; 1658 dev->started--; 1659 if (dev->started < 0) 1660 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, 1661 dev->started); 1662 spin_unlock_irqrestore(&dev->lock, flags); 1663 BUG_ON(dev->started < 0); 1664 } 1665 1666 static void hifn_process_ready(struct skcipher_request *req, int error) 1667 { 1668 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1669 1670 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1671 unsigned int nbytes = req->cryptlen; 1672 int idx = 0, err; 1673 struct scatterlist *dst, *t; 1674 void *saddr; 1675 1676 while (nbytes) { 1677 t = &rctx->walk.cache[idx]; 1678 dst = &req->dst[idx]; 1679 1680 pr_debug("\n%s: sg_page(t): %p, t->length: %u, " 1681 "sg_page(dst): %p, dst->length: %u, " 1682 "nbytes: %u.\n", 1683 __func__, sg_page(t), t->length, 1684 sg_page(dst), dst->length, nbytes); 1685 1686 if (!t->length) { 1687 nbytes -= min(dst->length, nbytes); 1688 idx++; 1689 continue; 1690 } 1691 1692 saddr = kmap_atomic(sg_page(t)); 1693 1694 err = skcipher_get(saddr, &t->length, t->offset, 1695 dst, nbytes, &nbytes); 1696 if (err < 0) { 1697 kunmap_atomic(saddr); 1698 break; 1699 } 1700 1701 idx += err; 1702 kunmap_atomic(saddr); 1703 } 1704 1705 hifn_cipher_walk_exit(&rctx->walk); 1706 } 1707 1708 skcipher_request_complete(req, error); 1709 } 1710 1711 static void hifn_clear_rings(struct hifn_device *dev, int error) 1712 { 1713 struct hifn_dma *dma = dev->desc_virt; 1714 int i, u; 1715 1716 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1717 "k: %d.%d.%d.%d.\n", 1718 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1719 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1720 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1721 1722 i = dma->resk; u = dma->resu; 1723 while (u != 0) { 1724 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1725 break; 1726 1727 if (dev->sa[i]) { 1728 dev->success++; 1729 dev->reset = 0; 1730 hifn_process_ready(dev->sa[i], error); 1731 hifn_complete_sa(dev, i); 1732 } 1733 1734 if (++i == HIFN_D_RES_RSIZE) 1735 i = 0; 1736 u--; 1737 } 1738 dma->resk = i; dma->resu = u; 1739 1740 i = dma->srck; u = dma->srcu; 1741 while (u != 0) { 1742 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1743 break; 1744 if (++i == HIFN_D_SRC_RSIZE) 1745 i = 0; 1746 u--; 1747 } 1748 dma->srck = i; dma->srcu = u; 1749 1750 i = dma->cmdk; u = dma->cmdu; 1751 while (u != 0) { 1752 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1753 break; 1754 if (++i == HIFN_D_CMD_RSIZE) 1755 i = 0; 1756 u--; 1757 } 1758 dma->cmdk = i; dma->cmdu = u; 1759 1760 i = dma->dstk; u = dma->dstu; 1761 while (u != 0) { 1762 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1763 break; 1764 if (++i == HIFN_D_DST_RSIZE) 1765 i = 0; 1766 u--; 1767 } 1768 dma->dstk = i; dma->dstu = u; 1769 1770 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1771 "k: %d.%d.%d.%d.\n", 1772 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1773 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1774 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1775 } 1776 1777 static void hifn_work(struct work_struct *work) 1778 { 1779 struct delayed_work *dw = to_delayed_work(work); 1780 struct hifn_device *dev = container_of(dw, struct hifn_device, work); 1781 unsigned long flags; 1782 int reset = 0; 1783 u32 r = 0; 1784 1785 spin_lock_irqsave(&dev->lock, flags); 1786 if (dev->active == 0) { 1787 struct hifn_dma *dma = dev->desc_virt; 1788 1789 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { 1790 dev->flags &= ~HIFN_FLAG_CMD_BUSY; 1791 r |= HIFN_DMACSR_C_CTRL_DIS; 1792 } 1793 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { 1794 dev->flags &= ~HIFN_FLAG_SRC_BUSY; 1795 r |= HIFN_DMACSR_S_CTRL_DIS; 1796 } 1797 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { 1798 dev->flags &= ~HIFN_FLAG_DST_BUSY; 1799 r |= HIFN_DMACSR_D_CTRL_DIS; 1800 } 1801 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { 1802 dev->flags &= ~HIFN_FLAG_RES_BUSY; 1803 r |= HIFN_DMACSR_R_CTRL_DIS; 1804 } 1805 if (r) 1806 hifn_write_1(dev, HIFN_1_DMA_CSR, r); 1807 } else 1808 dev->active--; 1809 1810 if ((dev->prev_success == dev->success) && dev->started) 1811 reset = 1; 1812 dev->prev_success = dev->success; 1813 spin_unlock_irqrestore(&dev->lock, flags); 1814 1815 if (reset) { 1816 if (++dev->reset >= 5) { 1817 int i; 1818 struct hifn_dma *dma = dev->desc_virt; 1819 1820 dev_info(&dev->pdev->dev, 1821 "r: %08x, active: %d, started: %d, " 1822 "success: %lu: qlen: %u/%u, reset: %d.\n", 1823 r, dev->active, dev->started, 1824 dev->success, dev->queue.qlen, dev->queue.max_qlen, 1825 reset); 1826 1827 dev_info(&dev->pdev->dev, "%s: res: ", __func__); 1828 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1829 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); 1830 if (dev->sa[i]) { 1831 hifn_process_ready(dev->sa[i], -ENODEV); 1832 hifn_complete_sa(dev, i); 1833 } 1834 } 1835 pr_info("\n"); 1836 1837 hifn_reset_dma(dev, 1); 1838 hifn_stop_device(dev); 1839 hifn_start_device(dev); 1840 dev->reset = 0; 1841 } 1842 1843 tasklet_schedule(&dev->tasklet); 1844 } 1845 1846 schedule_delayed_work(&dev->work, HZ); 1847 } 1848 1849 static irqreturn_t hifn_interrupt(int irq, void *data) 1850 { 1851 struct hifn_device *dev = data; 1852 struct hifn_dma *dma = dev->desc_virt; 1853 u32 dmacsr, restart; 1854 1855 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); 1856 1857 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " 1858 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", 1859 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, 1860 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1861 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1862 1863 if ((dmacsr & dev->dmareg) == 0) 1864 return IRQ_NONE; 1865 1866 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); 1867 1868 if (dmacsr & HIFN_DMACSR_ENGINE) 1869 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR)); 1870 if (dmacsr & HIFN_DMACSR_PUBDONE) 1871 hifn_write_1(dev, HIFN_1_PUB_STATUS, 1872 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1873 1874 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1875 if (restart) { 1876 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); 1877 1878 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", 1879 !!(dmacsr & HIFN_DMACSR_R_OVER), 1880 !!(dmacsr & HIFN_DMACSR_D_OVER), 1881 puisr, !!(puisr & HIFN_PUISR_DSTOVER)); 1882 if (!!(puisr & HIFN_PUISR_DSTOVER)) 1883 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1884 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | 1885 HIFN_DMACSR_D_OVER)); 1886 } 1887 1888 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1889 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1890 if (restart) { 1891 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", 1892 !!(dmacsr & HIFN_DMACSR_C_ABORT), 1893 !!(dmacsr & HIFN_DMACSR_S_ABORT), 1894 !!(dmacsr & HIFN_DMACSR_D_ABORT), 1895 !!(dmacsr & HIFN_DMACSR_R_ABORT)); 1896 hifn_reset_dma(dev, 1); 1897 hifn_init_dma(dev); 1898 hifn_init_registers(dev); 1899 } 1900 1901 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { 1902 dev_dbg(&dev->pdev->dev, "wait on command.\n"); 1903 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); 1904 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1905 } 1906 1907 tasklet_schedule(&dev->tasklet); 1908 1909 return IRQ_HANDLED; 1910 } 1911 1912 static void hifn_flush(struct hifn_device *dev) 1913 { 1914 unsigned long flags; 1915 struct crypto_async_request *async_req; 1916 struct skcipher_request *req; 1917 struct hifn_dma *dma = dev->desc_virt; 1918 int i; 1919 1920 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1921 struct hifn_desc *d = &dma->resr[i]; 1922 1923 if (dev->sa[i]) { 1924 hifn_process_ready(dev->sa[i], 1925 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); 1926 hifn_complete_sa(dev, i); 1927 } 1928 } 1929 1930 spin_lock_irqsave(&dev->lock, flags); 1931 while ((async_req = crypto_dequeue_request(&dev->queue))) { 1932 req = skcipher_request_cast(async_req); 1933 spin_unlock_irqrestore(&dev->lock, flags); 1934 1935 hifn_process_ready(req, -ENODEV); 1936 1937 spin_lock_irqsave(&dev->lock, flags); 1938 } 1939 spin_unlock_irqrestore(&dev->lock, flags); 1940 } 1941 1942 static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, 1943 unsigned int len) 1944 { 1945 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1946 struct hifn_device *dev = ctx->dev; 1947 int err; 1948 1949 err = verify_skcipher_des_key(cipher, key); 1950 if (err) 1951 return err; 1952 1953 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1954 1955 memcpy(ctx->key, key, len); 1956 ctx->keysize = len; 1957 1958 return 0; 1959 } 1960 1961 static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 1962 unsigned int len) 1963 { 1964 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1965 struct hifn_device *dev = ctx->dev; 1966 int err; 1967 1968 err = verify_skcipher_des3_key(cipher, key); 1969 if (err) 1970 return err; 1971 1972 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1973 1974 memcpy(ctx->key, key, len); 1975 ctx->keysize = len; 1976 1977 return 0; 1978 } 1979 1980 static int hifn_handle_req(struct skcipher_request *req) 1981 { 1982 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1983 struct hifn_device *dev = ctx->dev; 1984 int err = -EAGAIN; 1985 1986 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) 1987 err = hifn_setup_session(req); 1988 1989 if (err == -EAGAIN) { 1990 unsigned long flags; 1991 1992 spin_lock_irqsave(&dev->lock, flags); 1993 err = crypto_enqueue_request(&dev->queue, &req->base); 1994 spin_unlock_irqrestore(&dev->lock, flags); 1995 } 1996 1997 return err; 1998 } 1999 2000 static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, 2001 u8 type, u8 mode) 2002 { 2003 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2004 struct hifn_request_context *rctx = skcipher_request_ctx(req); 2005 unsigned ivsize; 2006 2007 ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); 2008 2009 if (req->iv && mode != ACRYPTO_MODE_ECB) { 2010 if (type == ACRYPTO_TYPE_AES_128) 2011 ivsize = HIFN_AES_IV_LENGTH; 2012 else if (type == ACRYPTO_TYPE_DES) 2013 ivsize = HIFN_DES_KEY_LENGTH; 2014 else if (type == ACRYPTO_TYPE_3DES) 2015 ivsize = HIFN_3DES_KEY_LENGTH; 2016 } 2017 2018 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { 2019 if (ctx->keysize == 24) 2020 type = ACRYPTO_TYPE_AES_192; 2021 else if (ctx->keysize == 32) 2022 type = ACRYPTO_TYPE_AES_256; 2023 } 2024 2025 rctx->op = op; 2026 rctx->mode = mode; 2027 rctx->type = type; 2028 rctx->iv = req->iv; 2029 rctx->ivsize = ivsize; 2030 2031 /* 2032 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2033 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2034 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2035 */ 2036 2037 return hifn_handle_req(req); 2038 } 2039 2040 static int hifn_process_queue(struct hifn_device *dev) 2041 { 2042 struct crypto_async_request *async_req, *backlog; 2043 struct skcipher_request *req; 2044 unsigned long flags; 2045 int err = 0; 2046 2047 while (dev->started < HIFN_QUEUE_LENGTH) { 2048 spin_lock_irqsave(&dev->lock, flags); 2049 backlog = crypto_get_backlog(&dev->queue); 2050 async_req = crypto_dequeue_request(&dev->queue); 2051 spin_unlock_irqrestore(&dev->lock, flags); 2052 2053 if (!async_req) 2054 break; 2055 2056 if (backlog) 2057 crypto_request_complete(backlog, -EINPROGRESS); 2058 2059 req = skcipher_request_cast(async_req); 2060 2061 err = hifn_handle_req(req); 2062 if (err) 2063 break; 2064 } 2065 2066 return err; 2067 } 2068 2069 static int hifn_setup_crypto(struct skcipher_request *req, u8 op, 2070 u8 type, u8 mode) 2071 { 2072 int err; 2073 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2074 struct hifn_device *dev = ctx->dev; 2075 2076 err = hifn_setup_crypto_req(req, op, type, mode); 2077 if (err) 2078 return err; 2079 2080 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2081 hifn_process_queue(dev); 2082 2083 return -EINPROGRESS; 2084 } 2085 2086 /* 2087 * AES ecryption functions. 2088 */ 2089 static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) 2090 { 2091 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2092 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2093 } 2094 static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) 2095 { 2096 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2097 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2098 } 2099 2100 /* 2101 * AES decryption functions. 2102 */ 2103 static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) 2104 { 2105 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2106 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2107 } 2108 static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) 2109 { 2110 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2111 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2112 } 2113 2114 /* 2115 * DES ecryption functions. 2116 */ 2117 static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) 2118 { 2119 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2120 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2121 } 2122 static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) 2123 { 2124 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2125 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2126 } 2127 2128 /* 2129 * DES decryption functions. 2130 */ 2131 static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) 2132 { 2133 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2134 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2135 } 2136 static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) 2137 { 2138 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2139 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2140 } 2141 2142 /* 2143 * 3DES ecryption functions. 2144 */ 2145 static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) 2146 { 2147 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2148 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2149 } 2150 static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) 2151 { 2152 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2153 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2154 } 2155 2156 /* 3DES decryption functions. */ 2157 static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) 2158 { 2159 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2160 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2161 } 2162 static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) 2163 { 2164 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2165 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2166 } 2167 2168 struct hifn_alg_template { 2169 char name[CRYPTO_MAX_ALG_NAME]; 2170 char drv_name[CRYPTO_MAX_ALG_NAME]; 2171 unsigned int bsize; 2172 struct skcipher_alg skcipher; 2173 }; 2174 2175 static const struct hifn_alg_template hifn_alg_templates[] = { 2176 /* 2177 * 3DES ECB and CBC modes. 2178 */ 2179 { 2180 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, 2181 .skcipher = { 2182 .ivsize = HIFN_IV_LENGTH, 2183 .min_keysize = HIFN_3DES_KEY_LENGTH, 2184 .max_keysize = HIFN_3DES_KEY_LENGTH, 2185 .setkey = hifn_des3_setkey, 2186 .encrypt = hifn_encrypt_3des_cbc, 2187 .decrypt = hifn_decrypt_3des_cbc, 2188 }, 2189 }, 2190 { 2191 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, 2192 .skcipher = { 2193 .min_keysize = HIFN_3DES_KEY_LENGTH, 2194 .max_keysize = HIFN_3DES_KEY_LENGTH, 2195 .setkey = hifn_des3_setkey, 2196 .encrypt = hifn_encrypt_3des_ecb, 2197 .decrypt = hifn_decrypt_3des_ecb, 2198 }, 2199 }, 2200 2201 /* 2202 * DES ECB and CBC modes. 2203 */ 2204 { 2205 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, 2206 .skcipher = { 2207 .ivsize = HIFN_IV_LENGTH, 2208 .min_keysize = HIFN_DES_KEY_LENGTH, 2209 .max_keysize = HIFN_DES_KEY_LENGTH, 2210 .setkey = hifn_setkey, 2211 .encrypt = hifn_encrypt_des_cbc, 2212 .decrypt = hifn_decrypt_des_cbc, 2213 }, 2214 }, 2215 { 2216 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, 2217 .skcipher = { 2218 .min_keysize = HIFN_DES_KEY_LENGTH, 2219 .max_keysize = HIFN_DES_KEY_LENGTH, 2220 .setkey = hifn_setkey, 2221 .encrypt = hifn_encrypt_des_ecb, 2222 .decrypt = hifn_decrypt_des_ecb, 2223 }, 2224 }, 2225 2226 /* 2227 * AES ECB and CBC modes. 2228 */ 2229 { 2230 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, 2231 .skcipher = { 2232 .min_keysize = AES_MIN_KEY_SIZE, 2233 .max_keysize = AES_MAX_KEY_SIZE, 2234 .setkey = hifn_setkey, 2235 .encrypt = hifn_encrypt_aes_ecb, 2236 .decrypt = hifn_decrypt_aes_ecb, 2237 }, 2238 }, 2239 { 2240 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, 2241 .skcipher = { 2242 .ivsize = HIFN_AES_IV_LENGTH, 2243 .min_keysize = AES_MIN_KEY_SIZE, 2244 .max_keysize = AES_MAX_KEY_SIZE, 2245 .setkey = hifn_setkey, 2246 .encrypt = hifn_encrypt_aes_cbc, 2247 .decrypt = hifn_decrypt_aes_cbc, 2248 }, 2249 }, 2250 }; 2251 2252 static int hifn_init_tfm(struct crypto_skcipher *tfm) 2253 { 2254 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2255 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg); 2256 struct hifn_context *ctx = crypto_skcipher_ctx(tfm); 2257 2258 ctx->dev = ha->dev; 2259 crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context)); 2260 2261 return 0; 2262 } 2263 2264 static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) 2265 { 2266 struct hifn_crypto_alg *alg; 2267 int err; 2268 2269 alg = kzalloc(sizeof(*alg), GFP_KERNEL); 2270 if (!alg) 2271 return -ENOMEM; 2272 2273 alg->alg = t->skcipher; 2274 alg->alg.init = hifn_init_tfm; 2275 2276 err = -EINVAL; 2277 if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 2278 "%s", t->name) >= CRYPTO_MAX_ALG_NAME) 2279 goto out_free_alg; 2280 if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 2281 "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME) 2282 goto out_free_alg; 2283 2284 alg->alg.base.cra_priority = 300; 2285 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 2286 alg->alg.base.cra_blocksize = t->bsize; 2287 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); 2288 alg->alg.base.cra_alignmask = 0; 2289 alg->alg.base.cra_module = THIS_MODULE; 2290 2291 alg->dev = dev; 2292 2293 list_add_tail(&alg->entry, &dev->alg_list); 2294 2295 err = crypto_register_skcipher(&alg->alg); 2296 if (err) { 2297 list_del(&alg->entry); 2298 out_free_alg: 2299 kfree(alg); 2300 } 2301 2302 return err; 2303 } 2304 2305 static void hifn_unregister_alg(struct hifn_device *dev) 2306 { 2307 struct hifn_crypto_alg *a, *n; 2308 2309 list_for_each_entry_safe(a, n, &dev->alg_list, entry) { 2310 list_del(&a->entry); 2311 crypto_unregister_skcipher(&a->alg); 2312 kfree(a); 2313 } 2314 } 2315 2316 static int hifn_register_alg(struct hifn_device *dev) 2317 { 2318 int i, err; 2319 2320 for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { 2321 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); 2322 if (err) 2323 goto err_out_exit; 2324 } 2325 2326 return 0; 2327 2328 err_out_exit: 2329 hifn_unregister_alg(dev); 2330 return err; 2331 } 2332 2333 static void hifn_tasklet_callback(unsigned long data) 2334 { 2335 struct hifn_device *dev = (struct hifn_device *)data; 2336 2337 /* 2338 * This is ok to call this without lock being held, 2339 * althogh it modifies some parameters used in parallel, 2340 * (like dev->success), but they are used in process 2341 * context or update is atomic (like setting dev->sa[i] to NULL). 2342 */ 2343 hifn_clear_rings(dev, 0); 2344 2345 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2346 hifn_process_queue(dev); 2347 } 2348 2349 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2350 { 2351 int err, i; 2352 struct hifn_device *dev; 2353 char name[8]; 2354 2355 err = pci_enable_device(pdev); 2356 if (err) 2357 return err; 2358 pci_set_master(pdev); 2359 2360 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 2361 if (err) 2362 goto err_out_disable_pci_device; 2363 2364 snprintf(name, sizeof(name), "hifn%d", 2365 atomic_inc_return(&hifn_dev_number) - 1); 2366 2367 err = pci_request_regions(pdev, name); 2368 if (err) 2369 goto err_out_disable_pci_device; 2370 2371 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || 2372 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || 2373 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { 2374 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); 2375 err = -ENODEV; 2376 goto err_out_free_regions; 2377 } 2378 2379 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg), 2380 GFP_KERNEL); 2381 if (!dev) { 2382 err = -ENOMEM; 2383 goto err_out_free_regions; 2384 } 2385 2386 INIT_LIST_HEAD(&dev->alg_list); 2387 2388 snprintf(dev->name, sizeof(dev->name), "%s", name); 2389 spin_lock_init(&dev->lock); 2390 2391 for (i = 0; i < 3; ++i) { 2392 unsigned long addr, size; 2393 2394 addr = pci_resource_start(pdev, i); 2395 size = pci_resource_len(pdev, i); 2396 2397 dev->bar[i] = ioremap(addr, size); 2398 if (!dev->bar[i]) { 2399 err = -ENOMEM; 2400 goto err_out_unmap_bars; 2401 } 2402 } 2403 2404 dev->desc_virt = dma_alloc_coherent(&pdev->dev, 2405 sizeof(struct hifn_dma), 2406 &dev->desc_dma, GFP_KERNEL); 2407 if (!dev->desc_virt) { 2408 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); 2409 err = -ENOMEM; 2410 goto err_out_unmap_bars; 2411 } 2412 2413 dev->pdev = pdev; 2414 dev->irq = pdev->irq; 2415 2416 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 2417 dev->sa[i] = NULL; 2418 2419 pci_set_drvdata(pdev, dev); 2420 2421 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); 2422 2423 crypto_init_queue(&dev->queue, 1); 2424 2425 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); 2426 if (err) { 2427 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", 2428 dev->irq, err); 2429 dev->irq = 0; 2430 goto err_out_free_desc; 2431 } 2432 2433 err = hifn_start_device(dev); 2434 if (err) 2435 goto err_out_free_irq; 2436 2437 err = hifn_register_rng(dev); 2438 if (err) 2439 goto err_out_stop_device; 2440 2441 err = hifn_register_alg(dev); 2442 if (err) 2443 goto err_out_unregister_rng; 2444 2445 INIT_DELAYED_WORK(&dev->work, hifn_work); 2446 schedule_delayed_work(&dev->work, HZ); 2447 2448 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " 2449 "successfully registered as %s.\n", 2450 pci_name(pdev), dev->name); 2451 2452 return 0; 2453 2454 err_out_unregister_rng: 2455 hifn_unregister_rng(dev); 2456 err_out_stop_device: 2457 hifn_reset_dma(dev, 1); 2458 hifn_stop_device(dev); 2459 err_out_free_irq: 2460 free_irq(dev->irq, dev); 2461 tasklet_kill(&dev->tasklet); 2462 err_out_free_desc: 2463 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, 2464 dev->desc_dma); 2465 2466 err_out_unmap_bars: 2467 for (i = 0; i < 3; ++i) 2468 if (dev->bar[i]) 2469 iounmap(dev->bar[i]); 2470 kfree(dev); 2471 2472 err_out_free_regions: 2473 pci_release_regions(pdev); 2474 2475 err_out_disable_pci_device: 2476 pci_disable_device(pdev); 2477 2478 return err; 2479 } 2480 2481 static void hifn_remove(struct pci_dev *pdev) 2482 { 2483 int i; 2484 struct hifn_device *dev; 2485 2486 dev = pci_get_drvdata(pdev); 2487 2488 if (dev) { 2489 cancel_delayed_work_sync(&dev->work); 2490 2491 hifn_unregister_rng(dev); 2492 hifn_unregister_alg(dev); 2493 hifn_reset_dma(dev, 1); 2494 hifn_stop_device(dev); 2495 2496 free_irq(dev->irq, dev); 2497 tasklet_kill(&dev->tasklet); 2498 2499 hifn_flush(dev); 2500 2501 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), 2502 dev->desc_virt, dev->desc_dma); 2503 for (i = 0; i < 3; ++i) 2504 if (dev->bar[i]) 2505 iounmap(dev->bar[i]); 2506 2507 kfree(dev); 2508 } 2509 2510 pci_release_regions(pdev); 2511 pci_disable_device(pdev); 2512 } 2513 2514 static struct pci_device_id hifn_pci_tbl[] = { 2515 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) }, 2516 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) }, 2517 { 0 } 2518 }; 2519 MODULE_DEVICE_TABLE(pci, hifn_pci_tbl); 2520 2521 static struct pci_driver hifn_pci_driver = { 2522 .name = "hifn795x", 2523 .id_table = hifn_pci_tbl, 2524 .probe = hifn_probe, 2525 .remove = hifn_remove, 2526 }; 2527 2528 static int __init hifn_init(void) 2529 { 2530 unsigned int freq; 2531 int err; 2532 2533 if (strncmp(hifn_pll_ref, "ext", 3) && 2534 strncmp(hifn_pll_ref, "pci", 3)) { 2535 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); 2536 return -EINVAL; 2537 } 2538 2539 /* 2540 * For the 7955/7956 the reference clock frequency must be in the 2541 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, 2542 * but this chip is currently not supported. 2543 */ 2544 if (hifn_pll_ref[3] != '\0') { 2545 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 2546 if (freq < 20 || freq > 100) { 2547 pr_err("hifn795x: invalid hifn_pll_ref frequency, must" 2548 "be in the range of 20-100"); 2549 return -EINVAL; 2550 } 2551 } 2552 2553 err = pci_register_driver(&hifn_pci_driver); 2554 if (err < 0) { 2555 pr_err("Failed to register PCI driver for %s device.\n", 2556 hifn_pci_driver.name); 2557 return -ENODEV; 2558 } 2559 2560 pr_info("Driver for HIFN 795x crypto accelerator chip " 2561 "has been successfully registered.\n"); 2562 2563 return 0; 2564 } 2565 2566 static void __exit hifn_fini(void) 2567 { 2568 pci_unregister_driver(&hifn_pci_driver); 2569 2570 pr_info("Driver for HIFN 795x crypto accelerator chip " 2571 "has been successfully unregistered.\n"); 2572 } 2573 2574 module_init(hifn_init); 2575 module_exit(hifn_fini); 2576 2577 MODULE_LICENSE("GPL"); 2578 MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>"); 2579 MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip."); 2580