1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 4 * All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/moduleparam.h> 10 #include <linux/mod_devicetable.h> 11 #include <linux/interrupt.h> 12 #include <linux/pci.h> 13 #include <linux/slab.h> 14 #include <linux/delay.h> 15 #include <linux/mm.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/scatterlist.h> 18 #include <linux/highmem.h> 19 #include <linux/crypto.h> 20 #include <linux/hw_random.h> 21 #include <linux/ktime.h> 22 23 #include <crypto/algapi.h> 24 #include <crypto/internal/des.h> 25 #include <crypto/internal/skcipher.h> 26 27 static char hifn_pll_ref[sizeof("extNNN")] = "ext"; 28 module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); 29 MODULE_PARM_DESC(hifn_pll_ref, 30 "PLL reference clock (pci[freq] or ext[freq], default ext)"); 31 32 static atomic_t hifn_dev_number; 33 34 #define ACRYPTO_OP_DECRYPT 0 35 #define ACRYPTO_OP_ENCRYPT 1 36 #define ACRYPTO_OP_HMAC 2 37 #define ACRYPTO_OP_RNG 3 38 39 #define ACRYPTO_MODE_ECB 0 40 #define ACRYPTO_MODE_CBC 1 41 #define ACRYPTO_MODE_CFB 2 42 #define ACRYPTO_MODE_OFB 3 43 44 #define ACRYPTO_TYPE_AES_128 0 45 #define ACRYPTO_TYPE_AES_192 1 46 #define ACRYPTO_TYPE_AES_256 2 47 #define ACRYPTO_TYPE_3DES 3 48 #define ACRYPTO_TYPE_DES 4 49 50 #define PCI_VENDOR_ID_HIFN 0x13A3 51 #define PCI_DEVICE_ID_HIFN_7955 0x0020 52 #define PCI_DEVICE_ID_HIFN_7956 0x001d 53 54 /* I/O region sizes */ 55 56 #define HIFN_BAR0_SIZE 0x1000 57 #define HIFN_BAR1_SIZE 0x2000 58 #define HIFN_BAR2_SIZE 0x8000 59 60 /* DMA registres */ 61 62 #define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ 63 #define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ 64 #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ 65 #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ 66 #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ 67 #define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ 68 #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ 69 #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ 70 #define HIFN_CHIP_ID 0x98 /* Chip ID */ 71 72 /* 73 * Processing Unit Registers (offset from BASEREG0) 74 */ 75 #define HIFN_0_PUDATA 0x00 /* Processing Unit Data */ 76 #define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */ 77 #define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */ 78 #define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */ 79 #define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */ 80 #define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */ 81 #define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */ 82 #define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */ 83 #define HIFN_0_SPACESIZE 0x20 /* Register space size */ 84 85 /* Processing Unit Control Register (HIFN_0_PUCTRL) */ 86 #define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */ 87 #define HIFN_PUCTRL_STOP 0x0008 /* stop pu */ 88 #define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */ 89 #define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */ 90 #define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */ 91 92 /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */ 93 #define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */ 94 #define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */ 95 #define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 96 #define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 97 #define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */ 98 #define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */ 99 #define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */ 100 #define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */ 101 #define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */ 102 #define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */ 103 104 /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */ 105 #define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */ 106 #define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */ 107 #define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */ 108 #define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */ 109 #define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */ 110 #define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */ 111 #define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */ 112 #define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */ 113 #define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */ 114 #define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */ 115 #define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */ 116 #define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */ 117 #define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */ 118 #define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */ 119 #define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */ 120 #define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */ 121 #define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */ 122 #define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */ 123 #define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */ 124 #define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */ 125 #define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */ 126 #define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */ 127 #define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */ 128 129 /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */ 130 #define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */ 131 #define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */ 132 #define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 133 #define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 134 #define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */ 135 #define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */ 136 #define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */ 137 #define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */ 138 #define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */ 139 #define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */ 140 141 /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */ 142 #define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */ 143 #define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */ 144 #define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 145 #define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 146 #define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */ 147 #define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */ 148 #define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */ 149 #define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */ 150 #define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */ 151 #define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */ 152 #define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */ 153 #define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */ 154 #define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */ 155 #define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */ 156 #define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */ 157 #define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */ 158 #define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */ 159 160 /* FIFO Status Register (HIFN_0_FIFOSTAT) */ 161 #define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */ 162 #define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */ 163 164 /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */ 165 #define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */ 166 167 /* 168 * DMA Interface Registers (offset from BASEREG1) 169 */ 170 #define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */ 171 #define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */ 172 #define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */ 173 #define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */ 174 #define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */ 175 #define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */ 176 #define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */ 177 #define HIFN_1_PLL 0x4c /* 795x: PLL config */ 178 #define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */ 179 #define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */ 180 #define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */ 181 #define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */ 182 #define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */ 183 #define HIFN_1_REVID 0x98 /* Revision ID */ 184 #define HIFN_1_UNLOCK_SECRET1 0xf4 185 #define HIFN_1_UNLOCK_SECRET2 0xfc 186 #define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */ 187 #define HIFN_1_PUB_BASE 0x300 /* Public Base Address */ 188 #define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */ 189 #define HIFN_1_PUB_OP 0x308 /* Public Operand */ 190 #define HIFN_1_PUB_STATUS 0x30c /* Public Status */ 191 #define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */ 192 #define HIFN_1_RNG_CONFIG 0x314 /* RNG config */ 193 #define HIFN_1_RNG_DATA 0x318 /* RNG data */ 194 #define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */ 195 #define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */ 196 197 /* DMA Status and Control Register (HIFN_1_DMA_CSR) */ 198 #define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */ 199 #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */ 200 #define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */ 201 #define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */ 202 #define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */ 203 #define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */ 204 #define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */ 205 #define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */ 206 #define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */ 207 #define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */ 208 #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */ 209 #define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */ 210 #define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */ 211 #define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 212 #define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */ 213 #define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */ 214 #define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */ 215 #define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */ 216 #define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */ 217 #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */ 218 #define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */ 219 #define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */ 220 #define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 221 #define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */ 222 #define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */ 223 #define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */ 224 #define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */ 225 #define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */ 226 #define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */ 227 #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */ 228 #define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */ 229 #define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */ 230 #define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 231 #define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */ 232 #define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */ 233 #define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */ 234 #define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */ 235 #define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */ 236 237 /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */ 238 #define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */ 239 #define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */ 240 #define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */ 241 #define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */ 242 #define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */ 243 #define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 244 #define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */ 245 #define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */ 246 #define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */ 247 #define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */ 248 #define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 249 #define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */ 250 #define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */ 251 #define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */ 252 #define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */ 253 #define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */ 254 #define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 255 #define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */ 256 #define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */ 257 #define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */ 258 #define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */ 259 #define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */ 260 261 /* DMA Configuration Register (HIFN_1_DMA_CNFG) */ 262 #define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */ 263 #define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */ 264 #define HIFN_DMACNFG_UNLOCK 0x00000800 265 #define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */ 266 #define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */ 267 #define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */ 268 #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ 269 #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ 270 271 /* PLL configuration register */ 272 #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ 273 #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ 274 #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ 275 #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ 276 #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ 277 #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ 278 #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ 279 #define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ 280 #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ 281 #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ 282 #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ 283 #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ 284 #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ 285 #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ 286 #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ 287 #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ 288 #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ 289 290 #define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ 291 292 /* Public key reset register (HIFN_1_PUB_RESET) */ 293 #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ 294 295 /* Public base address register (HIFN_1_PUB_BASE) */ 296 #define HIFN_PUBBASE_ADDR 0x00003fff /* base address */ 297 298 /* Public operand length register (HIFN_1_PUB_OPLEN) */ 299 #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ 300 #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ 301 #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ 302 #define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ 303 #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ 304 #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ 305 306 /* Public operation register (HIFN_1_PUB_OP) */ 307 #define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */ 308 #define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */ 309 #define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */ 310 #define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */ 311 #define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */ 312 #define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */ 313 #define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */ 314 #define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */ 315 #define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */ 316 #define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */ 317 #define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */ 318 #define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */ 319 #define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */ 320 #define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */ 321 #define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */ 322 #define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */ 323 #define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */ 324 #define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */ 325 #define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */ 326 #define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */ 327 328 /* Public status register (HIFN_1_PUB_STATUS) */ 329 #define HIFN_PUBSTS_DONE 0x00000001 /* operation done */ 330 #define HIFN_PUBSTS_CARRY 0x00000002 /* carry */ 331 332 /* Public interrupt enable register (HIFN_1_PUB_IEN) */ 333 #define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */ 334 335 /* Random number generator config register (HIFN_1_RNG_CONFIG) */ 336 #define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */ 337 338 #define HIFN_NAMESIZE 32 339 #define HIFN_MAX_RESULT_ORDER 5 340 341 #define HIFN_D_CMD_RSIZE (24 * 1) 342 #define HIFN_D_SRC_RSIZE (80 * 1) 343 #define HIFN_D_DST_RSIZE (80 * 1) 344 #define HIFN_D_RES_RSIZE (24 * 1) 345 346 #define HIFN_D_DST_DALIGN 4 347 348 #define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) 349 350 #define AES_MIN_KEY_SIZE 16 351 #define AES_MAX_KEY_SIZE 32 352 353 #define HIFN_DES_KEY_LENGTH 8 354 #define HIFN_3DES_KEY_LENGTH 24 355 #define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE 356 #define HIFN_IV_LENGTH 8 357 #define HIFN_AES_IV_LENGTH 16 358 #define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH 359 360 #define HIFN_MAC_KEY_LENGTH 64 361 #define HIFN_MD5_LENGTH 16 362 #define HIFN_SHA1_LENGTH 20 363 #define HIFN_MAC_TRUNC_LENGTH 12 364 365 #define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260) 366 #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) 367 #define HIFN_USED_RESULT 12 368 369 struct hifn_desc { 370 volatile __le32 l; 371 volatile __le32 p; 372 }; 373 374 struct hifn_dma { 375 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; 376 struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; 377 struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; 378 struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; 379 380 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; 381 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; 382 383 /* 384 * Our current positions for insertion and removal from the descriptor 385 * rings. 386 */ 387 volatile int cmdi, srci, dsti, resi; 388 volatile int cmdu, srcu, dstu, resu; 389 int cmdk, srck, dstk, resk; 390 }; 391 392 #define HIFN_FLAG_CMD_BUSY (1 << 0) 393 #define HIFN_FLAG_SRC_BUSY (1 << 1) 394 #define HIFN_FLAG_DST_BUSY (1 << 2) 395 #define HIFN_FLAG_RES_BUSY (1 << 3) 396 #define HIFN_FLAG_OLD_KEY (1 << 4) 397 398 #define HIFN_DEFAULT_ACTIVE_NUM 5 399 400 struct hifn_device { 401 char name[HIFN_NAMESIZE]; 402 403 int irq; 404 405 struct pci_dev *pdev; 406 void __iomem *bar[3]; 407 408 void *desc_virt; 409 dma_addr_t desc_dma; 410 411 u32 dmareg; 412 413 void *sa[HIFN_D_RES_RSIZE]; 414 415 spinlock_t lock; 416 417 u32 flags; 418 int active, started; 419 struct delayed_work work; 420 unsigned long reset; 421 unsigned long success; 422 unsigned long prev_success; 423 424 u8 snum; 425 426 struct tasklet_struct tasklet; 427 428 struct crypto_queue queue; 429 struct list_head alg_list; 430 431 unsigned int pk_clk_freq; 432 433 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 434 unsigned int rng_wait_time; 435 ktime_t rngtime; 436 struct hwrng rng; 437 #endif 438 }; 439 440 #define HIFN_D_LENGTH 0x0000ffff 441 #define HIFN_D_NOINVALID 0x01000000 442 #define HIFN_D_MASKDONEIRQ 0x02000000 443 #define HIFN_D_DESTOVER 0x04000000 444 #define HIFN_D_OVER 0x08000000 445 #define HIFN_D_LAST 0x20000000 446 #define HIFN_D_JUMP 0x40000000 447 #define HIFN_D_VALID 0x80000000 448 449 struct hifn_base_command { 450 volatile __le16 masks; 451 volatile __le16 session_num; 452 volatile __le16 total_source_count; 453 volatile __le16 total_dest_count; 454 }; 455 456 #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ 457 #define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */ 458 #define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */ 459 #define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */ 460 #define HIFN_BASE_CMD_DECODE 0x2000 461 #define HIFN_BASE_CMD_SRCLEN_M 0xc000 462 #define HIFN_BASE_CMD_SRCLEN_S 14 463 #define HIFN_BASE_CMD_DSTLEN_M 0x3000 464 #define HIFN_BASE_CMD_DSTLEN_S 12 465 #define HIFN_BASE_CMD_LENMASK_HI 0x30000 466 #define HIFN_BASE_CMD_LENMASK_LO 0x0ffff 467 468 /* 469 * Structure to help build up the command data structure. 470 */ 471 struct hifn_crypt_command { 472 volatile __le16 masks; 473 volatile __le16 header_skip; 474 volatile __le16 source_count; 475 volatile __le16 reserved; 476 }; 477 478 #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ 479 #define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */ 480 #define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */ 481 #define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */ 482 #define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */ 483 #define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */ 484 #define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */ 485 #define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */ 486 #define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */ 487 #define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */ 488 #define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */ 489 #define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */ 490 #define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */ 491 #define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */ 492 #define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */ 493 #define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */ 494 #define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */ 495 #define HIFN_CRYPT_CMD_SRCLEN_M 0xc000 496 #define HIFN_CRYPT_CMD_SRCLEN_S 14 497 498 #define HIFN_MAC_CMD_ALG_MASK 0x0001 499 #define HIFN_MAC_CMD_ALG_SHA1 0x0000 500 #define HIFN_MAC_CMD_ALG_MD5 0x0001 501 #define HIFN_MAC_CMD_MODE_MASK 0x000c 502 #define HIFN_MAC_CMD_MODE_HMAC 0x0000 503 #define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004 504 #define HIFN_MAC_CMD_MODE_HASH 0x0008 505 #define HIFN_MAC_CMD_MODE_FULL 0x0004 506 #define HIFN_MAC_CMD_TRUNC 0x0010 507 #define HIFN_MAC_CMD_RESULT 0x0020 508 #define HIFN_MAC_CMD_APPEND 0x0040 509 #define HIFN_MAC_CMD_SRCLEN_M 0xc000 510 #define HIFN_MAC_CMD_SRCLEN_S 14 511 512 /* 513 * MAC POS IPsec initiates authentication after encryption on encodes 514 * and before decryption on decodes. 515 */ 516 #define HIFN_MAC_CMD_POS_IPSEC 0x0200 517 #define HIFN_MAC_CMD_NEW_KEY 0x0800 518 519 #define HIFN_COMP_CMD_SRCLEN_M 0xc000 520 #define HIFN_COMP_CMD_SRCLEN_S 14 521 #define HIFN_COMP_CMD_ONE 0x0100 /* must be one */ 522 #define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */ 523 #define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */ 524 #define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */ 525 #define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */ 526 #define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */ 527 #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ 528 #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ 529 530 struct hifn_base_result { 531 volatile __le16 flags; 532 volatile __le16 session; 533 volatile __le16 src_cnt; /* 15:0 of source count */ 534 volatile __le16 dst_cnt; /* 15:0 of dest count */ 535 }; 536 537 #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ 538 #define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */ 539 #define HIFN_BASE_RES_SRCLEN_S 14 540 #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ 541 #define HIFN_BASE_RES_DSTLEN_S 12 542 543 struct hifn_comp_result { 544 volatile __le16 flags; 545 volatile __le16 crc; 546 }; 547 548 #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ 549 #define HIFN_COMP_RES_LCB_S 8 550 #define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */ 551 #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ 552 #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ 553 554 struct hifn_mac_result { 555 volatile __le16 flags; 556 volatile __le16 reserved; 557 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ 558 }; 559 560 #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ 561 #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ 562 563 struct hifn_crypt_result { 564 volatile __le16 flags; 565 volatile __le16 reserved; 566 }; 567 568 #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ 569 570 #ifndef HIFN_POLL_FREQUENCY 571 #define HIFN_POLL_FREQUENCY 0x1 572 #endif 573 574 #ifndef HIFN_POLL_SCALAR 575 #define HIFN_POLL_SCALAR 0x0 576 #endif 577 578 #define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ 579 #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ 580 581 struct hifn_crypto_alg { 582 struct list_head entry; 583 struct skcipher_alg alg; 584 struct hifn_device *dev; 585 }; 586 587 #define ASYNC_SCATTERLIST_CACHE 16 588 589 #define ASYNC_FLAGS_MISALIGNED (1 << 0) 590 591 struct hifn_cipher_walk { 592 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; 593 u32 flags; 594 int num; 595 }; 596 597 struct hifn_context { 598 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; 599 struct hifn_device *dev; 600 unsigned int keysize; 601 }; 602 603 struct hifn_request_context { 604 u8 *iv; 605 unsigned int ivsize; 606 u8 op, type, mode, unused; 607 struct hifn_cipher_walk walk; 608 }; 609 610 #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) 611 612 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) 613 { 614 return readl(dev->bar[0] + reg); 615 } 616 617 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) 618 { 619 return readl(dev->bar[1] + reg); 620 } 621 622 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) 623 { 624 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); 625 } 626 627 static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) 628 { 629 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); 630 } 631 632 static void hifn_wait_puc(struct hifn_device *dev) 633 { 634 int i; 635 u32 ret; 636 637 for (i = 10000; i > 0; --i) { 638 ret = hifn_read_0(dev, HIFN_0_PUCTRL); 639 if (!(ret & HIFN_PUCTRL_RESET)) 640 break; 641 642 udelay(1); 643 } 644 645 if (!i) 646 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); 647 } 648 649 static void hifn_reset_puc(struct hifn_device *dev) 650 { 651 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 652 hifn_wait_puc(dev); 653 } 654 655 static void hifn_stop_device(struct hifn_device *dev) 656 { 657 hifn_write_1(dev, HIFN_1_DMA_CSR, 658 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 659 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS); 660 hifn_write_0(dev, HIFN_0_PUIER, 0); 661 hifn_write_1(dev, HIFN_1_DMA_IER, 0); 662 } 663 664 static void hifn_reset_dma(struct hifn_device *dev, int full) 665 { 666 hifn_stop_device(dev); 667 668 /* 669 * Setting poll frequency and others to 0. 670 */ 671 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 672 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 673 mdelay(1); 674 675 /* 676 * Reset DMA. 677 */ 678 if (full) { 679 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 680 mdelay(1); 681 } else { 682 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE | 683 HIFN_DMACNFG_MSTRESET); 684 hifn_reset_puc(dev); 685 } 686 687 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 688 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 689 690 hifn_reset_puc(dev); 691 } 692 693 static u32 hifn_next_signature(u32 a, u_int cnt) 694 { 695 int i; 696 u32 v; 697 698 for (i = 0; i < cnt; i++) { 699 /* get the parity */ 700 v = a & 0x80080125; 701 v ^= v >> 16; 702 v ^= v >> 8; 703 v ^= v >> 4; 704 v ^= v >> 2; 705 v ^= v >> 1; 706 707 a = (v & 1) ^ (a << 1); 708 } 709 710 return a; 711 } 712 713 static struct pci2id { 714 u_short pci_vendor; 715 u_short pci_prod; 716 char card_id[13]; 717 } pci2id[] = { 718 { 719 PCI_VENDOR_ID_HIFN, 720 PCI_DEVICE_ID_HIFN_7955, 721 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 722 0x00, 0x00, 0x00, 0x00, 0x00 } 723 }, 724 { 725 PCI_VENDOR_ID_HIFN, 726 PCI_DEVICE_ID_HIFN_7956, 727 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 728 0x00, 0x00, 0x00, 0x00, 0x00 } 729 } 730 }; 731 732 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 733 static int hifn_rng_data_present(struct hwrng *rng, int wait) 734 { 735 struct hifn_device *dev = (struct hifn_device *)rng->priv; 736 s64 nsec; 737 738 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); 739 nsec -= dev->rng_wait_time; 740 if (nsec <= 0) 741 return 1; 742 if (!wait) 743 return 0; 744 ndelay(nsec); 745 return 1; 746 } 747 748 static int hifn_rng_data_read(struct hwrng *rng, u32 *data) 749 { 750 struct hifn_device *dev = (struct hifn_device *)rng->priv; 751 752 *data = hifn_read_1(dev, HIFN_1_RNG_DATA); 753 dev->rngtime = ktime_get(); 754 return 4; 755 } 756 757 static int hifn_register_rng(struct hifn_device *dev) 758 { 759 /* 760 * We must wait at least 256 Pk_clk cycles between two reads of the rng. 761 */ 762 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, 763 dev->pk_clk_freq) * 256; 764 765 dev->rng.name = dev->name; 766 dev->rng.data_present = hifn_rng_data_present; 767 dev->rng.data_read = hifn_rng_data_read; 768 dev->rng.priv = (unsigned long)dev; 769 770 return hwrng_register(&dev->rng); 771 } 772 773 static void hifn_unregister_rng(struct hifn_device *dev) 774 { 775 hwrng_unregister(&dev->rng); 776 } 777 #else 778 #define hifn_register_rng(dev) 0 779 #define hifn_unregister_rng(dev) 780 #endif 781 782 static int hifn_init_pubrng(struct hifn_device *dev) 783 { 784 int i; 785 786 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | 787 HIFN_PUBRST_RESET); 788 789 for (i = 100; i > 0; --i) { 790 mdelay(1); 791 792 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) 793 break; 794 } 795 796 if (!i) { 797 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); 798 } else { 799 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 800 dev->dmareg |= HIFN_DMAIER_PUBDONE; 801 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 802 803 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); 804 } 805 806 /* Enable RNG engine. */ 807 808 hifn_write_1(dev, HIFN_1_RNG_CONFIG, 809 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); 810 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); 811 812 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 813 /* First value must be discarded */ 814 hifn_read_1(dev, HIFN_1_RNG_DATA); 815 dev->rngtime = ktime_get(); 816 #endif 817 return 0; 818 } 819 820 static int hifn_enable_crypto(struct hifn_device *dev) 821 { 822 u32 dmacfg, addr; 823 char *offtbl = NULL; 824 int i; 825 826 for (i = 0; i < ARRAY_SIZE(pci2id); i++) { 827 if (pci2id[i].pci_vendor == dev->pdev->vendor && 828 pci2id[i].pci_prod == dev->pdev->device) { 829 offtbl = pci2id[i].card_id; 830 break; 831 } 832 } 833 834 if (!offtbl) { 835 dev_err(&dev->pdev->dev, "Unknown card!\n"); 836 return -ENODEV; 837 } 838 839 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG); 840 841 hifn_write_1(dev, HIFN_1_DMA_CNFG, 842 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET | 843 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 844 mdelay(1); 845 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1); 846 mdelay(1); 847 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); 848 mdelay(1); 849 850 for (i = 0; i < 12; ++i) { 851 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 852 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); 853 854 mdelay(1); 855 } 856 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); 857 858 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); 859 860 return 0; 861 } 862 863 static void hifn_init_dma(struct hifn_device *dev) 864 { 865 struct hifn_dma *dma = dev->desc_virt; 866 u32 dptr = dev->desc_dma; 867 int i; 868 869 for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) 870 dma->cmdr[i].p = __cpu_to_le32(dptr + 871 offsetof(struct hifn_dma, command_bufs[i][0])); 872 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 873 dma->resr[i].p = __cpu_to_le32(dptr + 874 offsetof(struct hifn_dma, result_bufs[i][0])); 875 876 /* Setup LAST descriptors. */ 877 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + 878 offsetof(struct hifn_dma, cmdr[0])); 879 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + 880 offsetof(struct hifn_dma, srcr[0])); 881 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + 882 offsetof(struct hifn_dma, dstr[0])); 883 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + 884 offsetof(struct hifn_dma, resr[0])); 885 886 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 887 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 888 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 889 } 890 891 /* 892 * Initialize the PLL. We need to know the frequency of the reference clock 893 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that 894 * allows us to operate without the risk of overclocking the chip. If it 895 * actually uses 33MHz, the chip will operate at half the speed, this can be 896 * overridden by specifying the frequency as module parameter (pci33). 897 * 898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a 899 * stable clock and the PCI clock frequency may vary, so the default is the 900 * external clock. There is no way to find out its frequency, we default to 901 * 66MHz since according to Mike Ham of HiFn, almost every board in existence 902 * has an external crystal populated at 66MHz. 903 */ 904 static void hifn_init_pll(struct hifn_device *dev) 905 { 906 unsigned int freq, m; 907 u32 pllcfg; 908 909 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; 910 911 if (strncmp(hifn_pll_ref, "ext", 3) == 0) 912 pllcfg |= HIFN_PLL_REF_CLK_PLL; 913 else 914 pllcfg |= HIFN_PLL_REF_CLK_HBI; 915 916 if (hifn_pll_ref[3] == '\0' || 917 kstrtouint(hifn_pll_ref + 3, 10, &freq)) { 918 freq = 66; 919 dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", 920 freq, hifn_pll_ref); 921 } 922 923 m = HIFN_PLL_FCK_MAX / freq; 924 925 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; 926 if (m <= 8) 927 pllcfg |= HIFN_PLL_IS_1_8; 928 else 929 pllcfg |= HIFN_PLL_IS_9_12; 930 931 /* Select clock source and enable clock bypass */ 932 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 933 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); 934 935 /* Let the chip lock to the input clock */ 936 mdelay(10); 937 938 /* Disable clock bypass */ 939 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 940 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); 941 942 /* Switch the engines to the PLL */ 943 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 944 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); 945 946 /* 947 * The Fpk_clk runs at half the total speed. Its frequency is needed to 948 * calculate the minimum time between two reads of the rng. Since 33MHz 949 * is actually 33.333... we overestimate the frequency here, resulting 950 * in slightly larger intervals. 951 */ 952 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; 953 } 954 955 static void hifn_init_registers(struct hifn_device *dev) 956 { 957 u32 dptr = dev->desc_dma; 958 959 /* Initialization magic... */ 960 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 961 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 962 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 963 964 /* write all 4 ring address registers */ 965 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + 966 offsetof(struct hifn_dma, cmdr[0])); 967 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + 968 offsetof(struct hifn_dma, srcr[0])); 969 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + 970 offsetof(struct hifn_dma, dstr[0])); 971 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + 972 offsetof(struct hifn_dma, resr[0])); 973 974 mdelay(2); 975 #if 0 976 hifn_write_1(dev, HIFN_1_DMA_CSR, 977 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 978 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 979 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 980 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 981 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 982 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 983 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 984 HIFN_DMACSR_S_WAIT | 985 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 986 HIFN_DMACSR_C_WAIT | 987 HIFN_DMACSR_ENGINE | 988 HIFN_DMACSR_PUBDONE); 989 #else 990 hifn_write_1(dev, HIFN_1_DMA_CSR, 991 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 992 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA | 993 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 994 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 995 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 996 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 997 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 998 HIFN_DMACSR_S_WAIT | 999 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1000 HIFN_DMACSR_C_WAIT | 1001 HIFN_DMACSR_ENGINE | 1002 HIFN_DMACSR_PUBDONE); 1003 #endif 1004 hifn_read_1(dev, HIFN_1_DMA_CSR); 1005 1006 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1007 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1008 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1009 HIFN_DMAIER_ENGINE; 1010 dev->dmareg &= ~HIFN_DMAIER_C_WAIT; 1011 1012 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1013 hifn_read_1(dev, HIFN_1_DMA_IER); 1014 #if 0 1015 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG | 1016 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1017 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1018 HIFN_PUCNFG_DRAM); 1019 #else 1020 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); 1021 #endif 1022 hifn_init_pll(dev); 1023 1024 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1025 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1026 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1027 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1028 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1029 } 1030 1031 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, 1032 unsigned dlen, unsigned slen, u16 mask, u8 snum) 1033 { 1034 struct hifn_base_command *base_cmd; 1035 u8 *buf_pos = buf; 1036 1037 base_cmd = (struct hifn_base_command *)buf_pos; 1038 base_cmd->masks = __cpu_to_le16(mask); 1039 base_cmd->total_source_count = 1040 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO); 1041 base_cmd->total_dest_count = 1042 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1043 1044 dlen >>= 16; 1045 slen >>= 16; 1046 base_cmd->session_num = __cpu_to_le16(snum | 1047 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1048 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1049 1050 return sizeof(struct hifn_base_command); 1051 } 1052 1053 static int hifn_setup_crypto_command(struct hifn_device *dev, 1054 u8 *buf, unsigned dlen, unsigned slen, 1055 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) 1056 { 1057 struct hifn_dma *dma = dev->desc_virt; 1058 struct hifn_crypt_command *cry_cmd; 1059 u8 *buf_pos = buf; 1060 u16 cmd_len; 1061 1062 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1063 1064 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); 1065 dlen >>= 16; 1066 cry_cmd->masks = __cpu_to_le16(mode | 1067 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & 1068 HIFN_CRYPT_CMD_SRCLEN_M)); 1069 cry_cmd->header_skip = 0; 1070 cry_cmd->reserved = 0; 1071 1072 buf_pos += sizeof(struct hifn_crypt_command); 1073 1074 dma->cmdu++; 1075 if (dma->cmdu > 1) { 1076 dev->dmareg |= HIFN_DMAIER_C_WAIT; 1077 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1078 } 1079 1080 if (keylen) { 1081 memcpy(buf_pos, key, keylen); 1082 buf_pos += keylen; 1083 } 1084 if (ivsize) { 1085 memcpy(buf_pos, iv, ivsize); 1086 buf_pos += ivsize; 1087 } 1088 1089 cmd_len = buf_pos - buf; 1090 1091 return cmd_len; 1092 } 1093 1094 static int hifn_setup_cmd_desc(struct hifn_device *dev, 1095 struct hifn_context *ctx, struct hifn_request_context *rctx, 1096 void *priv, unsigned int nbytes) 1097 { 1098 struct hifn_dma *dma = dev->desc_virt; 1099 int cmd_len, sa_idx; 1100 u8 *buf, *buf_pos; 1101 u16 mask; 1102 1103 sa_idx = dma->cmdi; 1104 buf_pos = buf = dma->command_bufs[dma->cmdi]; 1105 1106 mask = 0; 1107 switch (rctx->op) { 1108 case ACRYPTO_OP_DECRYPT: 1109 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; 1110 break; 1111 case ACRYPTO_OP_ENCRYPT: 1112 mask = HIFN_BASE_CMD_CRYPT; 1113 break; 1114 case ACRYPTO_OP_HMAC: 1115 mask = HIFN_BASE_CMD_MAC; 1116 break; 1117 default: 1118 goto err_out; 1119 } 1120 1121 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, 1122 nbytes, mask, dev->snum); 1123 1124 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { 1125 u16 md = 0; 1126 1127 if (ctx->keysize) 1128 md |= HIFN_CRYPT_CMD_NEW_KEY; 1129 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) 1130 md |= HIFN_CRYPT_CMD_NEW_IV; 1131 1132 switch (rctx->mode) { 1133 case ACRYPTO_MODE_ECB: 1134 md |= HIFN_CRYPT_CMD_MODE_ECB; 1135 break; 1136 case ACRYPTO_MODE_CBC: 1137 md |= HIFN_CRYPT_CMD_MODE_CBC; 1138 break; 1139 case ACRYPTO_MODE_CFB: 1140 md |= HIFN_CRYPT_CMD_MODE_CFB; 1141 break; 1142 case ACRYPTO_MODE_OFB: 1143 md |= HIFN_CRYPT_CMD_MODE_OFB; 1144 break; 1145 default: 1146 goto err_out; 1147 } 1148 1149 switch (rctx->type) { 1150 case ACRYPTO_TYPE_AES_128: 1151 if (ctx->keysize != 16) 1152 goto err_out; 1153 md |= HIFN_CRYPT_CMD_KSZ_128 | 1154 HIFN_CRYPT_CMD_ALG_AES; 1155 break; 1156 case ACRYPTO_TYPE_AES_192: 1157 if (ctx->keysize != 24) 1158 goto err_out; 1159 md |= HIFN_CRYPT_CMD_KSZ_192 | 1160 HIFN_CRYPT_CMD_ALG_AES; 1161 break; 1162 case ACRYPTO_TYPE_AES_256: 1163 if (ctx->keysize != 32) 1164 goto err_out; 1165 md |= HIFN_CRYPT_CMD_KSZ_256 | 1166 HIFN_CRYPT_CMD_ALG_AES; 1167 break; 1168 case ACRYPTO_TYPE_3DES: 1169 if (ctx->keysize != 24) 1170 goto err_out; 1171 md |= HIFN_CRYPT_CMD_ALG_3DES; 1172 break; 1173 case ACRYPTO_TYPE_DES: 1174 if (ctx->keysize != 8) 1175 goto err_out; 1176 md |= HIFN_CRYPT_CMD_ALG_DES; 1177 break; 1178 default: 1179 goto err_out; 1180 } 1181 1182 buf_pos += hifn_setup_crypto_command(dev, buf_pos, 1183 nbytes, nbytes, ctx->key, ctx->keysize, 1184 rctx->iv, rctx->ivsize, md); 1185 } 1186 1187 dev->sa[sa_idx] = priv; 1188 dev->started++; 1189 1190 cmd_len = buf_pos - buf; 1191 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | 1192 HIFN_D_LAST | HIFN_D_MASKDONEIRQ); 1193 1194 if (++dma->cmdi == HIFN_D_CMD_RSIZE) { 1195 dma->cmdr[dma->cmdi].l = __cpu_to_le32( 1196 HIFN_D_VALID | HIFN_D_LAST | 1197 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); 1198 dma->cmdi = 0; 1199 } else { 1200 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); 1201 } 1202 1203 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { 1204 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1205 dev->flags |= HIFN_FLAG_CMD_BUSY; 1206 } 1207 return 0; 1208 1209 err_out: 1210 return -EINVAL; 1211 } 1212 1213 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, 1214 unsigned int offset, unsigned int size, int last) 1215 { 1216 struct hifn_dma *dma = dev->desc_virt; 1217 int idx; 1218 dma_addr_t addr; 1219 1220 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1221 DMA_TO_DEVICE); 1222 1223 idx = dma->srci; 1224 1225 dma->srcr[idx].p = __cpu_to_le32(addr); 1226 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1227 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1228 1229 if (++idx == HIFN_D_SRC_RSIZE) { 1230 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1231 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1232 (last ? HIFN_D_LAST : 0)); 1233 idx = 0; 1234 } 1235 1236 dma->srci = idx; 1237 dma->srcu++; 1238 1239 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { 1240 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1241 dev->flags |= HIFN_FLAG_SRC_BUSY; 1242 } 1243 1244 return size; 1245 } 1246 1247 static void hifn_setup_res_desc(struct hifn_device *dev) 1248 { 1249 struct hifn_dma *dma = dev->desc_virt; 1250 1251 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | 1252 HIFN_D_VALID | HIFN_D_LAST); 1253 /* 1254 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | 1255 * HIFN_D_LAST); 1256 */ 1257 1258 if (++dma->resi == HIFN_D_RES_RSIZE) { 1259 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | 1260 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); 1261 dma->resi = 0; 1262 } 1263 1264 dma->resu++; 1265 1266 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { 1267 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1268 dev->flags |= HIFN_FLAG_RES_BUSY; 1269 } 1270 } 1271 1272 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, 1273 unsigned offset, unsigned size, int last) 1274 { 1275 struct hifn_dma *dma = dev->desc_virt; 1276 int idx; 1277 dma_addr_t addr; 1278 1279 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1280 DMA_FROM_DEVICE); 1281 1282 idx = dma->dsti; 1283 dma->dstr[idx].p = __cpu_to_le32(addr); 1284 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1285 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1286 1287 if (++idx == HIFN_D_DST_RSIZE) { 1288 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1289 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1290 (last ? HIFN_D_LAST : 0)); 1291 idx = 0; 1292 } 1293 dma->dsti = idx; 1294 dma->dstu++; 1295 1296 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { 1297 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1298 dev->flags |= HIFN_FLAG_DST_BUSY; 1299 } 1300 } 1301 1302 static int hifn_setup_dma(struct hifn_device *dev, 1303 struct hifn_context *ctx, struct hifn_request_context *rctx, 1304 struct scatterlist *src, struct scatterlist *dst, 1305 unsigned int nbytes, void *priv) 1306 { 1307 struct scatterlist *t; 1308 struct page *spage, *dpage; 1309 unsigned int soff, doff; 1310 unsigned int n, len; 1311 1312 n = nbytes; 1313 while (n) { 1314 spage = sg_page(src); 1315 soff = src->offset; 1316 len = min(src->length, n); 1317 1318 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); 1319 1320 src++; 1321 n -= len; 1322 } 1323 1324 t = &rctx->walk.cache[0]; 1325 n = nbytes; 1326 while (n) { 1327 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1328 BUG_ON(!sg_page(t)); 1329 dpage = sg_page(t); 1330 doff = 0; 1331 len = t->length; 1332 } else { 1333 BUG_ON(!sg_page(dst)); 1334 dpage = sg_page(dst); 1335 doff = dst->offset; 1336 len = dst->length; 1337 } 1338 len = min(len, n); 1339 1340 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); 1341 1342 dst++; 1343 t++; 1344 n -= len; 1345 } 1346 1347 hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); 1348 hifn_setup_res_desc(dev); 1349 return 0; 1350 } 1351 1352 static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, 1353 int num, gfp_t gfp_flags) 1354 { 1355 int i; 1356 1357 num = min(ASYNC_SCATTERLIST_CACHE, num); 1358 sg_init_table(w->cache, num); 1359 1360 w->num = 0; 1361 for (i = 0; i < num; ++i) { 1362 struct page *page = alloc_page(gfp_flags); 1363 struct scatterlist *s; 1364 1365 if (!page) 1366 break; 1367 1368 s = &w->cache[i]; 1369 1370 sg_set_page(s, page, PAGE_SIZE, 0); 1371 w->num++; 1372 } 1373 1374 return i; 1375 } 1376 1377 static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) 1378 { 1379 int i; 1380 1381 for (i = 0; i < w->num; ++i) { 1382 struct scatterlist *s = &w->cache[i]; 1383 1384 __free_page(sg_page(s)); 1385 1386 s->length = 0; 1387 } 1388 1389 w->num = 0; 1390 } 1391 1392 static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, 1393 unsigned int size, unsigned int *nbytesp) 1394 { 1395 unsigned int copy, drest = *drestp, nbytes = *nbytesp; 1396 int idx = 0; 1397 1398 if (drest < size || size > nbytes) 1399 return -EINVAL; 1400 1401 while (size) { 1402 copy = min3(drest, size, dst->length); 1403 1404 size -= copy; 1405 drest -= copy; 1406 nbytes -= copy; 1407 1408 pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", 1409 __func__, copy, size, drest, nbytes); 1410 1411 dst++; 1412 idx++; 1413 } 1414 1415 *nbytesp = nbytes; 1416 *drestp = drest; 1417 1418 return idx; 1419 } 1420 1421 static int hifn_cipher_walk(struct skcipher_request *req, 1422 struct hifn_cipher_walk *w) 1423 { 1424 struct scatterlist *dst, *t; 1425 unsigned int nbytes = req->cryptlen, offset, copy, diff; 1426 int idx, tidx, err; 1427 1428 tidx = idx = 0; 1429 offset = 0; 1430 while (nbytes) { 1431 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) 1432 return -EINVAL; 1433 1434 dst = &req->dst[idx]; 1435 1436 pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", 1437 __func__, dst->length, dst->offset, offset, nbytes); 1438 1439 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1440 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || 1441 offset) { 1442 unsigned slen = min(dst->length - offset, nbytes); 1443 unsigned dlen = PAGE_SIZE; 1444 1445 t = &w->cache[idx]; 1446 1447 err = skcipher_add(&dlen, dst, slen, &nbytes); 1448 if (err < 0) 1449 return err; 1450 1451 idx += err; 1452 1453 copy = slen & ~(HIFN_D_DST_DALIGN - 1); 1454 diff = slen & (HIFN_D_DST_DALIGN - 1); 1455 1456 if (dlen < nbytes) { 1457 /* 1458 * Destination page does not have enough space 1459 * to put there additional blocksized chunk, 1460 * so we mark that page as containing only 1461 * blocksize aligned chunks: 1462 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); 1463 * and increase number of bytes to be processed 1464 * in next chunk: 1465 * nbytes += diff; 1466 */ 1467 nbytes += diff; 1468 1469 /* 1470 * Temporary of course... 1471 * Kick author if you will catch this one. 1472 */ 1473 pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", 1474 __func__, dlen, nbytes, slen, offset); 1475 pr_err("%s: please contact author to fix this " 1476 "issue, generally you should not catch " 1477 "this path under any condition but who " 1478 "knows how did you use crypto code.\n" 1479 "Thank you.\n", __func__); 1480 BUG(); 1481 } else { 1482 copy += diff + nbytes; 1483 1484 dst = &req->dst[idx]; 1485 1486 err = skcipher_add(&dlen, dst, nbytes, &nbytes); 1487 if (err < 0) 1488 return err; 1489 1490 idx += err; 1491 } 1492 1493 t->length = copy; 1494 t->offset = offset; 1495 } else { 1496 nbytes -= min(dst->length, nbytes); 1497 idx++; 1498 } 1499 1500 tidx++; 1501 } 1502 1503 return tidx; 1504 } 1505 1506 static int hifn_setup_session(struct skcipher_request *req) 1507 { 1508 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1509 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1510 struct hifn_device *dev = ctx->dev; 1511 unsigned long dlen, flags; 1512 unsigned int nbytes = req->cryptlen, idx = 0; 1513 int err = -EINVAL, sg_num; 1514 struct scatterlist *dst; 1515 1516 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) 1517 goto err_out_exit; 1518 1519 rctx->walk.flags = 0; 1520 1521 while (nbytes) { 1522 dst = &req->dst[idx]; 1523 dlen = min(dst->length, nbytes); 1524 1525 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1526 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) 1527 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; 1528 1529 nbytes -= dlen; 1530 idx++; 1531 } 1532 1533 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1534 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); 1535 if (err < 0) 1536 return err; 1537 } 1538 1539 sg_num = hifn_cipher_walk(req, &rctx->walk); 1540 if (sg_num < 0) { 1541 err = sg_num; 1542 goto err_out_exit; 1543 } 1544 1545 spin_lock_irqsave(&dev->lock, flags); 1546 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { 1547 err = -EAGAIN; 1548 goto err_out; 1549 } 1550 1551 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); 1552 if (err) 1553 goto err_out; 1554 1555 dev->snum++; 1556 1557 dev->active = HIFN_DEFAULT_ACTIVE_NUM; 1558 spin_unlock_irqrestore(&dev->lock, flags); 1559 1560 return 0; 1561 1562 err_out: 1563 spin_unlock_irqrestore(&dev->lock, flags); 1564 err_out_exit: 1565 if (err) { 1566 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " 1567 "type: %u, err: %d.\n", 1568 rctx->iv, rctx->ivsize, 1569 ctx->key, ctx->keysize, 1570 rctx->mode, rctx->op, rctx->type, err); 1571 } 1572 1573 return err; 1574 } 1575 1576 static int hifn_start_device(struct hifn_device *dev) 1577 { 1578 int err; 1579 1580 dev->started = dev->active = 0; 1581 hifn_reset_dma(dev, 1); 1582 1583 err = hifn_enable_crypto(dev); 1584 if (err) 1585 return err; 1586 1587 hifn_reset_puc(dev); 1588 1589 hifn_init_dma(dev); 1590 1591 hifn_init_registers(dev); 1592 1593 hifn_init_pubrng(dev); 1594 1595 return 0; 1596 } 1597 1598 static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, 1599 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) 1600 { 1601 unsigned int srest = *srestp, nbytes = *nbytesp, copy; 1602 void *daddr; 1603 int idx = 0; 1604 1605 if (srest < size || size > nbytes) 1606 return -EINVAL; 1607 1608 while (size) { 1609 copy = min3(srest, dst->length, size); 1610 1611 daddr = kmap_atomic(sg_page(dst)); 1612 memcpy(daddr + dst->offset + offset, saddr, copy); 1613 kunmap_atomic(daddr); 1614 1615 nbytes -= copy; 1616 size -= copy; 1617 srest -= copy; 1618 saddr += copy; 1619 offset = 0; 1620 1621 pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", 1622 __func__, copy, size, srest, nbytes); 1623 1624 dst++; 1625 idx++; 1626 } 1627 1628 *nbytesp = nbytes; 1629 *srestp = srest; 1630 1631 return idx; 1632 } 1633 1634 static inline void hifn_complete_sa(struct hifn_device *dev, int i) 1635 { 1636 unsigned long flags; 1637 1638 spin_lock_irqsave(&dev->lock, flags); 1639 dev->sa[i] = NULL; 1640 dev->started--; 1641 if (dev->started < 0) 1642 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, 1643 dev->started); 1644 spin_unlock_irqrestore(&dev->lock, flags); 1645 BUG_ON(dev->started < 0); 1646 } 1647 1648 static void hifn_process_ready(struct skcipher_request *req, int error) 1649 { 1650 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1651 1652 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1653 unsigned int nbytes = req->cryptlen; 1654 int idx = 0, err; 1655 struct scatterlist *dst, *t; 1656 void *saddr; 1657 1658 while (nbytes) { 1659 t = &rctx->walk.cache[idx]; 1660 dst = &req->dst[idx]; 1661 1662 pr_debug("\n%s: sg_page(t): %p, t->length: %u, " 1663 "sg_page(dst): %p, dst->length: %u, " 1664 "nbytes: %u.\n", 1665 __func__, sg_page(t), t->length, 1666 sg_page(dst), dst->length, nbytes); 1667 1668 if (!t->length) { 1669 nbytes -= min(dst->length, nbytes); 1670 idx++; 1671 continue; 1672 } 1673 1674 saddr = kmap_atomic(sg_page(t)); 1675 1676 err = skcipher_get(saddr, &t->length, t->offset, 1677 dst, nbytes, &nbytes); 1678 if (err < 0) { 1679 kunmap_atomic(saddr); 1680 break; 1681 } 1682 1683 idx += err; 1684 kunmap_atomic(saddr); 1685 } 1686 1687 hifn_cipher_walk_exit(&rctx->walk); 1688 } 1689 1690 skcipher_request_complete(req, error); 1691 } 1692 1693 static void hifn_clear_rings(struct hifn_device *dev, int error) 1694 { 1695 struct hifn_dma *dma = dev->desc_virt; 1696 int i, u; 1697 1698 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1699 "k: %d.%d.%d.%d.\n", 1700 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1701 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1702 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1703 1704 i = dma->resk; u = dma->resu; 1705 while (u != 0) { 1706 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1707 break; 1708 1709 if (dev->sa[i]) { 1710 dev->success++; 1711 dev->reset = 0; 1712 hifn_process_ready(dev->sa[i], error); 1713 hifn_complete_sa(dev, i); 1714 } 1715 1716 if (++i == HIFN_D_RES_RSIZE) 1717 i = 0; 1718 u--; 1719 } 1720 dma->resk = i; dma->resu = u; 1721 1722 i = dma->srck; u = dma->srcu; 1723 while (u != 0) { 1724 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1725 break; 1726 if (++i == HIFN_D_SRC_RSIZE) 1727 i = 0; 1728 u--; 1729 } 1730 dma->srck = i; dma->srcu = u; 1731 1732 i = dma->cmdk; u = dma->cmdu; 1733 while (u != 0) { 1734 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1735 break; 1736 if (++i == HIFN_D_CMD_RSIZE) 1737 i = 0; 1738 u--; 1739 } 1740 dma->cmdk = i; dma->cmdu = u; 1741 1742 i = dma->dstk; u = dma->dstu; 1743 while (u != 0) { 1744 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1745 break; 1746 if (++i == HIFN_D_DST_RSIZE) 1747 i = 0; 1748 u--; 1749 } 1750 dma->dstk = i; dma->dstu = u; 1751 1752 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1753 "k: %d.%d.%d.%d.\n", 1754 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1755 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1756 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1757 } 1758 1759 static void hifn_work(struct work_struct *work) 1760 { 1761 struct delayed_work *dw = to_delayed_work(work); 1762 struct hifn_device *dev = container_of(dw, struct hifn_device, work); 1763 unsigned long flags; 1764 int reset = 0; 1765 u32 r = 0; 1766 1767 spin_lock_irqsave(&dev->lock, flags); 1768 if (dev->active == 0) { 1769 struct hifn_dma *dma = dev->desc_virt; 1770 1771 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { 1772 dev->flags &= ~HIFN_FLAG_CMD_BUSY; 1773 r |= HIFN_DMACSR_C_CTRL_DIS; 1774 } 1775 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { 1776 dev->flags &= ~HIFN_FLAG_SRC_BUSY; 1777 r |= HIFN_DMACSR_S_CTRL_DIS; 1778 } 1779 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { 1780 dev->flags &= ~HIFN_FLAG_DST_BUSY; 1781 r |= HIFN_DMACSR_D_CTRL_DIS; 1782 } 1783 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { 1784 dev->flags &= ~HIFN_FLAG_RES_BUSY; 1785 r |= HIFN_DMACSR_R_CTRL_DIS; 1786 } 1787 if (r) 1788 hifn_write_1(dev, HIFN_1_DMA_CSR, r); 1789 } else 1790 dev->active--; 1791 1792 if ((dev->prev_success == dev->success) && dev->started) 1793 reset = 1; 1794 dev->prev_success = dev->success; 1795 spin_unlock_irqrestore(&dev->lock, flags); 1796 1797 if (reset) { 1798 if (++dev->reset >= 5) { 1799 int i; 1800 struct hifn_dma *dma = dev->desc_virt; 1801 1802 dev_info(&dev->pdev->dev, 1803 "r: %08x, active: %d, started: %d, " 1804 "success: %lu: qlen: %u/%u, reset: %d.\n", 1805 r, dev->active, dev->started, 1806 dev->success, dev->queue.qlen, dev->queue.max_qlen, 1807 reset); 1808 1809 dev_info(&dev->pdev->dev, "%s: res: ", __func__); 1810 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1811 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); 1812 if (dev->sa[i]) { 1813 hifn_process_ready(dev->sa[i], -ENODEV); 1814 hifn_complete_sa(dev, i); 1815 } 1816 } 1817 pr_info("\n"); 1818 1819 hifn_reset_dma(dev, 1); 1820 hifn_stop_device(dev); 1821 hifn_start_device(dev); 1822 dev->reset = 0; 1823 } 1824 1825 tasklet_schedule(&dev->tasklet); 1826 } 1827 1828 schedule_delayed_work(&dev->work, HZ); 1829 } 1830 1831 static irqreturn_t hifn_interrupt(int irq, void *data) 1832 { 1833 struct hifn_device *dev = data; 1834 struct hifn_dma *dma = dev->desc_virt; 1835 u32 dmacsr, restart; 1836 1837 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); 1838 1839 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " 1840 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", 1841 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, 1842 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1843 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1844 1845 if ((dmacsr & dev->dmareg) == 0) 1846 return IRQ_NONE; 1847 1848 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); 1849 1850 if (dmacsr & HIFN_DMACSR_ENGINE) 1851 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR)); 1852 if (dmacsr & HIFN_DMACSR_PUBDONE) 1853 hifn_write_1(dev, HIFN_1_PUB_STATUS, 1854 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1855 1856 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1857 if (restart) { 1858 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); 1859 1860 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", 1861 !!(dmacsr & HIFN_DMACSR_R_OVER), 1862 !!(dmacsr & HIFN_DMACSR_D_OVER), 1863 puisr, !!(puisr & HIFN_PUISR_DSTOVER)); 1864 if (!!(puisr & HIFN_PUISR_DSTOVER)) 1865 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1866 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | 1867 HIFN_DMACSR_D_OVER)); 1868 } 1869 1870 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1871 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1872 if (restart) { 1873 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", 1874 !!(dmacsr & HIFN_DMACSR_C_ABORT), 1875 !!(dmacsr & HIFN_DMACSR_S_ABORT), 1876 !!(dmacsr & HIFN_DMACSR_D_ABORT), 1877 !!(dmacsr & HIFN_DMACSR_R_ABORT)); 1878 hifn_reset_dma(dev, 1); 1879 hifn_init_dma(dev); 1880 hifn_init_registers(dev); 1881 } 1882 1883 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { 1884 dev_dbg(&dev->pdev->dev, "wait on command.\n"); 1885 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); 1886 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1887 } 1888 1889 tasklet_schedule(&dev->tasklet); 1890 1891 return IRQ_HANDLED; 1892 } 1893 1894 static void hifn_flush(struct hifn_device *dev) 1895 { 1896 unsigned long flags; 1897 struct crypto_async_request *async_req; 1898 struct skcipher_request *req; 1899 struct hifn_dma *dma = dev->desc_virt; 1900 int i; 1901 1902 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1903 struct hifn_desc *d = &dma->resr[i]; 1904 1905 if (dev->sa[i]) { 1906 hifn_process_ready(dev->sa[i], 1907 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); 1908 hifn_complete_sa(dev, i); 1909 } 1910 } 1911 1912 spin_lock_irqsave(&dev->lock, flags); 1913 while ((async_req = crypto_dequeue_request(&dev->queue))) { 1914 req = skcipher_request_cast(async_req); 1915 spin_unlock_irqrestore(&dev->lock, flags); 1916 1917 hifn_process_ready(req, -ENODEV); 1918 1919 spin_lock_irqsave(&dev->lock, flags); 1920 } 1921 spin_unlock_irqrestore(&dev->lock, flags); 1922 } 1923 1924 static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, 1925 unsigned int len) 1926 { 1927 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1928 struct hifn_device *dev = ctx->dev; 1929 int err; 1930 1931 err = verify_skcipher_des_key(cipher, key); 1932 if (err) 1933 return err; 1934 1935 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1936 1937 memcpy(ctx->key, key, len); 1938 ctx->keysize = len; 1939 1940 return 0; 1941 } 1942 1943 static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 1944 unsigned int len) 1945 { 1946 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1947 struct hifn_device *dev = ctx->dev; 1948 int err; 1949 1950 err = verify_skcipher_des3_key(cipher, key); 1951 if (err) 1952 return err; 1953 1954 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1955 1956 memcpy(ctx->key, key, len); 1957 ctx->keysize = len; 1958 1959 return 0; 1960 } 1961 1962 static int hifn_handle_req(struct skcipher_request *req) 1963 { 1964 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1965 struct hifn_device *dev = ctx->dev; 1966 int err = -EAGAIN; 1967 1968 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) 1969 err = hifn_setup_session(req); 1970 1971 if (err == -EAGAIN) { 1972 unsigned long flags; 1973 1974 spin_lock_irqsave(&dev->lock, flags); 1975 err = crypto_enqueue_request(&dev->queue, &req->base); 1976 spin_unlock_irqrestore(&dev->lock, flags); 1977 } 1978 1979 return err; 1980 } 1981 1982 static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, 1983 u8 type, u8 mode) 1984 { 1985 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1986 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1987 unsigned ivsize; 1988 1989 ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); 1990 1991 if (req->iv && mode != ACRYPTO_MODE_ECB) { 1992 if (type == ACRYPTO_TYPE_AES_128) 1993 ivsize = HIFN_AES_IV_LENGTH; 1994 else if (type == ACRYPTO_TYPE_DES) 1995 ivsize = HIFN_DES_KEY_LENGTH; 1996 else if (type == ACRYPTO_TYPE_3DES) 1997 ivsize = HIFN_3DES_KEY_LENGTH; 1998 } 1999 2000 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { 2001 if (ctx->keysize == 24) 2002 type = ACRYPTO_TYPE_AES_192; 2003 else if (ctx->keysize == 32) 2004 type = ACRYPTO_TYPE_AES_256; 2005 } 2006 2007 rctx->op = op; 2008 rctx->mode = mode; 2009 rctx->type = type; 2010 rctx->iv = req->iv; 2011 rctx->ivsize = ivsize; 2012 2013 /* 2014 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2015 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2016 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2017 */ 2018 2019 return hifn_handle_req(req); 2020 } 2021 2022 static int hifn_process_queue(struct hifn_device *dev) 2023 { 2024 struct crypto_async_request *async_req, *backlog; 2025 struct skcipher_request *req; 2026 unsigned long flags; 2027 int err = 0; 2028 2029 while (dev->started < HIFN_QUEUE_LENGTH) { 2030 spin_lock_irqsave(&dev->lock, flags); 2031 backlog = crypto_get_backlog(&dev->queue); 2032 async_req = crypto_dequeue_request(&dev->queue); 2033 spin_unlock_irqrestore(&dev->lock, flags); 2034 2035 if (!async_req) 2036 break; 2037 2038 if (backlog) 2039 crypto_request_complete(backlog, -EINPROGRESS); 2040 2041 req = skcipher_request_cast(async_req); 2042 2043 err = hifn_handle_req(req); 2044 if (err) 2045 break; 2046 } 2047 2048 return err; 2049 } 2050 2051 static int hifn_setup_crypto(struct skcipher_request *req, u8 op, 2052 u8 type, u8 mode) 2053 { 2054 int err; 2055 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2056 struct hifn_device *dev = ctx->dev; 2057 2058 err = hifn_setup_crypto_req(req, op, type, mode); 2059 if (err) 2060 return err; 2061 2062 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2063 hifn_process_queue(dev); 2064 2065 return -EINPROGRESS; 2066 } 2067 2068 /* 2069 * AES ecryption functions. 2070 */ 2071 static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) 2072 { 2073 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2074 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2075 } 2076 static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) 2077 { 2078 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2079 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2080 } 2081 2082 /* 2083 * AES decryption functions. 2084 */ 2085 static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) 2086 { 2087 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2088 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2089 } 2090 static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) 2091 { 2092 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2093 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2094 } 2095 2096 /* 2097 * DES ecryption functions. 2098 */ 2099 static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) 2100 { 2101 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2102 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2103 } 2104 static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) 2105 { 2106 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2107 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2108 } 2109 2110 /* 2111 * DES decryption functions. 2112 */ 2113 static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) 2114 { 2115 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2116 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2117 } 2118 static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) 2119 { 2120 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2121 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2122 } 2123 2124 /* 2125 * 3DES ecryption functions. 2126 */ 2127 static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) 2128 { 2129 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2130 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2131 } 2132 static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) 2133 { 2134 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2135 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2136 } 2137 2138 /* 3DES decryption functions. */ 2139 static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) 2140 { 2141 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2142 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2143 } 2144 static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) 2145 { 2146 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2147 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2148 } 2149 2150 struct hifn_alg_template { 2151 char name[CRYPTO_MAX_ALG_NAME]; 2152 char drv_name[CRYPTO_MAX_ALG_NAME]; 2153 unsigned int bsize; 2154 struct skcipher_alg skcipher; 2155 }; 2156 2157 static const struct hifn_alg_template hifn_alg_templates[] = { 2158 /* 2159 * 3DES ECB and CBC modes. 2160 */ 2161 { 2162 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, 2163 .skcipher = { 2164 .ivsize = HIFN_IV_LENGTH, 2165 .min_keysize = HIFN_3DES_KEY_LENGTH, 2166 .max_keysize = HIFN_3DES_KEY_LENGTH, 2167 .setkey = hifn_des3_setkey, 2168 .encrypt = hifn_encrypt_3des_cbc, 2169 .decrypt = hifn_decrypt_3des_cbc, 2170 }, 2171 }, 2172 { 2173 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, 2174 .skcipher = { 2175 .min_keysize = HIFN_3DES_KEY_LENGTH, 2176 .max_keysize = HIFN_3DES_KEY_LENGTH, 2177 .setkey = hifn_des3_setkey, 2178 .encrypt = hifn_encrypt_3des_ecb, 2179 .decrypt = hifn_decrypt_3des_ecb, 2180 }, 2181 }, 2182 2183 /* 2184 * DES ECB and CBC modes. 2185 */ 2186 { 2187 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, 2188 .skcipher = { 2189 .ivsize = HIFN_IV_LENGTH, 2190 .min_keysize = HIFN_DES_KEY_LENGTH, 2191 .max_keysize = HIFN_DES_KEY_LENGTH, 2192 .setkey = hifn_setkey, 2193 .encrypt = hifn_encrypt_des_cbc, 2194 .decrypt = hifn_decrypt_des_cbc, 2195 }, 2196 }, 2197 { 2198 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, 2199 .skcipher = { 2200 .min_keysize = HIFN_DES_KEY_LENGTH, 2201 .max_keysize = HIFN_DES_KEY_LENGTH, 2202 .setkey = hifn_setkey, 2203 .encrypt = hifn_encrypt_des_ecb, 2204 .decrypt = hifn_decrypt_des_ecb, 2205 }, 2206 }, 2207 2208 /* 2209 * AES ECB and CBC modes. 2210 */ 2211 { 2212 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, 2213 .skcipher = { 2214 .min_keysize = AES_MIN_KEY_SIZE, 2215 .max_keysize = AES_MAX_KEY_SIZE, 2216 .setkey = hifn_setkey, 2217 .encrypt = hifn_encrypt_aes_ecb, 2218 .decrypt = hifn_decrypt_aes_ecb, 2219 }, 2220 }, 2221 { 2222 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, 2223 .skcipher = { 2224 .ivsize = HIFN_AES_IV_LENGTH, 2225 .min_keysize = AES_MIN_KEY_SIZE, 2226 .max_keysize = AES_MAX_KEY_SIZE, 2227 .setkey = hifn_setkey, 2228 .encrypt = hifn_encrypt_aes_cbc, 2229 .decrypt = hifn_decrypt_aes_cbc, 2230 }, 2231 }, 2232 }; 2233 2234 static int hifn_init_tfm(struct crypto_skcipher *tfm) 2235 { 2236 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2237 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg); 2238 struct hifn_context *ctx = crypto_skcipher_ctx(tfm); 2239 2240 ctx->dev = ha->dev; 2241 crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context)); 2242 2243 return 0; 2244 } 2245 2246 static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) 2247 { 2248 struct hifn_crypto_alg *alg; 2249 int err; 2250 2251 alg = kzalloc(sizeof(*alg), GFP_KERNEL); 2252 if (!alg) 2253 return -ENOMEM; 2254 2255 alg->alg = t->skcipher; 2256 alg->alg.init = hifn_init_tfm; 2257 2258 err = -EINVAL; 2259 if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 2260 "%s", t->name) >= CRYPTO_MAX_ALG_NAME) 2261 goto out_free_alg; 2262 if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 2263 "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME) 2264 goto out_free_alg; 2265 2266 alg->alg.base.cra_priority = 300; 2267 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 2268 alg->alg.base.cra_blocksize = t->bsize; 2269 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); 2270 alg->alg.base.cra_alignmask = 0; 2271 alg->alg.base.cra_module = THIS_MODULE; 2272 2273 alg->dev = dev; 2274 2275 list_add_tail(&alg->entry, &dev->alg_list); 2276 2277 err = crypto_register_skcipher(&alg->alg); 2278 if (err) { 2279 list_del(&alg->entry); 2280 out_free_alg: 2281 kfree(alg); 2282 } 2283 2284 return err; 2285 } 2286 2287 static void hifn_unregister_alg(struct hifn_device *dev) 2288 { 2289 struct hifn_crypto_alg *a, *n; 2290 2291 list_for_each_entry_safe(a, n, &dev->alg_list, entry) { 2292 list_del(&a->entry); 2293 crypto_unregister_skcipher(&a->alg); 2294 kfree(a); 2295 } 2296 } 2297 2298 static int hifn_register_alg(struct hifn_device *dev) 2299 { 2300 int i, err; 2301 2302 for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { 2303 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); 2304 if (err) 2305 goto err_out_exit; 2306 } 2307 2308 return 0; 2309 2310 err_out_exit: 2311 hifn_unregister_alg(dev); 2312 return err; 2313 } 2314 2315 static void hifn_tasklet_callback(unsigned long data) 2316 { 2317 struct hifn_device *dev = (struct hifn_device *)data; 2318 2319 /* 2320 * This is ok to call this without lock being held, 2321 * althogh it modifies some parameters used in parallel, 2322 * (like dev->success), but they are used in process 2323 * context or update is atomic (like setting dev->sa[i] to NULL). 2324 */ 2325 hifn_clear_rings(dev, 0); 2326 2327 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2328 hifn_process_queue(dev); 2329 } 2330 2331 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2332 { 2333 int err, i; 2334 struct hifn_device *dev; 2335 char name[8]; 2336 2337 err = pci_enable_device(pdev); 2338 if (err) 2339 return err; 2340 pci_set_master(pdev); 2341 2342 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 2343 if (err) 2344 goto err_out_disable_pci_device; 2345 2346 snprintf(name, sizeof(name), "hifn%d", 2347 atomic_inc_return(&hifn_dev_number) - 1); 2348 2349 err = pci_request_regions(pdev, name); 2350 if (err) 2351 goto err_out_disable_pci_device; 2352 2353 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || 2354 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || 2355 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { 2356 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); 2357 err = -ENODEV; 2358 goto err_out_free_regions; 2359 } 2360 2361 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg), 2362 GFP_KERNEL); 2363 if (!dev) { 2364 err = -ENOMEM; 2365 goto err_out_free_regions; 2366 } 2367 2368 INIT_LIST_HEAD(&dev->alg_list); 2369 2370 snprintf(dev->name, sizeof(dev->name), "%s", name); 2371 spin_lock_init(&dev->lock); 2372 2373 for (i = 0; i < 3; ++i) { 2374 unsigned long addr, size; 2375 2376 addr = pci_resource_start(pdev, i); 2377 size = pci_resource_len(pdev, i); 2378 2379 dev->bar[i] = ioremap(addr, size); 2380 if (!dev->bar[i]) { 2381 err = -ENOMEM; 2382 goto err_out_unmap_bars; 2383 } 2384 } 2385 2386 dev->desc_virt = dma_alloc_coherent(&pdev->dev, 2387 sizeof(struct hifn_dma), 2388 &dev->desc_dma, GFP_KERNEL); 2389 if (!dev->desc_virt) { 2390 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); 2391 err = -ENOMEM; 2392 goto err_out_unmap_bars; 2393 } 2394 2395 dev->pdev = pdev; 2396 dev->irq = pdev->irq; 2397 2398 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 2399 dev->sa[i] = NULL; 2400 2401 pci_set_drvdata(pdev, dev); 2402 2403 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); 2404 2405 crypto_init_queue(&dev->queue, 1); 2406 2407 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); 2408 if (err) { 2409 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", 2410 dev->irq, err); 2411 dev->irq = 0; 2412 goto err_out_free_desc; 2413 } 2414 2415 err = hifn_start_device(dev); 2416 if (err) 2417 goto err_out_free_irq; 2418 2419 err = hifn_register_rng(dev); 2420 if (err) 2421 goto err_out_stop_device; 2422 2423 err = hifn_register_alg(dev); 2424 if (err) 2425 goto err_out_unregister_rng; 2426 2427 INIT_DELAYED_WORK(&dev->work, hifn_work); 2428 schedule_delayed_work(&dev->work, HZ); 2429 2430 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " 2431 "successfully registered as %s.\n", 2432 pci_name(pdev), dev->name); 2433 2434 return 0; 2435 2436 err_out_unregister_rng: 2437 hifn_unregister_rng(dev); 2438 err_out_stop_device: 2439 hifn_reset_dma(dev, 1); 2440 hifn_stop_device(dev); 2441 err_out_free_irq: 2442 free_irq(dev->irq, dev); 2443 tasklet_kill(&dev->tasklet); 2444 err_out_free_desc: 2445 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, 2446 dev->desc_dma); 2447 2448 err_out_unmap_bars: 2449 for (i = 0; i < 3; ++i) 2450 if (dev->bar[i]) 2451 iounmap(dev->bar[i]); 2452 kfree(dev); 2453 2454 err_out_free_regions: 2455 pci_release_regions(pdev); 2456 2457 err_out_disable_pci_device: 2458 pci_disable_device(pdev); 2459 2460 return err; 2461 } 2462 2463 static void hifn_remove(struct pci_dev *pdev) 2464 { 2465 int i; 2466 struct hifn_device *dev; 2467 2468 dev = pci_get_drvdata(pdev); 2469 2470 if (dev) { 2471 cancel_delayed_work_sync(&dev->work); 2472 2473 hifn_unregister_rng(dev); 2474 hifn_unregister_alg(dev); 2475 hifn_reset_dma(dev, 1); 2476 hifn_stop_device(dev); 2477 2478 free_irq(dev->irq, dev); 2479 tasklet_kill(&dev->tasklet); 2480 2481 hifn_flush(dev); 2482 2483 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), 2484 dev->desc_virt, dev->desc_dma); 2485 for (i = 0; i < 3; ++i) 2486 if (dev->bar[i]) 2487 iounmap(dev->bar[i]); 2488 2489 kfree(dev); 2490 } 2491 2492 pci_release_regions(pdev); 2493 pci_disable_device(pdev); 2494 } 2495 2496 static struct pci_device_id hifn_pci_tbl[] = { 2497 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) }, 2498 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) }, 2499 { 0 } 2500 }; 2501 MODULE_DEVICE_TABLE(pci, hifn_pci_tbl); 2502 2503 static struct pci_driver hifn_pci_driver = { 2504 .name = "hifn795x", 2505 .id_table = hifn_pci_tbl, 2506 .probe = hifn_probe, 2507 .remove = hifn_remove, 2508 }; 2509 2510 static int __init hifn_init(void) 2511 { 2512 unsigned int freq; 2513 int err; 2514 2515 if (strncmp(hifn_pll_ref, "ext", 3) && 2516 strncmp(hifn_pll_ref, "pci", 3)) { 2517 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); 2518 return -EINVAL; 2519 } 2520 2521 /* 2522 * For the 7955/7956 the reference clock frequency must be in the 2523 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, 2524 * but this chip is currently not supported. 2525 */ 2526 if (hifn_pll_ref[3] != '\0') { 2527 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 2528 if (freq < 20 || freq > 100) { 2529 pr_err("hifn795x: invalid hifn_pll_ref frequency, must" 2530 "be in the range of 20-100"); 2531 return -EINVAL; 2532 } 2533 } 2534 2535 err = pci_register_driver(&hifn_pci_driver); 2536 if (err < 0) { 2537 pr_err("Failed to register PCI driver for %s device.\n", 2538 hifn_pci_driver.name); 2539 return -ENODEV; 2540 } 2541 2542 pr_info("Driver for HIFN 795x crypto accelerator chip " 2543 "has been successfully registered.\n"); 2544 2545 return 0; 2546 } 2547 2548 static void __exit hifn_fini(void) 2549 { 2550 pci_unregister_driver(&hifn_pci_driver); 2551 2552 pr_info("Driver for HIFN 795x crypto accelerator chip " 2553 "has been successfully unregistered.\n"); 2554 } 2555 2556 module_init(hifn_init); 2557 module_exit(hifn_fini); 2558 2559 MODULE_LICENSE("GPL"); 2560 MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>"); 2561 MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip."); 2562