linux/drivers/crypto/hifn_795x.c
<<
>>
Prefs
   1/*
   2 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
   3 * All rights reserved.
   4 *
   5 * This program is free software; you can redistribute it and/or modify
   6 * it under the terms of the GNU General Public License as published by
   7 * the Free Software Foundation; either version 2 of the License, or
   8 * (at your option) any later version.
   9 *
  10 * This program is distributed in the hope that it will be useful,
  11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 */
  15
  16#include <linux/kernel.h>
  17#include <linux/module.h>
  18#include <linux/moduleparam.h>
  19#include <linux/mod_devicetable.h>
  20#include <linux/interrupt.h>
  21#include <linux/pci.h>
  22#include <linux/slab.h>
  23#include <linux/delay.h>
  24#include <linux/mm.h>
  25#include <linux/dma-mapping.h>
  26#include <linux/scatterlist.h>
  27#include <linux/highmem.h>
  28#include <linux/crypto.h>
  29#include <linux/hw_random.h>
  30#include <linux/ktime.h>
  31
  32#include <crypto/algapi.h>
  33#include <crypto/des.h>
  34
  35static char hifn_pll_ref[sizeof("extNNN")] = "ext";
  36module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
  37MODULE_PARM_DESC(hifn_pll_ref,
  38                 "PLL reference clock (pci[freq] or ext[freq], default ext)");
  39
  40static atomic_t hifn_dev_number;
  41
  42#define ACRYPTO_OP_DECRYPT      0
  43#define ACRYPTO_OP_ENCRYPT      1
  44#define ACRYPTO_OP_HMAC         2
  45#define ACRYPTO_OP_RNG          3
  46
  47#define ACRYPTO_MODE_ECB                0
  48#define ACRYPTO_MODE_CBC                1
  49#define ACRYPTO_MODE_CFB                2
  50#define ACRYPTO_MODE_OFB                3
  51
  52#define ACRYPTO_TYPE_AES_128    0
  53#define ACRYPTO_TYPE_AES_192    1
  54#define ACRYPTO_TYPE_AES_256    2
  55#define ACRYPTO_TYPE_3DES       3
  56#define ACRYPTO_TYPE_DES        4
  57
  58#define PCI_VENDOR_ID_HIFN              0x13A3
  59#define PCI_DEVICE_ID_HIFN_7955         0x0020
  60#define PCI_DEVICE_ID_HIFN_7956         0x001d
  61
  62/* I/O region sizes */
  63
  64#define HIFN_BAR0_SIZE                  0x1000
  65#define HIFN_BAR1_SIZE                  0x2000
  66#define HIFN_BAR2_SIZE                  0x8000
  67
  68/* DMA registres */
  69
  70#define HIFN_DMA_CRA                    0x0C    /* DMA Command Ring Address */
  71#define HIFN_DMA_SDRA                   0x1C    /* DMA Source Data Ring Address */
  72#define HIFN_DMA_RRA                    0x2C    /* DMA Result Ring Address */
  73#define HIFN_DMA_DDRA                   0x3C    /* DMA Destination Data Ring Address */
  74#define HIFN_DMA_STCTL                  0x40    /* DMA Status and Control */
  75#define HIFN_DMA_INTREN                 0x44    /* DMA Interrupt Enable */
  76#define HIFN_DMA_CFG1                   0x48    /* DMA Configuration #1 */
  77#define HIFN_DMA_CFG2                   0x6C    /* DMA Configuration #2 */
  78#define HIFN_CHIP_ID                    0x98    /* Chip ID */
  79
  80/*
  81 * Processing Unit Registers (offset from BASEREG0)
  82 */
  83#define HIFN_0_PUDATA           0x00    /* Processing Unit Data */
  84#define HIFN_0_PUCTRL           0x04    /* Processing Unit Control */
  85#define HIFN_0_PUISR            0x08    /* Processing Unit Interrupt Status */
  86#define HIFN_0_PUCNFG           0x0c    /* Processing Unit Configuration */
  87#define HIFN_0_PUIER            0x10    /* Processing Unit Interrupt Enable */
  88#define HIFN_0_PUSTAT           0x14    /* Processing Unit Status/Chip ID */
  89#define HIFN_0_FIFOSTAT         0x18    /* FIFO Status */
  90#define HIFN_0_FIFOCNFG         0x1c    /* FIFO Configuration */
  91#define HIFN_0_SPACESIZE        0x20    /* Register space size */
  92
  93/* Processing Unit Control Register (HIFN_0_PUCTRL) */
  94#define HIFN_PUCTRL_CLRSRCFIFO  0x0010  /* clear source fifo */
  95#define HIFN_PUCTRL_STOP        0x0008  /* stop pu */
  96#define HIFN_PUCTRL_LOCKRAM     0x0004  /* lock ram */
  97#define HIFN_PUCTRL_DMAENA      0x0002  /* enable dma */
  98#define HIFN_PUCTRL_RESET       0x0001  /* Reset processing unit */
  99
 100/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
 101#define HIFN_PUISR_CMDINVAL     0x8000  /* Invalid command interrupt */
 102#define HIFN_PUISR_DATAERR      0x4000  /* Data error interrupt */
 103#define HIFN_PUISR_SRCFIFO      0x2000  /* Source FIFO ready interrupt */
 104#define HIFN_PUISR_DSTFIFO      0x1000  /* Destination FIFO ready interrupt */
 105#define HIFN_PUISR_DSTOVER      0x0200  /* Destination overrun interrupt */
 106#define HIFN_PUISR_SRCCMD       0x0080  /* Source command interrupt */
 107#define HIFN_PUISR_SRCCTX       0x0040  /* Source context interrupt */
 108#define HIFN_PUISR_SRCDATA      0x0020  /* Source data interrupt */
 109#define HIFN_PUISR_DSTDATA      0x0010  /* Destination data interrupt */
 110#define HIFN_PUISR_DSTRESULT    0x0004  /* Destination result interrupt */
 111
 112/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
 113#define HIFN_PUCNFG_DRAMMASK    0xe000  /* DRAM size mask */
 114#define HIFN_PUCNFG_DSZ_256K    0x0000  /* 256k dram */
 115#define HIFN_PUCNFG_DSZ_512K    0x2000  /* 512k dram */
 116#define HIFN_PUCNFG_DSZ_1M      0x4000  /* 1m dram */
 117#define HIFN_PUCNFG_DSZ_2M      0x6000  /* 2m dram */
 118#define HIFN_PUCNFG_DSZ_4M      0x8000  /* 4m dram */
 119#define HIFN_PUCNFG_DSZ_8M      0xa000  /* 8m dram */
 120#define HIFN_PUNCFG_DSZ_16M     0xc000  /* 16m dram */
 121#define HIFN_PUCNFG_DSZ_32M     0xe000  /* 32m dram */
 122#define HIFN_PUCNFG_DRAMREFRESH 0x1800  /* DRAM refresh rate mask */
 123#define HIFN_PUCNFG_DRFR_512    0x0000  /* 512 divisor of ECLK */
 124#define HIFN_PUCNFG_DRFR_256    0x0800  /* 256 divisor of ECLK */
 125#define HIFN_PUCNFG_DRFR_128    0x1000  /* 128 divisor of ECLK */
 126#define HIFN_PUCNFG_TCALLPHASES 0x0200  /* your guess is as good as mine... */
 127#define HIFN_PUCNFG_TCDRVTOTEM  0x0100  /* your guess is as good as mine... */
 128#define HIFN_PUCNFG_BIGENDIAN   0x0080  /* DMA big endian mode */
 129#define HIFN_PUCNFG_BUS32       0x0040  /* Bus width 32bits */
 130#define HIFN_PUCNFG_BUS16       0x0000  /* Bus width 16 bits */
 131#define HIFN_PUCNFG_CHIPID      0x0020  /* Allow chipid from PUSTAT */
 132#define HIFN_PUCNFG_DRAM        0x0010  /* Context RAM is DRAM */
 133#define HIFN_PUCNFG_SRAM        0x0000  /* Context RAM is SRAM */
 134#define HIFN_PUCNFG_COMPSING    0x0004  /* Enable single compression context */
 135#define HIFN_PUCNFG_ENCCNFG     0x0002  /* Encryption configuration */
 136
 137/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
 138#define HIFN_PUIER_CMDINVAL     0x8000  /* Invalid command interrupt */
 139#define HIFN_PUIER_DATAERR      0x4000  /* Data error interrupt */
 140#define HIFN_PUIER_SRCFIFO      0x2000  /* Source FIFO ready interrupt */
 141#define HIFN_PUIER_DSTFIFO      0x1000  /* Destination FIFO ready interrupt */
 142#define HIFN_PUIER_DSTOVER      0x0200  /* Destination overrun interrupt */
 143#define HIFN_PUIER_SRCCMD       0x0080  /* Source command interrupt */
 144#define HIFN_PUIER_SRCCTX       0x0040  /* Source context interrupt */
 145#define HIFN_PUIER_SRCDATA      0x0020  /* Source data interrupt */
 146#define HIFN_PUIER_DSTDATA      0x0010  /* Destination data interrupt */
 147#define HIFN_PUIER_DSTRESULT    0x0004  /* Destination result interrupt */
 148
 149/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
 150#define HIFN_PUSTAT_CMDINVAL    0x8000  /* Invalid command interrupt */
 151#define HIFN_PUSTAT_DATAERR     0x4000  /* Data error interrupt */
 152#define HIFN_PUSTAT_SRCFIFO     0x2000  /* Source FIFO ready interrupt */
 153#define HIFN_PUSTAT_DSTFIFO     0x1000  /* Destination FIFO ready interrupt */
 154#define HIFN_PUSTAT_DSTOVER     0x0200  /* Destination overrun interrupt */
 155#define HIFN_PUSTAT_SRCCMD      0x0080  /* Source command interrupt */
 156#define HIFN_PUSTAT_SRCCTX      0x0040  /* Source context interrupt */
 157#define HIFN_PUSTAT_SRCDATA     0x0020  /* Source data interrupt */
 158#define HIFN_PUSTAT_DSTDATA     0x0010  /* Destination data interrupt */
 159#define HIFN_PUSTAT_DSTRESULT   0x0004  /* Destination result interrupt */
 160#define HIFN_PUSTAT_CHIPREV     0x00ff  /* Chip revision mask */
 161#define HIFN_PUSTAT_CHIPENA     0xff00  /* Chip enabled mask */
 162#define HIFN_PUSTAT_ENA_2       0x1100  /* Level 2 enabled */
 163#define HIFN_PUSTAT_ENA_1       0x1000  /* Level 1 enabled */
 164#define HIFN_PUSTAT_ENA_0       0x3000  /* Level 0 enabled */
 165#define HIFN_PUSTAT_REV_2       0x0020  /* 7751 PT6/2 */
 166#define HIFN_PUSTAT_REV_3       0x0030  /* 7751 PT6/3 */
 167
 168/* FIFO Status Register (HIFN_0_FIFOSTAT) */
 169#define HIFN_FIFOSTAT_SRC       0x7f00  /* Source FIFO available */
 170#define HIFN_FIFOSTAT_DST       0x007f  /* Destination FIFO available */
 171
 172/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
 173#define HIFN_FIFOCNFG_THRESHOLD 0x0400  /* must be written as 1 */
 174
 175/*
 176 * DMA Interface Registers (offset from BASEREG1)
 177 */
 178#define HIFN_1_DMA_CRAR         0x0c    /* DMA Command Ring Address */
 179#define HIFN_1_DMA_SRAR         0x1c    /* DMA Source Ring Address */
 180#define HIFN_1_DMA_RRAR         0x2c    /* DMA Result Ring Address */
 181#define HIFN_1_DMA_DRAR         0x3c    /* DMA Destination Ring Address */
 182#define HIFN_1_DMA_CSR          0x40    /* DMA Status and Control */
 183#define HIFN_1_DMA_IER          0x44    /* DMA Interrupt Enable */
 184#define HIFN_1_DMA_CNFG         0x48    /* DMA Configuration */
 185#define HIFN_1_PLL              0x4c    /* 795x: PLL config */
 186#define HIFN_1_7811_RNGENA      0x60    /* 7811: rng enable */
 187#define HIFN_1_7811_RNGCFG      0x64    /* 7811: rng config */
 188#define HIFN_1_7811_RNGDAT      0x68    /* 7811: rng data */
 189#define HIFN_1_7811_RNGSTS      0x6c    /* 7811: rng status */
 190#define HIFN_1_7811_MIPSRST     0x94    /* 7811: MIPS reset */
 191#define HIFN_1_REVID            0x98    /* Revision ID */
 192#define HIFN_1_UNLOCK_SECRET1   0xf4
 193#define HIFN_1_UNLOCK_SECRET2   0xfc
 194#define HIFN_1_PUB_RESET        0x204   /* Public/RNG Reset */
 195#define HIFN_1_PUB_BASE         0x300   /* Public Base Address */
 196#define HIFN_1_PUB_OPLEN        0x304   /* Public Operand Length */
 197#define HIFN_1_PUB_OP           0x308   /* Public Operand */
 198#define HIFN_1_PUB_STATUS       0x30c   /* Public Status */
 199#define HIFN_1_PUB_IEN          0x310   /* Public Interrupt enable */
 200#define HIFN_1_RNG_CONFIG       0x314   /* RNG config */
 201#define HIFN_1_RNG_DATA         0x318   /* RNG data */
 202#define HIFN_1_PUB_MEM          0x400   /* start of Public key memory */
 203#define HIFN_1_PUB_MEMEND       0xbff   /* end of Public key memory */
 204
 205/* DMA Status and Control Register (HIFN_1_DMA_CSR) */
 206#define HIFN_DMACSR_D_CTRLMASK  0xc0000000      /* Destinition Ring Control */
 207#define HIFN_DMACSR_D_CTRL_NOP  0x00000000      /* Dest. Control: no-op */
 208#define HIFN_DMACSR_D_CTRL_DIS  0x40000000      /* Dest. Control: disable */
 209#define HIFN_DMACSR_D_CTRL_ENA  0x80000000      /* Dest. Control: enable */
 210#define HIFN_DMACSR_D_ABORT     0x20000000      /* Destinition Ring PCIAbort */
 211#define HIFN_DMACSR_D_DONE      0x10000000      /* Destinition Ring Done */
 212#define HIFN_DMACSR_D_LAST      0x08000000      /* Destinition Ring Last */
 213#define HIFN_DMACSR_D_WAIT      0x04000000      /* Destinition Ring Waiting */
 214#define HIFN_DMACSR_D_OVER      0x02000000      /* Destinition Ring Overflow */
 215#define HIFN_DMACSR_R_CTRL      0x00c00000      /* Result Ring Control */
 216#define HIFN_DMACSR_R_CTRL_NOP  0x00000000      /* Result Control: no-op */
 217#define HIFN_DMACSR_R_CTRL_DIS  0x00400000      /* Result Control: disable */
 218#define HIFN_DMACSR_R_CTRL_ENA  0x00800000      /* Result Control: enable */
 219#define HIFN_DMACSR_R_ABORT     0x00200000      /* Result Ring PCI Abort */
 220#define HIFN_DMACSR_R_DONE      0x00100000      /* Result Ring Done */
 221#define HIFN_DMACSR_R_LAST      0x00080000      /* Result Ring Last */
 222#define HIFN_DMACSR_R_WAIT      0x00040000      /* Result Ring Waiting */
 223#define HIFN_DMACSR_R_OVER      0x00020000      /* Result Ring Overflow */
 224#define HIFN_DMACSR_S_CTRL      0x0000c000      /* Source Ring Control */
 225#define HIFN_DMACSR_S_CTRL_NOP  0x00000000      /* Source Control: no-op */
 226#define HIFN_DMACSR_S_CTRL_DIS  0x00004000      /* Source Control: disable */
 227#define HIFN_DMACSR_S_CTRL_ENA  0x00008000      /* Source Control: enable */
 228#define HIFN_DMACSR_S_ABORT     0x00002000      /* Source Ring PCI Abort */
 229#define HIFN_DMACSR_S_DONE      0x00001000      /* Source Ring Done */
 230#define HIFN_DMACSR_S_LAST      0x00000800      /* Source Ring Last */
 231#define HIFN_DMACSR_S_WAIT      0x00000400      /* Source Ring Waiting */
 232#define HIFN_DMACSR_ILLW        0x00000200      /* Illegal write (7811 only) */
 233#define HIFN_DMACSR_ILLR        0x00000100      /* Illegal read (7811 only) */
 234#define HIFN_DMACSR_C_CTRL      0x000000c0      /* Command Ring Control */
 235#define HIFN_DMACSR_C_CTRL_NOP  0x00000000      /* Command Control: no-op */
 236#define HIFN_DMACSR_C_CTRL_DIS  0x00000040      /* Command Control: disable */
 237#define HIFN_DMACSR_C_CTRL_ENA  0x00000080      /* Command Control: enable */
 238#define HIFN_DMACSR_C_ABORT     0x00000020      /* Command Ring PCI Abort */
 239#define HIFN_DMACSR_C_DONE      0x00000010      /* Command Ring Done */
 240#define HIFN_DMACSR_C_LAST      0x00000008      /* Command Ring Last */
 241#define HIFN_DMACSR_C_WAIT      0x00000004      /* Command Ring Waiting */
 242#define HIFN_DMACSR_PUBDONE     0x00000002      /* Public op done (7951 only) */
 243#define HIFN_DMACSR_ENGINE      0x00000001      /* Command Ring Engine IRQ */
 244
 245/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
 246#define HIFN_DMAIER_D_ABORT     0x20000000      /* Destination Ring PCIAbort */
 247#define HIFN_DMAIER_D_DONE      0x10000000      /* Destination Ring Done */
 248#define HIFN_DMAIER_D_LAST      0x08000000      /* Destination Ring Last */
 249#define HIFN_DMAIER_D_WAIT      0x04000000      /* Destination Ring Waiting */
 250#define HIFN_DMAIER_D_OVER      0x02000000      /* Destination Ring Overflow */
 251#define HIFN_DMAIER_R_ABORT     0x00200000      /* Result Ring PCI Abort */
 252#define HIFN_DMAIER_R_DONE      0x00100000      /* Result Ring Done */
 253#define HIFN_DMAIER_R_LAST      0x00080000      /* Result Ring Last */
 254#define HIFN_DMAIER_R_WAIT      0x00040000      /* Result Ring Waiting */
 255#define HIFN_DMAIER_R_OVER      0x00020000      /* Result Ring Overflow */
 256#define HIFN_DMAIER_S_ABORT     0x00002000      /* Source Ring PCI Abort */
 257#define HIFN_DMAIER_S_DONE      0x00001000      /* Source Ring Done */
 258#define HIFN_DMAIER_S_LAST      0x00000800      /* Source Ring Last */
 259#define HIFN_DMAIER_S_WAIT      0x00000400      /* Source Ring Waiting */
 260#define HIFN_DMAIER_ILLW        0x00000200      /* Illegal write (7811 only) */
 261#define HIFN_DMAIER_ILLR        0x00000100      /* Illegal read (7811 only) */
 262#define HIFN_DMAIER_C_ABORT     0x00000020      /* Command Ring PCI Abort */
 263#define HIFN_DMAIER_C_DONE      0x00000010      /* Command Ring Done */
 264#define HIFN_DMAIER_C_LAST      0x00000008      /* Command Ring Last */
 265#define HIFN_DMAIER_C_WAIT      0x00000004      /* Command Ring Waiting */
 266#define HIFN_DMAIER_PUBDONE     0x00000002      /* public op done (7951 only) */
 267#define HIFN_DMAIER_ENGINE      0x00000001      /* Engine IRQ */
 268
 269/* DMA Configuration Register (HIFN_1_DMA_CNFG) */
 270#define HIFN_DMACNFG_BIGENDIAN  0x10000000      /* big endian mode */
 271#define HIFN_DMACNFG_POLLFREQ   0x00ff0000      /* Poll frequency mask */
 272#define HIFN_DMACNFG_UNLOCK     0x00000800
 273#define HIFN_DMACNFG_POLLINVAL  0x00000700      /* Invalid Poll Scalar */
 274#define HIFN_DMACNFG_LAST       0x00000010      /* Host control LAST bit */
 275#define HIFN_DMACNFG_MODE       0x00000004      /* DMA mode */
 276#define HIFN_DMACNFG_DMARESET   0x00000002      /* DMA Reset # */
 277#define HIFN_DMACNFG_MSTRESET   0x00000001      /* Master Reset # */
 278
 279/* PLL configuration register */
 280#define HIFN_PLL_REF_CLK_HBI    0x00000000      /* HBI reference clock */
 281#define HIFN_PLL_REF_CLK_PLL    0x00000001      /* PLL reference clock */
 282#define HIFN_PLL_BP             0x00000002      /* Reference clock bypass */
 283#define HIFN_PLL_PK_CLK_HBI     0x00000000      /* PK engine HBI clock */
 284#define HIFN_PLL_PK_CLK_PLL     0x00000008      /* PK engine PLL clock */
 285#define HIFN_PLL_PE_CLK_HBI     0x00000000      /* PE engine HBI clock */
 286#define HIFN_PLL_PE_CLK_PLL     0x00000010      /* PE engine PLL clock */
 287#define HIFN_PLL_RESERVED_1     0x00000400      /* Reserved bit, must be 1 */
 288#define HIFN_PLL_ND_SHIFT       11              /* Clock multiplier shift */
 289#define HIFN_PLL_ND_MULT_2      0x00000000      /* PLL clock multiplier 2 */
 290#define HIFN_PLL_ND_MULT_4      0x00000800      /* PLL clock multiplier 4 */
 291#define HIFN_PLL_ND_MULT_6      0x00001000      /* PLL clock multiplier 6 */
 292#define HIFN_PLL_ND_MULT_8      0x00001800      /* PLL clock multiplier 8 */
 293#define HIFN_PLL_ND_MULT_10     0x00002000      /* PLL clock multiplier 10 */
 294#define HIFN_PLL_ND_MULT_12     0x00002800      /* PLL clock multiplier 12 */
 295#define HIFN_PLL_IS_1_8         0x00000000      /* charge pump (mult. 1-8) */
 296#define HIFN_PLL_IS_9_12        0x00010000      /* charge pump (mult. 9-12) */
 297
 298#define HIFN_PLL_FCK_MAX        266             /* Maximum PLL frequency */
 299
 300/* Public key reset register (HIFN_1_PUB_RESET) */
 301#define HIFN_PUBRST_RESET       0x00000001      /* reset public/rng unit */
 302
 303/* Public base address register (HIFN_1_PUB_BASE) */
 304#define HIFN_PUBBASE_ADDR       0x00003fff      /* base address */
 305
 306/* Public operand length register (HIFN_1_PUB_OPLEN) */
 307#define HIFN_PUBOPLEN_MOD_M     0x0000007f      /* modulus length mask */
 308#define HIFN_PUBOPLEN_MOD_S     0               /* modulus length shift */
 309#define HIFN_PUBOPLEN_EXP_M     0x0003ff80      /* exponent length mask */
 310#define HIFN_PUBOPLEN_EXP_S     7               /* exponent length shift */
 311#define HIFN_PUBOPLEN_RED_M     0x003c0000      /* reducend length mask */
 312#define HIFN_PUBOPLEN_RED_S     18              /* reducend length shift */
 313
 314/* Public operation register (HIFN_1_PUB_OP) */
 315#define HIFN_PUBOP_AOFFSET_M    0x0000007f      /* A offset mask */
 316#define HIFN_PUBOP_AOFFSET_S    0               /* A offset shift */
 317#define HIFN_PUBOP_BOFFSET_M    0x00000f80      /* B offset mask */
 318#define HIFN_PUBOP_BOFFSET_S    7               /* B offset shift */
 319#define HIFN_PUBOP_MOFFSET_M    0x0003f000      /* M offset mask */
 320#define HIFN_PUBOP_MOFFSET_S    12              /* M offset shift */
 321#define HIFN_PUBOP_OP_MASK      0x003c0000      /* Opcode: */
 322#define HIFN_PUBOP_OP_NOP       0x00000000      /*  NOP */
 323#define HIFN_PUBOP_OP_ADD       0x00040000      /*  ADD */
 324#define HIFN_PUBOP_OP_ADDC      0x00080000      /*  ADD w/carry */
 325#define HIFN_PUBOP_OP_SUB       0x000c0000      /*  SUB */
 326#define HIFN_PUBOP_OP_SUBC      0x00100000      /*  SUB w/carry */
 327#define HIFN_PUBOP_OP_MODADD    0x00140000      /*  Modular ADD */
 328#define HIFN_PUBOP_OP_MODSUB    0x00180000      /*  Modular SUB */
 329#define HIFN_PUBOP_OP_INCA      0x001c0000      /*  INC A */
 330#define HIFN_PUBOP_OP_DECA      0x00200000      /*  DEC A */
 331#define HIFN_PUBOP_OP_MULT      0x00240000      /*  MULT */
 332#define HIFN_PUBOP_OP_MODMULT   0x00280000      /*  Modular MULT */
 333#define HIFN_PUBOP_OP_MODRED    0x002c0000      /*  Modular RED */
 334#define HIFN_PUBOP_OP_MODEXP    0x00300000      /*  Modular EXP */
 335
 336/* Public status register (HIFN_1_PUB_STATUS) */
 337#define HIFN_PUBSTS_DONE        0x00000001      /* operation done */
 338#define HIFN_PUBSTS_CARRY       0x00000002      /* carry */
 339
 340/* Public interrupt enable register (HIFN_1_PUB_IEN) */
 341#define HIFN_PUBIEN_DONE        0x00000001      /* operation done interrupt */
 342
 343/* Random number generator config register (HIFN_1_RNG_CONFIG) */
 344#define HIFN_RNGCFG_ENA         0x00000001      /* enable rng */
 345
 346#define HIFN_NAMESIZE                   32
 347#define HIFN_MAX_RESULT_ORDER           5
 348
 349#define HIFN_D_CMD_RSIZE                (24 * 1)
 350#define HIFN_D_SRC_RSIZE                (80 * 1)
 351#define HIFN_D_DST_RSIZE                (80 * 1)
 352#define HIFN_D_RES_RSIZE                (24 * 1)
 353
 354#define HIFN_D_DST_DALIGN               4
 355
 356#define HIFN_QUEUE_LENGTH               (HIFN_D_CMD_RSIZE - 1)
 357
 358#define AES_MIN_KEY_SIZE                16
 359#define AES_MAX_KEY_SIZE                32
 360
 361#define HIFN_DES_KEY_LENGTH             8
 362#define HIFN_3DES_KEY_LENGTH            24
 363#define HIFN_MAX_CRYPT_KEY_LENGTH       AES_MAX_KEY_SIZE
 364#define HIFN_IV_LENGTH                  8
 365#define HIFN_AES_IV_LENGTH              16
 366#define HIFN_MAX_IV_LENGTH              HIFN_AES_IV_LENGTH
 367
 368#define HIFN_MAC_KEY_LENGTH             64
 369#define HIFN_MD5_LENGTH                 16
 370#define HIFN_SHA1_LENGTH                20
 371#define HIFN_MAC_TRUNC_LENGTH           12
 372
 373#define HIFN_MAX_COMMAND                (8 + 8 + 8 + 64 + 260)
 374#define HIFN_MAX_RESULT                 (8 + 4 + 4 + 20 + 4)
 375#define HIFN_USED_RESULT                12
 376
 377struct hifn_desc {
 378        volatile __le32         l;
 379        volatile __le32         p;
 380};
 381
 382struct hifn_dma {
 383        struct hifn_desc        cmdr[HIFN_D_CMD_RSIZE + 1];
 384        struct hifn_desc        srcr[HIFN_D_SRC_RSIZE + 1];
 385        struct hifn_desc        dstr[HIFN_D_DST_RSIZE + 1];
 386        struct hifn_desc        resr[HIFN_D_RES_RSIZE + 1];
 387
 388        u8                      command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
 389        u8                      result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
 390
 391        /*
 392         *  Our current positions for insertion and removal from the descriptor
 393         *  rings.
 394         */
 395        volatile int            cmdi, srci, dsti, resi;
 396        volatile int            cmdu, srcu, dstu, resu;
 397        int                     cmdk, srck, dstk, resk;
 398};
 399
 400#define HIFN_FLAG_CMD_BUSY      (1 << 0)
 401#define HIFN_FLAG_SRC_BUSY      (1 << 1)
 402#define HIFN_FLAG_DST_BUSY      (1 << 2)
 403#define HIFN_FLAG_RES_BUSY      (1 << 3)
 404#define HIFN_FLAG_OLD_KEY       (1 << 4)
 405
 406#define HIFN_DEFAULT_ACTIVE_NUM 5
 407
 408struct hifn_device {
 409        char                    name[HIFN_NAMESIZE];
 410
 411        int                     irq;
 412
 413        struct pci_dev          *pdev;
 414        void __iomem            *bar[3];
 415
 416        void                    *desc_virt;
 417        dma_addr_t              desc_dma;
 418
 419        u32                     dmareg;
 420
 421        void                    *sa[HIFN_D_RES_RSIZE];
 422
 423        spinlock_t              lock;
 424
 425        u32                     flags;
 426        int                     active, started;
 427        struct delayed_work     work;
 428        unsigned long           reset;
 429        unsigned long           success;
 430        unsigned long           prev_success;
 431
 432        u8                      snum;
 433
 434        struct tasklet_struct   tasklet;
 435
 436        struct crypto_queue     queue;
 437        struct list_head        alg_list;
 438
 439        unsigned int            pk_clk_freq;
 440
 441#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 442        unsigned int            rng_wait_time;
 443        ktime_t                 rngtime;
 444        struct hwrng            rng;
 445#endif
 446};
 447
 448#define HIFN_D_LENGTH                   0x0000ffff
 449#define HIFN_D_NOINVALID                0x01000000
 450#define HIFN_D_MASKDONEIRQ              0x02000000
 451#define HIFN_D_DESTOVER                 0x04000000
 452#define HIFN_D_OVER                     0x08000000
 453#define HIFN_D_LAST                     0x20000000
 454#define HIFN_D_JUMP                     0x40000000
 455#define HIFN_D_VALID                    0x80000000
 456
 457struct hifn_base_command {
 458        volatile __le16         masks;
 459        volatile __le16         session_num;
 460        volatile __le16         total_source_count;
 461        volatile __le16         total_dest_count;
 462};
 463
 464#define HIFN_BASE_CMD_COMP              0x0100  /* enable compression engine */
 465#define HIFN_BASE_CMD_PAD               0x0200  /* enable padding engine */
 466#define HIFN_BASE_CMD_MAC               0x0400  /* enable MAC engine */
 467#define HIFN_BASE_CMD_CRYPT             0x0800  /* enable crypt engine */
 468#define HIFN_BASE_CMD_DECODE            0x2000
 469#define HIFN_BASE_CMD_SRCLEN_M          0xc000
 470#define HIFN_BASE_CMD_SRCLEN_S          14
 471#define HIFN_BASE_CMD_DSTLEN_M          0x3000
 472#define HIFN_BASE_CMD_DSTLEN_S          12
 473#define HIFN_BASE_CMD_LENMASK_HI        0x30000
 474#define HIFN_BASE_CMD_LENMASK_LO        0x0ffff
 475
 476/*
 477 * Structure to help build up the command data structure.
 478 */
 479struct hifn_crypt_command {
 480        volatile __le16         masks;
 481        volatile __le16         header_skip;
 482        volatile __le16         source_count;
 483        volatile __le16         reserved;
 484};
 485
 486#define HIFN_CRYPT_CMD_ALG_MASK         0x0003          /* algorithm: */
 487#define HIFN_CRYPT_CMD_ALG_DES          0x0000          /*   DES */
 488#define HIFN_CRYPT_CMD_ALG_3DES         0x0001          /*   3DES */
 489#define HIFN_CRYPT_CMD_ALG_RC4          0x0002          /*   RC4 */
 490#define HIFN_CRYPT_CMD_ALG_AES          0x0003          /*   AES */
 491#define HIFN_CRYPT_CMD_MODE_MASK        0x0018          /* Encrypt mode: */
 492#define HIFN_CRYPT_CMD_MODE_ECB         0x0000          /*   ECB */
 493#define HIFN_CRYPT_CMD_MODE_CBC         0x0008          /*   CBC */
 494#define HIFN_CRYPT_CMD_MODE_CFB         0x0010          /*   CFB */
 495#define HIFN_CRYPT_CMD_MODE_OFB         0x0018          /*   OFB */
 496#define HIFN_CRYPT_CMD_CLR_CTX          0x0040          /* clear context */
 497#define HIFN_CRYPT_CMD_KSZ_MASK         0x0600          /* AES key size: */
 498#define HIFN_CRYPT_CMD_KSZ_128          0x0000          /*  128 bit */
 499#define HIFN_CRYPT_CMD_KSZ_192          0x0200          /*  192 bit */
 500#define HIFN_CRYPT_CMD_KSZ_256          0x0400          /*  256 bit */
 501#define HIFN_CRYPT_CMD_NEW_KEY          0x0800          /* expect new key */
 502#define HIFN_CRYPT_CMD_NEW_IV           0x1000          /* expect new iv */
 503#define HIFN_CRYPT_CMD_SRCLEN_M         0xc000
 504#define HIFN_CRYPT_CMD_SRCLEN_S         14
 505
 506/*
 507 * Structure to help build up the command data structure.
 508 */
 509struct hifn_mac_command {
 510        volatile __le16 masks;
 511        volatile __le16 header_skip;
 512        volatile __le16 source_count;
 513        volatile __le16 reserved;
 514};
 515
 516#define HIFN_MAC_CMD_ALG_MASK           0x0001
 517#define HIFN_MAC_CMD_ALG_SHA1           0x0000
 518#define HIFN_MAC_CMD_ALG_MD5            0x0001
 519#define HIFN_MAC_CMD_MODE_MASK          0x000c
 520#define HIFN_MAC_CMD_MODE_HMAC          0x0000
 521#define HIFN_MAC_CMD_MODE_SSL_MAC       0x0004
 522#define HIFN_MAC_CMD_MODE_HASH          0x0008
 523#define HIFN_MAC_CMD_MODE_FULL          0x0004
 524#define HIFN_MAC_CMD_TRUNC              0x0010
 525#define HIFN_MAC_CMD_RESULT             0x0020
 526#define HIFN_MAC_CMD_APPEND             0x0040
 527#define HIFN_MAC_CMD_SRCLEN_M           0xc000
 528#define HIFN_MAC_CMD_SRCLEN_S           14
 529
 530/*
 531 * MAC POS IPsec initiates authentication after encryption on encodes
 532 * and before decryption on decodes.
 533 */
 534#define HIFN_MAC_CMD_POS_IPSEC          0x0200
 535#define HIFN_MAC_CMD_NEW_KEY            0x0800
 536
 537struct hifn_comp_command {
 538        volatile __le16         masks;
 539        volatile __le16         header_skip;
 540        volatile __le16         source_count;
 541        volatile __le16         reserved;
 542};
 543
 544#define HIFN_COMP_CMD_SRCLEN_M          0xc000
 545#define HIFN_COMP_CMD_SRCLEN_S          14
 546#define HIFN_COMP_CMD_ONE               0x0100  /* must be one */
 547#define HIFN_COMP_CMD_CLEARHIST         0x0010  /* clear history */
 548#define HIFN_COMP_CMD_UPDATEHIST        0x0008  /* update history */
 549#define HIFN_COMP_CMD_LZS_STRIP0        0x0004  /* LZS: strip zero */
 550#define HIFN_COMP_CMD_MPPC_RESTART      0x0004  /* MPPC: restart */
 551#define HIFN_COMP_CMD_ALG_MASK          0x0001  /* compression mode: */
 552#define HIFN_COMP_CMD_ALG_MPPC          0x0001  /*   MPPC */
 553#define HIFN_COMP_CMD_ALG_LZS           0x0000  /*   LZS */
 554
 555struct hifn_base_result {
 556        volatile __le16         flags;
 557        volatile __le16         session;
 558        volatile __le16         src_cnt;                /* 15:0 of source count */
 559        volatile __le16         dst_cnt;                /* 15:0 of dest count */
 560};
 561
 562#define HIFN_BASE_RES_DSTOVERRUN        0x0200  /* destination overrun */
 563#define HIFN_BASE_RES_SRCLEN_M          0xc000  /* 17:16 of source count */
 564#define HIFN_BASE_RES_SRCLEN_S          14
 565#define HIFN_BASE_RES_DSTLEN_M          0x3000  /* 17:16 of dest count */
 566#define HIFN_BASE_RES_DSTLEN_S          12
 567
 568struct hifn_comp_result {
 569        volatile __le16         flags;
 570        volatile __le16         crc;
 571};
 572
 573#define HIFN_COMP_RES_LCB_M             0xff00  /* longitudinal check byte */
 574#define HIFN_COMP_RES_LCB_S             8
 575#define HIFN_COMP_RES_RESTART           0x0004  /* MPPC: restart */
 576#define HIFN_COMP_RES_ENDMARKER         0x0002  /* LZS: end marker seen */
 577#define HIFN_COMP_RES_SRC_NOTZERO       0x0001  /* source expired */
 578
 579struct hifn_mac_result {
 580        volatile __le16         flags;
 581        volatile __le16         reserved;
 582        /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
 583};
 584
 585#define HIFN_MAC_RES_MISCOMPARE         0x0002  /* compare failed */
 586#define HIFN_MAC_RES_SRC_NOTZERO        0x0001  /* source expired */
 587
 588struct hifn_crypt_result {
 589        volatile __le16         flags;
 590        volatile __le16         reserved;
 591};
 592
 593#define HIFN_CRYPT_RES_SRC_NOTZERO      0x0001  /* source expired */
 594
 595#ifndef HIFN_POLL_FREQUENCY
 596#define HIFN_POLL_FREQUENCY     0x1
 597#endif
 598
 599#ifndef HIFN_POLL_SCALAR
 600#define HIFN_POLL_SCALAR        0x0
 601#endif
 602
 603#define HIFN_MAX_SEGLEN         0xffff          /* maximum dma segment len */
 604#define HIFN_MAX_DMALEN         0x3ffff         /* maximum dma length */
 605
 606struct hifn_crypto_alg {
 607        struct list_head        entry;
 608        struct crypto_alg       alg;
 609        struct hifn_device      *dev;
 610};
 611
 612#define ASYNC_SCATTERLIST_CACHE 16
 613
 614#define ASYNC_FLAGS_MISALIGNED  (1 << 0)
 615
 616struct hifn_cipher_walk {
 617        struct scatterlist      cache[ASYNC_SCATTERLIST_CACHE];
 618        u32                     flags;
 619        int                     num;
 620};
 621
 622struct hifn_context {
 623        u8                      key[HIFN_MAX_CRYPT_KEY_LENGTH];
 624        struct hifn_device      *dev;
 625        unsigned int            keysize;
 626};
 627
 628struct hifn_request_context {
 629        u8                      *iv;
 630        unsigned int            ivsize;
 631        u8                      op, type, mode, unused;
 632        struct hifn_cipher_walk walk;
 633};
 634
 635#define crypto_alg_to_hifn(a)   container_of(a, struct hifn_crypto_alg, alg)
 636
 637static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
 638{
 639        return readl(dev->bar[0] + reg);
 640}
 641
 642static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
 643{
 644        return readl(dev->bar[1] + reg);
 645}
 646
 647static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
 648{
 649        writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
 650}
 651
 652static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
 653{
 654        writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
 655}
 656
 657static void hifn_wait_puc(struct hifn_device *dev)
 658{
 659        int i;
 660        u32 ret;
 661
 662        for (i = 10000; i > 0; --i) {
 663                ret = hifn_read_0(dev, HIFN_0_PUCTRL);
 664                if (!(ret & HIFN_PUCTRL_RESET))
 665                        break;
 666
 667                udelay(1);
 668        }
 669
 670        if (!i)
 671                dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n");
 672}
 673
 674static void hifn_reset_puc(struct hifn_device *dev)
 675{
 676        hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 677        hifn_wait_puc(dev);
 678}
 679
 680static void hifn_stop_device(struct hifn_device *dev)
 681{
 682        hifn_write_1(dev, HIFN_1_DMA_CSR,
 683                HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 684                HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
 685        hifn_write_0(dev, HIFN_0_PUIER, 0);
 686        hifn_write_1(dev, HIFN_1_DMA_IER, 0);
 687}
 688
 689static void hifn_reset_dma(struct hifn_device *dev, int full)
 690{
 691        hifn_stop_device(dev);
 692
 693        /*
 694         * Setting poll frequency and others to 0.
 695         */
 696        hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 697                        HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 698        mdelay(1);
 699
 700        /*
 701         * Reset DMA.
 702         */
 703        if (full) {
 704                hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 705                mdelay(1);
 706        } else {
 707                hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
 708                                HIFN_DMACNFG_MSTRESET);
 709                hifn_reset_puc(dev);
 710        }
 711
 712        hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 713                        HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 714
 715        hifn_reset_puc(dev);
 716}
 717
 718static u32 hifn_next_signature(u32 a, u_int cnt)
 719{
 720        int i;
 721        u32 v;
 722
 723        for (i = 0; i < cnt; i++) {
 724                /* get the parity */
 725                v = a & 0x80080125;
 726                v ^= v >> 16;
 727                v ^= v >> 8;
 728                v ^= v >> 4;
 729                v ^= v >> 2;
 730                v ^= v >> 1;
 731
 732                a = (v & 1) ^ (a << 1);
 733        }
 734
 735        return a;
 736}
 737
 738static struct pci2id {
 739        u_short         pci_vendor;
 740        u_short         pci_prod;
 741        char            card_id[13];
 742} pci2id[] = {
 743        {
 744                PCI_VENDOR_ID_HIFN,
 745                PCI_DEVICE_ID_HIFN_7955,
 746                { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 747                  0x00, 0x00, 0x00, 0x00, 0x00 }
 748        },
 749        {
 750                PCI_VENDOR_ID_HIFN,
 751                PCI_DEVICE_ID_HIFN_7956,
 752                { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 753                  0x00, 0x00, 0x00, 0x00, 0x00 }
 754        }
 755};
 756
 757#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 758static int hifn_rng_data_present(struct hwrng *rng, int wait)
 759{
 760        struct hifn_device *dev = (struct hifn_device *)rng->priv;
 761        s64 nsec;
 762
 763        nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
 764        nsec -= dev->rng_wait_time;
 765        if (nsec <= 0)
 766                return 1;
 767        if (!wait)
 768                return 0;
 769        ndelay(nsec);
 770        return 1;
 771}
 772
 773static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
 774{
 775        struct hifn_device *dev = (struct hifn_device *)rng->priv;
 776
 777        *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
 778        dev->rngtime = ktime_get();
 779        return 4;
 780}
 781
 782static int hifn_register_rng(struct hifn_device *dev)
 783{
 784        /*
 785         * We must wait at least 256 Pk_clk cycles between two reads of the rng.
 786         */
 787        dev->rng_wait_time      = DIV_ROUND_UP_ULL(NSEC_PER_SEC,
 788                                                   dev->pk_clk_freq) * 256;
 789
 790        dev->rng.name           = dev->name;
 791        dev->rng.data_present   = hifn_rng_data_present,
 792        dev->rng.data_read      = hifn_rng_data_read,
 793        dev->rng.priv           = (unsigned long)dev;
 794
 795        return hwrng_register(&dev->rng);
 796}
 797
 798static void hifn_unregister_rng(struct hifn_device *dev)
 799{
 800        hwrng_unregister(&dev->rng);
 801}
 802#else
 803#define hifn_register_rng(dev)          0
 804#define hifn_unregister_rng(dev)
 805#endif
 806
 807static int hifn_init_pubrng(struct hifn_device *dev)
 808{
 809        int i;
 810
 811        hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
 812                        HIFN_PUBRST_RESET);
 813
 814        for (i = 100; i > 0; --i) {
 815                mdelay(1);
 816
 817                if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
 818                        break;
 819        }
 820
 821        if (!i) {
 822                dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n");
 823        } else {
 824                hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 825                dev->dmareg |= HIFN_DMAIER_PUBDONE;
 826                hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
 827
 828                dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n");
 829        }
 830
 831        /* Enable RNG engine. */
 832
 833        hifn_write_1(dev, HIFN_1_RNG_CONFIG,
 834                        hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
 835        dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n");
 836
 837#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 838        /* First value must be discarded */
 839        hifn_read_1(dev, HIFN_1_RNG_DATA);
 840        dev->rngtime = ktime_get();
 841#endif
 842        return 0;
 843}
 844
 845static int hifn_enable_crypto(struct hifn_device *dev)
 846{
 847        u32 dmacfg, addr;
 848        char *offtbl = NULL;
 849        int i;
 850
 851        for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
 852                if (pci2id[i].pci_vendor == dev->pdev->vendor &&
 853                                pci2id[i].pci_prod == dev->pdev->device) {
 854                        offtbl = pci2id[i].card_id;
 855                        break;
 856                }
 857        }
 858
 859        if (!offtbl) {
 860                dev_err(&dev->pdev->dev, "Unknown card!\n");
 861                return -ENODEV;
 862        }
 863
 864        dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
 865
 866        hifn_write_1(dev, HIFN_1_DMA_CNFG,
 867                        HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
 868                        HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 869        mdelay(1);
 870        addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
 871        mdelay(1);
 872        hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
 873        mdelay(1);
 874
 875        for (i = 0; i < 12; ++i) {
 876                addr = hifn_next_signature(addr, offtbl[i] + 0x101);
 877                hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
 878
 879                mdelay(1);
 880        }
 881        hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
 882
 883        dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev));
 884
 885        return 0;
 886}
 887
 888static void hifn_init_dma(struct hifn_device *dev)
 889{
 890        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
 891        u32 dptr = dev->desc_dma;
 892        int i;
 893
 894        for (i = 0; i < HIFN_D_CMD_RSIZE; ++i)
 895                dma->cmdr[i].p = __cpu_to_le32(dptr +
 896                                offsetof(struct hifn_dma, command_bufs[i][0]));
 897        for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
 898                dma->resr[i].p = __cpu_to_le32(dptr +
 899                                offsetof(struct hifn_dma, result_bufs[i][0]));
 900
 901        /* Setup LAST descriptors. */
 902        dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
 903                        offsetof(struct hifn_dma, cmdr[0]));
 904        dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
 905                        offsetof(struct hifn_dma, srcr[0]));
 906        dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
 907                        offsetof(struct hifn_dma, dstr[0]));
 908        dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
 909                        offsetof(struct hifn_dma, resr[0]));
 910
 911        dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
 912        dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
 913        dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
 914}
 915
 916/*
 917 * Initialize the PLL. We need to know the frequency of the reference clock
 918 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
 919 * allows us to operate without the risk of overclocking the chip. If it
 920 * actually uses 33MHz, the chip will operate at half the speed, this can be
 921 * overridden by specifying the frequency as module parameter (pci33).
 922 *
 923 * Unfortunately the PCI clock is not very suitable since the HIFN needs a
 924 * stable clock and the PCI clock frequency may vary, so the default is the
 925 * external clock. There is no way to find out its frequency, we default to
 926 * 66MHz since according to Mike Ham of HiFn, almost every board in existence
 927 * has an external crystal populated at 66MHz.
 928 */
 929static void hifn_init_pll(struct hifn_device *dev)
 930{
 931        unsigned int freq, m;
 932        u32 pllcfg;
 933
 934        pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
 935
 936        if (strncmp(hifn_pll_ref, "ext", 3) == 0)
 937                pllcfg |= HIFN_PLL_REF_CLK_PLL;
 938        else
 939                pllcfg |= HIFN_PLL_REF_CLK_HBI;
 940
 941        if (hifn_pll_ref[3] != '\0')
 942                freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
 943        else {
 944                freq = 66;
 945                dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n",
 946                         freq, hifn_pll_ref);
 947        }
 948
 949        m = HIFN_PLL_FCK_MAX / freq;
 950
 951        pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
 952        if (m <= 8)
 953                pllcfg |= HIFN_PLL_IS_1_8;
 954        else
 955                pllcfg |= HIFN_PLL_IS_9_12;
 956
 957        /* Select clock source and enable clock bypass */
 958        hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 959                     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
 960
 961        /* Let the chip lock to the input clock */
 962        mdelay(10);
 963
 964        /* Disable clock bypass */
 965        hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 966                     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
 967
 968        /* Switch the engines to the PLL */
 969        hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 970                     HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
 971
 972        /*
 973         * The Fpk_clk runs at half the total speed. Its frequency is needed to
 974         * calculate the minimum time between two reads of the rng. Since 33MHz
 975         * is actually 33.333... we overestimate the frequency here, resulting
 976         * in slightly larger intervals.
 977         */
 978        dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
 979}
 980
 981static void hifn_init_registers(struct hifn_device *dev)
 982{
 983        u32 dptr = dev->desc_dma;
 984
 985        /* Initialization magic... */
 986        hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 987        hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
 988        hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
 989
 990        /* write all 4 ring address registers */
 991        hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
 992                                offsetof(struct hifn_dma, cmdr[0]));
 993        hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
 994                                offsetof(struct hifn_dma, srcr[0]));
 995        hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
 996                                offsetof(struct hifn_dma, dstr[0]));
 997        hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
 998                                offsetof(struct hifn_dma, resr[0]));
 999
1000        mdelay(2);
1001#if 0
1002        hifn_write_1(dev, HIFN_1_DMA_CSR,
1003            HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1004            HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1005            HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1006            HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1007            HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1008            HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1009            HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1010            HIFN_DMACSR_S_WAIT |
1011            HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1012            HIFN_DMACSR_C_WAIT |
1013            HIFN_DMACSR_ENGINE |
1014            HIFN_DMACSR_PUBDONE);
1015#else
1016        hifn_write_1(dev, HIFN_1_DMA_CSR,
1017            HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1018            HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
1019            HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1020            HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1021            HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1022            HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1023            HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1024            HIFN_DMACSR_S_WAIT |
1025            HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1026            HIFN_DMACSR_C_WAIT |
1027            HIFN_DMACSR_ENGINE |
1028            HIFN_DMACSR_PUBDONE);
1029#endif
1030        hifn_read_1(dev, HIFN_1_DMA_CSR);
1031
1032        dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1033            HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1034            HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1035            HIFN_DMAIER_ENGINE;
1036        dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
1037
1038        hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1039        hifn_read_1(dev, HIFN_1_DMA_IER);
1040#if 0
1041        hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
1042                    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1043                    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1044                    HIFN_PUCNFG_DRAM);
1045#else
1046        hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
1047#endif
1048        hifn_init_pll(dev);
1049
1050        hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1051        hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1052            HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1053            ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1054            ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1055}
1056
1057static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
1058                unsigned dlen, unsigned slen, u16 mask, u8 snum)
1059{
1060        struct hifn_base_command *base_cmd;
1061        u8 *buf_pos = buf;
1062
1063        base_cmd = (struct hifn_base_command *)buf_pos;
1064        base_cmd->masks = __cpu_to_le16(mask);
1065        base_cmd->total_source_count =
1066                __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
1067        base_cmd->total_dest_count =
1068                __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1069
1070        dlen >>= 16;
1071        slen >>= 16;
1072        base_cmd->session_num = __cpu_to_le16(snum |
1073            ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1074            ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1075
1076        return sizeof(struct hifn_base_command);
1077}
1078
1079static int hifn_setup_crypto_command(struct hifn_device *dev,
1080                u8 *buf, unsigned dlen, unsigned slen,
1081                u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
1082{
1083        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1084        struct hifn_crypt_command *cry_cmd;
1085        u8 *buf_pos = buf;
1086        u16 cmd_len;
1087
1088        cry_cmd = (struct hifn_crypt_command *)buf_pos;
1089
1090        cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
1091        dlen >>= 16;
1092        cry_cmd->masks = __cpu_to_le16(mode |
1093                        ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
1094                         HIFN_CRYPT_CMD_SRCLEN_M));
1095        cry_cmd->header_skip = 0;
1096        cry_cmd->reserved = 0;
1097
1098        buf_pos += sizeof(struct hifn_crypt_command);
1099
1100        dma->cmdu++;
1101        if (dma->cmdu > 1) {
1102                dev->dmareg |= HIFN_DMAIER_C_WAIT;
1103                hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1104        }
1105
1106        if (keylen) {
1107                memcpy(buf_pos, key, keylen);
1108                buf_pos += keylen;
1109        }
1110        if (ivsize) {
1111                memcpy(buf_pos, iv, ivsize);
1112                buf_pos += ivsize;
1113        }
1114
1115        cmd_len = buf_pos - buf;
1116
1117        return cmd_len;
1118}
1119
1120static int hifn_setup_cmd_desc(struct hifn_device *dev,
1121                struct hifn_context *ctx, struct hifn_request_context *rctx,
1122                void *priv, unsigned int nbytes)
1123{
1124        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1125        int cmd_len, sa_idx;
1126        u8 *buf, *buf_pos;
1127        u16 mask;
1128
1129        sa_idx = dma->cmdi;
1130        buf_pos = buf = dma->command_bufs[dma->cmdi];
1131
1132        mask = 0;
1133        switch (rctx->op) {
1134        case ACRYPTO_OP_DECRYPT:
1135                mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1136                break;
1137        case ACRYPTO_OP_ENCRYPT:
1138                mask = HIFN_BASE_CMD_CRYPT;
1139                break;
1140        case ACRYPTO_OP_HMAC:
1141                mask = HIFN_BASE_CMD_MAC;
1142                break;
1143        default:
1144                goto err_out;
1145        }
1146
1147        buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1148                        nbytes, mask, dev->snum);
1149
1150        if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
1151                u16 md = 0;
1152
1153                if (ctx->keysize)
1154                        md |= HIFN_CRYPT_CMD_NEW_KEY;
1155                if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
1156                        md |= HIFN_CRYPT_CMD_NEW_IV;
1157
1158                switch (rctx->mode) {
1159                case ACRYPTO_MODE_ECB:
1160                        md |= HIFN_CRYPT_CMD_MODE_ECB;
1161                        break;
1162                case ACRYPTO_MODE_CBC:
1163                        md |= HIFN_CRYPT_CMD_MODE_CBC;
1164                        break;
1165                case ACRYPTO_MODE_CFB:
1166                        md |= HIFN_CRYPT_CMD_MODE_CFB;
1167                        break;
1168                case ACRYPTO_MODE_OFB:
1169                        md |= HIFN_CRYPT_CMD_MODE_OFB;
1170                        break;
1171                default:
1172                        goto err_out;
1173                }
1174
1175                switch (rctx->type) {
1176                case ACRYPTO_TYPE_AES_128:
1177                        if (ctx->keysize != 16)
1178                                goto err_out;
1179                        md |= HIFN_CRYPT_CMD_KSZ_128 |
1180                                HIFN_CRYPT_CMD_ALG_AES;
1181                        break;
1182                case ACRYPTO_TYPE_AES_192:
1183                        if (ctx->keysize != 24)
1184                                goto err_out;
1185                        md |= HIFN_CRYPT_CMD_KSZ_192 |
1186                                HIFN_CRYPT_CMD_ALG_AES;
1187                        break;
1188                case ACRYPTO_TYPE_AES_256:
1189                        if (ctx->keysize != 32)
1190                                goto err_out;
1191                        md |= HIFN_CRYPT_CMD_KSZ_256 |
1192                                HIFN_CRYPT_CMD_ALG_AES;
1193                        break;
1194                case ACRYPTO_TYPE_3DES:
1195                        if (ctx->keysize != 24)
1196                                goto err_out;
1197                        md |= HIFN_CRYPT_CMD_ALG_3DES;
1198                        break;
1199                case ACRYPTO_TYPE_DES:
1200                        if (ctx->keysize != 8)
1201                                goto err_out;
1202                        md |= HIFN_CRYPT_CMD_ALG_DES;
1203                        break;
1204                default:
1205                        goto err_out;
1206                }
1207
1208                buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1209                                nbytes, nbytes, ctx->key, ctx->keysize,
1210                                rctx->iv, rctx->ivsize, md);
1211        }
1212
1213        dev->sa[sa_idx] = priv;
1214        dev->started++;
1215
1216        cmd_len = buf_pos - buf;
1217        dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1218                        HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1219
1220        if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1221                dma->cmdr[dma->cmdi].l = __cpu_to_le32(
1222                        HIFN_D_VALID | HIFN_D_LAST |
1223                        HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1224                dma->cmdi = 0;
1225        } else {
1226                dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID);
1227        }
1228
1229        if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1230                hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1231                dev->flags |= HIFN_FLAG_CMD_BUSY;
1232        }
1233        return 0;
1234
1235err_out:
1236        return -EINVAL;
1237}
1238
1239static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1240                unsigned int offset, unsigned int size, int last)
1241{
1242        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1243        int idx;
1244        dma_addr_t addr;
1245
1246        addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1247
1248        idx = dma->srci;
1249
1250        dma->srcr[idx].p = __cpu_to_le32(addr);
1251        dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1252                        HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1253
1254        if (++idx == HIFN_D_SRC_RSIZE) {
1255                dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1256                                HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1257                                (last ? HIFN_D_LAST : 0));
1258                idx = 0;
1259        }
1260
1261        dma->srci = idx;
1262        dma->srcu++;
1263
1264        if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1265                hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1266                dev->flags |= HIFN_FLAG_SRC_BUSY;
1267        }
1268
1269        return size;
1270}
1271
1272static void hifn_setup_res_desc(struct hifn_device *dev)
1273{
1274        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1275
1276        dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1277                        HIFN_D_VALID | HIFN_D_LAST);
1278        /*
1279         * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1280         *                                      HIFN_D_LAST);
1281         */
1282
1283        if (++dma->resi == HIFN_D_RES_RSIZE) {
1284                dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1285                                HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1286                dma->resi = 0;
1287        }
1288
1289        dma->resu++;
1290
1291        if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1292                hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1293                dev->flags |= HIFN_FLAG_RES_BUSY;
1294        }
1295}
1296
1297static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1298                unsigned offset, unsigned size, int last)
1299{
1300        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1301        int idx;
1302        dma_addr_t addr;
1303
1304        addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1305
1306        idx = dma->dsti;
1307        dma->dstr[idx].p = __cpu_to_le32(addr);
1308        dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1309                        HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1310
1311        if (++idx == HIFN_D_DST_RSIZE) {
1312                dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1313                                HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1314                                (last ? HIFN_D_LAST : 0));
1315                idx = 0;
1316        }
1317        dma->dsti = idx;
1318        dma->dstu++;
1319
1320        if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1321                hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1322                dev->flags |= HIFN_FLAG_DST_BUSY;
1323        }
1324}
1325
1326static int hifn_setup_dma(struct hifn_device *dev,
1327                struct hifn_context *ctx, struct hifn_request_context *rctx,
1328                struct scatterlist *src, struct scatterlist *dst,
1329                unsigned int nbytes, void *priv)
1330{
1331        struct scatterlist *t;
1332        struct page *spage, *dpage;
1333        unsigned int soff, doff;
1334        unsigned int n, len;
1335
1336        n = nbytes;
1337        while (n) {
1338                spage = sg_page(src);
1339                soff = src->offset;
1340                len = min(src->length, n);
1341
1342                hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
1343
1344                src++;
1345                n -= len;
1346        }
1347
1348        t = &rctx->walk.cache[0];
1349        n = nbytes;
1350        while (n) {
1351                if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1352                        BUG_ON(!sg_page(t));
1353                        dpage = sg_page(t);
1354                        doff = 0;
1355                        len = t->length;
1356                } else {
1357                        BUG_ON(!sg_page(dst));
1358                        dpage = sg_page(dst);
1359                        doff = dst->offset;
1360                        len = dst->length;
1361                }
1362                len = min(len, n);
1363
1364                hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
1365
1366                dst++;
1367                t++;
1368                n -= len;
1369        }
1370
1371        hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
1372        hifn_setup_res_desc(dev);
1373        return 0;
1374}
1375
1376static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
1377                int num, gfp_t gfp_flags)
1378{
1379        int i;
1380
1381        num = min(ASYNC_SCATTERLIST_CACHE, num);
1382        sg_init_table(w->cache, num);
1383
1384        w->num = 0;
1385        for (i = 0; i < num; ++i) {
1386                struct page *page = alloc_page(gfp_flags);
1387                struct scatterlist *s;
1388
1389                if (!page)
1390                        break;
1391
1392                s = &w->cache[i];
1393
1394                sg_set_page(s, page, PAGE_SIZE, 0);
1395                w->num++;
1396        }
1397
1398        return i;
1399}
1400
1401static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
1402{
1403        int i;
1404
1405        for (i = 0; i < w->num; ++i) {
1406                struct scatterlist *s = &w->cache[i];
1407
1408                __free_page(sg_page(s));
1409
1410                s->length = 0;
1411        }
1412
1413        w->num = 0;
1414}
1415
1416static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst,
1417                unsigned int size, unsigned int *nbytesp)
1418{
1419        unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1420        int idx = 0;
1421
1422        if (drest < size || size > nbytes)
1423                return -EINVAL;
1424
1425        while (size) {
1426                copy = min3(drest, size, dst->length);
1427
1428                size -= copy;
1429                drest -= copy;
1430                nbytes -= copy;
1431
1432                pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1433                         __func__, copy, size, drest, nbytes);
1434
1435                dst++;
1436                idx++;
1437        }
1438
1439        *nbytesp = nbytes;
1440        *drestp = drest;
1441
1442        return idx;
1443}
1444
1445static int hifn_cipher_walk(struct ablkcipher_request *req,
1446                struct hifn_cipher_walk *w)
1447{
1448        struct scatterlist *dst, *t;
1449        unsigned int nbytes = req->nbytes, offset, copy, diff;
1450        int idx, tidx, err;
1451
1452        tidx = idx = 0;
1453        offset = 0;
1454        while (nbytes) {
1455                if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1456                        return -EINVAL;
1457
1458                dst = &req->dst[idx];
1459
1460                pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
1461                         __func__, dst->length, dst->offset, offset, nbytes);
1462
1463                if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1464                    !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
1465                    offset) {
1466                        unsigned slen = min(dst->length - offset, nbytes);
1467                        unsigned dlen = PAGE_SIZE;
1468
1469                        t = &w->cache[idx];
1470
1471                        err = ablkcipher_add(&dlen, dst, slen, &nbytes);
1472                        if (err < 0)
1473                                return err;
1474
1475                        idx += err;
1476
1477                        copy = slen & ~(HIFN_D_DST_DALIGN - 1);
1478                        diff = slen & (HIFN_D_DST_DALIGN - 1);
1479
1480                        if (dlen < nbytes) {
1481                                /*
1482                                 * Destination page does not have enough space
1483                                 * to put there additional blocksized chunk,
1484                                 * so we mark that page as containing only
1485                                 * blocksize aligned chunks:
1486                                 *      t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
1487                                 * and increase number of bytes to be processed
1488                                 * in next chunk:
1489                                 *      nbytes += diff;
1490                                 */
1491                                nbytes += diff;
1492
1493                                /*
1494                                 * Temporary of course...
1495                                 * Kick author if you will catch this one.
1496                                 */
1497                                pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n",
1498                                       __func__, dlen, nbytes, slen, offset);
1499                                pr_err("%s: please contact author to fix this "
1500                                       "issue, generally you should not catch "
1501                                       "this path under any condition but who "
1502                                       "knows how did you use crypto code.\n"
1503                                       "Thank you.\n",  __func__);
1504                                BUG();
1505                        } else {
1506                                copy += diff + nbytes;
1507
1508                                dst = &req->dst[idx];
1509
1510                                err = ablkcipher_add(&dlen, dst, nbytes, &nbytes);
1511                                if (err < 0)
1512                                        return err;
1513
1514                                idx += err;
1515                        }
1516
1517                        t->length = copy;
1518                        t->offset = offset;
1519                } else {
1520                        nbytes -= min(dst->length, nbytes);
1521                        idx++;
1522                }
1523
1524                tidx++;
1525        }
1526
1527        return tidx;
1528}
1529
1530static int hifn_setup_session(struct ablkcipher_request *req)
1531{
1532        struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1533        struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
1534        struct hifn_device *dev = ctx->dev;
1535        unsigned long dlen, flags;
1536        unsigned int nbytes = req->nbytes, idx = 0;
1537        int err = -EINVAL, sg_num;
1538        struct scatterlist *dst;
1539
1540        if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
1541                goto err_out_exit;
1542
1543        rctx->walk.flags = 0;
1544
1545        while (nbytes) {
1546                dst = &req->dst[idx];
1547                dlen = min(dst->length, nbytes);
1548
1549                if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1550                    !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
1551                        rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1552
1553                nbytes -= dlen;
1554                idx++;
1555        }
1556
1557        if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1558                err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
1559                if (err < 0)
1560                        return err;
1561        }
1562
1563        sg_num = hifn_cipher_walk(req, &rctx->walk);
1564        if (sg_num < 0) {
1565                err = sg_num;
1566                goto err_out_exit;
1567        }
1568
1569        spin_lock_irqsave(&dev->lock, flags);
1570        if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1571                err = -EAGAIN;
1572                goto err_out;
1573        }
1574
1575        err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req);
1576        if (err)
1577                goto err_out;
1578
1579        dev->snum++;
1580
1581        dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1582        spin_unlock_irqrestore(&dev->lock, flags);
1583
1584        return 0;
1585
1586err_out:
1587        spin_unlock_irqrestore(&dev->lock, flags);
1588err_out_exit:
1589        if (err) {
1590                dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1591                         "type: %u, err: %d.\n",
1592                         rctx->iv, rctx->ivsize,
1593                         ctx->key, ctx->keysize,
1594                         rctx->mode, rctx->op, rctx->type, err);
1595        }
1596
1597        return err;
1598}
1599
1600static int hifn_start_device(struct hifn_device *dev)
1601{
1602        int err;
1603
1604        dev->started = dev->active = 0;
1605        hifn_reset_dma(dev, 1);
1606
1607        err = hifn_enable_crypto(dev);
1608        if (err)
1609                return err;
1610
1611        hifn_reset_puc(dev);
1612
1613        hifn_init_dma(dev);
1614
1615        hifn_init_registers(dev);
1616
1617        hifn_init_pubrng(dev);
1618
1619        return 0;
1620}
1621
1622static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1623                struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1624{
1625        unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1626        void *daddr;
1627        int idx = 0;
1628
1629        if (srest < size || size > nbytes)
1630                return -EINVAL;
1631
1632        while (size) {
1633                copy = min3(srest, dst->length, size);
1634
1635                daddr = kmap_atomic(sg_page(dst));
1636                memcpy(daddr + dst->offset + offset, saddr, copy);
1637                kunmap_atomic(daddr);
1638
1639                nbytes -= copy;
1640                size -= copy;
1641                srest -= copy;
1642                saddr += copy;
1643                offset = 0;
1644
1645                pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1646                         __func__, copy, size, srest, nbytes);
1647
1648                dst++;
1649                idx++;
1650        }
1651
1652        *nbytesp = nbytes;
1653        *srestp = srest;
1654
1655        return idx;
1656}
1657
1658static inline void hifn_complete_sa(struct hifn_device *dev, int i)
1659{
1660        unsigned long flags;
1661
1662        spin_lock_irqsave(&dev->lock, flags);
1663        dev->sa[i] = NULL;
1664        dev->started--;
1665        if (dev->started < 0)
1666                dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__,
1667                         dev->started);
1668        spin_unlock_irqrestore(&dev->lock, flags);
1669        BUG_ON(dev->started < 0);
1670}
1671
1672static void hifn_process_ready(struct ablkcipher_request *req, int error)
1673{
1674        struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
1675
1676        if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1677                unsigned int nbytes = req->nbytes;
1678                int idx = 0, err;
1679                struct scatterlist *dst, *t;
1680                void *saddr;
1681
1682                while (nbytes) {
1683                        t = &rctx->walk.cache[idx];
1684                        dst = &req->dst[idx];
1685
1686                        pr_debug("\n%s: sg_page(t): %p, t->length: %u, "
1687                                "sg_page(dst): %p, dst->length: %u, "
1688                                "nbytes: %u.\n",
1689                                __func__, sg_page(t), t->length,
1690                                sg_page(dst), dst->length, nbytes);
1691
1692                        if (!t->length) {
1693                                nbytes -= min(dst->length, nbytes);
1694                                idx++;
1695                                continue;
1696                        }
1697
1698                        saddr = kmap_atomic(sg_page(t));
1699
1700                        err = ablkcipher_get(saddr, &t->length, t->offset,
1701                                        dst, nbytes, &nbytes);
1702                        if (err < 0) {
1703                                kunmap_atomic(saddr);
1704                                break;
1705                        }
1706
1707                        idx += err;
1708                        kunmap_atomic(saddr);
1709                }
1710
1711                hifn_cipher_walk_exit(&rctx->walk);
1712        }
1713
1714        req->base.complete(&req->base, error);
1715}
1716
1717static void hifn_clear_rings(struct hifn_device *dev, int error)
1718{
1719        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1720        int i, u;
1721
1722        dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1723                        "k: %d.%d.%d.%d.\n",
1724                        dma->cmdi, dma->srci, dma->dsti, dma->resi,
1725                        dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1726                        dma->cmdk, dma->srck, dma->dstk, dma->resk);
1727
1728        i = dma->resk; u = dma->resu;
1729        while (u != 0) {
1730                if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1731                        break;
1732
1733                if (dev->sa[i]) {
1734                        dev->success++;
1735                        dev->reset = 0;
1736                        hifn_process_ready(dev->sa[i], error);
1737                        hifn_complete_sa(dev, i);
1738                }
1739
1740                if (++i == HIFN_D_RES_RSIZE)
1741                        i = 0;
1742                u--;
1743        }
1744        dma->resk = i; dma->resu = u;
1745
1746        i = dma->srck; u = dma->srcu;
1747        while (u != 0) {
1748                if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1749                        break;
1750                if (++i == HIFN_D_SRC_RSIZE)
1751                        i = 0;
1752                u--;
1753        }
1754        dma->srck = i; dma->srcu = u;
1755
1756        i = dma->cmdk; u = dma->cmdu;
1757        while (u != 0) {
1758                if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1759                        break;
1760                if (++i == HIFN_D_CMD_RSIZE)
1761                        i = 0;
1762                u--;
1763        }
1764        dma->cmdk = i; dma->cmdu = u;
1765
1766        i = dma->dstk; u = dma->dstu;
1767        while (u != 0) {
1768                if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1769                        break;
1770                if (++i == HIFN_D_DST_RSIZE)
1771                        i = 0;
1772                u--;
1773        }
1774        dma->dstk = i; dma->dstu = u;
1775
1776        dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1777                        "k: %d.%d.%d.%d.\n",
1778                        dma->cmdi, dma->srci, dma->dsti, dma->resi,
1779                        dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1780                        dma->cmdk, dma->srck, dma->dstk, dma->resk);
1781}
1782
1783static void hifn_work(struct work_struct *work)
1784{
1785        struct delayed_work *dw = to_delayed_work(work);
1786        struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1787        unsigned long flags;
1788        int reset = 0;
1789        u32 r = 0;
1790
1791        spin_lock_irqsave(&dev->lock, flags);
1792        if (dev->active == 0) {
1793                struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1794
1795                if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1796                        dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1797                        r |= HIFN_DMACSR_C_CTRL_DIS;
1798                }
1799                if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1800                        dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1801                        r |= HIFN_DMACSR_S_CTRL_DIS;
1802                }
1803                if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1804                        dev->flags &= ~HIFN_FLAG_DST_BUSY;
1805                        r |= HIFN_DMACSR_D_CTRL_DIS;
1806                }
1807                if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1808                        dev->flags &= ~HIFN_FLAG_RES_BUSY;
1809                        r |= HIFN_DMACSR_R_CTRL_DIS;
1810                }
1811                if (r)
1812                        hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1813        } else
1814                dev->active--;
1815
1816        if ((dev->prev_success == dev->success) && dev->started)
1817                reset = 1;
1818        dev->prev_success = dev->success;
1819        spin_unlock_irqrestore(&dev->lock, flags);
1820
1821        if (reset) {
1822                if (++dev->reset >= 5) {
1823                        int i;
1824                        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1825
1826                        dev_info(&dev->pdev->dev,
1827                                 "r: %08x, active: %d, started: %d, "
1828                                 "success: %lu: qlen: %u/%u, reset: %d.\n",
1829                                 r, dev->active, dev->started,
1830                                 dev->success, dev->queue.qlen, dev->queue.max_qlen,
1831                                 reset);
1832
1833                        dev_info(&dev->pdev->dev, "%s: res: ", __func__);
1834                        for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1835                                pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]);
1836                                if (dev->sa[i]) {
1837                                        hifn_process_ready(dev->sa[i], -ENODEV);
1838                                        hifn_complete_sa(dev, i);
1839                                }
1840                        }
1841                        pr_info("\n");
1842
1843                        hifn_reset_dma(dev, 1);
1844                        hifn_stop_device(dev);
1845                        hifn_start_device(dev);
1846                        dev->reset = 0;
1847                }
1848
1849                tasklet_schedule(&dev->tasklet);
1850        }
1851
1852        schedule_delayed_work(&dev->work, HZ);
1853}
1854
1855static irqreturn_t hifn_interrupt(int irq, void *data)
1856{
1857        struct hifn_device *dev = (struct hifn_device *)data;
1858        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1859        u32 dmacsr, restart;
1860
1861        dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
1862
1863        dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
1864                        "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
1865                dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
1866                dma->cmdi, dma->srci, dma->dsti, dma->resi,
1867                dma->cmdu, dma->srcu, dma->dstu, dma->resu);
1868
1869        if ((dmacsr & dev->dmareg) == 0)
1870                return IRQ_NONE;
1871
1872        hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
1873
1874        if (dmacsr & HIFN_DMACSR_ENGINE)
1875                hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
1876        if (dmacsr & HIFN_DMACSR_PUBDONE)
1877                hifn_write_1(dev, HIFN_1_PUB_STATUS,
1878                        hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1879
1880        restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1881        if (restart) {
1882                u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
1883
1884                dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
1885                         !!(dmacsr & HIFN_DMACSR_R_OVER),
1886                         !!(dmacsr & HIFN_DMACSR_D_OVER),
1887                        puisr, !!(puisr & HIFN_PUISR_DSTOVER));
1888                if (!!(puisr & HIFN_PUISR_DSTOVER))
1889                        hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1890                hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
1891                                        HIFN_DMACSR_D_OVER));
1892        }
1893
1894        restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1895                        HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1896        if (restart) {
1897                dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n",
1898                         !!(dmacsr & HIFN_DMACSR_C_ABORT),
1899                         !!(dmacsr & HIFN_DMACSR_S_ABORT),
1900                         !!(dmacsr & HIFN_DMACSR_D_ABORT),
1901                         !!(dmacsr & HIFN_DMACSR_R_ABORT));
1902                hifn_reset_dma(dev, 1);
1903                hifn_init_dma(dev);
1904                hifn_init_registers(dev);
1905        }
1906
1907        if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
1908                dev_dbg(&dev->pdev->dev, "wait on command.\n");
1909                dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
1910                hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1911        }
1912
1913        tasklet_schedule(&dev->tasklet);
1914
1915        return IRQ_HANDLED;
1916}
1917
1918static void hifn_flush(struct hifn_device *dev)
1919{
1920        unsigned long flags;
1921        struct crypto_async_request *async_req;
1922        struct ablkcipher_request *req;
1923        struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1924        int i;
1925
1926        for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1927                struct hifn_desc *d = &dma->resr[i];
1928
1929                if (dev->sa[i]) {
1930                        hifn_process_ready(dev->sa[i],
1931                                (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0);
1932                        hifn_complete_sa(dev, i);
1933                }
1934        }
1935
1936        spin_lock_irqsave(&dev->lock, flags);
1937        while ((async_req = crypto_dequeue_request(&dev->queue))) {
1938                req = ablkcipher_request_cast(async_req);
1939                spin_unlock_irqrestore(&dev->lock, flags);
1940
1941                hifn_process_ready(req, -ENODEV);
1942
1943                spin_lock_irqsave(&dev->lock, flags);
1944        }
1945        spin_unlock_irqrestore(&dev->lock, flags);
1946}
1947
1948static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1949                unsigned int len)
1950{
1951        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
1952        struct hifn_context *ctx = crypto_tfm_ctx(tfm);
1953        struct hifn_device *dev = ctx->dev;
1954
1955        if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
1956                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1957                return -1;
1958        }
1959
1960        if (len == HIFN_DES_KEY_LENGTH) {
1961                u32 tmp[DES_EXPKEY_WORDS];
1962                int ret = des_ekey(tmp, key);
1963
1964                if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
1965                        tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
1966                        return -EINVAL;
1967                }
1968        }
1969
1970        dev->flags &= ~HIFN_FLAG_OLD_KEY;
1971
1972        memcpy(ctx->key, key, len);
1973        ctx->keysize = len;
1974
1975        return 0;
1976}
1977
1978static int hifn_handle_req(struct ablkcipher_request *req)
1979{
1980        struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1981        struct hifn_device *dev = ctx->dev;
1982        int err = -EAGAIN;
1983
1984        if (dev->started + DIV_ROUND_UP(req->nbytes, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
1985                err = hifn_setup_session(req);
1986
1987        if (err == -EAGAIN) {
1988                unsigned long flags;
1989
1990                spin_lock_irqsave(&dev->lock, flags);
1991                err = ablkcipher_enqueue_request(&dev->queue, req);
1992                spin_unlock_irqrestore(&dev->lock, flags);
1993        }
1994
1995        return err;
1996}
1997
1998static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op,
1999                u8 type, u8 mode)
2000{
2001        struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2002        struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
2003        unsigned ivsize;
2004
2005        ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
2006
2007        if (req->info && mode != ACRYPTO_MODE_ECB) {
2008                if (type == ACRYPTO_TYPE_AES_128)
2009                        ivsize = HIFN_AES_IV_LENGTH;
2010                else if (type == ACRYPTO_TYPE_DES)
2011                        ivsize = HIFN_DES_KEY_LENGTH;
2012                else if (type == ACRYPTO_TYPE_3DES)
2013                        ivsize = HIFN_3DES_KEY_LENGTH;
2014        }
2015
2016        if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
2017                if (ctx->keysize == 24)
2018                        type = ACRYPTO_TYPE_AES_192;
2019                else if (ctx->keysize == 32)
2020                        type = ACRYPTO_TYPE_AES_256;
2021        }
2022
2023        rctx->op = op;
2024        rctx->mode = mode;
2025        rctx->type = type;
2026        rctx->iv = req->info;
2027        rctx->ivsize = ivsize;
2028
2029        /*
2030         * HEAVY TODO: needs to kick Herbert XU to write documentation.
2031         * HEAVY TODO: needs to kick Herbert XU to write documentation.
2032         * HEAVY TODO: needs to kick Herbert XU to write documentation.
2033         */
2034
2035        return hifn_handle_req(req);
2036}
2037
2038static int hifn_process_queue(struct hifn_device *dev)
2039{
2040        struct crypto_async_request *async_req, *backlog;
2041        struct ablkcipher_request *req;
2042        unsigned long flags;
2043        int err = 0;
2044
2045        while (dev->started < HIFN_QUEUE_LENGTH) {
2046                spin_lock_irqsave(&dev->lock, flags);
2047                backlog = crypto_get_backlog(&dev->queue);
2048                async_req = crypto_dequeue_request(&dev->queue);
2049                spin_unlock_irqrestore(&dev->lock, flags);
2050
2051                if (!async_req)
2052                        break;
2053
2054                if (backlog)
2055                        backlog->complete(backlog, -EINPROGRESS);
2056
2057                req = ablkcipher_request_cast(async_req);
2058
2059                err = hifn_handle_req(req);
2060                if (err)
2061                        break;
2062        }
2063
2064        return err;
2065}
2066
2067static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
2068                u8 type, u8 mode)
2069{
2070        int err;
2071        struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2072        struct hifn_device *dev = ctx->dev;
2073
2074        err = hifn_setup_crypto_req(req, op, type, mode);
2075        if (err)
2076                return err;
2077
2078        if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2079                hifn_process_queue(dev);
2080
2081        return -EINPROGRESS;
2082}
2083
2084/*
2085 * AES ecryption functions.
2086 */
2087static inline int hifn_encrypt_aes_ecb(struct ablkcipher_request *req)
2088{
2089        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2090                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2091}
2092static inline int hifn_encrypt_aes_cbc(struct ablkcipher_request *req)
2093{
2094        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2095                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2096}
2097static inline int hifn_encrypt_aes_cfb(struct ablkcipher_request *req)
2098{
2099        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2100                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2101}
2102static inline int hifn_encrypt_aes_ofb(struct ablkcipher_request *req)
2103{
2104        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2105                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2106}
2107
2108/*
2109 * AES decryption functions.
2110 */
2111static inline int hifn_decrypt_aes_ecb(struct ablkcipher_request *req)
2112{
2113        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2114                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2115}
2116static inline int hifn_decrypt_aes_cbc(struct ablkcipher_request *req)
2117{
2118        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2119                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2120}
2121static inline int hifn_decrypt_aes_cfb(struct ablkcipher_request *req)
2122{
2123        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2124                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2125}
2126static inline int hifn_decrypt_aes_ofb(struct ablkcipher_request *req)
2127{
2128        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2129                        ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2130}
2131
2132/*
2133 * DES ecryption functions.
2134 */
2135static inline int hifn_encrypt_des_ecb(struct ablkcipher_request *req)
2136{
2137        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2138                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2139}
2140static inline int hifn_encrypt_des_cbc(struct ablkcipher_request *req)
2141{
2142        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2143                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2144}
2145static inline int hifn_encrypt_des_cfb(struct ablkcipher_request *req)
2146{
2147        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2148                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2149}
2150static inline int hifn_encrypt_des_ofb(struct ablkcipher_request *req)
2151{
2152        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2153                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2154}
2155
2156/*
2157 * DES decryption functions.
2158 */
2159static inline int hifn_decrypt_des_ecb(struct ablkcipher_request *req)
2160{
2161        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2162                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2163}
2164static inline int hifn_decrypt_des_cbc(struct ablkcipher_request *req)
2165{
2166        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2167                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2168}
2169static inline int hifn_decrypt_des_cfb(struct ablkcipher_request *req)
2170{
2171        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2172                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2173}
2174static inline int hifn_decrypt_des_ofb(struct ablkcipher_request *req)
2175{
2176        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2177                        ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2178}
2179
2180/*
2181 * 3DES ecryption functions.
2182 */
2183static inline int hifn_encrypt_3des_ecb(struct ablkcipher_request *req)
2184{
2185        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2186                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2187}
2188static inline int hifn_encrypt_3des_cbc(struct ablkcipher_request *req)
2189{
2190        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2191                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2192}
2193static inline int hifn_encrypt_3des_cfb(struct ablkcipher_request *req)
2194{
2195        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2196                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2197}
2198static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req)
2199{
2200        return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2201                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2202}
2203
2204/* 3DES decryption functions. */
2205static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req)
2206{
2207        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2208                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2209}
2210static inline int hifn_decrypt_3des_cbc(struct ablkcipher_request *req)
2211{
2212        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2213                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2214}
2215static inline int hifn_decrypt_3des_cfb(struct ablkcipher_request *req)
2216{
2217        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2218                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2219}
2220static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req)
2221{
2222        return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2223                        ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2224}
2225
2226struct hifn_alg_template {
2227        char name[CRYPTO_MAX_ALG_NAME];
2228        char drv_name[CRYPTO_MAX_ALG_NAME];
2229        unsigned int bsize;
2230        struct ablkcipher_alg ablkcipher;
2231};
2232
2233static struct hifn_alg_template hifn_alg_templates[] = {
2234        /*
2235         * 3DES ECB, CBC, CFB and OFB modes.
2236         */
2237        {
2238                .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
2239                .ablkcipher = {
2240                        .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2241                        .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2242                        .setkey         =       hifn_setkey,
2243                        .encrypt        =       hifn_encrypt_3des_cfb,
2244                        .decrypt        =       hifn_decrypt_3des_cfb,
2245                },
2246        },
2247        {
2248                .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
2249                .ablkcipher = {
2250                        .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2251                        .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2252                        .setkey         =       hifn_setkey,
2253                        .encrypt        =       hifn_encrypt_3des_ofb,
2254                        .decrypt        =       hifn_decrypt_3des_ofb,
2255                },
2256        },
2257        {
2258                .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2259                .ablkcipher = {
2260                        .ivsize         =       HIFN_IV_LENGTH,
2261                        .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2262                        .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2263                        .setkey         =       hifn_setkey,
2264                        .encrypt        =       hifn_encrypt_3des_cbc,
2265                        .decrypt        =       hifn_decrypt_3des_cbc,
2266                },
2267        },
2268        {
2269                .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2270                .ablkcipher = {
2271                        .min_keysize    =       HIFN_3DES_KEY_LENGTH,
2272                        .max_keysize    =       HIFN_3DES_KEY_LENGTH,
2273                        .setkey         =       hifn_setkey,
2274                        .encrypt        =       hifn_encrypt_3des_ecb,
2275                        .decrypt        =       hifn_decrypt_3des_ecb,
2276                },
2277        },
2278
2279        /*
2280         * DES ECB, CBC, CFB and OFB modes.
2281         */
2282        {
2283                .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
2284                .ablkcipher = {
2285                        .min_keysize    =       HIFN_DES_KEY_LENGTH,
2286                        .max_keysize    =       HIFN_DES_KEY_LENGTH,
2287                        .setkey         =       hifn_setkey,
2288                        .encrypt        =       hifn_encrypt_des_cfb,
2289                        .decrypt        =       hifn_decrypt_des_cfb,
2290                },
2291        },
2292        {
2293                .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
2294                .ablkcipher = {
2295                        .min_keysize    =       HIFN_DES_KEY_LENGTH,
2296                        .max_keysize    =       HIFN_DES_KEY_LENGTH,
2297                        .setkey         =       hifn_setkey,
2298                        .encrypt        =       hifn_encrypt_des_ofb,
2299                        .decrypt        =       hifn_decrypt_des_ofb,
2300                },
2301        },
2302        {
2303                .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2304                .ablkcipher = {
2305                        .ivsize         =       HIFN_IV_LENGTH,
2306                        .min_keysize    =       HIFN_DES_KEY_LENGTH,
2307                        .max_keysize    =       HIFN_DES_KEY_LENGTH,
2308                        .setkey         =       hifn_setkey,
2309                        .encrypt        =       hifn_encrypt_des_cbc,
2310                        .decrypt        =       hifn_decrypt_des_cbc,
2311                },
2312        },
2313        {
2314                .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2315                .ablkcipher = {
2316                        .min_keysize    =       HIFN_DES_KEY_LENGTH,
2317                        .max_keysize    =       HIFN_DES_KEY_LENGTH,
2318                        .setkey         =       hifn_setkey,
2319                        .encrypt        =       hifn_encrypt_des_ecb,
2320                        .decrypt        =       hifn_decrypt_des_ecb,
2321                },
2322        },
2323
2324        /*
2325         * AES ECB, CBC, CFB and OFB modes.
2326         */
2327        {
2328                .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2329                .ablkcipher = {
2330                        .min_keysize    =       AES_MIN_KEY_SIZE,
2331                        .max_keysize    =       AES_MAX_KEY_SIZE,
2332                        .setkey         =       hifn_setkey,
2333                        .encrypt        =       hifn_encrypt_aes_ecb,
2334                        .decrypt        =       hifn_decrypt_aes_ecb,
2335                },
2336        },
2337        {
2338                .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2339                .ablkcipher = {
2340                        .ivsize         =       HIFN_AES_IV_LENGTH,
2341                        .min_keysize    =       AES_MIN_KEY_SIZE,
2342                        .max_keysize    =       AES_MAX_KEY_SIZE,
2343                        .setkey         =       hifn_setkey,
2344                        .encrypt        =       hifn_encrypt_aes_cbc,
2345                        .decrypt        =       hifn_decrypt_aes_cbc,
2346                },
2347        },
2348        {
2349                .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
2350                .ablkcipher = {
2351                        .min_keysize    =       AES_MIN_KEY_SIZE,
2352                        .max_keysize    =       AES_MAX_KEY_SIZE,
2353                        .setkey         =       hifn_setkey,
2354                        .encrypt        =       hifn_encrypt_aes_cfb,
2355                        .decrypt        =       hifn_decrypt_aes_cfb,
2356                },
2357        },
2358        {
2359                .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
2360                .ablkcipher = {
2361                        .min_keysize    =       AES_MIN_KEY_SIZE,
2362                        .max_keysize    =       AES_MAX_KEY_SIZE,
2363                        .setkey         =       hifn_setkey,
2364                        .encrypt        =       hifn_encrypt_aes_ofb,
2365                        .decrypt        =       hifn_decrypt_aes_ofb,
2366                },
2367        },
2368};
2369
2370static int hifn_cra_init(struct crypto_tfm *tfm)
2371{
2372        struct crypto_alg *alg = tfm->__crt_alg;
2373        struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2374        struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2375
2376        ctx->dev = ha->dev;
2377        tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context);
2378        return 0;
2379}
2380
2381static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
2382{
2383        struct hifn_crypto_alg *alg;
2384        int err;
2385
2386        alg = kzalloc(sizeof(*alg), GFP_KERNEL);
2387        if (!alg)
2388                return -ENOMEM;
2389
2390        snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
2391        snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
2392                 t->drv_name, dev->name);
2393
2394        alg->alg.cra_priority = 300;
2395        alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2396                                CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
2397        alg->alg.cra_blocksize = t->bsize;
2398        alg->alg.cra_ctxsize = sizeof(struct hifn_context);
2399        alg->alg.cra_alignmask = 0;
2400        alg->alg.cra_type = &crypto_ablkcipher_type;
2401        alg->alg.cra_module = THIS_MODULE;
2402        alg->alg.cra_u.ablkcipher = t->ablkcipher;
2403        alg->alg.cra_init = hifn_cra_init;
2404
2405        alg->dev = dev;
2406
2407        list_add_tail(&alg->entry, &dev->alg_list);
2408
2409        err = crypto_register_alg(&alg->alg);
2410        if (err) {
2411                list_del(&alg->entry);
2412                kfree(alg);
2413        }
2414
2415        return err;
2416}
2417
2418static void hifn_unregister_alg(struct hifn_device *dev)
2419{
2420        struct hifn_crypto_alg *a, *n;
2421
2422        list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2423                list_del(&a->entry);
2424                crypto_unregister_alg(&a->alg);
2425                kfree(a);
2426        }
2427}
2428
2429static int hifn_register_alg(struct hifn_device *dev)
2430{
2431        int i, err;
2432
2433        for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) {
2434                err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2435                if (err)
2436                        goto err_out_exit;
2437        }
2438
2439        return 0;
2440
2441err_out_exit:
2442        hifn_unregister_alg(dev);
2443        return err;
2444}
2445
2446static void hifn_tasklet_callback(unsigned long data)
2447{
2448        struct hifn_device *dev = (struct hifn_device *)data;
2449
2450        /*
2451         * This is ok to call this without lock being held,
2452         * althogh it modifies some parameters used in parallel,
2453         * (like dev->success), but they are used in process
2454         * context or update is atomic (like setting dev->sa[i] to NULL).
2455         */
2456        hifn_clear_rings(dev, 0);
2457
2458        if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2459                hifn_process_queue(dev);
2460}
2461
2462static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2463{
2464        int err, i;
2465        struct hifn_device *dev;
2466        char name[8];
2467
2468        err = pci_enable_device(pdev);
2469        if (err)
2470                return err;
2471        pci_set_master(pdev);
2472
2473        err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2474        if (err)
2475                goto err_out_disable_pci_device;
2476
2477        snprintf(name, sizeof(name), "hifn%d",
2478                        atomic_inc_return(&hifn_dev_number) - 1);
2479
2480        err = pci_request_regions(pdev, name);
2481        if (err)
2482                goto err_out_disable_pci_device;
2483
2484        if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2485            pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2486            pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2487                dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n");
2488                err = -ENODEV;
2489                goto err_out_free_regions;
2490        }
2491
2492        dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2493                        GFP_KERNEL);
2494        if (!dev) {
2495                err = -ENOMEM;
2496                goto err_out_free_regions;
2497        }
2498
2499        INIT_LIST_HEAD(&dev->alg_list);
2500
2501        snprintf(dev->name, sizeof(dev->name), "%s", name);
2502        spin_lock_init(&dev->lock);
2503
2504        for (i = 0; i < 3; ++i) {
2505                unsigned long addr, size;
2506
2507                addr = pci_resource_start(pdev, i);
2508                size = pci_resource_len(pdev, i);
2509
2510                dev->bar[i] = ioremap_nocache(addr, size);
2511                if (!dev->bar[i]) {
2512                        err = -ENOMEM;
2513                        goto err_out_unmap_bars;
2514                }
2515        }
2516
2517        dev->desc_virt = pci_zalloc_consistent(pdev, sizeof(struct hifn_dma),
2518                                               &dev->desc_dma);
2519        if (!dev->desc_virt) {
2520                dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");
2521                err = -ENOMEM;
2522                goto err_out_unmap_bars;
2523        }
2524
2525        dev->pdev = pdev;
2526        dev->irq = pdev->irq;
2527
2528        for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
2529                dev->sa[i] = NULL;
2530
2531        pci_set_drvdata(pdev, dev);
2532
2533        tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
2534
2535        crypto_init_queue(&dev->queue, 1);
2536
2537        err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2538        if (err) {
2539                dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n",
2540                        dev->irq, err);
2541                dev->irq = 0;
2542                goto err_out_free_desc;
2543        }
2544
2545        err = hifn_start_device(dev);
2546        if (err)
2547                goto err_out_free_irq;
2548
2549        err = hifn_register_rng(dev);
2550        if (err)
2551                goto err_out_stop_device;
2552
2553        err = hifn_register_alg(dev);
2554        if (err)
2555                goto err_out_unregister_rng;
2556
2557        INIT_DELAYED_WORK(&dev->work, hifn_work);
2558        schedule_delayed_work(&dev->work, HZ);
2559
2560        dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been "
2561                "successfully registered as %s.\n",
2562                pci_name(pdev), dev->name);
2563
2564        return 0;
2565
2566err_out_unregister_rng:
2567        hifn_unregister_rng(dev);
2568err_out_stop_device:
2569        hifn_reset_dma(dev, 1);
2570        hifn_stop_device(dev);
2571err_out_free_irq:
2572        free_irq(dev->irq, dev);
2573        tasklet_kill(&dev->tasklet);
2574err_out_free_desc:
2575        pci_free_consistent(pdev, sizeof(struct hifn_dma),
2576                        dev->desc_virt, dev->desc_dma);
2577
2578err_out_unmap_bars:
2579        for (i = 0; i < 3; ++i)
2580                if (dev->bar[i])
2581                        iounmap(dev->bar[i]);
2582        kfree(dev);
2583
2584err_out_free_regions:
2585        pci_release_regions(pdev);
2586
2587err_out_disable_pci_device:
2588        pci_disable_device(pdev);
2589
2590        return err;
2591}
2592
2593static void hifn_remove(struct pci_dev *pdev)
2594{
2595        int i;
2596        struct hifn_device *dev;
2597
2598        dev = pci_get_drvdata(pdev);
2599
2600        if (dev) {
2601                cancel_delayed_work_sync(&dev->work);
2602
2603                hifn_unregister_rng(dev);
2604                hifn_unregister_alg(dev);
2605                hifn_reset_dma(dev, 1);
2606                hifn_stop_device(dev);
2607
2608                free_irq(dev->irq, dev);
2609                tasklet_kill(&dev->tasklet);
2610
2611                hifn_flush(dev);
2612
2613                pci_free_consistent(pdev, sizeof(struct hifn_dma),
2614                                dev->desc_virt, dev->desc_dma);
2615                for (i = 0; i < 3; ++i)
2616                        if (dev->bar[i])
2617                                iounmap(dev->bar[i]);
2618
2619                kfree(dev);
2620        }
2621
2622        pci_release_regions(pdev);
2623        pci_disable_device(pdev);
2624}
2625
2626static struct pci_device_id hifn_pci_tbl[] = {
2627        { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2628        { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2629        { 0 }
2630};
2631MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2632
2633static struct pci_driver hifn_pci_driver = {
2634        .name     = "hifn795x",
2635        .id_table = hifn_pci_tbl,
2636        .probe    = hifn_probe,
2637        .remove   = hifn_remove,
2638};
2639
2640static int __init hifn_init(void)
2641{
2642        unsigned int freq;
2643        int err;
2644
2645        /* HIFN supports only 32-bit addresses */
2646        BUILD_BUG_ON(sizeof(dma_addr_t) != 4);
2647
2648        if (strncmp(hifn_pll_ref, "ext", 3) &&
2649            strncmp(hifn_pll_ref, "pci", 3)) {
2650                pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");
2651                return -EINVAL;
2652        }
2653
2654        /*
2655         * For the 7955/7956 the reference clock frequency must be in the
2656         * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
2657         * but this chip is currently not supported.
2658         */
2659        if (hifn_pll_ref[3] != '\0') {
2660                freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
2661                if (freq < 20 || freq > 100) {
2662                        pr_err("hifn795x: invalid hifn_pll_ref frequency, must"
2663                               "be in the range of 20-100");
2664                        return -EINVAL;
2665                }
2666        }
2667
2668        err = pci_register_driver(&hifn_pci_driver);
2669        if (err < 0) {
2670                pr_err("Failed to register PCI driver for %s device.\n",
2671                       hifn_pci_driver.name);
2672                return -ENODEV;
2673        }
2674
2675        pr_info("Driver for HIFN 795x crypto accelerator chip "
2676                "has been successfully registered.\n");
2677
2678        return 0;
2679}
2680
2681static void __exit hifn_fini(void)
2682{
2683        pci_unregister_driver(&hifn_pci_driver);
2684
2685        pr_info("Driver for HIFN 795x crypto accelerator chip "
2686                "has been successfully unregistered.\n");
2687}
2688
2689module_init(hifn_init);
2690module_exit(hifn_fini);
2691
2692MODULE_LICENSE("GPL");
2693MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2694MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");
2695