109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
20a625fd2SDavid S. Miller /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
30a625fd2SDavid S. Miller *
4eb7caf35SDavid S. Miller * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
50a625fd2SDavid S. Miller */
60a625fd2SDavid S. Miller
70a625fd2SDavid S. Miller #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
80a625fd2SDavid S. Miller
90a625fd2SDavid S. Miller #include <linux/kernel.h>
100a625fd2SDavid S. Miller #include <linux/module.h>
110a625fd2SDavid S. Miller #include <linux/of.h>
12ee174e26SRob Herring #include <linux/of_address.h>
13b0cc7491SRob Herring #include <linux/platform_device.h>
140a625fd2SDavid S. Miller #include <linux/cpumask.h>
150a625fd2SDavid S. Miller #include <linux/slab.h>
160a625fd2SDavid S. Miller #include <linux/interrupt.h>
170a625fd2SDavid S. Miller #include <linux/crypto.h>
180a625fd2SDavid S. Miller #include <crypto/md5.h>
19a24d22b2SEric Biggers #include <crypto/sha1.h>
20a24d22b2SEric Biggers #include <crypto/sha2.h>
210a625fd2SDavid S. Miller #include <crypto/aes.h>
22d4b90dbcSArd Biesheuvel #include <crypto/internal/des.h>
230a625fd2SDavid S. Miller #include <linux/mutex.h>
240a625fd2SDavid S. Miller #include <linux/delay.h>
250a625fd2SDavid S. Miller #include <linux/sched.h>
260a625fd2SDavid S. Miller
270a625fd2SDavid S. Miller #include <crypto/internal/hash.h>
2823a6564aSArd Biesheuvel #include <crypto/internal/skcipher.h>
290a625fd2SDavid S. Miller #include <crypto/scatterwalk.h>
300a625fd2SDavid S. Miller #include <crypto/algapi.h>
310a625fd2SDavid S. Miller
320a625fd2SDavid S. Miller #include <asm/hypervisor.h>
330a625fd2SDavid S. Miller #include <asm/mdesc.h>
340a625fd2SDavid S. Miller
350a625fd2SDavid S. Miller #include "n2_core.h"
360a625fd2SDavid S. Miller
370a625fd2SDavid S. Miller #define DRV_MODULE_NAME "n2_crypto"
38eb7caf35SDavid S. Miller #define DRV_MODULE_VERSION "0.2"
39eb7caf35SDavid S. Miller #define DRV_MODULE_RELDATE "July 28, 2011"
400a625fd2SDavid S. Miller
4150826874SLABBE Corentin static const char version[] =
420a625fd2SDavid S. Miller DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
430a625fd2SDavid S. Miller
446a57a219SAhelenia Ziemiańska MODULE_AUTHOR("David S. Miller <davem@davemloft.net>");
450a625fd2SDavid S. Miller MODULE_DESCRIPTION("Niagara2 Crypto driver");
460a625fd2SDavid S. Miller MODULE_LICENSE("GPL");
470a625fd2SDavid S. Miller MODULE_VERSION(DRV_MODULE_VERSION);
480a625fd2SDavid S. Miller
4910803624SDavid S. Miller #define N2_CRA_PRIORITY 200
500a625fd2SDavid S. Miller
510a625fd2SDavid S. Miller static DEFINE_MUTEX(spu_lock);
520a625fd2SDavid S. Miller
530a625fd2SDavid S. Miller struct spu_queue {
540a625fd2SDavid S. Miller cpumask_t sharing;
550a625fd2SDavid S. Miller unsigned long qhandle;
560a625fd2SDavid S. Miller
570a625fd2SDavid S. Miller spinlock_t lock;
580a625fd2SDavid S. Miller u8 q_type;
590a625fd2SDavid S. Miller void *q;
600a625fd2SDavid S. Miller unsigned long head;
610a625fd2SDavid S. Miller unsigned long tail;
620a625fd2SDavid S. Miller struct list_head jobs;
630a625fd2SDavid S. Miller
640a625fd2SDavid S. Miller unsigned long devino;
650a625fd2SDavid S. Miller
660a625fd2SDavid S. Miller char irq_name[32];
670a625fd2SDavid S. Miller unsigned int irq;
680a625fd2SDavid S. Miller
690a625fd2SDavid S. Miller struct list_head list;
700a625fd2SDavid S. Miller };
710a625fd2SDavid S. Miller
7273810a06SThomas Gleixner struct spu_qreg {
7373810a06SThomas Gleixner struct spu_queue *queue;
7473810a06SThomas Gleixner unsigned long type;
7573810a06SThomas Gleixner };
7673810a06SThomas Gleixner
770a625fd2SDavid S. Miller static struct spu_queue **cpu_to_cwq;
780a625fd2SDavid S. Miller static struct spu_queue **cpu_to_mau;
790a625fd2SDavid S. Miller
spu_next_offset(struct spu_queue * q,unsigned long off)800a625fd2SDavid S. Miller static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
810a625fd2SDavid S. Miller {
820a625fd2SDavid S. Miller if (q->q_type == HV_NCS_QTYPE_MAU) {
830a625fd2SDavid S. Miller off += MAU_ENTRY_SIZE;
840a625fd2SDavid S. Miller if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
850a625fd2SDavid S. Miller off = 0;
860a625fd2SDavid S. Miller } else {
870a625fd2SDavid S. Miller off += CWQ_ENTRY_SIZE;
880a625fd2SDavid S. Miller if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
890a625fd2SDavid S. Miller off = 0;
900a625fd2SDavid S. Miller }
910a625fd2SDavid S. Miller return off;
920a625fd2SDavid S. Miller }
930a625fd2SDavid S. Miller
940a625fd2SDavid S. Miller struct n2_request_common {
950a625fd2SDavid S. Miller struct list_head entry;
960a625fd2SDavid S. Miller unsigned int offset;
970a625fd2SDavid S. Miller };
980a625fd2SDavid S. Miller #define OFFSET_NOT_RUNNING (~(unsigned int)0)
990a625fd2SDavid S. Miller
1000a625fd2SDavid S. Miller /* An async job request records the final tail value it used in
1010a625fd2SDavid S. Miller * n2_request_common->offset, test to see if that offset is in
1020a625fd2SDavid S. Miller * the range old_head, new_head, inclusive.
1030a625fd2SDavid S. Miller */
job_finished(struct spu_queue * q,unsigned int offset,unsigned long old_head,unsigned long new_head)1040a625fd2SDavid S. Miller static inline bool job_finished(struct spu_queue *q, unsigned int offset,
1050a625fd2SDavid S. Miller unsigned long old_head, unsigned long new_head)
1060a625fd2SDavid S. Miller {
1070a625fd2SDavid S. Miller if (old_head <= new_head) {
1080a625fd2SDavid S. Miller if (offset > old_head && offset <= new_head)
1090a625fd2SDavid S. Miller return true;
1100a625fd2SDavid S. Miller } else {
1110a625fd2SDavid S. Miller if (offset > old_head || offset <= new_head)
1120a625fd2SDavid S. Miller return true;
1130a625fd2SDavid S. Miller }
1140a625fd2SDavid S. Miller return false;
1150a625fd2SDavid S. Miller }
1160a625fd2SDavid S. Miller
1170a625fd2SDavid S. Miller /* When the HEAD marker is unequal to the actual HEAD, we get
1180a625fd2SDavid S. Miller * a virtual device INO interrupt. We should process the
1190a625fd2SDavid S. Miller * completed CWQ entries and adjust the HEAD marker to clear
1200a625fd2SDavid S. Miller * the IRQ.
1210a625fd2SDavid S. Miller */
cwq_intr(int irq,void * dev_id)1220a625fd2SDavid S. Miller static irqreturn_t cwq_intr(int irq, void *dev_id)
1230a625fd2SDavid S. Miller {
1240a625fd2SDavid S. Miller unsigned long off, new_head, hv_ret;
1250a625fd2SDavid S. Miller struct spu_queue *q = dev_id;
1260a625fd2SDavid S. Miller
1270a625fd2SDavid S. Miller pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
1280a625fd2SDavid S. Miller smp_processor_id(), q->qhandle);
1290a625fd2SDavid S. Miller
1300a625fd2SDavid S. Miller spin_lock(&q->lock);
1310a625fd2SDavid S. Miller
1320a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
1330a625fd2SDavid S. Miller
1340a625fd2SDavid S. Miller pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
1350a625fd2SDavid S. Miller smp_processor_id(), new_head, hv_ret);
1360a625fd2SDavid S. Miller
1370a625fd2SDavid S. Miller for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
1380a625fd2SDavid S. Miller /* XXX ... XXX */
1390a625fd2SDavid S. Miller }
1400a625fd2SDavid S. Miller
1410a625fd2SDavid S. Miller hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
1420a625fd2SDavid S. Miller if (hv_ret == HV_EOK)
1430a625fd2SDavid S. Miller q->head = new_head;
1440a625fd2SDavid S. Miller
1450a625fd2SDavid S. Miller spin_unlock(&q->lock);
1460a625fd2SDavid S. Miller
1470a625fd2SDavid S. Miller return IRQ_HANDLED;
1480a625fd2SDavid S. Miller }
1490a625fd2SDavid S. Miller
mau_intr(int irq,void * dev_id)1500a625fd2SDavid S. Miller static irqreturn_t mau_intr(int irq, void *dev_id)
1510a625fd2SDavid S. Miller {
1520a625fd2SDavid S. Miller struct spu_queue *q = dev_id;
1530a625fd2SDavid S. Miller unsigned long head, hv_ret;
1540a625fd2SDavid S. Miller
1550a625fd2SDavid S. Miller spin_lock(&q->lock);
1560a625fd2SDavid S. Miller
1570a625fd2SDavid S. Miller pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
1580a625fd2SDavid S. Miller smp_processor_id(), q->qhandle);
1590a625fd2SDavid S. Miller
1600a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
1610a625fd2SDavid S. Miller
1620a625fd2SDavid S. Miller pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
1630a625fd2SDavid S. Miller smp_processor_id(), head, hv_ret);
1640a625fd2SDavid S. Miller
1650a625fd2SDavid S. Miller sun4v_ncs_sethead_marker(q->qhandle, head);
1660a625fd2SDavid S. Miller
1670a625fd2SDavid S. Miller spin_unlock(&q->lock);
1680a625fd2SDavid S. Miller
1690a625fd2SDavid S. Miller return IRQ_HANDLED;
1700a625fd2SDavid S. Miller }
1710a625fd2SDavid S. Miller
spu_queue_next(struct spu_queue * q,void * cur)1720a625fd2SDavid S. Miller static void *spu_queue_next(struct spu_queue *q, void *cur)
1730a625fd2SDavid S. Miller {
1740a625fd2SDavid S. Miller return q->q + spu_next_offset(q, cur - q->q);
1750a625fd2SDavid S. Miller }
1760a625fd2SDavid S. Miller
spu_queue_num_free(struct spu_queue * q)1770a625fd2SDavid S. Miller static int spu_queue_num_free(struct spu_queue *q)
1780a625fd2SDavid S. Miller {
1790a625fd2SDavid S. Miller unsigned long head = q->head;
1800a625fd2SDavid S. Miller unsigned long tail = q->tail;
1810a625fd2SDavid S. Miller unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
1820a625fd2SDavid S. Miller unsigned long diff;
1830a625fd2SDavid S. Miller
1840a625fd2SDavid S. Miller if (head > tail)
1850a625fd2SDavid S. Miller diff = head - tail;
1860a625fd2SDavid S. Miller else
1870a625fd2SDavid S. Miller diff = (end - tail) + head;
1880a625fd2SDavid S. Miller
1890a625fd2SDavid S. Miller return (diff / CWQ_ENTRY_SIZE) - 1;
1900a625fd2SDavid S. Miller }
1910a625fd2SDavid S. Miller
spu_queue_alloc(struct spu_queue * q,int num_entries)1920a625fd2SDavid S. Miller static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
1930a625fd2SDavid S. Miller {
1940a625fd2SDavid S. Miller int avail = spu_queue_num_free(q);
1950a625fd2SDavid S. Miller
1960a625fd2SDavid S. Miller if (avail >= num_entries)
1970a625fd2SDavid S. Miller return q->q + q->tail;
1980a625fd2SDavid S. Miller
1990a625fd2SDavid S. Miller return NULL;
2000a625fd2SDavid S. Miller }
2010a625fd2SDavid S. Miller
spu_queue_submit(struct spu_queue * q,void * last)2020a625fd2SDavid S. Miller static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
2030a625fd2SDavid S. Miller {
2040a625fd2SDavid S. Miller unsigned long hv_ret, new_tail;
2050a625fd2SDavid S. Miller
2060a625fd2SDavid S. Miller new_tail = spu_next_offset(q, last - q->q);
2070a625fd2SDavid S. Miller
2080a625fd2SDavid S. Miller hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
2090a625fd2SDavid S. Miller if (hv_ret == HV_EOK)
2100a625fd2SDavid S. Miller q->tail = new_tail;
2110a625fd2SDavid S. Miller return hv_ret;
2120a625fd2SDavid S. Miller }
2130a625fd2SDavid S. Miller
control_word_base(unsigned int len,unsigned int hmac_key_len,int enc_type,int auth_type,unsigned int hash_len,bool sfas,bool sob,bool eob,bool encrypt,int opcode)2140a625fd2SDavid S. Miller static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
2150a625fd2SDavid S. Miller int enc_type, int auth_type,
2160a625fd2SDavid S. Miller unsigned int hash_len,
2170a625fd2SDavid S. Miller bool sfas, bool sob, bool eob, bool encrypt,
2180a625fd2SDavid S. Miller int opcode)
2190a625fd2SDavid S. Miller {
2200a625fd2SDavid S. Miller u64 word = (len - 1) & CONTROL_LEN;
2210a625fd2SDavid S. Miller
2220a625fd2SDavid S. Miller word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
2230a625fd2SDavid S. Miller word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
2240a625fd2SDavid S. Miller word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
2250a625fd2SDavid S. Miller if (sfas)
2260a625fd2SDavid S. Miller word |= CONTROL_STORE_FINAL_AUTH_STATE;
2270a625fd2SDavid S. Miller if (sob)
2280a625fd2SDavid S. Miller word |= CONTROL_START_OF_BLOCK;
2290a625fd2SDavid S. Miller if (eob)
2300a625fd2SDavid S. Miller word |= CONTROL_END_OF_BLOCK;
2310a625fd2SDavid S. Miller if (encrypt)
2320a625fd2SDavid S. Miller word |= CONTROL_ENCRYPT;
2330a625fd2SDavid S. Miller if (hmac_key_len)
2340a625fd2SDavid S. Miller word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
2350a625fd2SDavid S. Miller if (hash_len)
2360a625fd2SDavid S. Miller word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
2370a625fd2SDavid S. Miller
2380a625fd2SDavid S. Miller return word;
2390a625fd2SDavid S. Miller }
2400a625fd2SDavid S. Miller
2410a625fd2SDavid S. Miller #if 0
2420a625fd2SDavid S. Miller static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
2430a625fd2SDavid S. Miller {
2440a625fd2SDavid S. Miller if (this_len >= 64 ||
2450a625fd2SDavid S. Miller qp->head != qp->tail)
2460a625fd2SDavid S. Miller return true;
2470a625fd2SDavid S. Miller return false;
2480a625fd2SDavid S. Miller }
2490a625fd2SDavid S. Miller #endif
2500a625fd2SDavid S. Miller
2513a2c0346SDavid S. Miller struct n2_ahash_alg {
2523a2c0346SDavid S. Miller struct list_head entry;
2538054b800SLABBE Corentin const u8 *hash_zero;
2546d77282fSHerbert Xu const u8 *hash_init;
2553a2c0346SDavid S. Miller u8 hw_op_hashsz;
2563a2c0346SDavid S. Miller u8 digest_size;
2573a2c0346SDavid S. Miller u8 auth_type;
258dc4ccfd1SDavid S. Miller u8 hmac_type;
2593a2c0346SDavid S. Miller struct ahash_alg alg;
2603a2c0346SDavid S. Miller };
2613a2c0346SDavid S. Miller
n2_ahash_alg(struct crypto_tfm * tfm)2623a2c0346SDavid S. Miller static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
2633a2c0346SDavid S. Miller {
2643a2c0346SDavid S. Miller struct crypto_alg *alg = tfm->__crt_alg;
2653a2c0346SDavid S. Miller struct ahash_alg *ahash_alg;
2663a2c0346SDavid S. Miller
2673a2c0346SDavid S. Miller ahash_alg = container_of(alg, struct ahash_alg, halg.base);
2683a2c0346SDavid S. Miller
2693a2c0346SDavid S. Miller return container_of(ahash_alg, struct n2_ahash_alg, alg);
2703a2c0346SDavid S. Miller }
2713a2c0346SDavid S. Miller
272dc4ccfd1SDavid S. Miller struct n2_hmac_alg {
273dc4ccfd1SDavid S. Miller const char *child_alg;
274dc4ccfd1SDavid S. Miller struct n2_ahash_alg derived;
275dc4ccfd1SDavid S. Miller };
276dc4ccfd1SDavid S. Miller
n2_hmac_alg(struct crypto_tfm * tfm)277dc4ccfd1SDavid S. Miller static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
278dc4ccfd1SDavid S. Miller {
279dc4ccfd1SDavid S. Miller struct crypto_alg *alg = tfm->__crt_alg;
280dc4ccfd1SDavid S. Miller struct ahash_alg *ahash_alg;
281dc4ccfd1SDavid S. Miller
282dc4ccfd1SDavid S. Miller ahash_alg = container_of(alg, struct ahash_alg, halg.base);
283dc4ccfd1SDavid S. Miller
284dc4ccfd1SDavid S. Miller return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
285dc4ccfd1SDavid S. Miller }
286dc4ccfd1SDavid S. Miller
2870a625fd2SDavid S. Miller struct n2_hash_ctx {
288c9aa55e5SDavid S. Miller struct crypto_ahash *fallback_tfm;
289c9aa55e5SDavid S. Miller };
2900a625fd2SDavid S. Miller
291dc4ccfd1SDavid S. Miller #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */
292dc4ccfd1SDavid S. Miller
293dc4ccfd1SDavid S. Miller struct n2_hmac_ctx {
294dc4ccfd1SDavid S. Miller struct n2_hash_ctx base;
295dc4ccfd1SDavid S. Miller
296dc4ccfd1SDavid S. Miller struct crypto_shash *child_shash;
297dc4ccfd1SDavid S. Miller
298dc4ccfd1SDavid S. Miller int hash_key_len;
299dc4ccfd1SDavid S. Miller unsigned char hash_key[N2_HASH_KEY_MAX];
300dc4ccfd1SDavid S. Miller };
301dc4ccfd1SDavid S. Miller
302c9aa55e5SDavid S. Miller struct n2_hash_req_ctx {
3030a625fd2SDavid S. Miller union {
3040a625fd2SDavid S. Miller struct md5_state md5;
3050a625fd2SDavid S. Miller struct sha1_state sha1;
3060a625fd2SDavid S. Miller struct sha256_state sha256;
3070a625fd2SDavid S. Miller } u;
3080a625fd2SDavid S. Miller
309c9aa55e5SDavid S. Miller struct ahash_request fallback_req;
3100a625fd2SDavid S. Miller };
3110a625fd2SDavid S. Miller
n2_hash_async_init(struct ahash_request * req)3120a625fd2SDavid S. Miller static int n2_hash_async_init(struct ahash_request *req)
3130a625fd2SDavid S. Miller {
314c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
3150a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
3160a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
3170a625fd2SDavid S. Miller
318c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
319c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
3200a625fd2SDavid S. Miller
321c9aa55e5SDavid S. Miller return crypto_ahash_init(&rctx->fallback_req);
3220a625fd2SDavid S. Miller }
3230a625fd2SDavid S. Miller
n2_hash_async_update(struct ahash_request * req)3240a625fd2SDavid S. Miller static int n2_hash_async_update(struct ahash_request *req)
3250a625fd2SDavid S. Miller {
326c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
3270a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
3280a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
3290a625fd2SDavid S. Miller
330c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
331c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
332c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes;
333c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src;
3340a625fd2SDavid S. Miller
335c9aa55e5SDavid S. Miller return crypto_ahash_update(&rctx->fallback_req);
3360a625fd2SDavid S. Miller }
3370a625fd2SDavid S. Miller
n2_hash_async_final(struct ahash_request * req)3380a625fd2SDavid S. Miller static int n2_hash_async_final(struct ahash_request *req)
3390a625fd2SDavid S. Miller {
340c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
3410a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
3420a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
3430a625fd2SDavid S. Miller
344c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
345c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
346c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result;
3470a625fd2SDavid S. Miller
348c9aa55e5SDavid S. Miller return crypto_ahash_final(&rctx->fallback_req);
3490a625fd2SDavid S. Miller }
3500a625fd2SDavid S. Miller
n2_hash_async_finup(struct ahash_request * req)3510a625fd2SDavid S. Miller static int n2_hash_async_finup(struct ahash_request *req)
3520a625fd2SDavid S. Miller {
353c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
3540a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
3550a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
3560a625fd2SDavid S. Miller
357c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
358c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
359c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes;
360c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src;
361c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result;
3620a625fd2SDavid S. Miller
363c9aa55e5SDavid S. Miller return crypto_ahash_finup(&rctx->fallback_req);
3640a625fd2SDavid S. Miller }
3650a625fd2SDavid S. Miller
n2_hash_async_noimport(struct ahash_request * req,const void * in)366378fe6fbSKamil Konieczny static int n2_hash_async_noimport(struct ahash_request *req, const void *in)
367378fe6fbSKamil Konieczny {
368378fe6fbSKamil Konieczny return -ENOSYS;
369378fe6fbSKamil Konieczny }
370378fe6fbSKamil Konieczny
n2_hash_async_noexport(struct ahash_request * req,void * out)371378fe6fbSKamil Konieczny static int n2_hash_async_noexport(struct ahash_request *req, void *out)
372378fe6fbSKamil Konieczny {
373378fe6fbSKamil Konieczny return -ENOSYS;
374378fe6fbSKamil Konieczny }
375378fe6fbSKamil Konieczny
n2_hash_cra_init(struct crypto_tfm * tfm)3760a625fd2SDavid S. Miller static int n2_hash_cra_init(struct crypto_tfm *tfm)
3770a625fd2SDavid S. Miller {
3785837af01SMarek Vasut const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
3790a625fd2SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
3800a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3810a625fd2SDavid S. Miller struct crypto_ahash *fallback_tfm;
3820a625fd2SDavid S. Miller int err;
3830a625fd2SDavid S. Miller
3840a625fd2SDavid S. Miller fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
3850a625fd2SDavid S. Miller CRYPTO_ALG_NEED_FALLBACK);
3860a625fd2SDavid S. Miller if (IS_ERR(fallback_tfm)) {
387cacc594bSKefeng Wang pr_warn("Fallback driver '%s' could not be loaded!\n",
3880a625fd2SDavid S. Miller fallback_driver_name);
3890a625fd2SDavid S. Miller err = PTR_ERR(fallback_tfm);
3900a625fd2SDavid S. Miller goto out;
3910a625fd2SDavid S. Miller }
3920a625fd2SDavid S. Miller
393c9aa55e5SDavid S. Miller crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
394c9aa55e5SDavid S. Miller crypto_ahash_reqsize(fallback_tfm)));
395c9aa55e5SDavid S. Miller
396c9aa55e5SDavid S. Miller ctx->fallback_tfm = fallback_tfm;
3970a625fd2SDavid S. Miller return 0;
3980a625fd2SDavid S. Miller
3990a625fd2SDavid S. Miller out:
4000a625fd2SDavid S. Miller return err;
4010a625fd2SDavid S. Miller }
4020a625fd2SDavid S. Miller
n2_hash_cra_exit(struct crypto_tfm * tfm)4030a625fd2SDavid S. Miller static void n2_hash_cra_exit(struct crypto_tfm *tfm)
4040a625fd2SDavid S. Miller {
4050a625fd2SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4060a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4070a625fd2SDavid S. Miller
408c9aa55e5SDavid S. Miller crypto_free_ahash(ctx->fallback_tfm);
4090a625fd2SDavid S. Miller }
4100a625fd2SDavid S. Miller
n2_hmac_cra_init(struct crypto_tfm * tfm)411dc4ccfd1SDavid S. Miller static int n2_hmac_cra_init(struct crypto_tfm *tfm)
412dc4ccfd1SDavid S. Miller {
4135837af01SMarek Vasut const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
414dc4ccfd1SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
415dc4ccfd1SDavid S. Miller struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
416dc4ccfd1SDavid S. Miller struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
417dc4ccfd1SDavid S. Miller struct crypto_ahash *fallback_tfm;
418dc4ccfd1SDavid S. Miller struct crypto_shash *child_shash;
419dc4ccfd1SDavid S. Miller int err;
420dc4ccfd1SDavid S. Miller
421dc4ccfd1SDavid S. Miller fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
422dc4ccfd1SDavid S. Miller CRYPTO_ALG_NEED_FALLBACK);
423dc4ccfd1SDavid S. Miller if (IS_ERR(fallback_tfm)) {
424cacc594bSKefeng Wang pr_warn("Fallback driver '%s' could not be loaded!\n",
425dc4ccfd1SDavid S. Miller fallback_driver_name);
426dc4ccfd1SDavid S. Miller err = PTR_ERR(fallback_tfm);
427dc4ccfd1SDavid S. Miller goto out;
428dc4ccfd1SDavid S. Miller }
429dc4ccfd1SDavid S. Miller
430dc4ccfd1SDavid S. Miller child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
431dc4ccfd1SDavid S. Miller if (IS_ERR(child_shash)) {
432cacc594bSKefeng Wang pr_warn("Child shash '%s' could not be loaded!\n",
433dc4ccfd1SDavid S. Miller n2alg->child_alg);
434dc4ccfd1SDavid S. Miller err = PTR_ERR(child_shash);
435dc4ccfd1SDavid S. Miller goto out_free_fallback;
436dc4ccfd1SDavid S. Miller }
437dc4ccfd1SDavid S. Miller
438dc4ccfd1SDavid S. Miller crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
439dc4ccfd1SDavid S. Miller crypto_ahash_reqsize(fallback_tfm)));
440dc4ccfd1SDavid S. Miller
441dc4ccfd1SDavid S. Miller ctx->child_shash = child_shash;
442dc4ccfd1SDavid S. Miller ctx->base.fallback_tfm = fallback_tfm;
443dc4ccfd1SDavid S. Miller return 0;
444dc4ccfd1SDavid S. Miller
445dc4ccfd1SDavid S. Miller out_free_fallback:
446dc4ccfd1SDavid S. Miller crypto_free_ahash(fallback_tfm);
447dc4ccfd1SDavid S. Miller
448dc4ccfd1SDavid S. Miller out:
449dc4ccfd1SDavid S. Miller return err;
450dc4ccfd1SDavid S. Miller }
451dc4ccfd1SDavid S. Miller
n2_hmac_cra_exit(struct crypto_tfm * tfm)452dc4ccfd1SDavid S. Miller static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
453dc4ccfd1SDavid S. Miller {
454dc4ccfd1SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
455dc4ccfd1SDavid S. Miller struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
456dc4ccfd1SDavid S. Miller
457dc4ccfd1SDavid S. Miller crypto_free_ahash(ctx->base.fallback_tfm);
458dc4ccfd1SDavid S. Miller crypto_free_shash(ctx->child_shash);
459dc4ccfd1SDavid S. Miller }
460dc4ccfd1SDavid S. Miller
n2_hmac_async_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)461dc4ccfd1SDavid S. Miller static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
462dc4ccfd1SDavid S. Miller unsigned int keylen)
463dc4ccfd1SDavid S. Miller {
464dc4ccfd1SDavid S. Miller struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
465dc4ccfd1SDavid S. Miller struct crypto_shash *child_shash = ctx->child_shash;
466dc4ccfd1SDavid S. Miller struct crypto_ahash *fallback_tfm;
467dc4ccfd1SDavid S. Miller int err, bs, ds;
468dc4ccfd1SDavid S. Miller
469dc4ccfd1SDavid S. Miller fallback_tfm = ctx->base.fallback_tfm;
470dc4ccfd1SDavid S. Miller err = crypto_ahash_setkey(fallback_tfm, key, keylen);
471dc4ccfd1SDavid S. Miller if (err)
472dc4ccfd1SDavid S. Miller return err;
473dc4ccfd1SDavid S. Miller
474dc4ccfd1SDavid S. Miller bs = crypto_shash_blocksize(child_shash);
475dc4ccfd1SDavid S. Miller ds = crypto_shash_digestsize(child_shash);
476dc4ccfd1SDavid S. Miller BUG_ON(ds > N2_HASH_KEY_MAX);
477dc4ccfd1SDavid S. Miller if (keylen > bs) {
478ce8e0488SEric Biggers err = crypto_shash_tfm_digest(child_shash, key, keylen,
479dc4ccfd1SDavid S. Miller ctx->hash_key);
480dc4ccfd1SDavid S. Miller if (err)
481dc4ccfd1SDavid S. Miller return err;
482dc4ccfd1SDavid S. Miller keylen = ds;
483dc4ccfd1SDavid S. Miller } else if (keylen <= N2_HASH_KEY_MAX)
484dc4ccfd1SDavid S. Miller memcpy(ctx->hash_key, key, keylen);
485dc4ccfd1SDavid S. Miller
486dc4ccfd1SDavid S. Miller ctx->hash_key_len = keylen;
487dc4ccfd1SDavid S. Miller
488dc4ccfd1SDavid S. Miller return err;
489dc4ccfd1SDavid S. Miller }
490dc4ccfd1SDavid S. Miller
wait_for_tail(struct spu_queue * qp)4910a625fd2SDavid S. Miller static unsigned long wait_for_tail(struct spu_queue *qp)
4920a625fd2SDavid S. Miller {
4930a625fd2SDavid S. Miller unsigned long head, hv_ret;
4940a625fd2SDavid S. Miller
4950a625fd2SDavid S. Miller do {
4960a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
4970a625fd2SDavid S. Miller if (hv_ret != HV_EOK) {
4980a625fd2SDavid S. Miller pr_err("Hypervisor error on gethead\n");
4990a625fd2SDavid S. Miller break;
5000a625fd2SDavid S. Miller }
5010a625fd2SDavid S. Miller if (head == qp->tail) {
5020a625fd2SDavid S. Miller qp->head = head;
5030a625fd2SDavid S. Miller break;
5040a625fd2SDavid S. Miller }
5050a625fd2SDavid S. Miller } while (1);
5060a625fd2SDavid S. Miller return hv_ret;
5070a625fd2SDavid S. Miller }
5080a625fd2SDavid S. Miller
submit_and_wait_for_tail(struct spu_queue * qp,struct cwq_initial_entry * ent)5090a625fd2SDavid S. Miller static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
5100a625fd2SDavid S. Miller struct cwq_initial_entry *ent)
5110a625fd2SDavid S. Miller {
5120a625fd2SDavid S. Miller unsigned long hv_ret = spu_queue_submit(qp, ent);
5130a625fd2SDavid S. Miller
5140a625fd2SDavid S. Miller if (hv_ret == HV_EOK)
5150a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp);
5160a625fd2SDavid S. Miller
5170a625fd2SDavid S. Miller return hv_ret;
5180a625fd2SDavid S. Miller }
5190a625fd2SDavid S. Miller
n2_do_async_digest(struct ahash_request * req,unsigned int auth_type,unsigned int digest_size,unsigned int result_size,void * hash_loc,unsigned long auth_key,unsigned int auth_key_len)5203a2c0346SDavid S. Miller static int n2_do_async_digest(struct ahash_request *req,
5210a625fd2SDavid S. Miller unsigned int auth_type, unsigned int digest_size,
522dc4ccfd1SDavid S. Miller unsigned int result_size, void *hash_loc,
523dc4ccfd1SDavid S. Miller unsigned long auth_key, unsigned int auth_key_len)
5240a625fd2SDavid S. Miller {
5250a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
5260a625fd2SDavid S. Miller struct cwq_initial_entry *ent;
5270a625fd2SDavid S. Miller struct crypto_hash_walk walk;
5280a625fd2SDavid S. Miller struct spu_queue *qp;
5290a625fd2SDavid S. Miller unsigned long flags;
5300a625fd2SDavid S. Miller int err = -ENODEV;
5310a625fd2SDavid S. Miller int nbytes, cpu;
5320a625fd2SDavid S. Miller
5330a625fd2SDavid S. Miller /* The total effective length of the operation may not
5340a625fd2SDavid S. Miller * exceed 2^16.
5350a625fd2SDavid S. Miller */
5360a625fd2SDavid S. Miller if (unlikely(req->nbytes > (1 << 16))) {
537c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
53865a23d67SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
5390a625fd2SDavid S. Miller
540c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
541c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags =
542c9aa55e5SDavid S. Miller req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
543c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes;
544c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src;
545c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result;
546c9aa55e5SDavid S. Miller
547c9aa55e5SDavid S. Miller return crypto_ahash_digest(&rctx->fallback_req);
5480a625fd2SDavid S. Miller }
5490a625fd2SDavid S. Miller
5500a625fd2SDavid S. Miller nbytes = crypto_hash_walk_first(req, &walk);
5510a625fd2SDavid S. Miller
5520a625fd2SDavid S. Miller cpu = get_cpu();
5530a625fd2SDavid S. Miller qp = cpu_to_cwq[cpu];
5540a625fd2SDavid S. Miller if (!qp)
5550a625fd2SDavid S. Miller goto out;
5560a625fd2SDavid S. Miller
5570a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags);
5580a625fd2SDavid S. Miller
5590a625fd2SDavid S. Miller /* XXX can do better, improve this later by doing a by-hand scatterlist
5600a625fd2SDavid S. Miller * XXX walk, etc.
5610a625fd2SDavid S. Miller */
5620a625fd2SDavid S. Miller ent = qp->q + qp->tail;
5630a625fd2SDavid S. Miller
564dc4ccfd1SDavid S. Miller ent->control = control_word_base(nbytes, auth_key_len, 0,
5650a625fd2SDavid S. Miller auth_type, digest_size,
5660a625fd2SDavid S. Miller false, true, false, false,
5670a625fd2SDavid S. Miller OPCODE_INPLACE_BIT |
5680a625fd2SDavid S. Miller OPCODE_AUTH_MAC);
5690a625fd2SDavid S. Miller ent->src_addr = __pa(walk.data);
570dc4ccfd1SDavid S. Miller ent->auth_key_addr = auth_key;
5710a625fd2SDavid S. Miller ent->auth_iv_addr = __pa(hash_loc);
5720a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL;
5730a625fd2SDavid S. Miller ent->enc_key_addr = 0UL;
5740a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL;
5750a625fd2SDavid S. Miller ent->dest_addr = __pa(hash_loc);
5760a625fd2SDavid S. Miller
5770a625fd2SDavid S. Miller nbytes = crypto_hash_walk_done(&walk, 0);
5780a625fd2SDavid S. Miller while (nbytes > 0) {
5790a625fd2SDavid S. Miller ent = spu_queue_next(qp, ent);
5800a625fd2SDavid S. Miller
5810a625fd2SDavid S. Miller ent->control = (nbytes - 1);
5820a625fd2SDavid S. Miller ent->src_addr = __pa(walk.data);
5830a625fd2SDavid S. Miller ent->auth_key_addr = 0UL;
5840a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL;
5850a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL;
5860a625fd2SDavid S. Miller ent->enc_key_addr = 0UL;
5870a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL;
5880a625fd2SDavid S. Miller ent->dest_addr = 0UL;
5890a625fd2SDavid S. Miller
5900a625fd2SDavid S. Miller nbytes = crypto_hash_walk_done(&walk, 0);
5910a625fd2SDavid S. Miller }
5920a625fd2SDavid S. Miller ent->control |= CONTROL_END_OF_BLOCK;
5930a625fd2SDavid S. Miller
5940a625fd2SDavid S. Miller if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
5950a625fd2SDavid S. Miller err = -EINVAL;
5960a625fd2SDavid S. Miller else
5970a625fd2SDavid S. Miller err = 0;
5980a625fd2SDavid S. Miller
5990a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags);
6000a625fd2SDavid S. Miller
6010a625fd2SDavid S. Miller if (!err)
6020a625fd2SDavid S. Miller memcpy(req->result, hash_loc, result_size);
6030a625fd2SDavid S. Miller out:
6040a625fd2SDavid S. Miller put_cpu();
6050a625fd2SDavid S. Miller
6060a625fd2SDavid S. Miller return err;
6070a625fd2SDavid S. Miller }
6080a625fd2SDavid S. Miller
n2_hash_async_digest(struct ahash_request * req)6093a2c0346SDavid S. Miller static int n2_hash_async_digest(struct ahash_request *req)
6100a625fd2SDavid S. Miller {
6113a2c0346SDavid S. Miller struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
612c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
6133a2c0346SDavid S. Miller int ds;
6140a625fd2SDavid S. Miller
6153a2c0346SDavid S. Miller ds = n2alg->digest_size;
6160a625fd2SDavid S. Miller if (unlikely(req->nbytes == 0)) {
6173a2c0346SDavid S. Miller memcpy(req->result, n2alg->hash_zero, ds);
6180a625fd2SDavid S. Miller return 0;
6190a625fd2SDavid S. Miller }
6203a2c0346SDavid S. Miller memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
6210a625fd2SDavid S. Miller
6223a2c0346SDavid S. Miller return n2_do_async_digest(req, n2alg->auth_type,
6233a2c0346SDavid S. Miller n2alg->hw_op_hashsz, ds,
624dc4ccfd1SDavid S. Miller &rctx->u, 0UL, 0);
625dc4ccfd1SDavid S. Miller }
626dc4ccfd1SDavid S. Miller
n2_hmac_async_digest(struct ahash_request * req)627dc4ccfd1SDavid S. Miller static int n2_hmac_async_digest(struct ahash_request *req)
628dc4ccfd1SDavid S. Miller {
629dc4ccfd1SDavid S. Miller struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
630dc4ccfd1SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
631dc4ccfd1SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
632dc4ccfd1SDavid S. Miller struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
633dc4ccfd1SDavid S. Miller int ds;
634dc4ccfd1SDavid S. Miller
635dc4ccfd1SDavid S. Miller ds = n2alg->derived.digest_size;
636dc4ccfd1SDavid S. Miller if (unlikely(req->nbytes == 0) ||
637dc4ccfd1SDavid S. Miller unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
638dc4ccfd1SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
639dc4ccfd1SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
640dc4ccfd1SDavid S. Miller
641dc4ccfd1SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
642dc4ccfd1SDavid S. Miller rctx->fallback_req.base.flags =
643dc4ccfd1SDavid S. Miller req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
644dc4ccfd1SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes;
645dc4ccfd1SDavid S. Miller rctx->fallback_req.src = req->src;
646dc4ccfd1SDavid S. Miller rctx->fallback_req.result = req->result;
647dc4ccfd1SDavid S. Miller
648dc4ccfd1SDavid S. Miller return crypto_ahash_digest(&rctx->fallback_req);
649dc4ccfd1SDavid S. Miller }
650dc4ccfd1SDavid S. Miller memcpy(&rctx->u, n2alg->derived.hash_init,
651dc4ccfd1SDavid S. Miller n2alg->derived.hw_op_hashsz);
652dc4ccfd1SDavid S. Miller
653dc4ccfd1SDavid S. Miller return n2_do_async_digest(req, n2alg->derived.hmac_type,
654dc4ccfd1SDavid S. Miller n2alg->derived.hw_op_hashsz, ds,
655dc4ccfd1SDavid S. Miller &rctx->u,
656dc4ccfd1SDavid S. Miller __pa(&ctx->hash_key),
657dc4ccfd1SDavid S. Miller ctx->hash_key_len);
6580a625fd2SDavid S. Miller }
6590a625fd2SDavid S. Miller
66023a6564aSArd Biesheuvel struct n2_skcipher_context {
6610a625fd2SDavid S. Miller int key_len;
6620a625fd2SDavid S. Miller int enc_type;
6630a625fd2SDavid S. Miller union {
6640a625fd2SDavid S. Miller u8 aes[AES_MAX_KEY_SIZE];
6650a625fd2SDavid S. Miller u8 des[DES_KEY_SIZE];
6660a625fd2SDavid S. Miller u8 des3[3 * DES_KEY_SIZE];
6670a625fd2SDavid S. Miller } key;
6680a625fd2SDavid S. Miller };
6690a625fd2SDavid S. Miller
6700a625fd2SDavid S. Miller #define N2_CHUNK_ARR_LEN 16
6710a625fd2SDavid S. Miller
6720a625fd2SDavid S. Miller struct n2_crypto_chunk {
6730a625fd2SDavid S. Miller struct list_head entry;
6740a625fd2SDavid S. Miller unsigned long iv_paddr : 44;
6750a625fd2SDavid S. Miller unsigned long arr_len : 20;
6760a625fd2SDavid S. Miller unsigned long dest_paddr;
6770a625fd2SDavid S. Miller unsigned long dest_final;
6780a625fd2SDavid S. Miller struct {
6790a625fd2SDavid S. Miller unsigned long src_paddr : 44;
6800a625fd2SDavid S. Miller unsigned long src_len : 20;
6810a625fd2SDavid S. Miller } arr[N2_CHUNK_ARR_LEN];
6820a625fd2SDavid S. Miller };
6830a625fd2SDavid S. Miller
6840a625fd2SDavid S. Miller struct n2_request_context {
68523a6564aSArd Biesheuvel struct skcipher_walk walk;
6860a625fd2SDavid S. Miller struct list_head chunk_list;
6870a625fd2SDavid S. Miller struct n2_crypto_chunk chunk;
6880a625fd2SDavid S. Miller u8 temp_iv[16];
6890a625fd2SDavid S. Miller };
6900a625fd2SDavid S. Miller
6910a625fd2SDavid S. Miller /* The SPU allows some level of flexibility for partial cipher blocks
6920a625fd2SDavid S. Miller * being specified in a descriptor.
6930a625fd2SDavid S. Miller *
6940a625fd2SDavid S. Miller * It merely requires that every descriptor's length field is at least
6950a625fd2SDavid S. Miller * as large as the cipher block size. This means that a cipher block
6960a625fd2SDavid S. Miller * can span at most 2 descriptors. However, this does not allow a
6970a625fd2SDavid S. Miller * partial block to span into the final descriptor as that would
6980a625fd2SDavid S. Miller * violate the rule (since every descriptor's length must be at lest
6990a625fd2SDavid S. Miller * the block size). So, for example, assuming an 8 byte block size:
7000a625fd2SDavid S. Miller *
7010a625fd2SDavid S. Miller * 0xe --> 0xa --> 0x8
7020a625fd2SDavid S. Miller *
7030a625fd2SDavid S. Miller * is a valid length sequence, whereas:
7040a625fd2SDavid S. Miller *
7050a625fd2SDavid S. Miller * 0xe --> 0xb --> 0x7
7060a625fd2SDavid S. Miller *
7070a625fd2SDavid S. Miller * is not a valid sequence.
7080a625fd2SDavid S. Miller */
7090a625fd2SDavid S. Miller
71023a6564aSArd Biesheuvel struct n2_skcipher_alg {
7110a625fd2SDavid S. Miller struct list_head entry;
7120a625fd2SDavid S. Miller u8 enc_type;
71323a6564aSArd Biesheuvel struct skcipher_alg skcipher;
7140a625fd2SDavid S. Miller };
7150a625fd2SDavid S. Miller
n2_skcipher_alg(struct crypto_skcipher * tfm)71623a6564aSArd Biesheuvel static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm)
7170a625fd2SDavid S. Miller {
71823a6564aSArd Biesheuvel struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
7190a625fd2SDavid S. Miller
72023a6564aSArd Biesheuvel return container_of(alg, struct n2_skcipher_alg, skcipher);
7210a625fd2SDavid S. Miller }
7220a625fd2SDavid S. Miller
n2_aes_setkey(struct crypto_skcipher * skcipher,const u8 * key,unsigned int keylen)72323a6564aSArd Biesheuvel static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key,
7240a625fd2SDavid S. Miller unsigned int keylen)
7250a625fd2SDavid S. Miller {
72623a6564aSArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
72723a6564aSArd Biesheuvel struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
72823a6564aSArd Biesheuvel struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
7290a625fd2SDavid S. Miller
7300a625fd2SDavid S. Miller ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
7310a625fd2SDavid S. Miller
7320a625fd2SDavid S. Miller switch (keylen) {
7330a625fd2SDavid S. Miller case AES_KEYSIZE_128:
7340a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES128;
7350a625fd2SDavid S. Miller break;
7360a625fd2SDavid S. Miller case AES_KEYSIZE_192:
7370a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES192;
7380a625fd2SDavid S. Miller break;
7390a625fd2SDavid S. Miller case AES_KEYSIZE_256:
7400a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES256;
7410a625fd2SDavid S. Miller break;
7420a625fd2SDavid S. Miller default:
7430a625fd2SDavid S. Miller return -EINVAL;
7440a625fd2SDavid S. Miller }
7450a625fd2SDavid S. Miller
7460a625fd2SDavid S. Miller ctx->key_len = keylen;
7470a625fd2SDavid S. Miller memcpy(ctx->key.aes, key, keylen);
7480a625fd2SDavid S. Miller return 0;
7490a625fd2SDavid S. Miller }
7500a625fd2SDavid S. Miller
n2_des_setkey(struct crypto_skcipher * skcipher,const u8 * key,unsigned int keylen)75123a6564aSArd Biesheuvel static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key,
7520a625fd2SDavid S. Miller unsigned int keylen)
7530a625fd2SDavid S. Miller {
75423a6564aSArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
75523a6564aSArd Biesheuvel struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
75623a6564aSArd Biesheuvel struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
7570a625fd2SDavid S. Miller int err;
7580a625fd2SDavid S. Miller
75923a6564aSArd Biesheuvel err = verify_skcipher_des_key(skcipher, key);
760d4b90dbcSArd Biesheuvel if (err)
761d4b90dbcSArd Biesheuvel return err;
762d4b90dbcSArd Biesheuvel
7630a625fd2SDavid S. Miller ctx->enc_type = n2alg->enc_type;
7640a625fd2SDavid S. Miller
7650a625fd2SDavid S. Miller ctx->key_len = keylen;
7660a625fd2SDavid S. Miller memcpy(ctx->key.des, key, keylen);
7670a625fd2SDavid S. Miller return 0;
7680a625fd2SDavid S. Miller }
7690a625fd2SDavid S. Miller
n2_3des_setkey(struct crypto_skcipher * skcipher,const u8 * key,unsigned int keylen)77023a6564aSArd Biesheuvel static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key,
7710a625fd2SDavid S. Miller unsigned int keylen)
7720a625fd2SDavid S. Miller {
77323a6564aSArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
77423a6564aSArd Biesheuvel struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
77523a6564aSArd Biesheuvel struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
776e4fffa5fSHerbert Xu int err;
777e4fffa5fSHerbert Xu
77823a6564aSArd Biesheuvel err = verify_skcipher_des3_key(skcipher, key);
779d4b90dbcSArd Biesheuvel if (err)
780e4fffa5fSHerbert Xu return err;
7810a625fd2SDavid S. Miller
7820a625fd2SDavid S. Miller ctx->enc_type = n2alg->enc_type;
7830a625fd2SDavid S. Miller
7840a625fd2SDavid S. Miller ctx->key_len = keylen;
7850a625fd2SDavid S. Miller memcpy(ctx->key.des3, key, keylen);
7860a625fd2SDavid S. Miller return 0;
7870a625fd2SDavid S. Miller }
7880a625fd2SDavid S. Miller
skcipher_descriptor_len(int nbytes,unsigned int block_size)78923a6564aSArd Biesheuvel static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size)
7900a625fd2SDavid S. Miller {
7910a625fd2SDavid S. Miller int this_len = nbytes;
7920a625fd2SDavid S. Miller
7930a625fd2SDavid S. Miller this_len -= (nbytes & (block_size - 1));
7940a625fd2SDavid S. Miller return this_len > (1 << 16) ? (1 << 16) : this_len;
7950a625fd2SDavid S. Miller }
7960a625fd2SDavid S. Miller
__n2_crypt_chunk(struct crypto_skcipher * skcipher,struct n2_crypto_chunk * cp,struct spu_queue * qp,bool encrypt)79723a6564aSArd Biesheuvel static int __n2_crypt_chunk(struct crypto_skcipher *skcipher,
79823a6564aSArd Biesheuvel struct n2_crypto_chunk *cp,
7990a625fd2SDavid S. Miller struct spu_queue *qp, bool encrypt)
8000a625fd2SDavid S. Miller {
80123a6564aSArd Biesheuvel struct n2_skcipher_context *ctx = crypto_skcipher_ctx(skcipher);
8020a625fd2SDavid S. Miller struct cwq_initial_entry *ent;
8030a625fd2SDavid S. Miller bool in_place;
8040a625fd2SDavid S. Miller int i;
8050a625fd2SDavid S. Miller
8060a625fd2SDavid S. Miller ent = spu_queue_alloc(qp, cp->arr_len);
8070a625fd2SDavid S. Miller if (!ent) {
8080a625fd2SDavid S. Miller pr_info("queue_alloc() of %d fails\n",
8090a625fd2SDavid S. Miller cp->arr_len);
8100a625fd2SDavid S. Miller return -EBUSY;
8110a625fd2SDavid S. Miller }
8120a625fd2SDavid S. Miller
8130a625fd2SDavid S. Miller in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
8140a625fd2SDavid S. Miller
8150a625fd2SDavid S. Miller ent->control = control_word_base(cp->arr[0].src_len,
8160a625fd2SDavid S. Miller 0, ctx->enc_type, 0, 0,
8170a625fd2SDavid S. Miller false, true, false, encrypt,
8180a625fd2SDavid S. Miller OPCODE_ENCRYPT |
8190a625fd2SDavid S. Miller (in_place ? OPCODE_INPLACE_BIT : 0));
8200a625fd2SDavid S. Miller ent->src_addr = cp->arr[0].src_paddr;
8210a625fd2SDavid S. Miller ent->auth_key_addr = 0UL;
8220a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL;
8230a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL;
8240a625fd2SDavid S. Miller ent->enc_key_addr = __pa(&ctx->key);
8250a625fd2SDavid S. Miller ent->enc_iv_addr = cp->iv_paddr;
8260a625fd2SDavid S. Miller ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
8270a625fd2SDavid S. Miller
8280a625fd2SDavid S. Miller for (i = 1; i < cp->arr_len; i++) {
8290a625fd2SDavid S. Miller ent = spu_queue_next(qp, ent);
8300a625fd2SDavid S. Miller
8310a625fd2SDavid S. Miller ent->control = cp->arr[i].src_len - 1;
8320a625fd2SDavid S. Miller ent->src_addr = cp->arr[i].src_paddr;
8330a625fd2SDavid S. Miller ent->auth_key_addr = 0UL;
8340a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL;
8350a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL;
8360a625fd2SDavid S. Miller ent->enc_key_addr = 0UL;
8370a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL;
8380a625fd2SDavid S. Miller ent->dest_addr = 0UL;
8390a625fd2SDavid S. Miller }
8400a625fd2SDavid S. Miller ent->control |= CONTROL_END_OF_BLOCK;
8410a625fd2SDavid S. Miller
8420a625fd2SDavid S. Miller return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
8430a625fd2SDavid S. Miller }
8440a625fd2SDavid S. Miller
n2_compute_chunks(struct skcipher_request * req)84523a6564aSArd Biesheuvel static int n2_compute_chunks(struct skcipher_request *req)
8460a625fd2SDavid S. Miller {
84723a6564aSArd Biesheuvel struct n2_request_context *rctx = skcipher_request_ctx(req);
84823a6564aSArd Biesheuvel struct skcipher_walk *walk = &rctx->walk;
8490a625fd2SDavid S. Miller struct n2_crypto_chunk *chunk;
8500a625fd2SDavid S. Miller unsigned long dest_prev;
8510a625fd2SDavid S. Miller unsigned int tot_len;
8520a625fd2SDavid S. Miller bool prev_in_place;
8530a625fd2SDavid S. Miller int err, nbytes;
8540a625fd2SDavid S. Miller
85523a6564aSArd Biesheuvel err = skcipher_walk_async(walk, req);
8560a625fd2SDavid S. Miller if (err)
8570a625fd2SDavid S. Miller return err;
8580a625fd2SDavid S. Miller
8590a625fd2SDavid S. Miller INIT_LIST_HEAD(&rctx->chunk_list);
8600a625fd2SDavid S. Miller
8610a625fd2SDavid S. Miller chunk = &rctx->chunk;
8620a625fd2SDavid S. Miller INIT_LIST_HEAD(&chunk->entry);
8630a625fd2SDavid S. Miller
8640a625fd2SDavid S. Miller chunk->iv_paddr = 0UL;
8650a625fd2SDavid S. Miller chunk->arr_len = 0;
8660a625fd2SDavid S. Miller chunk->dest_paddr = 0UL;
8670a625fd2SDavid S. Miller
8680a625fd2SDavid S. Miller prev_in_place = false;
8690a625fd2SDavid S. Miller dest_prev = ~0UL;
8700a625fd2SDavid S. Miller tot_len = 0;
8710a625fd2SDavid S. Miller
8720a625fd2SDavid S. Miller while ((nbytes = walk->nbytes) != 0) {
8730a625fd2SDavid S. Miller unsigned long dest_paddr, src_paddr;
8740a625fd2SDavid S. Miller bool in_place;
8750a625fd2SDavid S. Miller int this_len;
8760a625fd2SDavid S. Miller
87723a6564aSArd Biesheuvel src_paddr = (page_to_phys(walk->src.phys.page) +
87823a6564aSArd Biesheuvel walk->src.phys.offset);
87923a6564aSArd Biesheuvel dest_paddr = (page_to_phys(walk->dst.phys.page) +
88023a6564aSArd Biesheuvel walk->dst.phys.offset);
8810a625fd2SDavid S. Miller in_place = (src_paddr == dest_paddr);
88223a6564aSArd Biesheuvel this_len = skcipher_descriptor_len(nbytes, walk->blocksize);
8830a625fd2SDavid S. Miller
8840a625fd2SDavid S. Miller if (chunk->arr_len != 0) {
8850a625fd2SDavid S. Miller if (in_place != prev_in_place ||
8860a625fd2SDavid S. Miller (!prev_in_place &&
8870a625fd2SDavid S. Miller dest_paddr != dest_prev) ||
8880a625fd2SDavid S. Miller chunk->arr_len == N2_CHUNK_ARR_LEN ||
8890a625fd2SDavid S. Miller tot_len + this_len > (1 << 16)) {
8900a625fd2SDavid S. Miller chunk->dest_final = dest_prev;
8910a625fd2SDavid S. Miller list_add_tail(&chunk->entry,
8920a625fd2SDavid S. Miller &rctx->chunk_list);
8930a625fd2SDavid S. Miller chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
8940a625fd2SDavid S. Miller if (!chunk) {
8950a625fd2SDavid S. Miller err = -ENOMEM;
8960a625fd2SDavid S. Miller break;
8970a625fd2SDavid S. Miller }
8980a625fd2SDavid S. Miller INIT_LIST_HEAD(&chunk->entry);
8990a625fd2SDavid S. Miller }
9000a625fd2SDavid S. Miller }
9010a625fd2SDavid S. Miller if (chunk->arr_len == 0) {
9020a625fd2SDavid S. Miller chunk->dest_paddr = dest_paddr;
9030a625fd2SDavid S. Miller tot_len = 0;
9040a625fd2SDavid S. Miller }
9050a625fd2SDavid S. Miller chunk->arr[chunk->arr_len].src_paddr = src_paddr;
9060a625fd2SDavid S. Miller chunk->arr[chunk->arr_len].src_len = this_len;
9070a625fd2SDavid S. Miller chunk->arr_len++;
9080a625fd2SDavid S. Miller
9090a625fd2SDavid S. Miller dest_prev = dest_paddr + this_len;
9100a625fd2SDavid S. Miller prev_in_place = in_place;
9110a625fd2SDavid S. Miller tot_len += this_len;
9120a625fd2SDavid S. Miller
91323a6564aSArd Biesheuvel err = skcipher_walk_done(walk, nbytes - this_len);
9140a625fd2SDavid S. Miller if (err)
9150a625fd2SDavid S. Miller break;
9160a625fd2SDavid S. Miller }
9170a625fd2SDavid S. Miller if (!err && chunk->arr_len != 0) {
9180a625fd2SDavid S. Miller chunk->dest_final = dest_prev;
9190a625fd2SDavid S. Miller list_add_tail(&chunk->entry, &rctx->chunk_list);
9200a625fd2SDavid S. Miller }
9210a625fd2SDavid S. Miller
9220a625fd2SDavid S. Miller return err;
9230a625fd2SDavid S. Miller }
9240a625fd2SDavid S. Miller
n2_chunk_complete(struct skcipher_request * req,void * final_iv)92523a6564aSArd Biesheuvel static void n2_chunk_complete(struct skcipher_request *req, void *final_iv)
9260a625fd2SDavid S. Miller {
92723a6564aSArd Biesheuvel struct n2_request_context *rctx = skcipher_request_ctx(req);
9280a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp;
9290a625fd2SDavid S. Miller
9300a625fd2SDavid S. Miller if (final_iv)
9310a625fd2SDavid S. Miller memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
9320a625fd2SDavid S. Miller
9330a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
9340a625fd2SDavid S. Miller list_del(&c->entry);
9350a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk))
9360a625fd2SDavid S. Miller kfree(c);
9370a625fd2SDavid S. Miller }
9380a625fd2SDavid S. Miller
9390a625fd2SDavid S. Miller }
9400a625fd2SDavid S. Miller
n2_do_ecb(struct skcipher_request * req,bool encrypt)94123a6564aSArd Biesheuvel static int n2_do_ecb(struct skcipher_request *req, bool encrypt)
9420a625fd2SDavid S. Miller {
94323a6564aSArd Biesheuvel struct n2_request_context *rctx = skcipher_request_ctx(req);
94423a6564aSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
9450a625fd2SDavid S. Miller int err = n2_compute_chunks(req);
9460a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp;
9470a625fd2SDavid S. Miller unsigned long flags, hv_ret;
9480a625fd2SDavid S. Miller struct spu_queue *qp;
9490a625fd2SDavid S. Miller
9500a625fd2SDavid S. Miller if (err)
9510a625fd2SDavid S. Miller return err;
9520a625fd2SDavid S. Miller
9530a625fd2SDavid S. Miller qp = cpu_to_cwq[get_cpu()];
9540a625fd2SDavid S. Miller err = -ENODEV;
9550a625fd2SDavid S. Miller if (!qp)
9560a625fd2SDavid S. Miller goto out;
9570a625fd2SDavid S. Miller
9580a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags);
9590a625fd2SDavid S. Miller
9600a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
9610a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, encrypt);
9620a625fd2SDavid S. Miller if (err)
9630a625fd2SDavid S. Miller break;
9640a625fd2SDavid S. Miller list_del(&c->entry);
9650a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk))
9660a625fd2SDavid S. Miller kfree(c);
9670a625fd2SDavid S. Miller }
9680a625fd2SDavid S. Miller if (!err) {
9690a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp);
9700a625fd2SDavid S. Miller if (hv_ret != HV_EOK)
9710a625fd2SDavid S. Miller err = -EINVAL;
9720a625fd2SDavid S. Miller }
9730a625fd2SDavid S. Miller
9740a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags);
9750a625fd2SDavid S. Miller
976e27303b2SThomas Meyer out:
9770a625fd2SDavid S. Miller put_cpu();
9780a625fd2SDavid S. Miller
9790a625fd2SDavid S. Miller n2_chunk_complete(req, NULL);
9800a625fd2SDavid S. Miller return err;
9810a625fd2SDavid S. Miller }
9820a625fd2SDavid S. Miller
n2_encrypt_ecb(struct skcipher_request * req)98323a6564aSArd Biesheuvel static int n2_encrypt_ecb(struct skcipher_request *req)
9840a625fd2SDavid S. Miller {
9850a625fd2SDavid S. Miller return n2_do_ecb(req, true);
9860a625fd2SDavid S. Miller }
9870a625fd2SDavid S. Miller
n2_decrypt_ecb(struct skcipher_request * req)98823a6564aSArd Biesheuvel static int n2_decrypt_ecb(struct skcipher_request *req)
9890a625fd2SDavid S. Miller {
9900a625fd2SDavid S. Miller return n2_do_ecb(req, false);
9910a625fd2SDavid S. Miller }
9920a625fd2SDavid S. Miller
n2_do_chaining(struct skcipher_request * req,bool encrypt)99323a6564aSArd Biesheuvel static int n2_do_chaining(struct skcipher_request *req, bool encrypt)
9940a625fd2SDavid S. Miller {
99523a6564aSArd Biesheuvel struct n2_request_context *rctx = skcipher_request_ctx(req);
99623a6564aSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
9970a625fd2SDavid S. Miller unsigned long flags, hv_ret, iv_paddr;
9980a625fd2SDavid S. Miller int err = n2_compute_chunks(req);
9990a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp;
10000a625fd2SDavid S. Miller struct spu_queue *qp;
10010a625fd2SDavid S. Miller void *final_iv_addr;
10020a625fd2SDavid S. Miller
10030a625fd2SDavid S. Miller final_iv_addr = NULL;
10040a625fd2SDavid S. Miller
10050a625fd2SDavid S. Miller if (err)
10060a625fd2SDavid S. Miller return err;
10070a625fd2SDavid S. Miller
10080a625fd2SDavid S. Miller qp = cpu_to_cwq[get_cpu()];
10090a625fd2SDavid S. Miller err = -ENODEV;
10100a625fd2SDavid S. Miller if (!qp)
10110a625fd2SDavid S. Miller goto out;
10120a625fd2SDavid S. Miller
10130a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags);
10140a625fd2SDavid S. Miller
10150a625fd2SDavid S. Miller if (encrypt) {
10160a625fd2SDavid S. Miller iv_paddr = __pa(rctx->walk.iv);
10170a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
10180a625fd2SDavid S. Miller entry) {
10190a625fd2SDavid S. Miller c->iv_paddr = iv_paddr;
10200a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, true);
10210a625fd2SDavid S. Miller if (err)
10220a625fd2SDavid S. Miller break;
10230a625fd2SDavid S. Miller iv_paddr = c->dest_final - rctx->walk.blocksize;
10240a625fd2SDavid S. Miller list_del(&c->entry);
10250a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk))
10260a625fd2SDavid S. Miller kfree(c);
10270a625fd2SDavid S. Miller }
10280a625fd2SDavid S. Miller final_iv_addr = __va(iv_paddr);
10290a625fd2SDavid S. Miller } else {
10300a625fd2SDavid S. Miller list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
10310a625fd2SDavid S. Miller entry) {
10320a625fd2SDavid S. Miller if (c == &rctx->chunk) {
10330a625fd2SDavid S. Miller iv_paddr = __pa(rctx->walk.iv);
10340a625fd2SDavid S. Miller } else {
10350a625fd2SDavid S. Miller iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
10360a625fd2SDavid S. Miller tmp->arr[tmp->arr_len-1].src_len -
10370a625fd2SDavid S. Miller rctx->walk.blocksize);
10380a625fd2SDavid S. Miller }
10390a625fd2SDavid S. Miller if (!final_iv_addr) {
10400a625fd2SDavid S. Miller unsigned long pa;
10410a625fd2SDavid S. Miller
10420a625fd2SDavid S. Miller pa = (c->arr[c->arr_len-1].src_paddr +
10430a625fd2SDavid S. Miller c->arr[c->arr_len-1].src_len -
10440a625fd2SDavid S. Miller rctx->walk.blocksize);
10450a625fd2SDavid S. Miller final_iv_addr = rctx->temp_iv;
10460a625fd2SDavid S. Miller memcpy(rctx->temp_iv, __va(pa),
10470a625fd2SDavid S. Miller rctx->walk.blocksize);
10480a625fd2SDavid S. Miller }
10490a625fd2SDavid S. Miller c->iv_paddr = iv_paddr;
10500a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, false);
10510a625fd2SDavid S. Miller if (err)
10520a625fd2SDavid S. Miller break;
10530a625fd2SDavid S. Miller list_del(&c->entry);
10540a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk))
10550a625fd2SDavid S. Miller kfree(c);
10560a625fd2SDavid S. Miller }
10570a625fd2SDavid S. Miller }
10580a625fd2SDavid S. Miller if (!err) {
10590a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp);
10600a625fd2SDavid S. Miller if (hv_ret != HV_EOK)
10610a625fd2SDavid S. Miller err = -EINVAL;
10620a625fd2SDavid S. Miller }
10630a625fd2SDavid S. Miller
10640a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags);
10650a625fd2SDavid S. Miller
1066e27303b2SThomas Meyer out:
10670a625fd2SDavid S. Miller put_cpu();
10680a625fd2SDavid S. Miller
10690a625fd2SDavid S. Miller n2_chunk_complete(req, err ? NULL : final_iv_addr);
10700a625fd2SDavid S. Miller return err;
10710a625fd2SDavid S. Miller }
10720a625fd2SDavid S. Miller
n2_encrypt_chaining(struct skcipher_request * req)107323a6564aSArd Biesheuvel static int n2_encrypt_chaining(struct skcipher_request *req)
10740a625fd2SDavid S. Miller {
10750a625fd2SDavid S. Miller return n2_do_chaining(req, true);
10760a625fd2SDavid S. Miller }
10770a625fd2SDavid S. Miller
n2_decrypt_chaining(struct skcipher_request * req)107823a6564aSArd Biesheuvel static int n2_decrypt_chaining(struct skcipher_request *req)
10790a625fd2SDavid S. Miller {
10800a625fd2SDavid S. Miller return n2_do_chaining(req, false);
10810a625fd2SDavid S. Miller }
10820a625fd2SDavid S. Miller
108323a6564aSArd Biesheuvel struct n2_skcipher_tmpl {
10840a625fd2SDavid S. Miller const char *name;
10850a625fd2SDavid S. Miller const char *drv_name;
10860a625fd2SDavid S. Miller u8 block_size;
10870a625fd2SDavid S. Miller u8 enc_type;
108823a6564aSArd Biesheuvel struct skcipher_alg skcipher;
10890a625fd2SDavid S. Miller };
10900a625fd2SDavid S. Miller
109123a6564aSArd Biesheuvel static const struct n2_skcipher_tmpl skcipher_tmpls[] = {
10920a625fd2SDavid S. Miller /* DES: ECB CBC and CFB are supported */
10930a625fd2SDavid S. Miller { .name = "ecb(des)",
10940a625fd2SDavid S. Miller .drv_name = "ecb-des",
10950a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE,
10960a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_DES |
10970a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB),
109823a6564aSArd Biesheuvel .skcipher = {
10990a625fd2SDavid S. Miller .min_keysize = DES_KEY_SIZE,
11000a625fd2SDavid S. Miller .max_keysize = DES_KEY_SIZE,
11010a625fd2SDavid S. Miller .setkey = n2_des_setkey,
11020a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb,
11030a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb,
11040a625fd2SDavid S. Miller },
11050a625fd2SDavid S. Miller },
11060a625fd2SDavid S. Miller { .name = "cbc(des)",
11070a625fd2SDavid S. Miller .drv_name = "cbc-des",
11080a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE,
11090a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_DES |
11100a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC),
111123a6564aSArd Biesheuvel .skcipher = {
11120a625fd2SDavid S. Miller .ivsize = DES_BLOCK_SIZE,
11130a625fd2SDavid S. Miller .min_keysize = DES_KEY_SIZE,
11140a625fd2SDavid S. Miller .max_keysize = DES_KEY_SIZE,
11150a625fd2SDavid S. Miller .setkey = n2_des_setkey,
11160a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining,
11170a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining,
11180a625fd2SDavid S. Miller },
11190a625fd2SDavid S. Miller },
11200a625fd2SDavid S. Miller
11210a625fd2SDavid S. Miller /* 3DES: ECB CBC and CFB are supported */
11220a625fd2SDavid S. Miller { .name = "ecb(des3_ede)",
11230a625fd2SDavid S. Miller .drv_name = "ecb-3des",
11240a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE,
11250a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_3DES |
11260a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB),
112723a6564aSArd Biesheuvel .skcipher = {
11280a625fd2SDavid S. Miller .min_keysize = 3 * DES_KEY_SIZE,
11290a625fd2SDavid S. Miller .max_keysize = 3 * DES_KEY_SIZE,
11300a625fd2SDavid S. Miller .setkey = n2_3des_setkey,
11310a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb,
11320a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb,
11330a625fd2SDavid S. Miller },
11340a625fd2SDavid S. Miller },
11350a625fd2SDavid S. Miller { .name = "cbc(des3_ede)",
11360a625fd2SDavid S. Miller .drv_name = "cbc-3des",
11370a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE,
11380a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_3DES |
11390a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC),
114023a6564aSArd Biesheuvel .skcipher = {
11410a625fd2SDavid S. Miller .ivsize = DES_BLOCK_SIZE,
11420a625fd2SDavid S. Miller .min_keysize = 3 * DES_KEY_SIZE,
11430a625fd2SDavid S. Miller .max_keysize = 3 * DES_KEY_SIZE,
11440a625fd2SDavid S. Miller .setkey = n2_3des_setkey,
11450a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining,
11460a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining,
11470a625fd2SDavid S. Miller },
11480a625fd2SDavid S. Miller },
114966c465c6SHerbert Xu
11500a625fd2SDavid S. Miller /* AES: ECB CBC and CTR are supported */
11510a625fd2SDavid S. Miller { .name = "ecb(aes)",
11520a625fd2SDavid S. Miller .drv_name = "ecb-aes",
11530a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE,
11540a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 |
11550a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB),
115623a6564aSArd Biesheuvel .skcipher = {
11570a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE,
11580a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE,
11590a625fd2SDavid S. Miller .setkey = n2_aes_setkey,
11600a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb,
11610a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb,
11620a625fd2SDavid S. Miller },
11630a625fd2SDavid S. Miller },
11640a625fd2SDavid S. Miller { .name = "cbc(aes)",
11650a625fd2SDavid S. Miller .drv_name = "cbc-aes",
11660a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE,
11670a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 |
11680a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC),
116923a6564aSArd Biesheuvel .skcipher = {
11700a625fd2SDavid S. Miller .ivsize = AES_BLOCK_SIZE,
11710a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE,
11720a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE,
11730a625fd2SDavid S. Miller .setkey = n2_aes_setkey,
11740a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining,
11750a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining,
11760a625fd2SDavid S. Miller },
11770a625fd2SDavid S. Miller },
11780a625fd2SDavid S. Miller { .name = "ctr(aes)",
11790a625fd2SDavid S. Miller .drv_name = "ctr-aes",
11800a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE,
11810a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 |
11820a625fd2SDavid S. Miller ENC_TYPE_CHAINING_COUNTER),
118323a6564aSArd Biesheuvel .skcipher = {
11840a625fd2SDavid S. Miller .ivsize = AES_BLOCK_SIZE,
11850a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE,
11860a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE,
11870a625fd2SDavid S. Miller .setkey = n2_aes_setkey,
11880a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining,
11890a625fd2SDavid S. Miller .decrypt = n2_encrypt_chaining,
11900a625fd2SDavid S. Miller },
11910a625fd2SDavid S. Miller },
11920a625fd2SDavid S. Miller
11930a625fd2SDavid S. Miller };
119423a6564aSArd Biesheuvel #define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls)
11950a625fd2SDavid S. Miller
119623a6564aSArd Biesheuvel static LIST_HEAD(skcipher_algs);
11970a625fd2SDavid S. Miller
11980a625fd2SDavid S. Miller struct n2_hash_tmpl {
11990a625fd2SDavid S. Miller const char *name;
12008054b800SLABBE Corentin const u8 *hash_zero;
12016d77282fSHerbert Xu const u8 *hash_init;
12023a2c0346SDavid S. Miller u8 hw_op_hashsz;
12030a625fd2SDavid S. Miller u8 digest_size;
120476a4e874SCorentin Labbe u8 statesize;
12050a625fd2SDavid S. Miller u8 block_size;
12063a2c0346SDavid S. Miller u8 auth_type;
1207dc4ccfd1SDavid S. Miller u8 hmac_type;
12080a625fd2SDavid S. Miller };
12093a2c0346SDavid S. Miller
12106d77282fSHerbert Xu static const __le32 n2_md5_init[MD5_HASH_WORDS] = {
1211d0bb9ee3SLABBE Corentin cpu_to_le32(MD5_H0),
1212d0bb9ee3SLABBE Corentin cpu_to_le32(MD5_H1),
1213d0bb9ee3SLABBE Corentin cpu_to_le32(MD5_H2),
1214d0bb9ee3SLABBE Corentin cpu_to_le32(MD5_H3),
12153a2c0346SDavid S. Miller };
1216527aa895SHans de Goede static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = {
12173a2c0346SDavid S. Miller SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
12183a2c0346SDavid S. Miller };
1219527aa895SHans de Goede static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = {
12203a2c0346SDavid S. Miller SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
12213a2c0346SDavid S. Miller SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
12223a2c0346SDavid S. Miller };
1223527aa895SHans de Goede static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = {
12243a2c0346SDavid S. Miller SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
12253a2c0346SDavid S. Miller SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
12263a2c0346SDavid S. Miller };
12273a2c0346SDavid S. Miller
12280a625fd2SDavid S. Miller static const struct n2_hash_tmpl hash_tmpls[] = {
12290a625fd2SDavid S. Miller { .name = "md5",
12308054b800SLABBE Corentin .hash_zero = md5_zero_message_hash,
12316d77282fSHerbert Xu .hash_init = (u8 *)n2_md5_init,
12323a2c0346SDavid S. Miller .auth_type = AUTH_TYPE_MD5,
1233dc4ccfd1SDavid S. Miller .hmac_type = AUTH_TYPE_HMAC_MD5,
12343a2c0346SDavid S. Miller .hw_op_hashsz = MD5_DIGEST_SIZE,
12350a625fd2SDavid S. Miller .digest_size = MD5_DIGEST_SIZE,
123676a4e874SCorentin Labbe .statesize = sizeof(struct md5_state),
12370a625fd2SDavid S. Miller .block_size = MD5_HMAC_BLOCK_SIZE },
12380a625fd2SDavid S. Miller { .name = "sha1",
12398054b800SLABBE Corentin .hash_zero = sha1_zero_message_hash,
12406d77282fSHerbert Xu .hash_init = (u8 *)n2_sha1_init,
12413a2c0346SDavid S. Miller .auth_type = AUTH_TYPE_SHA1,
1242dc4ccfd1SDavid S. Miller .hmac_type = AUTH_TYPE_HMAC_SHA1,
12433a2c0346SDavid S. Miller .hw_op_hashsz = SHA1_DIGEST_SIZE,
12440a625fd2SDavid S. Miller .digest_size = SHA1_DIGEST_SIZE,
124576a4e874SCorentin Labbe .statesize = sizeof(struct sha1_state),
12460a625fd2SDavid S. Miller .block_size = SHA1_BLOCK_SIZE },
12470a625fd2SDavid S. Miller { .name = "sha256",
12488054b800SLABBE Corentin .hash_zero = sha256_zero_message_hash,
12496d77282fSHerbert Xu .hash_init = (u8 *)n2_sha256_init,
12503a2c0346SDavid S. Miller .auth_type = AUTH_TYPE_SHA256,
1251dc4ccfd1SDavid S. Miller .hmac_type = AUTH_TYPE_HMAC_SHA256,
12523a2c0346SDavid S. Miller .hw_op_hashsz = SHA256_DIGEST_SIZE,
12530a625fd2SDavid S. Miller .digest_size = SHA256_DIGEST_SIZE,
125476a4e874SCorentin Labbe .statesize = sizeof(struct sha256_state),
12550a625fd2SDavid S. Miller .block_size = SHA256_BLOCK_SIZE },
12560a625fd2SDavid S. Miller { .name = "sha224",
12578054b800SLABBE Corentin .hash_zero = sha224_zero_message_hash,
12586d77282fSHerbert Xu .hash_init = (u8 *)n2_sha224_init,
12593a2c0346SDavid S. Miller .auth_type = AUTH_TYPE_SHA256,
1260dc4ccfd1SDavid S. Miller .hmac_type = AUTH_TYPE_RESERVED,
12613a2c0346SDavid S. Miller .hw_op_hashsz = SHA256_DIGEST_SIZE,
12620a625fd2SDavid S. Miller .digest_size = SHA224_DIGEST_SIZE,
126376a4e874SCorentin Labbe .statesize = sizeof(struct sha256_state),
12640a625fd2SDavid S. Miller .block_size = SHA224_BLOCK_SIZE },
12650a625fd2SDavid S. Miller };
12660a625fd2SDavid S. Miller #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
12670a625fd2SDavid S. Miller
12680a625fd2SDavid S. Miller static LIST_HEAD(ahash_algs);
1269dc4ccfd1SDavid S. Miller static LIST_HEAD(hmac_algs);
12700a625fd2SDavid S. Miller
12710a625fd2SDavid S. Miller static int algs_registered;
12720a625fd2SDavid S. Miller
__n2_unregister_algs(void)12730a625fd2SDavid S. Miller static void __n2_unregister_algs(void)
12740a625fd2SDavid S. Miller {
127523a6564aSArd Biesheuvel struct n2_skcipher_alg *skcipher, *skcipher_tmp;
12760a625fd2SDavid S. Miller struct n2_ahash_alg *alg, *alg_tmp;
1277dc4ccfd1SDavid S. Miller struct n2_hmac_alg *hmac, *hmac_tmp;
12780a625fd2SDavid S. Miller
127923a6564aSArd Biesheuvel list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) {
128023a6564aSArd Biesheuvel crypto_unregister_skcipher(&skcipher->skcipher);
128123a6564aSArd Biesheuvel list_del(&skcipher->entry);
128223a6564aSArd Biesheuvel kfree(skcipher);
12830a625fd2SDavid S. Miller }
1284dc4ccfd1SDavid S. Miller list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1285dc4ccfd1SDavid S. Miller crypto_unregister_ahash(&hmac->derived.alg);
1286dc4ccfd1SDavid S. Miller list_del(&hmac->derived.entry);
1287dc4ccfd1SDavid S. Miller kfree(hmac);
1288dc4ccfd1SDavid S. Miller }
12890a625fd2SDavid S. Miller list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
12900a625fd2SDavid S. Miller crypto_unregister_ahash(&alg->alg);
12910a625fd2SDavid S. Miller list_del(&alg->entry);
12920a625fd2SDavid S. Miller kfree(alg);
12930a625fd2SDavid S. Miller }
12940a625fd2SDavid S. Miller }
12950a625fd2SDavid S. Miller
n2_skcipher_init_tfm(struct crypto_skcipher * tfm)129623a6564aSArd Biesheuvel static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm)
12970a625fd2SDavid S. Miller {
129823a6564aSArd Biesheuvel crypto_skcipher_set_reqsize(tfm, sizeof(struct n2_request_context));
12990a625fd2SDavid S. Miller return 0;
13000a625fd2SDavid S. Miller }
13010a625fd2SDavid S. Miller
__n2_register_one_skcipher(const struct n2_skcipher_tmpl * tmpl)130223a6564aSArd Biesheuvel static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl)
13030a625fd2SDavid S. Miller {
130423a6564aSArd Biesheuvel struct n2_skcipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
130523a6564aSArd Biesheuvel struct skcipher_alg *alg;
13060a625fd2SDavid S. Miller int err;
13070a625fd2SDavid S. Miller
13080a625fd2SDavid S. Miller if (!p)
13090a625fd2SDavid S. Miller return -ENOMEM;
13100a625fd2SDavid S. Miller
131123a6564aSArd Biesheuvel alg = &p->skcipher;
131223a6564aSArd Biesheuvel *alg = tmpl->skcipher;
13130a625fd2SDavid S. Miller
131423a6564aSArd Biesheuvel snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
131523a6564aSArd Biesheuvel snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
131623a6564aSArd Biesheuvel alg->base.cra_priority = N2_CRA_PRIORITY;
1317b8aa7dc5SMikulas Patocka alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC |
1318b8aa7dc5SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY;
131923a6564aSArd Biesheuvel alg->base.cra_blocksize = tmpl->block_size;
13200a625fd2SDavid S. Miller p->enc_type = tmpl->enc_type;
132123a6564aSArd Biesheuvel alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context);
132223a6564aSArd Biesheuvel alg->base.cra_module = THIS_MODULE;
132323a6564aSArd Biesheuvel alg->init = n2_skcipher_init_tfm;
13240a625fd2SDavid S. Miller
132523a6564aSArd Biesheuvel list_add(&p->entry, &skcipher_algs);
132623a6564aSArd Biesheuvel err = crypto_register_skcipher(alg);
13270a625fd2SDavid S. Miller if (err) {
132823a6564aSArd Biesheuvel pr_err("%s alg registration failed\n", alg->base.cra_name);
13290a625fd2SDavid S. Miller list_del(&p->entry);
13300a625fd2SDavid S. Miller kfree(p);
133138511108SDavid S. Miller } else {
133223a6564aSArd Biesheuvel pr_info("%s alg registered\n", alg->base.cra_name);
13330a625fd2SDavid S. Miller }
13340a625fd2SDavid S. Miller return err;
13350a625fd2SDavid S. Miller }
13360a625fd2SDavid S. Miller
__n2_register_one_hmac(struct n2_ahash_alg * n2ahash)133749cfe4dbSGreg Kroah-Hartman static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
1338dc4ccfd1SDavid S. Miller {
1339dc4ccfd1SDavid S. Miller struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1340dc4ccfd1SDavid S. Miller struct ahash_alg *ahash;
1341dc4ccfd1SDavid S. Miller struct crypto_alg *base;
1342dc4ccfd1SDavid S. Miller int err;
1343dc4ccfd1SDavid S. Miller
1344dc4ccfd1SDavid S. Miller if (!p)
1345dc4ccfd1SDavid S. Miller return -ENOMEM;
1346dc4ccfd1SDavid S. Miller
1347dc4ccfd1SDavid S. Miller p->child_alg = n2ahash->alg.halg.base.cra_name;
1348dc4ccfd1SDavid S. Miller memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1349dc4ccfd1SDavid S. Miller INIT_LIST_HEAD(&p->derived.entry);
1350dc4ccfd1SDavid S. Miller
1351dc4ccfd1SDavid S. Miller ahash = &p->derived.alg;
1352dc4ccfd1SDavid S. Miller ahash->digest = n2_hmac_async_digest;
1353dc4ccfd1SDavid S. Miller ahash->setkey = n2_hmac_async_setkey;
1354dc4ccfd1SDavid S. Miller
1355dc4ccfd1SDavid S. Miller base = &ahash->halg.base;
1356*ce212d2aSHerbert Xu err = -EINVAL;
13578c20982cSHerbert Xu if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)",
13588c20982cSHerbert Xu p->child_alg) >= CRYPTO_MAX_ALG_NAME)
13598c20982cSHerbert Xu goto out_free_p;
13608c20982cSHerbert Xu if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2",
13618c20982cSHerbert Xu p->child_alg) >= CRYPTO_MAX_ALG_NAME)
13628c20982cSHerbert Xu goto out_free_p;
1363dc4ccfd1SDavid S. Miller
1364dc4ccfd1SDavid S. Miller base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1365dc4ccfd1SDavid S. Miller base->cra_init = n2_hmac_cra_init;
1366dc4ccfd1SDavid S. Miller base->cra_exit = n2_hmac_cra_exit;
1367dc4ccfd1SDavid S. Miller
1368dc4ccfd1SDavid S. Miller list_add(&p->derived.entry, &hmac_algs);
1369dc4ccfd1SDavid S. Miller err = crypto_register_ahash(ahash);
1370dc4ccfd1SDavid S. Miller if (err) {
1371dc4ccfd1SDavid S. Miller pr_err("%s alg registration failed\n", base->cra_name);
1372dc4ccfd1SDavid S. Miller list_del(&p->derived.entry);
13738c20982cSHerbert Xu out_free_p:
1374dc4ccfd1SDavid S. Miller kfree(p);
1375dc4ccfd1SDavid S. Miller } else {
1376dc4ccfd1SDavid S. Miller pr_info("%s alg registered\n", base->cra_name);
1377dc4ccfd1SDavid S. Miller }
1378dc4ccfd1SDavid S. Miller return err;
1379dc4ccfd1SDavid S. Miller }
1380dc4ccfd1SDavid S. Miller
__n2_register_one_ahash(const struct n2_hash_tmpl * tmpl)138149cfe4dbSGreg Kroah-Hartman static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
13820a625fd2SDavid S. Miller {
13830a625fd2SDavid S. Miller struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
13840a625fd2SDavid S. Miller struct hash_alg_common *halg;
13850a625fd2SDavid S. Miller struct crypto_alg *base;
13860a625fd2SDavid S. Miller struct ahash_alg *ahash;
13870a625fd2SDavid S. Miller int err;
13880a625fd2SDavid S. Miller
13890a625fd2SDavid S. Miller if (!p)
13900a625fd2SDavid S. Miller return -ENOMEM;
13910a625fd2SDavid S. Miller
13923a2c0346SDavid S. Miller p->hash_zero = tmpl->hash_zero;
13933a2c0346SDavid S. Miller p->hash_init = tmpl->hash_init;
13943a2c0346SDavid S. Miller p->auth_type = tmpl->auth_type;
1395dc4ccfd1SDavid S. Miller p->hmac_type = tmpl->hmac_type;
13963a2c0346SDavid S. Miller p->hw_op_hashsz = tmpl->hw_op_hashsz;
13973a2c0346SDavid S. Miller p->digest_size = tmpl->digest_size;
13983a2c0346SDavid S. Miller
13990a625fd2SDavid S. Miller ahash = &p->alg;
14000a625fd2SDavid S. Miller ahash->init = n2_hash_async_init;
14010a625fd2SDavid S. Miller ahash->update = n2_hash_async_update;
14020a625fd2SDavid S. Miller ahash->final = n2_hash_async_final;
14030a625fd2SDavid S. Miller ahash->finup = n2_hash_async_finup;
14043a2c0346SDavid S. Miller ahash->digest = n2_hash_async_digest;
1405378fe6fbSKamil Konieczny ahash->export = n2_hash_async_noexport;
1406378fe6fbSKamil Konieczny ahash->import = n2_hash_async_noimport;
14070a625fd2SDavid S. Miller
14080a625fd2SDavid S. Miller halg = &ahash->halg;
14090a625fd2SDavid S. Miller halg->digestsize = tmpl->digest_size;
141076a4e874SCorentin Labbe halg->statesize = tmpl->statesize;
14110a625fd2SDavid S. Miller
14120a625fd2SDavid S. Miller base = &halg->base;
14130a625fd2SDavid S. Miller snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
14140a625fd2SDavid S. Miller snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
14150a625fd2SDavid S. Miller base->cra_priority = N2_CRA_PRIORITY;
14166a38f622SEric Biggers base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1417d912bb76SNikos Mavrogiannopoulos CRYPTO_ALG_NEED_FALLBACK;
14180a625fd2SDavid S. Miller base->cra_blocksize = tmpl->block_size;
14190a625fd2SDavid S. Miller base->cra_ctxsize = sizeof(struct n2_hash_ctx);
14200a625fd2SDavid S. Miller base->cra_module = THIS_MODULE;
14210a625fd2SDavid S. Miller base->cra_init = n2_hash_cra_init;
14220a625fd2SDavid S. Miller base->cra_exit = n2_hash_cra_exit;
14230a625fd2SDavid S. Miller
14240a625fd2SDavid S. Miller list_add(&p->entry, &ahash_algs);
14250a625fd2SDavid S. Miller err = crypto_register_ahash(ahash);
14260a625fd2SDavid S. Miller if (err) {
142738511108SDavid S. Miller pr_err("%s alg registration failed\n", base->cra_name);
14280a625fd2SDavid S. Miller list_del(&p->entry);
14290a625fd2SDavid S. Miller kfree(p);
143038511108SDavid S. Miller } else {
143138511108SDavid S. Miller pr_info("%s alg registered\n", base->cra_name);
14320a625fd2SDavid S. Miller }
1433dc4ccfd1SDavid S. Miller if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1434dc4ccfd1SDavid S. Miller err = __n2_register_one_hmac(p);
14350a625fd2SDavid S. Miller return err;
14360a625fd2SDavid S. Miller }
14370a625fd2SDavid S. Miller
n2_register_algs(void)143849cfe4dbSGreg Kroah-Hartman static int n2_register_algs(void)
14390a625fd2SDavid S. Miller {
14400a625fd2SDavid S. Miller int i, err = 0;
14410a625fd2SDavid S. Miller
14420a625fd2SDavid S. Miller mutex_lock(&spu_lock);
14430a625fd2SDavid S. Miller if (algs_registered++)
14440a625fd2SDavid S. Miller goto out;
14450a625fd2SDavid S. Miller
14460a625fd2SDavid S. Miller for (i = 0; i < NUM_HASH_TMPLS; i++) {
14470a625fd2SDavid S. Miller err = __n2_register_one_ahash(&hash_tmpls[i]);
14480a625fd2SDavid S. Miller if (err) {
14490a625fd2SDavid S. Miller __n2_unregister_algs();
14500a625fd2SDavid S. Miller goto out;
14510a625fd2SDavid S. Miller }
14520a625fd2SDavid S. Miller }
14530a625fd2SDavid S. Miller for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
145423a6564aSArd Biesheuvel err = __n2_register_one_skcipher(&skcipher_tmpls[i]);
14550a625fd2SDavid S. Miller if (err) {
14560a625fd2SDavid S. Miller __n2_unregister_algs();
14570a625fd2SDavid S. Miller goto out;
14580a625fd2SDavid S. Miller }
14590a625fd2SDavid S. Miller }
14600a625fd2SDavid S. Miller
14610a625fd2SDavid S. Miller out:
14620a625fd2SDavid S. Miller mutex_unlock(&spu_lock);
14630a625fd2SDavid S. Miller return err;
14640a625fd2SDavid S. Miller }
14650a625fd2SDavid S. Miller
n2_unregister_algs(void)146649cfe4dbSGreg Kroah-Hartman static void n2_unregister_algs(void)
14670a625fd2SDavid S. Miller {
14680a625fd2SDavid S. Miller mutex_lock(&spu_lock);
14690a625fd2SDavid S. Miller if (!--algs_registered)
14700a625fd2SDavid S. Miller __n2_unregister_algs();
14710a625fd2SDavid S. Miller mutex_unlock(&spu_lock);
14720a625fd2SDavid S. Miller }
14730a625fd2SDavid S. Miller
14740a625fd2SDavid S. Miller /* To map CWQ queues to interrupt sources, the hypervisor API provides
14750a625fd2SDavid S. Miller * a devino. This isn't very useful to us because all of the
14762dc11581SGrant Likely * interrupts listed in the device_node have been translated to
14770a625fd2SDavid S. Miller * Linux virtual IRQ cookie numbers.
14780a625fd2SDavid S. Miller *
14790a625fd2SDavid S. Miller * So we have to back-translate, going through the 'intr' and 'ino'
14800a625fd2SDavid S. Miller * property tables of the n2cp MDESC node, matching it with the OF
14818c8e5b6aSwangjianli * 'interrupts' property entries, in order to figure out which
14820a625fd2SDavid S. Miller * devino goes to which already-translated IRQ.
14830a625fd2SDavid S. Miller */
find_devino_index(struct platform_device * dev,struct spu_mdesc_info * ip,unsigned long dev_ino)14842dc11581SGrant Likely static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
14850a625fd2SDavid S. Miller unsigned long dev_ino)
14860a625fd2SDavid S. Miller {
14870a625fd2SDavid S. Miller const unsigned int *dev_intrs;
14880a625fd2SDavid S. Miller unsigned int intr;
14890a625fd2SDavid S. Miller int i;
14900a625fd2SDavid S. Miller
14910a625fd2SDavid S. Miller for (i = 0; i < ip->num_intrs; i++) {
14920a625fd2SDavid S. Miller if (ip->ino_table[i].ino == dev_ino)
14930a625fd2SDavid S. Miller break;
14940a625fd2SDavid S. Miller }
14950a625fd2SDavid S. Miller if (i == ip->num_intrs)
14960a625fd2SDavid S. Miller return -ENODEV;
14970a625fd2SDavid S. Miller
14980a625fd2SDavid S. Miller intr = ip->ino_table[i].intr;
14990a625fd2SDavid S. Miller
1500ff6c7341SDavid S. Miller dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
15010a625fd2SDavid S. Miller if (!dev_intrs)
15020a625fd2SDavid S. Miller return -ENODEV;
15030a625fd2SDavid S. Miller
150419e4875fSGrant Likely for (i = 0; i < dev->archdata.num_irqs; i++) {
15050a625fd2SDavid S. Miller if (dev_intrs[i] == intr)
15060a625fd2SDavid S. Miller return i;
15070a625fd2SDavid S. Miller }
15080a625fd2SDavid S. Miller
15090a625fd2SDavid S. Miller return -ENODEV;
15100a625fd2SDavid S. Miller }
15110a625fd2SDavid S. Miller
spu_map_ino(struct platform_device * dev,struct spu_mdesc_info * ip,const char * irq_name,struct spu_queue * p,irq_handler_t handler)15122dc11581SGrant Likely static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
15130a625fd2SDavid S. Miller const char *irq_name, struct spu_queue *p,
15140a625fd2SDavid S. Miller irq_handler_t handler)
15150a625fd2SDavid S. Miller {
15160a625fd2SDavid S. Miller unsigned long herr;
15170a625fd2SDavid S. Miller int index;
15180a625fd2SDavid S. Miller
15190a625fd2SDavid S. Miller herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
15200a625fd2SDavid S. Miller if (herr)
15210a625fd2SDavid S. Miller return -EINVAL;
15220a625fd2SDavid S. Miller
15230a625fd2SDavid S. Miller index = find_devino_index(dev, ip, p->devino);
15240a625fd2SDavid S. Miller if (index < 0)
15250a625fd2SDavid S. Miller return index;
15260a625fd2SDavid S. Miller
152719e4875fSGrant Likely p->irq = dev->archdata.irqs[index];
15280a625fd2SDavid S. Miller
15290a625fd2SDavid S. Miller sprintf(p->irq_name, "%s-%d", irq_name, index);
15300a625fd2SDavid S. Miller
15319751bfd1STheodore Ts'o return request_irq(p->irq, handler, 0, p->irq_name, p);
15320a625fd2SDavid S. Miller }
15330a625fd2SDavid S. Miller
15340a625fd2SDavid S. Miller static struct kmem_cache *queue_cache[2];
15350a625fd2SDavid S. Miller
new_queue(unsigned long q_type)15360a625fd2SDavid S. Miller static void *new_queue(unsigned long q_type)
15370a625fd2SDavid S. Miller {
15380a625fd2SDavid S. Miller return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
15390a625fd2SDavid S. Miller }
15400a625fd2SDavid S. Miller
free_queue(void * p,unsigned long q_type)15410a625fd2SDavid S. Miller static void free_queue(void *p, unsigned long q_type)
15420a625fd2SDavid S. Miller {
1543150f6d45SAmitoj Kaur Chawla kmem_cache_free(queue_cache[q_type - 1], p);
15440a625fd2SDavid S. Miller }
15450a625fd2SDavid S. Miller
queue_cache_init(void)15460a625fd2SDavid S. Miller static int queue_cache_init(void)
15470a625fd2SDavid S. Miller {
15480a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
15490a625fd2SDavid S. Miller queue_cache[HV_NCS_QTYPE_MAU - 1] =
1550527b9525SDavid S. Miller kmem_cache_create("mau_queue",
15510a625fd2SDavid S. Miller (MAU_NUM_ENTRIES *
15520a625fd2SDavid S. Miller MAU_ENTRY_SIZE),
15530a625fd2SDavid S. Miller MAU_ENTRY_SIZE, 0, NULL);
15540a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
15550a625fd2SDavid S. Miller return -ENOMEM;
15560a625fd2SDavid S. Miller
15570a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
15580a625fd2SDavid S. Miller queue_cache[HV_NCS_QTYPE_CWQ - 1] =
15590a625fd2SDavid S. Miller kmem_cache_create("cwq_queue",
15600a625fd2SDavid S. Miller (CWQ_NUM_ENTRIES *
15610a625fd2SDavid S. Miller CWQ_ENTRY_SIZE),
15620a625fd2SDavid S. Miller CWQ_ENTRY_SIZE, 0, NULL);
15630a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
15640a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1565203f4500SJan Engelhardt queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL;
15660a625fd2SDavid S. Miller return -ENOMEM;
15670a625fd2SDavid S. Miller }
15680a625fd2SDavid S. Miller return 0;
15690a625fd2SDavid S. Miller }
15700a625fd2SDavid S. Miller
queue_cache_destroy(void)15710a625fd2SDavid S. Miller static void queue_cache_destroy(void)
15720a625fd2SDavid S. Miller {
15730a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
15740a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1575203f4500SJan Engelhardt queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL;
1576203f4500SJan Engelhardt queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL;
15770a625fd2SDavid S. Miller }
15780a625fd2SDavid S. Miller
spu_queue_register_workfn(void * arg)157973810a06SThomas Gleixner static long spu_queue_register_workfn(void *arg)
15800a625fd2SDavid S. Miller {
158173810a06SThomas Gleixner struct spu_qreg *qr = arg;
158273810a06SThomas Gleixner struct spu_queue *p = qr->queue;
158373810a06SThomas Gleixner unsigned long q_type = qr->type;
15840a625fd2SDavid S. Miller unsigned long hv_ret;
15850a625fd2SDavid S. Miller
15860a625fd2SDavid S. Miller hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
15870a625fd2SDavid S. Miller CWQ_NUM_ENTRIES, &p->qhandle);
15880a625fd2SDavid S. Miller if (!hv_ret)
15890a625fd2SDavid S. Miller sun4v_ncs_sethead_marker(p->qhandle, 0);
15900a625fd2SDavid S. Miller
159173810a06SThomas Gleixner return hv_ret ? -EINVAL : 0;
159273810a06SThomas Gleixner }
15930a625fd2SDavid S. Miller
spu_queue_register(struct spu_queue * p,unsigned long q_type)159473810a06SThomas Gleixner static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
159573810a06SThomas Gleixner {
159673810a06SThomas Gleixner int cpu = cpumask_any_and(&p->sharing, cpu_online_mask);
159773810a06SThomas Gleixner struct spu_qreg qr = { .queue = p, .type = q_type };
15980a625fd2SDavid S. Miller
159973810a06SThomas Gleixner return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr);
16000a625fd2SDavid S. Miller }
16010a625fd2SDavid S. Miller
spu_queue_setup(struct spu_queue * p)16020a625fd2SDavid S. Miller static int spu_queue_setup(struct spu_queue *p)
16030a625fd2SDavid S. Miller {
16040a625fd2SDavid S. Miller int err;
16050a625fd2SDavid S. Miller
16060a625fd2SDavid S. Miller p->q = new_queue(p->q_type);
16070a625fd2SDavid S. Miller if (!p->q)
16080a625fd2SDavid S. Miller return -ENOMEM;
16090a625fd2SDavid S. Miller
16100a625fd2SDavid S. Miller err = spu_queue_register(p, p->q_type);
16110a625fd2SDavid S. Miller if (err) {
16120a625fd2SDavid S. Miller free_queue(p->q, p->q_type);
16130a625fd2SDavid S. Miller p->q = NULL;
16140a625fd2SDavid S. Miller }
16150a625fd2SDavid S. Miller
16160a625fd2SDavid S. Miller return err;
16170a625fd2SDavid S. Miller }
16180a625fd2SDavid S. Miller
spu_queue_destroy(struct spu_queue * p)16190a625fd2SDavid S. Miller static void spu_queue_destroy(struct spu_queue *p)
16200a625fd2SDavid S. Miller {
16210a625fd2SDavid S. Miller unsigned long hv_ret;
16220a625fd2SDavid S. Miller
16230a625fd2SDavid S. Miller if (!p->q)
16240a625fd2SDavid S. Miller return;
16250a625fd2SDavid S. Miller
16260a625fd2SDavid S. Miller hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
16270a625fd2SDavid S. Miller
16280a625fd2SDavid S. Miller if (!hv_ret)
16290a625fd2SDavid S. Miller free_queue(p->q, p->q_type);
16300a625fd2SDavid S. Miller }
16310a625fd2SDavid S. Miller
spu_list_destroy(struct list_head * list)16320a625fd2SDavid S. Miller static void spu_list_destroy(struct list_head *list)
16330a625fd2SDavid S. Miller {
16340a625fd2SDavid S. Miller struct spu_queue *p, *n;
16350a625fd2SDavid S. Miller
16360a625fd2SDavid S. Miller list_for_each_entry_safe(p, n, list, list) {
16370a625fd2SDavid S. Miller int i;
16380a625fd2SDavid S. Miller
16390a625fd2SDavid S. Miller for (i = 0; i < NR_CPUS; i++) {
16400a625fd2SDavid S. Miller if (cpu_to_cwq[i] == p)
16410a625fd2SDavid S. Miller cpu_to_cwq[i] = NULL;
16420a625fd2SDavid S. Miller }
16430a625fd2SDavid S. Miller
16440a625fd2SDavid S. Miller if (p->irq) {
16450a625fd2SDavid S. Miller free_irq(p->irq, p);
16460a625fd2SDavid S. Miller p->irq = 0;
16470a625fd2SDavid S. Miller }
16480a625fd2SDavid S. Miller spu_queue_destroy(p);
16490a625fd2SDavid S. Miller list_del(&p->list);
16500a625fd2SDavid S. Miller kfree(p);
16510a625fd2SDavid S. Miller }
16520a625fd2SDavid S. Miller }
16530a625fd2SDavid S. Miller
16540a625fd2SDavid S. Miller /* Walk the backward arcs of a CWQ 'exec-unit' node,
16550a625fd2SDavid S. Miller * gathering cpu membership information.
16560a625fd2SDavid S. Miller */
spu_mdesc_walk_arcs(struct mdesc_handle * mdesc,struct platform_device * dev,u64 node,struct spu_queue * p,struct spu_queue ** table)16570a625fd2SDavid S. Miller static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
16582dc11581SGrant Likely struct platform_device *dev,
16590a625fd2SDavid S. Miller u64 node, struct spu_queue *p,
16600a625fd2SDavid S. Miller struct spu_queue **table)
16610a625fd2SDavid S. Miller {
16620a625fd2SDavid S. Miller u64 arc;
16630a625fd2SDavid S. Miller
16640a625fd2SDavid S. Miller mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
16650a625fd2SDavid S. Miller u64 tgt = mdesc_arc_target(mdesc, arc);
16660a625fd2SDavid S. Miller const char *name = mdesc_node_name(mdesc, tgt);
16670a625fd2SDavid S. Miller const u64 *id;
16680a625fd2SDavid S. Miller
16690a625fd2SDavid S. Miller if (strcmp(name, "cpu"))
16700a625fd2SDavid S. Miller continue;
16710a625fd2SDavid S. Miller id = mdesc_get_property(mdesc, tgt, "id", NULL);
16720a625fd2SDavid S. Miller if (table[*id] != NULL) {
1673b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n",
1674b2e870f2SRob Herring dev->dev.of_node);
16750a625fd2SDavid S. Miller return -EINVAL;
16760a625fd2SDavid S. Miller }
1677f9b531feSRusty Russell cpumask_set_cpu(*id, &p->sharing);
16780a625fd2SDavid S. Miller table[*id] = p;
16790a625fd2SDavid S. Miller }
16800a625fd2SDavid S. Miller return 0;
16810a625fd2SDavid S. Miller }
16820a625fd2SDavid S. Miller
16830a625fd2SDavid S. Miller /* Process an 'exec-unit' MDESC node of type 'cwq'. */
handle_exec_unit(struct spu_mdesc_info * ip,struct list_head * list,struct platform_device * dev,struct mdesc_handle * mdesc,u64 node,const char * iname,unsigned long q_type,irq_handler_t handler,struct spu_queue ** table)16840a625fd2SDavid S. Miller static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
16852dc11581SGrant Likely struct platform_device *dev, struct mdesc_handle *mdesc,
16860a625fd2SDavid S. Miller u64 node, const char *iname, unsigned long q_type,
16870a625fd2SDavid S. Miller irq_handler_t handler, struct spu_queue **table)
16880a625fd2SDavid S. Miller {
16890a625fd2SDavid S. Miller struct spu_queue *p;
16900a625fd2SDavid S. Miller int err;
16910a625fd2SDavid S. Miller
16920a625fd2SDavid S. Miller p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
16930a625fd2SDavid S. Miller if (!p) {
1694b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n",
1695b2e870f2SRob Herring dev->dev.of_node);
16960a625fd2SDavid S. Miller return -ENOMEM;
16970a625fd2SDavid S. Miller }
16980a625fd2SDavid S. Miller
1699f9b531feSRusty Russell cpumask_clear(&p->sharing);
17000a625fd2SDavid S. Miller spin_lock_init(&p->lock);
17010a625fd2SDavid S. Miller p->q_type = q_type;
17020a625fd2SDavid S. Miller INIT_LIST_HEAD(&p->jobs);
17030a625fd2SDavid S. Miller list_add(&p->list, list);
17040a625fd2SDavid S. Miller
17050a625fd2SDavid S. Miller err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
17060a625fd2SDavid S. Miller if (err)
17070a625fd2SDavid S. Miller return err;
17080a625fd2SDavid S. Miller
17090a625fd2SDavid S. Miller err = spu_queue_setup(p);
17100a625fd2SDavid S. Miller if (err)
17110a625fd2SDavid S. Miller return err;
17120a625fd2SDavid S. Miller
17130a625fd2SDavid S. Miller return spu_map_ino(dev, ip, iname, p, handler);
17140a625fd2SDavid S. Miller }
17150a625fd2SDavid S. Miller
spu_mdesc_scan(struct mdesc_handle * mdesc,struct platform_device * dev,struct spu_mdesc_info * ip,struct list_head * list,const char * exec_name,unsigned long q_type,irq_handler_t handler,struct spu_queue ** table)17162dc11581SGrant Likely static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
17170a625fd2SDavid S. Miller struct spu_mdesc_info *ip, struct list_head *list,
17180a625fd2SDavid S. Miller const char *exec_name, unsigned long q_type,
17190a625fd2SDavid S. Miller irq_handler_t handler, struct spu_queue **table)
17200a625fd2SDavid S. Miller {
17210a625fd2SDavid S. Miller int err = 0;
17220a625fd2SDavid S. Miller u64 node;
17230a625fd2SDavid S. Miller
17240a625fd2SDavid S. Miller mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
17250a625fd2SDavid S. Miller const char *type;
17260a625fd2SDavid S. Miller
17270a625fd2SDavid S. Miller type = mdesc_get_property(mdesc, node, "type", NULL);
17280a625fd2SDavid S. Miller if (!type || strcmp(type, exec_name))
17290a625fd2SDavid S. Miller continue;
17300a625fd2SDavid S. Miller
17310a625fd2SDavid S. Miller err = handle_exec_unit(ip, list, dev, mdesc, node,
17320a625fd2SDavid S. Miller exec_name, q_type, handler, table);
17330a625fd2SDavid S. Miller if (err) {
17340a625fd2SDavid S. Miller spu_list_destroy(list);
17350a625fd2SDavid S. Miller break;
17360a625fd2SDavid S. Miller }
17370a625fd2SDavid S. Miller }
17380a625fd2SDavid S. Miller
17390a625fd2SDavid S. Miller return err;
17400a625fd2SDavid S. Miller }
17410a625fd2SDavid S. Miller
get_irq_props(struct mdesc_handle * mdesc,u64 node,struct spu_mdesc_info * ip)174249cfe4dbSGreg Kroah-Hartman static int get_irq_props(struct mdesc_handle *mdesc, u64 node,
17430a625fd2SDavid S. Miller struct spu_mdesc_info *ip)
17440a625fd2SDavid S. Miller {
1745eb7caf35SDavid S. Miller const u64 *ino;
1746eb7caf35SDavid S. Miller int ino_len;
17470a625fd2SDavid S. Miller int i;
17480a625fd2SDavid S. Miller
17490a625fd2SDavid S. Miller ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1750eb7caf35SDavid S. Miller if (!ino) {
1751eb7caf35SDavid S. Miller printk("NO 'ino'\n");
17520a625fd2SDavid S. Miller return -ENODEV;
1753eb7caf35SDavid S. Miller }
17540a625fd2SDavid S. Miller
1755eb7caf35SDavid S. Miller ip->num_intrs = ino_len / sizeof(u64);
17560a625fd2SDavid S. Miller ip->ino_table = kzalloc((sizeof(struct ino_blob) *
17570a625fd2SDavid S. Miller ip->num_intrs),
17580a625fd2SDavid S. Miller GFP_KERNEL);
17590a625fd2SDavid S. Miller if (!ip->ino_table)
17600a625fd2SDavid S. Miller return -ENOMEM;
17610a625fd2SDavid S. Miller
17620a625fd2SDavid S. Miller for (i = 0; i < ip->num_intrs; i++) {
17630a625fd2SDavid S. Miller struct ino_blob *b = &ip->ino_table[i];
1764eb7caf35SDavid S. Miller b->intr = i + 1;
17650a625fd2SDavid S. Miller b->ino = ino[i];
17660a625fd2SDavid S. Miller }
17670a625fd2SDavid S. Miller
17680a625fd2SDavid S. Miller return 0;
17690a625fd2SDavid S. Miller }
17700a625fd2SDavid S. Miller
grab_mdesc_irq_props(struct mdesc_handle * mdesc,struct platform_device * dev,struct spu_mdesc_info * ip,const char * node_name)177149cfe4dbSGreg Kroah-Hartman static int grab_mdesc_irq_props(struct mdesc_handle *mdesc,
17722dc11581SGrant Likely struct platform_device *dev,
17730a625fd2SDavid S. Miller struct spu_mdesc_info *ip,
17740a625fd2SDavid S. Miller const char *node_name)
17750a625fd2SDavid S. Miller {
1776ee174e26SRob Herring u64 node, reg;
17770a625fd2SDavid S. Miller
1778ee174e26SRob Herring if (of_property_read_reg(dev->dev.of_node, 0, ®, NULL) < 0)
17790a625fd2SDavid S. Miller return -ENODEV;
17800a625fd2SDavid S. Miller
17810a625fd2SDavid S. Miller mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
17820a625fd2SDavid S. Miller const char *name;
17830a625fd2SDavid S. Miller const u64 *chdl;
17840a625fd2SDavid S. Miller
17850a625fd2SDavid S. Miller name = mdesc_get_property(mdesc, node, "name", NULL);
17860a625fd2SDavid S. Miller if (!name || strcmp(name, node_name))
17870a625fd2SDavid S. Miller continue;
17880a625fd2SDavid S. Miller chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1789ee174e26SRob Herring if (!chdl || (*chdl != reg))
17900a625fd2SDavid S. Miller continue;
17910a625fd2SDavid S. Miller ip->cfg_handle = *chdl;
17920a625fd2SDavid S. Miller return get_irq_props(mdesc, node, ip);
17930a625fd2SDavid S. Miller }
17940a625fd2SDavid S. Miller
17950a625fd2SDavid S. Miller return -ENODEV;
17960a625fd2SDavid S. Miller }
17970a625fd2SDavid S. Miller
17980a625fd2SDavid S. Miller static unsigned long n2_spu_hvapi_major;
17990a625fd2SDavid S. Miller static unsigned long n2_spu_hvapi_minor;
18000a625fd2SDavid S. Miller
n2_spu_hvapi_register(void)180149cfe4dbSGreg Kroah-Hartman static int n2_spu_hvapi_register(void)
18020a625fd2SDavid S. Miller {
18030a625fd2SDavid S. Miller int err;
18040a625fd2SDavid S. Miller
18050a625fd2SDavid S. Miller n2_spu_hvapi_major = 2;
18060a625fd2SDavid S. Miller n2_spu_hvapi_minor = 0;
18070a625fd2SDavid S. Miller
18080a625fd2SDavid S. Miller err = sun4v_hvapi_register(HV_GRP_NCS,
18090a625fd2SDavid S. Miller n2_spu_hvapi_major,
18100a625fd2SDavid S. Miller &n2_spu_hvapi_minor);
18110a625fd2SDavid S. Miller
18120a625fd2SDavid S. Miller if (!err)
18130a625fd2SDavid S. Miller pr_info("Registered NCS HVAPI version %lu.%lu\n",
18140a625fd2SDavid S. Miller n2_spu_hvapi_major,
18150a625fd2SDavid S. Miller n2_spu_hvapi_minor);
18160a625fd2SDavid S. Miller
18170a625fd2SDavid S. Miller return err;
18180a625fd2SDavid S. Miller }
18190a625fd2SDavid S. Miller
n2_spu_hvapi_unregister(void)18200a625fd2SDavid S. Miller static void n2_spu_hvapi_unregister(void)
18210a625fd2SDavid S. Miller {
18220a625fd2SDavid S. Miller sun4v_hvapi_unregister(HV_GRP_NCS);
18230a625fd2SDavid S. Miller }
18240a625fd2SDavid S. Miller
18250a625fd2SDavid S. Miller static int global_ref;
18260a625fd2SDavid S. Miller
grab_global_resources(void)182749cfe4dbSGreg Kroah-Hartman static int grab_global_resources(void)
18280a625fd2SDavid S. Miller {
18290a625fd2SDavid S. Miller int err = 0;
18300a625fd2SDavid S. Miller
18310a625fd2SDavid S. Miller mutex_lock(&spu_lock);
18320a625fd2SDavid S. Miller
18330a625fd2SDavid S. Miller if (global_ref++)
18340a625fd2SDavid S. Miller goto out;
18350a625fd2SDavid S. Miller
18360a625fd2SDavid S. Miller err = n2_spu_hvapi_register();
18370a625fd2SDavid S. Miller if (err)
18380a625fd2SDavid S. Miller goto out;
18390a625fd2SDavid S. Miller
18400a625fd2SDavid S. Miller err = queue_cache_init();
18410a625fd2SDavid S. Miller if (err)
18420a625fd2SDavid S. Miller goto out_hvapi_release;
18430a625fd2SDavid S. Miller
18440a625fd2SDavid S. Miller err = -ENOMEM;
18456396bb22SKees Cook cpu_to_cwq = kcalloc(NR_CPUS, sizeof(struct spu_queue *),
18460a625fd2SDavid S. Miller GFP_KERNEL);
18470a625fd2SDavid S. Miller if (!cpu_to_cwq)
18480a625fd2SDavid S. Miller goto out_queue_cache_destroy;
18490a625fd2SDavid S. Miller
18506396bb22SKees Cook cpu_to_mau = kcalloc(NR_CPUS, sizeof(struct spu_queue *),
18510a625fd2SDavid S. Miller GFP_KERNEL);
18520a625fd2SDavid S. Miller if (!cpu_to_mau)
18530a625fd2SDavid S. Miller goto out_free_cwq_table;
18540a625fd2SDavid S. Miller
18550a625fd2SDavid S. Miller err = 0;
18560a625fd2SDavid S. Miller
18570a625fd2SDavid S. Miller out:
18580a625fd2SDavid S. Miller if (err)
18590a625fd2SDavid S. Miller global_ref--;
18600a625fd2SDavid S. Miller mutex_unlock(&spu_lock);
18610a625fd2SDavid S. Miller return err;
18620a625fd2SDavid S. Miller
18630a625fd2SDavid S. Miller out_free_cwq_table:
18640a625fd2SDavid S. Miller kfree(cpu_to_cwq);
18650a625fd2SDavid S. Miller cpu_to_cwq = NULL;
18660a625fd2SDavid S. Miller
18670a625fd2SDavid S. Miller out_queue_cache_destroy:
18680a625fd2SDavid S. Miller queue_cache_destroy();
18690a625fd2SDavid S. Miller
18700a625fd2SDavid S. Miller out_hvapi_release:
18710a625fd2SDavid S. Miller n2_spu_hvapi_unregister();
18720a625fd2SDavid S. Miller goto out;
18730a625fd2SDavid S. Miller }
18740a625fd2SDavid S. Miller
release_global_resources(void)18750a625fd2SDavid S. Miller static void release_global_resources(void)
18760a625fd2SDavid S. Miller {
18770a625fd2SDavid S. Miller mutex_lock(&spu_lock);
18780a625fd2SDavid S. Miller if (!--global_ref) {
18790a625fd2SDavid S. Miller kfree(cpu_to_cwq);
18800a625fd2SDavid S. Miller cpu_to_cwq = NULL;
18810a625fd2SDavid S. Miller
18820a625fd2SDavid S. Miller kfree(cpu_to_mau);
18830a625fd2SDavid S. Miller cpu_to_mau = NULL;
18840a625fd2SDavid S. Miller
18850a625fd2SDavid S. Miller queue_cache_destroy();
18860a625fd2SDavid S. Miller n2_spu_hvapi_unregister();
18870a625fd2SDavid S. Miller }
18880a625fd2SDavid S. Miller mutex_unlock(&spu_lock);
18890a625fd2SDavid S. Miller }
18900a625fd2SDavid S. Miller
alloc_n2cp(void)189149cfe4dbSGreg Kroah-Hartman static struct n2_crypto *alloc_n2cp(void)
18920a625fd2SDavid S. Miller {
18930a625fd2SDavid S. Miller struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
18940a625fd2SDavid S. Miller
18950a625fd2SDavid S. Miller if (np)
18960a625fd2SDavid S. Miller INIT_LIST_HEAD(&np->cwq_list);
18970a625fd2SDavid S. Miller
18980a625fd2SDavid S. Miller return np;
18990a625fd2SDavid S. Miller }
19000a625fd2SDavid S. Miller
free_n2cp(struct n2_crypto * np)19010a625fd2SDavid S. Miller static void free_n2cp(struct n2_crypto *np)
19020a625fd2SDavid S. Miller {
19030a625fd2SDavid S. Miller kfree(np->cwq_info.ino_table);
19040a625fd2SDavid S. Miller np->cwq_info.ino_table = NULL;
19050a625fd2SDavid S. Miller
19060a625fd2SDavid S. Miller kfree(np);
19070a625fd2SDavid S. Miller }
19080a625fd2SDavid S. Miller
n2_spu_driver_version(void)190949cfe4dbSGreg Kroah-Hartman static void n2_spu_driver_version(void)
19100a625fd2SDavid S. Miller {
19110a625fd2SDavid S. Miller static int n2_spu_version_printed;
19120a625fd2SDavid S. Miller
19130a625fd2SDavid S. Miller if (n2_spu_version_printed++ == 0)
19140a625fd2SDavid S. Miller pr_info("%s", version);
19150a625fd2SDavid S. Miller }
19160a625fd2SDavid S. Miller
n2_crypto_probe(struct platform_device * dev)191749cfe4dbSGreg Kroah-Hartman static int n2_crypto_probe(struct platform_device *dev)
19180a625fd2SDavid S. Miller {
19190a625fd2SDavid S. Miller struct mdesc_handle *mdesc;
19200a625fd2SDavid S. Miller struct n2_crypto *np;
19210a625fd2SDavid S. Miller int err;
19220a625fd2SDavid S. Miller
19230a625fd2SDavid S. Miller n2_spu_driver_version();
19240a625fd2SDavid S. Miller
1925b2e870f2SRob Herring pr_info("Found N2CP at %pOF\n", dev->dev.of_node);
19260a625fd2SDavid S. Miller
19270a625fd2SDavid S. Miller np = alloc_n2cp();
19280a625fd2SDavid S. Miller if (!np) {
1929b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n",
1930b2e870f2SRob Herring dev->dev.of_node);
19310a625fd2SDavid S. Miller return -ENOMEM;
19320a625fd2SDavid S. Miller }
19330a625fd2SDavid S. Miller
19340a625fd2SDavid S. Miller err = grab_global_resources();
19350a625fd2SDavid S. Miller if (err) {
1936b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
1937b2e870f2SRob Herring dev->dev.of_node);
19380a625fd2SDavid S. Miller goto out_free_n2cp;
19390a625fd2SDavid S. Miller }
19400a625fd2SDavid S. Miller
19410a625fd2SDavid S. Miller mdesc = mdesc_grab();
19420a625fd2SDavid S. Miller
19430a625fd2SDavid S. Miller if (!mdesc) {
1944b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
1945b2e870f2SRob Herring dev->dev.of_node);
19460a625fd2SDavid S. Miller err = -ENODEV;
19470a625fd2SDavid S. Miller goto out_free_global;
19480a625fd2SDavid S. Miller }
19490a625fd2SDavid S. Miller err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
19500a625fd2SDavid S. Miller if (err) {
1951b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
1952b2e870f2SRob Herring dev->dev.of_node);
19530a625fd2SDavid S. Miller mdesc_release(mdesc);
19540a625fd2SDavid S. Miller goto out_free_global;
19550a625fd2SDavid S. Miller }
19560a625fd2SDavid S. Miller
19570a625fd2SDavid S. Miller err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
19580a625fd2SDavid S. Miller "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
19590a625fd2SDavid S. Miller cpu_to_cwq);
19600a625fd2SDavid S. Miller mdesc_release(mdesc);
19610a625fd2SDavid S. Miller
19620a625fd2SDavid S. Miller if (err) {
1963b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n",
1964b2e870f2SRob Herring dev->dev.of_node);
19650a625fd2SDavid S. Miller goto out_free_global;
19660a625fd2SDavid S. Miller }
19670a625fd2SDavid S. Miller
19680a625fd2SDavid S. Miller err = n2_register_algs();
19690a625fd2SDavid S. Miller if (err) {
1970b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n",
1971b2e870f2SRob Herring dev->dev.of_node);
19720a625fd2SDavid S. Miller goto out_free_spu_list;
19730a625fd2SDavid S. Miller }
19740a625fd2SDavid S. Miller
19750a625fd2SDavid S. Miller dev_set_drvdata(&dev->dev, np);
19760a625fd2SDavid S. Miller
19770a625fd2SDavid S. Miller return 0;
19780a625fd2SDavid S. Miller
19790a625fd2SDavid S. Miller out_free_spu_list:
19800a625fd2SDavid S. Miller spu_list_destroy(&np->cwq_list);
19810a625fd2SDavid S. Miller
19820a625fd2SDavid S. Miller out_free_global:
19830a625fd2SDavid S. Miller release_global_resources();
19840a625fd2SDavid S. Miller
19850a625fd2SDavid S. Miller out_free_n2cp:
19860a625fd2SDavid S. Miller free_n2cp(np);
19870a625fd2SDavid S. Miller
19880a625fd2SDavid S. Miller return err;
19890a625fd2SDavid S. Miller }
19900a625fd2SDavid S. Miller
n2_crypto_remove(struct platform_device * dev)1991a0061b93SUwe Kleine-König static void n2_crypto_remove(struct platform_device *dev)
19920a625fd2SDavid S. Miller {
19930a625fd2SDavid S. Miller struct n2_crypto *np = dev_get_drvdata(&dev->dev);
19940a625fd2SDavid S. Miller
19950a625fd2SDavid S. Miller n2_unregister_algs();
19960a625fd2SDavid S. Miller
19970a625fd2SDavid S. Miller spu_list_destroy(&np->cwq_list);
19980a625fd2SDavid S. Miller
19990a625fd2SDavid S. Miller release_global_resources();
20000a625fd2SDavid S. Miller
20010a625fd2SDavid S. Miller free_n2cp(np);
20020a625fd2SDavid S. Miller }
20030a625fd2SDavid S. Miller
alloc_ncp(void)200449cfe4dbSGreg Kroah-Hartman static struct n2_mau *alloc_ncp(void)
20050a625fd2SDavid S. Miller {
20060a625fd2SDavid S. Miller struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
20070a625fd2SDavid S. Miller
20080a625fd2SDavid S. Miller if (mp)
20090a625fd2SDavid S. Miller INIT_LIST_HEAD(&mp->mau_list);
20100a625fd2SDavid S. Miller
20110a625fd2SDavid S. Miller return mp;
20120a625fd2SDavid S. Miller }
20130a625fd2SDavid S. Miller
free_ncp(struct n2_mau * mp)20140a625fd2SDavid S. Miller static void free_ncp(struct n2_mau *mp)
20150a625fd2SDavid S. Miller {
20160a625fd2SDavid S. Miller kfree(mp->mau_info.ino_table);
20170a625fd2SDavid S. Miller mp->mau_info.ino_table = NULL;
20180a625fd2SDavid S. Miller
20190a625fd2SDavid S. Miller kfree(mp);
20200a625fd2SDavid S. Miller }
20210a625fd2SDavid S. Miller
n2_mau_probe(struct platform_device * dev)202249cfe4dbSGreg Kroah-Hartman static int n2_mau_probe(struct platform_device *dev)
20230a625fd2SDavid S. Miller {
20240a625fd2SDavid S. Miller struct mdesc_handle *mdesc;
20250a625fd2SDavid S. Miller struct n2_mau *mp;
20260a625fd2SDavid S. Miller int err;
20270a625fd2SDavid S. Miller
20280a625fd2SDavid S. Miller n2_spu_driver_version();
20290a625fd2SDavid S. Miller
2030b2e870f2SRob Herring pr_info("Found NCP at %pOF\n", dev->dev.of_node);
20310a625fd2SDavid S. Miller
20320a625fd2SDavid S. Miller mp = alloc_ncp();
20330a625fd2SDavid S. Miller if (!mp) {
2034b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n",
2035b2e870f2SRob Herring dev->dev.of_node);
20360a625fd2SDavid S. Miller return -ENOMEM;
20370a625fd2SDavid S. Miller }
20380a625fd2SDavid S. Miller
20390a625fd2SDavid S. Miller err = grab_global_resources();
20400a625fd2SDavid S. Miller if (err) {
2041b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
2042b2e870f2SRob Herring dev->dev.of_node);
20430a625fd2SDavid S. Miller goto out_free_ncp;
20440a625fd2SDavid S. Miller }
20450a625fd2SDavid S. Miller
20460a625fd2SDavid S. Miller mdesc = mdesc_grab();
20470a625fd2SDavid S. Miller
20480a625fd2SDavid S. Miller if (!mdesc) {
2049b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
2050b2e870f2SRob Herring dev->dev.of_node);
20510a625fd2SDavid S. Miller err = -ENODEV;
20520a625fd2SDavid S. Miller goto out_free_global;
20530a625fd2SDavid S. Miller }
20540a625fd2SDavid S. Miller
20550a625fd2SDavid S. Miller err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
20560a625fd2SDavid S. Miller if (err) {
2057b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
2058b2e870f2SRob Herring dev->dev.of_node);
20590a625fd2SDavid S. Miller mdesc_release(mdesc);
20600a625fd2SDavid S. Miller goto out_free_global;
20610a625fd2SDavid S. Miller }
20620a625fd2SDavid S. Miller
20630a625fd2SDavid S. Miller err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
20640a625fd2SDavid S. Miller "mau", HV_NCS_QTYPE_MAU, mau_intr,
20650a625fd2SDavid S. Miller cpu_to_mau);
20660a625fd2SDavid S. Miller mdesc_release(mdesc);
20670a625fd2SDavid S. Miller
20680a625fd2SDavid S. Miller if (err) {
2069b2e870f2SRob Herring dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n",
2070b2e870f2SRob Herring dev->dev.of_node);
20710a625fd2SDavid S. Miller goto out_free_global;
20720a625fd2SDavid S. Miller }
20730a625fd2SDavid S. Miller
20740a625fd2SDavid S. Miller dev_set_drvdata(&dev->dev, mp);
20750a625fd2SDavid S. Miller
20760a625fd2SDavid S. Miller return 0;
20770a625fd2SDavid S. Miller
20780a625fd2SDavid S. Miller out_free_global:
20790a625fd2SDavid S. Miller release_global_resources();
20800a625fd2SDavid S. Miller
20810a625fd2SDavid S. Miller out_free_ncp:
20820a625fd2SDavid S. Miller free_ncp(mp);
20830a625fd2SDavid S. Miller
20840a625fd2SDavid S. Miller return err;
20850a625fd2SDavid S. Miller }
20860a625fd2SDavid S. Miller
n2_mau_remove(struct platform_device * dev)2087a0061b93SUwe Kleine-König static void n2_mau_remove(struct platform_device *dev)
20880a625fd2SDavid S. Miller {
20890a625fd2SDavid S. Miller struct n2_mau *mp = dev_get_drvdata(&dev->dev);
20900a625fd2SDavid S. Miller
20910a625fd2SDavid S. Miller spu_list_destroy(&mp->mau_list);
20920a625fd2SDavid S. Miller
20930a625fd2SDavid S. Miller release_global_resources();
20940a625fd2SDavid S. Miller
20950a625fd2SDavid S. Miller free_ncp(mp);
20960a625fd2SDavid S. Miller }
20970a625fd2SDavid S. Miller
20984914b90bSArvind Yadav static const struct of_device_id n2_crypto_match[] = {
20990a625fd2SDavid S. Miller {
21000a625fd2SDavid S. Miller .name = "n2cp",
21010a625fd2SDavid S. Miller .compatible = "SUNW,n2-cwq",
21020a625fd2SDavid S. Miller },
21030a625fd2SDavid S. Miller {
21040a625fd2SDavid S. Miller .name = "n2cp",
21050a625fd2SDavid S. Miller .compatible = "SUNW,vf-cwq",
21060a625fd2SDavid S. Miller },
2107eb7caf35SDavid S. Miller {
2108eb7caf35SDavid S. Miller .name = "n2cp",
2109eb7caf35SDavid S. Miller .compatible = "SUNW,kt-cwq",
2110eb7caf35SDavid S. Miller },
21110a625fd2SDavid S. Miller {},
21120a625fd2SDavid S. Miller };
21130a625fd2SDavid S. Miller
21140a625fd2SDavid S. Miller MODULE_DEVICE_TABLE(of, n2_crypto_match);
21150a625fd2SDavid S. Miller
21164ebb24f7SGrant Likely static struct platform_driver n2_crypto_driver = {
2117ff6c7341SDavid S. Miller .driver = {
21180a625fd2SDavid S. Miller .name = "n2cp",
2119ff6c7341SDavid S. Miller .of_match_table = n2_crypto_match,
2120ff6c7341SDavid S. Miller },
21210a625fd2SDavid S. Miller .probe = n2_crypto_probe,
2122a0061b93SUwe Kleine-König .remove_new = n2_crypto_remove,
21230a625fd2SDavid S. Miller };
21240a625fd2SDavid S. Miller
21254914b90bSArvind Yadav static const struct of_device_id n2_mau_match[] = {
21260a625fd2SDavid S. Miller {
21270a625fd2SDavid S. Miller .name = "ncp",
21280a625fd2SDavid S. Miller .compatible = "SUNW,n2-mau",
21290a625fd2SDavid S. Miller },
21300a625fd2SDavid S. Miller {
21310a625fd2SDavid S. Miller .name = "ncp",
21320a625fd2SDavid S. Miller .compatible = "SUNW,vf-mau",
21330a625fd2SDavid S. Miller },
2134eb7caf35SDavid S. Miller {
2135eb7caf35SDavid S. Miller .name = "ncp",
2136eb7caf35SDavid S. Miller .compatible = "SUNW,kt-mau",
2137eb7caf35SDavid S. Miller },
21380a625fd2SDavid S. Miller {},
21390a625fd2SDavid S. Miller };
21400a625fd2SDavid S. Miller
21410a625fd2SDavid S. Miller MODULE_DEVICE_TABLE(of, n2_mau_match);
21420a625fd2SDavid S. Miller
21434ebb24f7SGrant Likely static struct platform_driver n2_mau_driver = {
2144ff6c7341SDavid S. Miller .driver = {
21450a625fd2SDavid S. Miller .name = "ncp",
2146ff6c7341SDavid S. Miller .of_match_table = n2_mau_match,
2147ff6c7341SDavid S. Miller },
21480a625fd2SDavid S. Miller .probe = n2_mau_probe,
2149a0061b93SUwe Kleine-König .remove_new = n2_mau_remove,
21500a625fd2SDavid S. Miller };
21510a625fd2SDavid S. Miller
2152a103a75aSThierry Reding static struct platform_driver * const drivers[] = {
2153a103a75aSThierry Reding &n2_crypto_driver,
2154a103a75aSThierry Reding &n2_mau_driver,
2155a103a75aSThierry Reding };
2156a103a75aSThierry Reding
n2_init(void)21570a625fd2SDavid S. Miller static int __init n2_init(void)
21580a625fd2SDavid S. Miller {
2159a103a75aSThierry Reding return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
21600a625fd2SDavid S. Miller }
21610a625fd2SDavid S. Miller
n2_exit(void)21620a625fd2SDavid S. Miller static void __exit n2_exit(void)
21630a625fd2SDavid S. Miller {
2164a103a75aSThierry Reding platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
21650a625fd2SDavid S. Miller }
21660a625fd2SDavid S. Miller
21670a625fd2SDavid S. Miller module_init(n2_init);
21680a625fd2SDavid S. Miller module_exit(n2_exit);
2169